From 4210a61acbdfd197bf00a48728cd575fb1e0dcda Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 15 Feb 2024 17:08:59 -0500 Subject: [PATCH 001/904] misc updates for the pam_debug routines --- .../eam/src/physics/crm/pam/pam_debug.h | 117 ++++++++++++------ 1 file changed, 82 insertions(+), 35 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_debug.h b/components/eam/src/physics/crm/pam/pam_debug.h index 5186d51c6a9d..f4d8cdb1eda2 100644 --- a/components/eam/src/physics/crm/pam/pam_debug.h +++ b/components/eam/src/physics/crm/pam/pam_debug.h @@ -16,29 +16,33 @@ inline void pam_debug_init( pam::PamCoupler &coupler ) { auto nens = coupler.get_option("ncrms"); //------------------------------------------------------------------------------------------------ dm_device.register_and_allocate("debug_save_temp", "saved temp for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); - dm_device.register_and_allocate("debug_save_rhod", "saved rhod for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); dm_device.register_and_allocate("debug_save_rhov", "saved rhov for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); dm_device.register_and_allocate("debug_save_rhoc", "saved rhoc for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); dm_device.register_and_allocate("debug_save_rhoi", "saved rhoi for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); + dm_device.register_and_allocate("debug_save_uvel", "saved uvel for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); + dm_device.register_and_allocate("debug_save_wvel", "saved wvel for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); auto debug_save_temp = dm_device.get("debug_save_temp"); - auto debug_save_rhod = dm_device.get("debug_save_rhod"); auto debug_save_rhov = dm_device.get("debug_save_rhov"); auto debug_save_rhoc = dm_device.get("debug_save_rhoc"); auto debug_save_rhoi = dm_device.get("debug_save_rhoi"); + auto debug_save_uvel = dm_device.get("debug_save_uvel"); + auto debug_save_wvel = dm_device.get("debug_save_wvel"); //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp"); - auto rhod = dm_device.get("density_dry"); auto rhov = dm_device.get("water_vapor"); auto rhoc = dm_device.get("cloud_water"); auto rhoi = dm_device.get("ice"); + auto uvel = dm_device.get("uvel"); + auto wvel = dm_device.get("wvel"); //------------------------------------------------------------------------------------------------ parallel_for("copy data to saved debug variables", SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { debug_save_temp(k,j,i,iens) = temp(k,j,i,iens); - debug_save_rhod(k,j,i,iens) = rhod(k,j,i,iens); debug_save_rhov(k,j,i,iens) = rhov(k,j,i,iens); debug_save_rhoc(k,j,i,iens) = rhoc(k,j,i,iens); debug_save_rhoi(k,j,i,iens) = rhoi(k,j,i,iens); + debug_save_uvel(k,j,i,iens) = uvel(k,j,i,iens); + debug_save_wvel(k,j,i,iens) = wvel(k,j,i,iens); }); //------------------------------------------------------------------------------------------------ } @@ -48,81 +52,124 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { using yakl::c::parallel_for; using yakl::c::SimpleBounds; auto &dm_device = coupler.get_data_manager_device_readwrite(); - auto &dm_host = coupler.get_data_manager_host_readwrite(); + auto &dm_host = coupler.get_data_manager_host_readonly(); auto nx = coupler.get_option("crm_nx"); auto ny = coupler.get_option("crm_ny"); auto nz = coupler.get_option("crm_nz"); auto nens = coupler.get_option("ncrms"); - auto temp = dm_device.get("temp"); - auto rhod = dm_device.get("density_dry"); - auto rhov = dm_device.get("water_vapor"); - auto rhoc = dm_device.get("cloud_water"); - auto rhoi = dm_device.get("ice"); + auto temp = dm_device.get("temp"); + auto rhov = dm_device.get("water_vapor"); + auto rhoc = dm_device.get("cloud_water"); + auto rhoi = dm_device.get("ice"); + auto uvel = dm_device.get("uvel"); + auto wvel = dm_device.get("wvel"); auto debug_save_temp = dm_device.get("debug_save_temp"); - auto debug_save_rhod = dm_device.get("debug_save_rhod"); auto debug_save_rhov = dm_device.get("debug_save_rhov"); auto debug_save_rhoc = dm_device.get("debug_save_rhoc"); auto debug_save_rhoi = dm_device.get("debug_save_rhoi"); - real grav = 9.80616; - auto input_phis = dm_host.get("input_phis").createDeviceCopy(); + auto debug_save_uvel = dm_device.get("debug_save_uvel"); + auto debug_save_wvel = dm_device.get("debug_save_wvel"); auto lat = dm_host.get("latitude" ).createDeviceCopy(); auto lon = dm_host.get("longitude" ).createDeviceCopy(); + auto input_phis = dm_host.get("input_phis").createDeviceCopy(); + real grav = 9.80616; //------------------------------------------------------------------------------------------------ // Check for invalid values parallel_for("", SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { + auto phis = input_phis(iens)/grav; // Check for NaNs const auto is_nan_t_atm = isnan( temp(k,j,i,iens) ); - const auto is_nan_r_atm = isnan( rhod(k,j,i,iens) ); const auto is_nan_q_atm = isnan( rhov(k,j,i,iens) ); - if ( is_nan_t_atm || is_nan_r_atm || is_nan_q_atm ) { - auto phis = input_phis(iens)/grav; - printf("PAM-DEBUG nan-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g \n", + if ( is_nan_t_atm || is_nan_q_atm ) { + printf("PAM-DEBUG nan-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", nstep,id,k,i,iens,lat(iens),lon(iens),phis, temp(k,j,i,iens), - rhod(k,j,i,iens), rhov(k,j,i,iens), rhoc(k,j,i,iens), rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), debug_save_temp(k,j,i,iens), - debug_save_rhod(k,j,i,iens), debug_save_rhov(k,j,i,iens), debug_save_rhoc(k,j,i,iens), - debug_save_rhoi(k,j,i,iens) + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) ); } // Check for negative values const auto is_neg_t_atm = temp(k,j,i,iens)<0; - const auto is_neg_r_atm = rhod(k,j,i,iens)<0; const auto is_neg_q_atm = rhov(k,j,i,iens)<0; - if ( is_neg_t_atm || is_neg_r_atm || is_neg_q_atm ) { - auto phis = input_phis(iens)/grav; - printf("PAM-DEBUG neg-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g \n", + if ( is_neg_t_atm || is_neg_q_atm ) { + printf("PAM-DEBUG neg-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", nstep,id,k,i,iens,lat(iens),lon(iens),phis, temp(k,j,i,iens), - rhod(k,j,i,iens), rhov(k,j,i,iens), rhoc(k,j,i,iens), rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), debug_save_temp(k,j,i,iens), - debug_save_rhod(k,j,i,iens), debug_save_rhov(k,j,i,iens), debug_save_rhoc(k,j,i,iens), - debug_save_rhoi(k,j,i,iens) + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) + ); + } + // Check for low temperature + const auto is_low_t = temp(k,j,i,iens)<100; + if ( is_low_t ) { + printf("PAM-DEBUG low-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + nstep,id,k,i,iens,lat(iens),lon(iens),phis, + temp(k,j,i,iens), + rhov(k,j,i,iens), + rhoc(k,j,i,iens), + rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), + debug_save_temp(k,j,i,iens), + debug_save_rhov(k,j,i,iens), + debug_save_rhoc(k,j,i,iens), + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) + ); + } + // Check for large vertical velocity + const auto is_large_pos_w = wvel(k,j,i,iens)> 40; + const auto is_large_neg_w = wvel(k,j,i,iens)<-40; + if ( is_large_pos_w || is_large_neg_w ) { + printf("PAM-DEBUG large-W - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + nstep,id,k,i,iens,lat(iens),lon(iens),phis, + temp(k,j,i,iens), + rhov(k,j,i,iens), + rhoc(k,j,i,iens), + rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), + debug_save_temp(k,j,i,iens), + debug_save_rhov(k,j,i,iens), + debug_save_rhoc(k,j,i,iens), + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) ); } // update saved previous values debug_save_temp(k,j,i,iens) = temp(k,j,i,iens); - debug_save_rhod(k,j,i,iens) = rhod(k,j,i,iens); debug_save_rhov(k,j,i,iens) = rhov(k,j,i,iens); debug_save_rhoc(k,j,i,iens) = rhoc(k,j,i,iens); debug_save_rhoi(k,j,i,iens) = rhoi(k,j,i,iens); + debug_save_uvel(k,j,i,iens) = uvel(k,j,i,iens); + debug_save_wvel(k,j,i,iens) = wvel(k,j,i,iens); }); //------------------------------------------------------------------------------------------------ } // print the min and max of PAM state variables to help look for problems // void print_state_min_max( pam::PamCoupler &coupler, std::string id ) { - // auto &dm_device = coupler.get_data_manager_device_readwrite(); + // auto &dm_device = coupler.get_data_manager_device_readonly(); // auto nz = coupler.get_option("crm_nz"); // auto nx = coupler.get_option("crm_nx"); // auto ny = coupler.get_option("crm_ny"); @@ -184,18 +231,18 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { void pam_debug_print_state( pam::PamCoupler &coupler, int id ) { using yakl::c::parallel_for; using yakl::c::SimpleBounds; - auto &dm_device = coupler.get_data_manager_device_readwrite(); + auto &dm_device = coupler.get_data_manager_device_readonly(); auto nz = coupler.get_option("crm_nz"); auto nx = coupler.get_option("crm_nx"); auto ny = coupler.get_option("crm_ny"); auto nens = coupler.get_option("ncrms"); // auto pmid = coupler.compute_pressure_array(); // auto zmid = dm_device.get("vertical_midpoint_height" ); - auto temp = dm_device.get("temp"); - auto rho_v = dm_device.get("water_vapor"); - auto rho_c = dm_device.get("cloud_water"); - auto rho_d = dm_device.get("density_dry"); - auto rho_i = dm_device.get("ice"); + auto temp = dm_device.get("temp"); + auto rho_v = dm_device.get("water_vapor"); + auto rho_c = dm_device.get("cloud_water"); + auto rho_d = dm_device.get("density_dry"); + auto rho_i = dm_device.get("ice"); // parallel_for("", SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { parallel_for("pam_debug_print_state", SimpleBounds<2>(nz,nx), YAKL_LAMBDA (int k, int i) { int j = 0; From 0def774fe8d97cc5a890f48604062dcabea96635 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 15 Feb 2024 17:19:56 -0500 Subject: [PATCH 002/904] add MMF_PAM_dyn_per_phys namelist parameter --- components/eam/bld/build-namelist | 3 +++ .../eam/bld/namelist_files/namelist_defaults_eam.xml | 2 +- components/eam/bld/namelist_files/namelist_definition.xml | 6 ++++++ components/eam/src/physics/cam/phys_control.F90 | 8 ++++++-- components/eam/src/physics/crm/crm_physics.F90 | 7 +++++++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index 5a29d02bba48..c94bb5054d57 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -3885,6 +3885,9 @@ if ($cfg->get('use_MMF')) { # MMF CRM domain orientation add_default($nl, 'MMF_orientation_angle'); + + # PAM dynamics sub-stepping parameter + add_default($nl, 'MMF_PAM_dyn_per_phys'); } add_default($nl, 'do_aerocom_ind3'); diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index c5a61b500d21..4768f1799a4e 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -879,7 +879,7 @@ 0 .false. .true. - + 2 90.0 0.0 diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index fb5049df2110..dc63a488a6b6 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -4769,6 +4769,12 @@ Defaults: 3D CRM: 0 + +Number of PAM dycor calls per physics time step. +Default: 2 + + Wavenumber cutoff for filtered MMF variance transport. diff --git a/components/eam/src/physics/cam/phys_control.F90 b/components/eam/src/physics/cam/phys_control.F90 index c4d4b7fba7fa..41cd05e2714e 100644 --- a/components/eam/src/physics/cam/phys_control.F90 +++ b/components/eam/src/physics/cam/phys_control.F90 @@ -72,6 +72,7 @@ module phys_control logical :: use_crm_accel = .false. ! true => use MMF CRM mean-state acceleration (MSA) real(r8) :: crm_accel_factor = 2.D0 ! CRM acceleration factor logical :: crm_accel_uv = .true. ! true => apply MMF CRM MSA to momentum fields +integer :: MMF_PAM_dyn_per_phys = 2 ! PAM CRM dynamics steps per CRM physics steps logical :: use_subcol_microp = .false. ! if .true. then use sub-columns in microphysics @@ -229,7 +230,7 @@ subroutine phys_ctl_readnl(nlfile) eddy_scheme, microp_scheme, macrop_scheme, radiation_scheme, srf_flux_avg, & MMF_microphysics_scheme, MMF_orientation_angle, use_MMF, use_ECPP, & use_MMF_VT, MMF_VT_wn_max, use_MMF_ESMT, & - use_crm_accel, crm_accel_factor, crm_accel_uv, & + use_crm_accel, crm_accel_factor, crm_accel_uv, MMF_PAM_dyn_per_phys, & use_subcol_microp, atm_dep_flux, history_amwg, history_verbose, history_vdiag, & get_presc_aero_data,history_aerosol, history_aero_optics, & is_output_interactive_volc, & @@ -314,6 +315,7 @@ subroutine phys_ctl_readnl(nlfile) call mpibcast(use_crm_accel, 1 , mpilog, 0, mpicom) call mpibcast(crm_accel_factor, 1 , mpir8, 0, mpicom) call mpibcast(crm_accel_uv, 1 , mpilog, 0, mpicom) + call mpibcast(MMF_PAM_dyn_per_phys, 1 , mpiint, 0, mpicom) call mpibcast(use_subcol_microp, 1 , mpilog, 0, mpicom) call mpibcast(atm_dep_flux, 1 , mpilog, 0, mpicom) call mpibcast(history_amwg, 1 , mpilog, 0, mpicom) @@ -598,7 +600,7 @@ subroutine phys_getopts(deep_scheme_out, shallow_scheme_out, eddy_scheme_out, & prog_modal_aero_out, macrop_scheme_out, ideal_phys_option_out, & use_MMF_out, use_ECPP_out, MMF_microphysics_scheme_out, & MMF_orientation_angle_out, use_MMF_VT_out, MMF_VT_wn_max_out, use_MMF_ESMT_out, & - use_crm_accel_out, crm_accel_factor_out, crm_accel_uv_out, & + use_crm_accel_out, crm_accel_factor_out, crm_accel_uv_out, MMF_PAM_dyn_per_phys_out, & do_clubb_sgs_out, do_shoc_sgs_out, do_tms_out, state_debug_checks_out, & linearize_pbl_winds_out, export_gustiness_out, & do_aerocom_ind3_out, & @@ -641,6 +643,7 @@ subroutine phys_getopts(deep_scheme_out, shallow_scheme_out, eddy_scheme_out, & logical, intent(out), optional :: use_crm_accel_out real(r8), intent(out), optional :: crm_accel_factor_out logical, intent(out), optional :: crm_accel_uv_out + integer, intent(out), optional :: MMF_PAM_dyn_per_phys_out logical, intent(out), optional :: use_subcol_microp_out logical, intent(out), optional :: atm_dep_flux_out logical, intent(out), optional :: history_amwg_out @@ -745,6 +748,7 @@ subroutine phys_getopts(deep_scheme_out, shallow_scheme_out, eddy_scheme_out, & if ( present(use_crm_accel_out ) ) use_crm_accel_out = use_crm_accel if ( present(crm_accel_factor_out ) ) crm_accel_factor_out = crm_accel_factor if ( present(crm_accel_uv_out ) ) crm_accel_uv_out = crm_accel_uv + if ( present(MMF_PAM_dyn_per_phys ) ) MMF_PAM_dyn_per_phys_out = MMF_PAM_dyn_per_phys if ( present(use_subcol_microp_out ) ) use_subcol_microp_out = use_subcol_microp if ( present(macrop_scheme_out ) ) macrop_scheme_out = macrop_scheme diff --git a/components/eam/src/physics/crm/crm_physics.F90 b/components/eam/src/physics/crm/crm_physics.F90 index 66e5b57e28eb..31654e2bb7ac 100644 --- a/components/eam/src/physics/crm/crm_physics.F90 +++ b/components/eam/src/physics/crm/crm_physics.F90 @@ -620,6 +620,8 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, logical :: use_MMF_VT_tmp ! flag for MMF variance transport (for Fortran CRM) logical :: use_MMF_ESMT_tmp ! flag for MMF scalar momentum transport (for Fortran CRM) integer :: MMF_VT_wn_max ! wavenumber cutoff for filtered variance transport + + integer :: MMF_PAM_dyn_per_phys ! PAM CRM dynamics steps per CRM physics steps real(r8) :: tmp_e_sat ! temporary saturation vapor pressure real(r8) :: tmp_q_sat ! temporary saturation specific humidity @@ -732,6 +734,9 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, use_crm_accel = use_crm_accel_tmp crm_accel_uv = crm_accel_uv_tmp + ! PAM namelist options + call phys_getopts(MMF_PAM_dyn_per_phys_out = MMF_PAM_dyn_per_phys) + nstep = get_nstep() itim = pbuf_old_tim_idx() ! "Old" pbuf time index (what does all this mean?) @@ -1459,6 +1464,8 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, call pam_set_option('enable_physics_tend_stats', .false. ) + call pam_set_option('crm_dyn_per_phys', MMF_PAM_dyn_per_phys ) + call pam_set_option('is_first_step', (nstep<=1) ) call pam_set_option('is_restart', (nsrest>0) ) call pam_set_option('am_i_root', masterproc ) From 9a25fc689c29fcbcf6725f09859fcb91c3c46a2d Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 15 Feb 2024 17:58:22 -0500 Subject: [PATCH 003/904] pam driver updates --- .../eam/src/physics/crm/pam/pam_driver.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index f131bbd44f09..128974a856e4 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -24,7 +24,7 @@ #include "p3_f90.hpp" #include "pam_debug.h" -bool constexpr enable_check_state = false; +bool constexpr enable_check_state = true; extern "C" void pam_driver() { //------------------------------------------------------------------------------------------------ @@ -50,12 +50,6 @@ extern "C" void pam_driver() { //------------------------------------------------------------------------------------------------ // set various coupler options coupler.set_option("gcm_physics_dt",gcm_dt); - #ifdef MMF_PAM_DPP - // this is leftover from debugging, but it might still be useful for testing values of crm_per_phys - coupler.set_option("crm_per_phys",MMF_PAM_DPP); - #else - coupler.set_option("crm_per_phys",2); // # of PAM-C dynamics steps per physics - #endif coupler.set_option("sponge_num_layers",crm_nz*0.3); // depth of sponge layer coupler.set_option("sponge_time_scale",60); // minimum damping timescale at top coupler.set_option("crm_acceleration_ceaseflag",false); @@ -143,6 +137,9 @@ extern "C" void pam_driver() { dycore.declare_current_profile_as_hydrostatic(coupler,/*use_gcm_data=*/true); #endif + #ifdef MMF_DISABLE_DENSITY_RECALL + do_density_save_recall = false; + #endif //------------------------------------------------------------------------------------------------ //------------------------------------------------------------------------------------------------ //------------------------------------------------------------------------------------------------ @@ -168,8 +165,9 @@ extern "C" void pam_driver() { // run a PAM time step coupler.run_module( "apply_gcm_forcing_tendencies" , modules::apply_gcm_forcing_tendencies ); - coupler.run_module( "radiation" , [&] (pam::PamCoupler &coupler) {rad .timeStep(coupler);} ); if (enable_check_state) { pam_debug_check_state(coupler, 2, nstep); } + coupler.run_module( "radiation" , [&] (pam::PamCoupler &coupler) {rad .timeStep(coupler);} ); + if (enable_check_state) { pam_debug_check_state(coupler, 3, nstep); } // Dynamics if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } @@ -177,13 +175,13 @@ extern "C" void pam_driver() { coupler.run_module( "dycore", [&] (pam::PamCoupler &coupler) {dycore.timeStep(coupler);} ); if (do_density_save_recall) { pam_state_recall_dry_density(coupler); } if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"dycor"); } - if (enable_check_state) { pam_debug_check_state(coupler, 3, nstep); } + if (enable_check_state) { pam_debug_check_state(coupler, 4, nstep); } // Sponge layer damping if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } coupler.run_module( "sponge_layer", modules::sponge_layer ); if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"sponge"); } - if (enable_check_state) { pam_debug_check_state(coupler, 4, nstep); } + // if (enable_check_state) { pam_debug_check_state(coupler, 4, nstep); } // Apply hyperdiffusion to account for lack of horizontal mixing in SHOC pam_hyperdiffusion(coupler); From 0e93989e805d8ae7ecd59d2ffb955d0c10bcba30 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 15 Feb 2024 17:59:05 -0500 Subject: [PATCH 004/904] add temporary MMF_PAM_HDT for debugging --- components/eam/src/physics/crm/pam/pam_hyperdiffusion.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h b/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h index 83dc83252c68..4452c583afe3 100644 --- a/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h +++ b/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h @@ -25,7 +25,11 @@ inline void pam_hyperdiffusion( pam::PamCoupler &coupler ) { auto uvel = dm_device.get("uvel" ); auto vvel = dm_device.get("vvel" ); //------------------------------------------------------------------------------------------------ + #ifdef MMF_PAM_HDT + real constexpr hd_timescale = MMF_PAM_HDT; // damping time scale [sec] + #else real constexpr hd_timescale = 10.0; // damping time scale [sec] + #endif //------------------------------------------------------------------------------------------------ real4d hd_temp("hd_temp",nz,ny,nx,nens); real4d hd_rhod("hd_rhod",nz,ny,nx,nens); From 047f01af0a3025d1b64d63afce65a90df2bededb Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 15 Feb 2024 17:59:38 -0500 Subject: [PATCH 005/904] Add temporary MMF_ALT_DENSITY_RECALL for debugging --- .../eam/src/physics/crm/pam/pam_state.h | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_state.h b/components/eam/src/physics/crm/pam/pam_state.h index e8f6e4ab668a..34cc42a4a788 100644 --- a/components/eam/src/physics/crm/pam/pam_state.h +++ b/components/eam/src/physics/crm/pam/pam_state.h @@ -261,6 +261,7 @@ inline void pam_state_save_dry_density( pam::PamCoupler &coupler ) { inline void pam_state_recall_dry_density( pam::PamCoupler &coupler ) { using yakl::c::parallel_for; using yakl::c::SimpleBounds; + using yakl::atomicAdd; auto &dm_device = coupler.get_data_manager_device_readwrite(); auto nens = coupler.get_option("ncrms"); auto nz = coupler.get_option("crm_nz"); @@ -269,11 +270,29 @@ inline void pam_state_recall_dry_density( pam::PamCoupler &coupler ) { auto crm_rho_d = dm_device.get("density_dry"); auto tmp_rho_d = dm_device.get("density_dry_save"); //------------------------------------------------------------------------------------------------ - parallel_for( "Recall CRM dry density", - SimpleBounds<4>(nz,ny,nx,nens), - YAKL_LAMBDA (int k_crm, int j, int i, int iens) { - crm_rho_d(k_crm,j,i,iens) = tmp_rho_d(k_crm,j,i,iens); - }); + #ifdef MMF_ALT_DENSITY_RECALL + // initialize horizontal mean of previous CRM dry density + real2d old_hmean_rho_d("old_hmean_rho_d" ,nz,nens); + real2d new_hmean_rho_d("new_hmean_rho_d" ,nz,nens); + parallel_for(SimpleBounds<2>(nz,nens), YAKL_LAMBDA (int k, int iens) { + old_hmean_rho_d(k,iens) = 0; + new_hmean_rho_d(k,iens) = 0; + }); + real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions + // calculate horizontal mean of previous CRM dry density + parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { + atomicAdd( old_hmean_rho_d(k,iens), tmp_rho_d(k,j,i,iens) * r_nx_ny ); + atomicAdd( new_hmean_rho_d(k,iens), crm_rho_d(k,j,i,iens) * r_nx_ny ); + }); + // replace horizontal mean dry density with prevoius value + parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { + crm_rho_d(k,j,i,iens) = crm_rho_d(k,j,i,iens) - new_hmean_rho_d(k,iens) + old_hmean_rho_d(k,iens); + }); + #else + parallel_for( "Recall CRM dry density",SimpleBounds<4>(nz,ny,nx,nens),YAKL_LAMBDA (int k_crm, int j, int i, int iens) { + crm_rho_d(k_crm,j,i,iens) = tmp_rho_d(k_crm,j,i,iens); + }); + #endif //------------------------------------------------------------------------------------------------ } From ecc0b523ccd5d6ccdea50dc7574b9e5b6c4893c5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 19 Feb 2024 19:48:50 -0800 Subject: [PATCH 006/904] Minimal set of additions for drydep interface --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- ...x_mam_dry_deposition_process_interface.cpp | 93 +++++++++++++++++++ ...x_mam_dry_deposition_process_interface.hpp | 64 +++++++++++++ .../eamxx/src/physics/register_physics.hpp | 2 + 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index a4e095bf5f2a..fc48acce3f23 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(mam eamxx_mam_microphysics_process_interface.cpp - eamxx_mam_optics_process_interface.cpp) + eamxx_mam_optics_process_interface.cpp + eamxx_mam_dry_deposition_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx_proj) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp new file mode 100644 index 000000000000..7a8adf5d087f --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -0,0 +1,93 @@ +#include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" + +/* +Future work: +Wirte comments +write in/outs for all variables clearly +*/ + +namespace scream { + +// ========================================================================================= +MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ========================================================================================= +void MAMDryDep::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + + // Layout for 2D (2d horiz) variable defined at mid-level and + // interfaces + const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + + +} + +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMDryDep::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMDryDep."); +} + +// ========================================================================================= +void MAMDryDep::initialize_impl(const RunType run_type) { + // Gather runtime options + //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + +} + +// ========================================================================================= +void MAMDryDep::run_impl(const double dt) { + + std::cout << "End of derydep run" << std::endl; +} + +// ========================================================================================= +} // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp new file mode 100644 index 000000000000..7692205dbafb --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -0,0 +1,64 @@ +#ifndef EAMXX_MAM_DRYDEP_HPP +#define EAMXX_MAM_DRYDEP_HPP + +#include + +// For declaring wetscav class derived from atm process class +#include + +// For MAM4 aerosol configuration +#include + +// For component name +#include + +namespace scream { + +// The process responsible for handling MAM4 dry deposition. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMDryDep final : public scream::AtmosphereProcess { + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + public: + // Constructor + MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_dry_deposition"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMDryDep + +} // namespace scream + +#endif // EAMXX_MAM_DRYDEP_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 329b7022756c..decab942c849 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -26,6 +26,7 @@ #ifdef EAMXX_HAS_MAM #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" +#include "physics/mam/eamxx_mam_dry_deposition_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -62,6 +63,7 @@ inline void register_physics () { #ifdef EAMXX_HAS_MAM proc_factory.register_product("mam4_micro",&create_atmosphere_process); proc_factory.register_product("mam4_optics",&create_atmosphere_process); + proc_factory.register_product("mam4_drydep",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 0f58dcaae2cd35beec2dc0533f2ca130d625dbd1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 20 Feb 2024 11:35:00 -0800 Subject: [PATCH 007/904] Adds a simple minimal test and it works --- .../eamxx_mam_dry_deposition_process_interface.cpp | 11 ++++------- .../eamxx_mam_dry_deposition_process_interface.hpp | 2 -- components/eamxx/src/physics/register_physics.hpp | 2 +- components/eamxx/tests/uncoupled/CMakeLists.txt | 2 +- .../tests/uncoupled/mam4/{ => drydep}/CMakeLists.txt | 5 +---- .../tests/uncoupled/mam4/{ => drydep}/input.yaml | 10 +--------- .../tests/uncoupled/mam4/{ => drydep}/output.yaml | 6 +----- 7 files changed, 9 insertions(+), 29 deletions(-) rename components/eamxx/tests/uncoupled/mam4/{ => drydep}/CMakeLists.txt (80%) rename components/eamxx/tests/uncoupled/mam4/{ => drydep}/input.yaml (63%) rename components/eamxx/tests/uncoupled/mam4/{ => drydep}/output.yaml (59%) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 7a8adf5d087f..f27e5d6a1ee4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -29,7 +29,7 @@ void MAMDryDep::set_grids( auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -49,8 +49,8 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - - + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] } // ========================================================================================= @@ -79,15 +79,12 @@ void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { void MAMDryDep::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - - } // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - std::cout << "End of derydep run" << std::endl; } // ========================================================================================= -} // namespace scream \ No newline at end of file +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 7692205dbafb..47be070c4228 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -1,8 +1,6 @@ #ifndef EAMXX_MAM_DRYDEP_HPP #define EAMXX_MAM_DRYDEP_HPP -#include - // For declaring wetscav class derived from atm process class #include diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index decab942c849..bfb3428b7afc 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -26,7 +26,7 @@ #ifdef EAMXX_HAS_MAM #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" -#include "physics/mam/eamxx_mam_dry_deposition_interface.hpp" +#include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" diff --git a/components/eamxx/tests/uncoupled/CMakeLists.txt b/components/eamxx/tests/uncoupled/CMakeLists.txt index dfac92140396..b16d6ac8ccc1 100644 --- a/components/eamxx/tests/uncoupled/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/CMakeLists.txt @@ -21,7 +21,7 @@ if (NOT SCREAM_BASELINES_ONLY) # Once the mam4xx aerosol microphysics AtmosphereProcess is running, the # corresponding test here needs to be reworked with valid aerosol # initial conditions. - #add_subdirectory(mam4) + add_subdirectory(mam4/drydep) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4/drydep/CMakeLists.txt similarity index 80% rename from components/eamxx/tests/uncoupled/mam4/CMakeLists.txt rename to components/eamxx/tests/uncoupled/mam4/drydep/CMakeLists.txt index 18448d025667..17cee4b0fe03 100644 --- a/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/mam4/drydep/CMakeLists.txt @@ -1,10 +1,7 @@ include (ScreamUtils) -# This test needs to be reworked for microphysics -- currently it's still using -# input for nucleation. - # Create the test -CreateADUnitTest(mam4_microphysics_standalone +CreateADUnitTest(mam4_drydep_standalone LABELS mam4 physics LIBS mam MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} diff --git a/components/eamxx/tests/uncoupled/mam4/input.yaml b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml similarity index 63% rename from components/eamxx/tests/uncoupled/mam4/input.yaml rename to components/eamxx/tests/uncoupled/mam4/drydep/input.yaml index 80571b614f0d..c6499e85b508 100644 --- a/components/eamxx/tests/uncoupled/mam4/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml @@ -9,9 +9,7 @@ time_stepping: number_of_steps: ${NUM_STEPS} atmosphere_processes: - atm_procs_list: [mam4_micro] - mam4_micro: - compute_tendencies: [q_aitken_so4, n_aitken, q_h2so4] + atm_procs_list: [mam4_drydep] grids_manager: Type: Mesh Free @@ -24,13 +22,7 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} - q_aitken_so4: 0.0 - n_aitken: 0.0 - q_h2so4: 1.9186478479542893e-011 # 0.65e-10 is from namelist, but this is what gets to nucleation - pbl_height: 1100.0 T_mid: 273.0 - p_mid: 1.e5 - qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/uncoupled/mam4/output.yaml b/components/eamxx/tests/uncoupled/mam4/drydep/output.yaml similarity index 59% rename from components/eamxx/tests/uncoupled/mam4/output.yaml rename to components/eamxx/tests/uncoupled/mam4/drydep/output.yaml index 4e6ad2ac9128..e900bb804340 100644 --- a/components/eamxx/tests/uncoupled/mam4/output.yaml +++ b/components/eamxx/tests/uncoupled/mam4/drydep/output.yaml @@ -1,13 +1,9 @@ %YAML 1.1 --- -filename_prefix: mam4_nucleation_standalone_output +filename_prefix: mam4_drydep_standalone_output Averaging Type: Instant Field Names: - T_mid - - p_mid - - q_aitken_so4 - - n_aitken - - q_h2so4 output_control: Frequency: 1 frequency_units: nsteps From 7eeb63cf2158bbbd90d21da6241d2087cd00f212 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 20 Feb 2024 18:06:00 -0800 Subject: [PATCH 008/904] Adds required variables and the initial condition file --- ...x_mam_dry_deposition_process_interface.cpp | 94 ++++++++++++++++++- ...x_mam_dry_deposition_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 14 +-- .../eamxx/src/share/field/field_tag.hpp | 7 +- components/eamxx/tests/CMakeLists.txt | 1 + .../tests/uncoupled/mam4/drydep/input.yaml | 11 ++- 6 files changed, 116 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index f27e5d6a1ee4..fb71e4708c52 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -29,6 +29,8 @@ void MAMDryDep::set_grids( auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); + auto m3 = m * m * m; // meter cubed + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); @@ -41,16 +43,99 @@ void MAMDryDep::set_grids( // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // Layout for 2D (2d horiz) variable defined at mid-level and - // interfaces - const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + const int num_aero_modes = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_layout_mid{ + {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa] + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + add_field("obklen", scalar2d_layout, m, grid_name); + add_field("surfric", scalar2d_layout, m / s, grid_name); + + auto nondim = ekat::units::Units::nondimensional(); + add_field("landfrac", scalar2d_layout, nondim, grid_name); + add_field("icefrac", scalar2d_layout, nondim, grid_name); + add_field("ocnfrac", scalar2d_layout, nondim, grid_name); + add_field("fv", scalar2d_layout, m / s, grid_name); + add_field("ram1", scalar2d_layout, s / m, grid_name); + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + // printf("%s \n", int_nmr_field_name); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + + /*dgncur_awet, wetdens, obklen, surfric, + cam_in%landfrac, cam_in%icefrac, cam_in%ocnfrac, & cam_in%fv, + cam_in%ram1,*/ + + /*state%t, state%pmid, state%pint, state%pdel, & + state%q, dgncur_awet, wetdens, qqcw, obklen, surfric, + cam_in%landfrac, cam_in%icefrac, cam_in%ocnfrac, & cam_in%fv, cam_in%ram1, + ztodt, cam_out, ptend */ } // ========================================================================================= @@ -84,6 +169,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMDryDep::run_impl(const double dt) { std::cout << "End of derydep run" << std::endl; + + + } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 47be070c4228..7b054657a18f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -1,7 +1,7 @@ #ifndef EAMXX_MAM_DRYDEP_HPP #define EAMXX_MAM_DRYDEP_HPP -// For declaring wetscav class derived from atm process class +// For declaring dry deposition class derived from atm process class #include // For MAM4 aerosol configuration diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index cc627643914e..830743cd3f63 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -102,12 +102,12 @@ const char* aero_species_name(const int species_id) { KOKKOS_INLINE_FUNCTION const char* gas_species_name(const int gas_id) { static const char *species_names[num_aero_gases()] = { - "o3", - "h2o2", - "h2so4", - "so2", - "dms", - "soag" + "O3", + "H2O2", + "H2SO4", + "SO2", + "DMS", + "SOAG" }; return species_names[gas_id]; } @@ -245,7 +245,7 @@ const char* cld_aero_mmr_field_name(const int mode, const int species) { KOKKOS_INLINE_FUNCTION const char* gas_mmr_field_name(const int gas) { if (!gas_mmr_names(gas)[0]) { - concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); + concat_2_strings("", gas_species_name(gas), gas_mmr_names(gas)); } return const_cast(gas_mmr_names(gas)); } diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index a96f781a465c..2e8014273b95 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -40,7 +40,8 @@ enum class FieldTag { LongWaveBand, LongWaveGpoint, IsccpTau, - IsccpPrs + IsccpPrs, + num_modes }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -65,6 +66,7 @@ namespace ShortFieldTagsNames { constexpr auto LWGPT = FieldTag::LongWaveGpoint; constexpr auto ISCCPTAU = FieldTag::IsccpTau; constexpr auto ISCCPPRS = FieldTag::IsccpPrs; + constexpr auto NMODES = FieldTag::num_modes; } inline std::string e2str (const FieldTag ft) { @@ -117,6 +119,9 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::IsccpPrs: name = "ISCCPPRS"; break; + case FieldTag::num_modes: + name = "num_modes"; + break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index ab168e67e79f..6df2d89dd5c4 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -12,6 +12,7 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) # Initial condition files used in the tests set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") + set(EAMxx_tests_IC_FILE_MAM "scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc") set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") # Testing individual atm processes diff --git a/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml index c6499e85b508..3d76ca948282 100644 --- a/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml @@ -21,8 +21,17 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM} T_mid: 273.0 + dgncur_awet: 1e-5 + wetdens: 1e-5 + obklen: 1e-5 + surfric: 1e-5 + landfrac: 1.0 + icefrac: 0.0 + ocnfrac: 0.0 + fv: 1e-5 + ram1: 1e-5 # The parameters for I/O control Scorpio: From 2dd3d73b29762be5c0eb2dd84a4e2b964dc81a18 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 20 Feb 2024 21:39:04 -0800 Subject: [PATCH 009/904] Adds variables to the init and run functions --- ...x_mam_dry_deposition_process_interface.cpp | 130 ++++++++++++++++-- ...x_mam_dry_deposition_process_interface.hpp | 55 ++++++++ 2 files changed, 176 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index fb71e4708c52..1eb1aa95b57c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -74,6 +74,10 @@ void MAMDryDep::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); add_field("obklen", scalar2d_layout, m, grid_name); @@ -127,15 +131,6 @@ void MAMDryDep::set_grids( add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - - /*dgncur_awet, wetdens, obklen, surfric, - cam_in%landfrac, cam_in%icefrac, cam_in%ocnfrac, & cam_in%fv, - cam_in%ram1,*/ - - /*state%t, state%pmid, state%pint, state%pdel, & - state%q, dgncur_awet, wetdens, qqcw, obklen, surfric, - cam_in%landfrac, cam_in%icefrac, cam_in%ocnfrac, & cam_in%fv, cam_in%ram1, - ztodt, cam_out, ptend */ } // ========================================================================================= @@ -164,10 +159,127 @@ void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { void MAMDryDep::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = + get_field_in("pseudo_density").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // set up our preprocess functor + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, + dry_atm_, dry_aero_); } // ========================================================================================= void MAMDryDep::run_impl(const double dt) { + +const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + + + /* + + Rough notes: + + tair == T_mid + pmid == p_mid + pint ==p_int + pdel == p_del + + state_q: It can be obtained using dry_aero_ and dry_atm_, there is an example of this in the optics code + qqcw can also be obtained from wet_aero.There is an example in optics code + Feel free to improvise input variables by adding them to the input.yaml file. + + + + Function to call in mam4xx/drydep.hpp: + void aero_model_drydep( + const ThreadTeam &team, + const Real fraction_landuse[DryDeposition::n_land_type] , + const haero::ConstColumnView tair, const haero::ConstColumnView pmid, + const haero::ConstColumnView pint, const haero::ConstColumnView pdel, + const Diagnostics::ColumnTracerView state_q, + const ColumnView dgncur_awet[AeroConfig::num_modes()], + const ColumnView wetdens[AeroConfig::num_modes()], + const Kokkos::View qqcw[aero_model::pcnst], const Real obklen, + const Real ustar, const Real landfrac, const Real icefrac, + const Real ocnfrac, const Real fricvelin, const Real ram1in, + const Diagnostics::ColumnTracerView ptend_q, + bool ptend_lq[aero_model::pcnst], const Real dt, + const ColumnView aerdepdrycw, const ColumnView aerdepdryis, + + const ColumnView rho, + const Kokkos::View vlc_dry[AeroConfig::num_modes()] + [DryDeposition::aerosol_categories], + const Kokkos::View vlc_trb[AeroConfig::num_modes()] + [DryDeposition::aerosol_categories], + const Kokkos::View vlc_grv[AeroConfig::num_modes()] + [DryDeposition::aerosol_categories], + const Kokkos::View dqdt_tmp[aero_model::pcnst]) + + */ + + + std::cout << "End of derydep run" << std::endl; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 7b054657a18f..7ab45d7fd195 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -15,9 +15,19 @@ namespace scream { // The process responsible for handling MAM4 dry deposition. The AD // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { + + using KT = ekat::KokkosTypes; + // number of horizontal columns and vertical levels int ncol_, nlev_; + //Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + // buffer for sotring temporary variables mam_coupling::Buffer buffer_; @@ -55,6 +65,51 @@ class MAMDryDep final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); + team.team_barrier(); + + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMAci::Preprocess + + private: + // pre- and postprocessing scratch pads + Preprocess preprocess_; + }; // MAMDryDep } // namespace scream From 6df3ae38c9cbb25897ef9341639c8d7a220ef957 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 22 Feb 2024 08:01:34 -0800 Subject: [PATCH 010/904] fix typo --- components/eam/src/physics/cam/phys_control.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/cam/phys_control.F90 b/components/eam/src/physics/cam/phys_control.F90 index 543fd29753f2..b87ea442c4e7 100644 --- a/components/eam/src/physics/cam/phys_control.F90 +++ b/components/eam/src/physics/cam/phys_control.F90 @@ -745,7 +745,7 @@ subroutine phys_getopts(deep_scheme_out, shallow_scheme_out, eddy_scheme_out, & if ( present(use_crm_accel_out ) ) use_crm_accel_out = use_crm_accel if ( present(crm_accel_factor_out ) ) crm_accel_factor_out = crm_accel_factor if ( present(crm_accel_uv_out ) ) crm_accel_uv_out = crm_accel_uv - if ( present(MMF_PAM_dyn_per_phys ) ) MMF_PAM_dyn_per_phys_out = MMF_PAM_dyn_per_phys + if ( present(MMF_PAM_dyn_per_phys_out) ) MMF_PAM_dyn_per_phys_out = MMF_PAM_dyn_per_phys if ( present(use_subcol_microp_out ) ) use_subcol_microp_out = use_subcol_microp if ( present(macrop_scheme_out ) ) macrop_scheme_out = macrop_scheme From 8b89b6fc87e9a0dbbd272d786e18f550b314a151 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Mon, 26 Feb 2024 16:05:17 -0800 Subject: [PATCH 011/904] adjust default MMF2 configuration --- components/eam/cime_config/config_component.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index c7585072ac94..16ff080ae839 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -75,10 +75,11 @@ -crm pam -pam_dycor spam -crm_dt 8 -crm pam -pam_dycor awfl -crm_dt 8 -use_MMF -nlev 60 -crm_nz 50 - -crm_dx 2000 -crm_nx 64 -crm_ny 1 + -crm_dx 2000 -crm_nx 64 -crm_ny 1 -crm_nx_rad 4 -crm_ny_rad 1 + -crm_dx 2000 -crm_nx 45 -crm_ny 1 -crm_nx_rad 5 -crm_ny_rad 1 -MMF_microphysics_scheme sam1mom -chem none -MMF_microphysics_scheme p3 -chem none - -crm_nx_rad 4 -crm_ny_rad 1 -rad rrtmgp -rrtmgpxx + -rad rrtmgp -rrtmgpxx -use_MMF_VT -use_MMF_ESMT -aquaplanet From ef9d3281c9051dadc2627f6abcf9d4e0308647bd Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 27 Feb 2024 08:18:08 -0800 Subject: [PATCH 012/904] add PAM coupler options to override SPAM parameter defaults --- components/eam/src/physics/crm/pam/pam_driver.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index 128974a856e4..c325945f8803 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -54,6 +54,14 @@ extern "C" void pam_driver() { coupler.set_option("sponge_time_scale",60); // minimum damping timescale at top coupler.set_option("crm_acceleration_ceaseflag",false); //------------------------------------------------------------------------------------------------ + // coupler options for SPAM dycor + coupler.set_option("spam_couple_wind_exact_inverse",true); + coupler.set_option("spam_clip_negative_densities",true); + coupler.set_option("spam_clip_vertical_velocities",true); + coupler.set_option("spam_adjust_crm_per_phys_using_vert_cfl",true); + coupler.set_option("spam_target_cfl",0.7); + coupler.set_option("spam_max_w",50.0); + //------------------------------------------------------------------------------------------------ // Allocate the coupler state and retrieve host/device data managers coupler.allocate_coupler_state( crm_nz , crm_ny , crm_nx , nens ); From 644bf4c00c6dc3295fc3996455318e0fdd6f9c51 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 27 Feb 2024 08:24:59 -0800 Subject: [PATCH 013/904] update pam_state_recall_dry_density --- .../eam/src/physics/crm/pam/pam_state.h | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_state.h b/components/eam/src/physics/crm/pam/pam_state.h index 34cc42a4a788..3266899ba217 100644 --- a/components/eam/src/physics/crm/pam/pam_state.h +++ b/components/eam/src/physics/crm/pam/pam_state.h @@ -258,6 +258,13 @@ inline void pam_state_save_dry_density( pam::PamCoupler &coupler ) { // recall CRM dry density saved previously - only for anelastic case +// Note - The need for this arises because the SPAM dycor diagnoses the dry +// density in a way that preserves the total density to match the reference +// density. However, this is problematic in the presence of the GCM forcing +// which naturally changes the total density. Therefore, we can recall the +// previous dry density and use it to replace the horizontal mean returned from +// the dycor, which ensures that the impact of GCM forcing is preserved while +// also retaining the dry density perturbations created by the dycor. inline void pam_state_recall_dry_density( pam::PamCoupler &coupler ) { using yakl::c::parallel_for; using yakl::c::SimpleBounds; @@ -270,29 +277,30 @@ inline void pam_state_recall_dry_density( pam::PamCoupler &coupler ) { auto crm_rho_d = dm_device.get("density_dry"); auto tmp_rho_d = dm_device.get("density_dry_save"); //------------------------------------------------------------------------------------------------ - #ifdef MMF_ALT_DENSITY_RECALL - // initialize horizontal mean of previous CRM dry density - real2d old_hmean_rho_d("old_hmean_rho_d" ,nz,nens); - real2d new_hmean_rho_d("new_hmean_rho_d" ,nz,nens); - parallel_for(SimpleBounds<2>(nz,nens), YAKL_LAMBDA (int k, int iens) { - old_hmean_rho_d(k,iens) = 0; - new_hmean_rho_d(k,iens) = 0; - }); - real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions - // calculate horizontal mean of previous CRM dry density - parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { - atomicAdd( old_hmean_rho_d(k,iens), tmp_rho_d(k,j,i,iens) * r_nx_ny ); - atomicAdd( new_hmean_rho_d(k,iens), crm_rho_d(k,j,i,iens) * r_nx_ny ); - }); - // replace horizontal mean dry density with prevoius value - parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { - crm_rho_d(k,j,i,iens) = crm_rho_d(k,j,i,iens) - new_hmean_rho_d(k,iens) + old_hmean_rho_d(k,iens); - }); - #else - parallel_for( "Recall CRM dry density",SimpleBounds<4>(nz,ny,nx,nens),YAKL_LAMBDA (int k_crm, int j, int i, int iens) { - crm_rho_d(k_crm,j,i,iens) = tmp_rho_d(k_crm,j,i,iens); - }); - #endif + // initialize horizontal mean of previous CRM dry density + real2d old_hmean_rho_d("old_hmean_rho_d" ,nz,nens); + real2d new_hmean_rho_d("new_hmean_rho_d" ,nz,nens); + parallel_for(SimpleBounds<2>(nz,nens), YAKL_LAMBDA (int k, int iens) { + old_hmean_rho_d(k,iens) = 0; + new_hmean_rho_d(k,iens) = 0; + }); + real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions + // calculate horizontal mean of previous CRM dry density + parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { + atomicAdd( old_hmean_rho_d(k,iens), tmp_rho_d(k,j,i,iens) * r_nx_ny ); + atomicAdd( new_hmean_rho_d(k,iens), crm_rho_d(k,j,i,iens) * r_nx_ny ); + }); + // replace horizontal mean dry density with previous value + parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { + crm_rho_d(k,j,i,iens) = crm_rho_d(k,j,i,iens) - new_hmean_rho_d(k,iens) + old_hmean_rho_d(k,iens); + }); + //------------------------------------------------------------------------------------------------ + // Original simple appraoch to completely reinstate the previous dry density field + // (this was shown to negatively impact the model stability) + // This was used initially, but the above + // parallel_for( "Recall CRM dry density",SimpleBounds<4>(nz,ny,nx,nens),YAKL_LAMBDA (int k_crm, int j, int i, int iens) { + // crm_rho_d(k_crm,j,i,iens) = tmp_rho_d(k_crm,j,i,iens); + // }); //------------------------------------------------------------------------------------------------ } From 6b7dcbeae6ac486a59041eb78e882ae06e1a8ea0 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 27 Feb 2024 08:37:57 -0800 Subject: [PATCH 014/904] update PAM submodule --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 87731d56aeee..9d692dccdd67 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 87731d56aeee4bfae4750e17732828ba186367a7 +Subproject commit 9d692dccdd67d9c4dc278c76290504a2af8cd187 From 9c7e8f5ba5fe268b3f40164347c5668f6f5ce57f Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 5 Mar 2024 15:00:28 -0700 Subject: [PATCH 015/904] Implement the MAMDryDep::run_impl() function. This is an interface to the mam4::DryDeposition::compute_tendencies() function. This means taking Scream Kokkos views of data and mapping them to mam4 Kokkos views of data. Most of the time it is just a sub-view so memory is not created or destroyed. There were a couple of choices for mapping which Scream data to the mam4 data that need to be reviewed. The resulting tendencies are in a local Kokkos view called tendencies. These need to be made available to Scream. This needs to be compiled and tested on a GPU. --- ...x_mam_dry_deposition_process_interface.cpp | 288 ++++++++++++++---- ...x_mam_dry_deposition_process_interface.hpp | 42 ++- .../mam/impl/compute_water_content.cpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 6 +- .../share/grid/remap/identity_remapper.hpp | 4 +- externals/haero | 2 +- externals/mam4xx | 2 +- 7 files changed, 270 insertions(+), 76 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 1eb1aa95b57c..65f0e280b877 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -1,5 +1,7 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" +#include "mam4xx/drydep.hpp" + /* Future work: Wirte comments @@ -14,6 +16,27 @@ MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. */ + std::string nname = "Obukhov_length"; + if(params.isParameter(nname)) + parameters_.Obukhov_length_ = params.get(nname); + nname = "surface_friction_velocty"; + if(params.isParameter(nname)) + parameters_.surface_friction_velocty_ = params.get(nname); + nname = "land_fraction"; + if(params.isParameter(nname)) + parameters_.land_fraction_ = params.get(nname); + nname = "ice_fraction"; + if(params.isParameter(nname)) + parameters_.ice_fraction_ = params.get(nname); + nname = "ocean_fraction"; + if(params.isParameter(nname)) + parameters_.ocean_fraction_ = params.get(nname); + nname = "friction_velocity"; + if(params.isParameter(nname)) + parameters_.friction_velocity_ = params.get(nname); + nname = "aerodynamical_resistance"; + if(params.isParameter(nname)) + parameters_.aerodynamical_resistance_ = params.get(nname); } // ========================================================================================= @@ -40,22 +63,42 @@ void MAMDryDep::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; + auto nondim = ekat::units::Units::nondimensional(); + + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ - {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + {NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; // mid points + + // Layout for tracers. + const int pcnst = mam4::aero_model::pcnst; + FieldLayout scalar4d_layout_q{ + {COL, LEV, NMODES}, {ncol_, nlev_, pcnst}}; + FieldLayout scalar4d_layout_qqcw_tends{ + {COL, NMODES, LEV}, {ncol_, pcnst, nlev_}}; + + // Memory to format teh wet_aero.int_aero_nmr and wet_aero.int_aero_mmr + // views into the order expected by mam4xx. + add_field("qtracers", scalar4d_layout_q, kg / kg, grid_name); + add_field("d_qtracers_dt", scalar4d_layout_q, kg / kg / s, grid_name); + // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); + // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_interstitial_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); + add_field("Tendencies", scalar4d_layout_qqcw_tends, kg / kg / s, grid_name); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- + add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, @@ -77,13 +120,14 @@ void MAMDryDep::set_grids( add_field( "omega", scalar3d_layout_mid, Pa / s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); - add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + // + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); add_field("obklen", scalar2d_layout, m, grid_name); add_field("surfric", scalar2d_layout, m / s, grid_name); - auto nondim = ekat::units::Units::nondimensional(); add_field("landfrac", scalar2d_layout, nondim, grid_name); add_field("icefrac", scalar2d_layout, nondim, grid_name); add_field("ocnfrac", scalar2d_layout, nondim, grid_name); @@ -92,7 +136,7 @@ void MAMDryDep::set_grids( // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < num_aero_modes; ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, @@ -108,7 +152,7 @@ void MAMDryDep::set_grids( } } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < num_aero_modes; ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); // printf("%s \n", int_nmr_field_name); @@ -130,7 +174,7 @@ void MAMDryDep::set_grids( const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } + } } // ========================================================================================= @@ -169,14 +213,28 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = - get_field_in("pseudo_density").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; dry_atm_.qv = buffer_.qv_dry; dry_atm_.qc = buffer_.qc_dry; dry_atm_.nc = buffer_.nc_dry; dry_atm_.qi = buffer_.qi_dry; dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + + dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); + qtracers_ = get_field_out("qtracers").get_view(); + d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols").get_view(); + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols").get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -223,67 +281,179 @@ void MAMDryDep::initialize_impl(const RunType run_type) { preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); } +namespace { +KOKKOS_INLINE_FUNCTION +void compute_tendencies( + const MAMDryDep::KT::MemberType &team, + const MAMDryDep::Parameters ¶meters, + const mam4::DryDeposition &dry_deposition, + const double dt, + MAMDryDep::view_3d qtracers, + MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::view_2d dgncur_awet[mam_coupling::num_aero_modes()], + const MAMDryDep::view_2d wet_dens[mam_coupling::num_aero_modes()], + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero, + const mam_coupling::AerosolState &wet_aero, + MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_2d aerdepdryis, + MAMDryDep::view_3d tendencies) +{ + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); + const Real t = 0; + const int icol = team.league_rank(); + + const mam4::AeroConfig aero_config; + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + atm.interface_pressure = atm.pressure; + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + mam4::Surface surf; + mam4::Diagnostics diags; + mam4::Tendencies tends; + + diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); + diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); + + for (int i=0; i::get_default_team_policy(ncol, nlev); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { + compute_tendencies(team, parameters, dry_deposition, dt, qtracers, d_qtracers_dt, + dgncur_awet, wet_dens, dry_atm, dry_aero, wet_aero, aerdepdrycw, aerdepdryis, tendencies); + }); +} + +void fill_tracer_views( + const int ncol, + const int nlev, + mam_coupling::DryAtmosphere dry_atm, + mam_coupling::AerosolState dry_aero, + mam_coupling::AerosolState wet_aero, + MAMDryDep::view_3d qtracers) +{ + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlev); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { + const int column = team.league_rank(); + compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, column); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int lev) { + for (int mode = 0; mode < num_aero_modes; ++mode) { + int icnst = mam4::ConvProc::numptrcw_amode(mode); + qtracers(column, lev, icnst) = wet_aero.int_aero_nmr[mode](column,lev); + for (int species = 0; species < num_aero_species; ++species) { + icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); + if (-1 < icnst) { + qtracers(column, lev, icnst) = wet_aero.int_aero_mmr[mode][species](column,lev); + } + } + } + }); + }); +} +} // ========================================================================================= void MAMDryDep::run_impl(const double dt) { -const auto scan_policy = ekat::ExeSpaceUtils< + // Include this but update when scream is updated to use the newer DryDeposition class: + using DryDep = mam4::DryDeposition; // mam4::DryDeposition; + + using MemberType = KT::MemberType; + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + std::cout<<__FILE__<<":"<<__LINE__<<" In MAMDryDep::run_impl"< qqcw[aero_model::pcnst], const Real obklen, - const Real ustar, const Real landfrac, const Real icefrac, - const Real ocnfrac, const Real fricvelin, const Real ram1in, - const Diagnostics::ColumnTracerView ptend_q, - bool ptend_lq[aero_model::pcnst], const Real dt, - const ColumnView aerdepdrycw, const ColumnView aerdepdryis, - - const ColumnView rho, - const Kokkos::View vlc_dry[AeroConfig::num_modes()] - [DryDeposition::aerosol_categories], - const Kokkos::View vlc_trb[AeroConfig::num_modes()] - [DryDeposition::aerosol_categories], - const Kokkos::View vlc_grv[AeroConfig::num_modes()] - [DryDeposition::aerosol_categories], - const Kokkos::View dqdt_tmp[aero_model::pcnst]) + const DryDep::Config process_config; + // Future enhancement: + // This is where we can set the fraction of land use parameters: + // if (do_soilw) { + // const Real *landuse = get_landuse_from_file(do_soilw); + // for (int i=0; i::get_default_team_policy(ncol_, nlev_); + // Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + // dry_deposition.validate( aero_config, team, atm, progs); + // } + // Kokkos::fence(); + + view_2d dgncur_awet[num_aero_modes]; + for (int i=0; i; - +public: + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); +private: // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -34,7 +38,18 @@ class MAMDryDep final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - public: + view_3d qtracers_; + view_3d d_qtracers_dt_; + view_3d dgncur_awet_; + view_3d wet_dens_; + view_3d tendencies_; + + view_2d aerdepdrycw_; + view_2d aerdepdryis_; + +public: + using KT = ekat::KokkosTypes; + // Constructor MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -43,10 +58,10 @@ class MAMDryDep final : public scream::AtmosphereProcess { // -------------------------------------------------------------------------- // The type of subcomponent - AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type() const override { return AtmosphereProcessType::Physics; } // The name of the subcomponent - std::string name() const { return "mam_dry_deposition"; } + std::string name() const override { return "mam_dry_deposition"; } // grid void set_grids( @@ -63,7 +78,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { void run_impl(const double dt) override; // Finalize - void finalize_impl(){/*Do nothing*/}; + void finalize_impl() override {/*Do nothing*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. @@ -106,12 +121,21 @@ class MAMDryDep final : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess + struct Parameters { + Real Obukhov_length_ = 0.20723257141035126e+03; + Real surface_friction_velocty_ = 0.39900396673305327; + Real land_fraction_ = 0.1; + Real ice_fraction_ = 0.6; + Real ocean_fraction_ = 0.3; + Real friction_velocity_ = 0.46637129718055864; + Real aerodynamical_resistance_ = 0.91147859222259044e+02; + }; private: // pre- and postprocessing scratch pads Preprocess preprocess_; - + Parameters parameters_; }; // MAMDryDep } // namespace scream -#endif // EAMXX_MAM_DRYDEP_HPP \ No newline at end of file +#endif // EAMXX_MAM_DRYDEP_HPP diff --git a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp index a31d81f2408a..673fe06292e1 100644 --- a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp +++ b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp @@ -23,7 +23,7 @@ void compute_water_content(const mam4::Prognostics &progs, int k, // extract aerosol tracers for this level into state_q, which is needed // for computing dry aerosol properties below // FIXME: we should eliminate this index translation stuff - constexpr int nvars = mam4::water_uptake::nvars; + constexpr int nvars = mam4::aero_model::pcnst; Real state_q[nvars]; // aerosol tracers for level k for (int imode = 0; imode < num_modes; ++imode) { int la, lc; // interstitial and cloudborne indices within state_q diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 830743cd3f63..388ff9e27329 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -512,6 +512,7 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, "cldfrac not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, "w_updraft not defined for dry atmosphere state!"); + haero::ConstColumnView interface_pressure; return haero::Atmosphere(mam4::nlev, ekat::subview(dry_atm.T_mid, column_index), ekat::subview(dry_atm.p_mid, column_index), @@ -522,6 +523,7 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, ekat::subview(dry_atm.ni, column_index), ekat::subview(dry_atm.z_mid, column_index), ekat::subview(dry_atm.p_del, column_index), + interface_pressure, ekat::subview(dry_atm.cldfrac, column_index), ekat::subview(dry_atm.w_updraft, column_index), dry_atm.pblh(column_index)); @@ -720,7 +722,7 @@ void compute_wet_mixing_ratios(const Team& team, const auto PC = mam4::ModeIndex::PrimaryCarbon; \ const auto NoMode = mam4::ModeIndex::None; \ static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ @@ -817,7 +819,6 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; @@ -850,7 +851,6 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; diff --git a/components/eamxx/src/share/grid/remap/identity_remapper.hpp b/components/eamxx/src/share/grid/remap/identity_remapper.hpp index 047bd90ef835..d281ce4bc6e7 100644 --- a/components/eamxx/src/share/grid/remap/identity_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/identity_remapper.hpp @@ -63,7 +63,7 @@ class IdentityRemapper : public AbstractRemapper return src_layout; } - void register_field_from_src (const field_type& src) { + void register_field_from_src (const field_type& src) override { EKAT_REQUIRE_MSG (m_aliasing!=SrcAliasTgt, "Error! Makes no sense to register from src and ask that src alias tgt.\n"); if (m_aliasing==TgtAliasSrc) { @@ -72,7 +72,7 @@ class IdentityRemapper : public AbstractRemapper AbstractRemapper::register_field_from_src(src); } } - void register_field_from_tgt (const field_type& tgt) { + void register_field_from_tgt (const field_type& tgt) override { EKAT_REQUIRE_MSG (m_aliasing!=TgtAliasSrc, "Error! Makes no sense to register from tgt and ask that tgt alias src.\n"); if (m_aliasing==SrcAliasTgt) { diff --git a/externals/haero b/externals/haero index ba86d80dc740..d3a953d43979 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit ba86d80dc740dc2dbb8220ab5e501f2547355fa0 +Subproject commit d3a953d43979aa2fc3ae6f72ebc899e9a756ad89 diff --git a/externals/mam4xx b/externals/mam4xx index 209126e50476..d43fe41047a8 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 209126e50476c1d7e0aafd77cec4caea1c56a880 +Subproject commit d43fe41047a8936d2649b8d4087bb820c04919f7 From a6b550cef2fa99dba45f891caad877b0d8f639a8 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 6 Mar 2024 11:00:54 -0700 Subject: [PATCH 016/904] Forgot to add updated test file. --- components/eamxx/tests/uncoupled/mam4/drydep/input.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml index 3d76ca948282..d36e6fe8a55c 100644 --- a/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml @@ -32,6 +32,8 @@ initial_conditions: ocnfrac: 0.0 fv: 1e-5 ram1: 1e-5 + w_updraft: 1e-5 + pbl_height: 0.0 # The parameters for I/O control Scorpio: From 9f59dcb2dbdd4497a1c23b5195488917e0ec76b0 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 6 Mar 2024 12:30:12 -0700 Subject: [PATCH 017/904] Fixes for Cuda build. --- ...x_mam_dry_deposition_process_interface.cpp | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 65f0e280b877..03d44474bc86 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -154,7 +154,6 @@ void MAMDryDep::set_grids( // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios for(int m = 0; m < num_aero_modes; ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - // printf("%s \n", int_nmr_field_name); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); @@ -315,7 +314,7 @@ void compute_tendencies( diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - for (int i=0; i::get_default_team_policy(ncol, nlev); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { + const int num_aero_modes = mam_coupling::num_aero_modes(); + MAMDryDep::view_2d dgncur_awet[num_aero_modes]; + MAMDryDep::view_2d wet_dens[num_aero_modes]; + + for (int i=0; i::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); @@ -437,19 +445,11 @@ void MAMDryDep::run_impl(const double dt) { // } // Kokkos::fence(); - view_2d dgncur_awet[num_aero_modes]; - for (int i=0; i Date: Wed, 6 Mar 2024 12:31:48 -0700 Subject: [PATCH 018/904] Fixes for Cuda build. --- .../physics/mam/eamxx_mam_dry_deposition_process_interface.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 03d44474bc86..aecea66a6c4a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -418,7 +418,6 @@ void MAMDryDep::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - const int num_aero_modes = mam_coupling::num_aero_modes(); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); From afb3c3221f6c439cfbbe7c4e9725fa014fa44867 Mon Sep 17 00:00:00 2001 From: daleihao Date: Wed, 6 Dec 2023 13:32:49 -0800 Subject: [PATCH 019/904] add code for making TOP_rad parameters code debug code debug debug debug debug debug add new mappings & debug fix bug --- .../bld/namelist_files/namelist_defaults.xml | 4 +- .../namelist_defaults_tools.xml | 7 + .../namelist_files/namelist_definition.xml | 9 +- .../elm/tools/mksurfdata_map/mksurfdata.pl | 6 +- .../elm/tools/mksurfdata_map/src/Srcfiles | 1 + .../tools/mksurfdata_map/src/mkfileMod.F90 | 7 + .../tools/mksurfdata_map/src/mksurfdat.F90 | 48 +++- .../tools/mksurfdata_map/src/mktopradMod.F90 | 239 ++++++++++++++++++ .../elm/tools/mksurfdata_map/src/mkvarctl.F90 | 4 +- 9 files changed, 316 insertions(+), 9 deletions(-) create mode 100644 components/elm/tools/mksurfdata_map/src/mktopradMod.F90 diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 78193b74ebe0..8d7ad4fba900 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -1687,7 +1687,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/0.5x0.5/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.5x0.5_nomask_aave_da_c190417.nc lnd/clm2/mappingdata/maps/0.5x0.5/map_0.1x0.1_nomask_to_0.5x0.5_nomask_aave_da_c120706.nc +>lnd/clm2/mappingdata/maps/0.5x0.5/map_0.1x0.1_nomask_to_0.5x0.5_nomask_aave_da_c240308.nc lnd/clm2/mappingdata/maps/0.5x0.5/map_ne240np4_nomask_to_0.5x0.5_nomask_aave_da_c120711.nc lnd/clm2/mappingdata/maps/ne30np4/map_1km-merge-10min_HYDRO1K-merge-nomask_to_ne30np4_nomask_aave_da_c130405.nc lnd/clm2/mappingdata/maps/ne30np4/map_0.5x0.5_GSDTG2000_to_ne30np4_nomask_aave_da_c161010.nc +lnd/clm2/mappingdata/maps/ne30np4/map_0.1x0.1_nomask_to_ne30np4_nomask_aave_da_c231218.nc lnd/clm2/mappingdata/maps/ne30np4/map_ne30np4_to_0.5x0.5rtm_aave_da_110320.nc diff --git a/components/elm/bld/namelist_files/namelist_defaults_tools.xml b/components/elm/bld/namelist_files/namelist_defaults_tools.xml index 113f949ffe82..d948b16c95a5 100644 --- a/components/elm/bld/namelist_files/namelist_defaults_tools.xml +++ b/components/elm/bld/namelist_files/namelist_defaults_tools.xml @@ -147,6 +147,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). AVHRR AVHRR ISRIC-WISE +nomask 3x3min @@ -176,6 +177,8 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 0.5x0.5 0.5x0.5 5x5min +0.1x0.1 + mksrf_flakwat @@ -203,6 +206,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). mksrf_fslp10 mksrf_fero mksrf_ffert +mksrf_ftoprad lnd/clm2/rawdata/mksrf_navyoro_20min.c010129.nc @@ -313,6 +317,9 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/rawdata/mksrf_fert.c220309.nc +lnd/clm2/rawdata/mksrf_toprad_0.1x0.1.c231218.nc + diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 8182c6d90666..10a9f98ee8e5 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -772,7 +772,7 @@ in its attributes. (Only used if scripgriddata_src_type = UGRID.) + valid_values="mksrf_fsoitex,mksrf_forganic,mksrf_flakwat,mksrf_fwetlnd,mksrf_fmax,mksrf_fmax,mksrf_fglacier,mksrf_fvocef,mksrf_furbtopo,mksrf_flndtopo,firrig,mksrf_furban,mksrf_fvegtyp,mksrf_fsoicol,mksrf_fsoiord,mksrf_flai,mksrf_fgdp,mksrf_fpeat,mksrf_fabm,mksrf_ftopostats,mksrf_fvic,mksrf_fch4,mksrf_fgrvl,mksrf_fslp10,mksrf_fero,mksrf_ffert,mksrf_ftoprad" > Filename for mksurfdata_map to remap raw data into the output surface dataset @@ -931,10 +931,15 @@ ELM-Erosion parameters dataset + input_pathname="abs" group="elmexp" valid_values="" > ELM-Fertilizer dataset + +Topography parameters dataset for TOP solar radiation parameterization + + If TRUE, output variables in double precision for mksurfdata diff --git a/components/elm/tools/mksurfdata_map/mksurfdata.pl b/components/elm/tools/mksurfdata_map/mksurfdata.pl index 222a9c51bd15..05389d0d579d 100755 --- a/components/elm/tools/mksurfdata_map/mksurfdata.pl +++ b/components/elm/tools/mksurfdata_map/mksurfdata.pl @@ -421,8 +421,8 @@ ($) my $mkopts = "-csmdata $CSMDATA -silent -justvalue -namelist elmexp $usrnam"; foreach my $typ ( "lak", "veg", "voc", "top", "tex", "col","ord", "fmx", "lai", "urb", "org", "glc", "utp", "wet", - "gdp", "peat","abm", "topostats" , "vic", "ch4", - "pho", "grvl", "slp10", "ero", "fert") { + "gdp", "peat","abm", "topostats" , "vic", "ch4", + "pho", "grvl", "slp10", "ero", "fert", "toprad") { my $lmask = `$scrdir/../../bld/queryDefaultNamelist.pl $mopts -silent -options type=$typ,mergeGIS=$merge_gis,hirespft=$hirespft -var lmask`; $lmask = trim($lmask); my $hgrid = `$scrdir/../../bld/queryDefaultNamelist.pl $mopts -options type=$typ,hirespft=$hirespft -var hgrid`; @@ -539,6 +539,7 @@ ($) map_fslp10 = '$map{'slp10'}' map_fero = '$map{'ero'}' map_ffert = '$map{'fert'}' + map_ftoprad = '$map{'toprad'}' mksrf_fsoitex = '$datfil{'tex'}' mksrf_forganic = '$datfil{'org'}' mksrf_flakwat = '$datfil{'lak'}' @@ -563,6 +564,7 @@ ($) mksrf_fslp10 = '$datfil{'slp10'}' mksrf_fero = '$datfil{'ero'}' mksrf_ffert = '$datfil{'fert'}' + mksrf_ftoprad = '$datfil{'toprad'}' EOF my $urbdesc = "urb3den"; my $rcp_option= ""; diff --git a/components/elm/tools/mksurfdata_map/src/Srcfiles b/components/elm/tools/mksurfdata_map/src/Srcfiles index f8bda5dcf61b..96cec841a941 100644 --- a/components/elm/tools/mksurfdata_map/src/Srcfiles +++ b/components/elm/tools/mksurfdata_map/src/Srcfiles @@ -28,6 +28,7 @@ mkVICparamsMod.F90 mkCH4inversionMod.F90 mkSedMod.F90 mkFertMod.F90 +mktopradMod.F90 nanMod.F90 shr_file_mod.F90 shr_sys_mod.F90 diff --git a/components/elm/tools/mksurfdata_map/src/mkfileMod.F90 b/components/elm/tools/mksurfdata_map/src/mkfileMod.F90 index c3f67cc7a12c..50b029efc7d5 100644 --- a/components/elm/tools/mksurfdata_map/src/mkfileMod.F90 +++ b/components/elm/tools/mksurfdata_map/src/mkfileMod.F90 @@ -15,6 +15,7 @@ subroutine mkfile(domain, fname, dynlanduse) use mkpftMod , only : mkpftAtt use mksoilMod , only : mksoilAtt use mkSedMod , only : mksedAtt + use mktopradMod , only : mktopradAtt use mkharvestMod , only : mkharvest_fieldname, mkharvest_numtypes, mkharvest_longname use mkncdio , only : check_ret, ncd_defvar use mkdomainMod @@ -297,6 +298,10 @@ subroutine mkfile(domain, fname, dynlanduse) call check_ret(nf_put_att_text(ncid, NF_GLOBAL, & 'map_fertilizer_file', len_trim(str), trim(str)), subname) + str = get_filename(map_ftoprad) + call check_ret(nf_put_att_text(ncid, NF_GLOBAL, & + 'map_ftoprad_file', len_trim(str), trim(str)), subname) + ! ---------------------------------------------------------------------- ! Define variables ! ---------------------------------------------------------------------- @@ -313,6 +318,8 @@ subroutine mkfile(domain, fname, dynlanduse) call mksedAtt( ncid, dynlanduse, xtype ) + call mktopradAtt( ncid, dynlanduse, xtype ) + if (outnc_1d) then call ncd_defvar(ncid=ncid, varname='AREA' , xtype=nf_double, & dim1name='gridcell',& diff --git a/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 b/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 index d062907cdccf..a936c43052b9 100644 --- a/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 +++ b/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 @@ -44,7 +44,9 @@ program mksurfdat use mkCH4inversionMod , only : mkCH4inversion use mksoilphosphorusMod, only : mksoilphosphorus use mkSedMod , only : mkgrvl, mkslp10, mkEROparams - use mkFertMod , only : mkfert! + use mkFertMod , only : mkfert + use mktopradMod , only : mktoprad +! ! !ARGUMENTS: implicit none @@ -152,6 +154,10 @@ program mksurfdat real(r8), allocatable :: litho(:) ! lithology erodiblity index (unitless) real(r8), allocatable :: nfert(:,:) ! crop nitrogen fertilizer (g/m^2) real(r8), allocatable :: pfert(:,:) ! crop phosphorus fertilizer (g/m^2) + real(r8), allocatable :: sinsl_sinas(:) ! sin(slope)*sin(aspect) / cos(slope) + real(r8), allocatable :: sinsl_cosas(:) ! sin(slope)*cos(aspect) / cos(slope) + real(r8), allocatable :: sky_view(:) ! mean of (sky view factor / cos(slope)) + real(r8), allocatable :: terrain_config(:) ! mean of (terrain configuration factor / cos(slope)) type(domain_type) :: ldomain @@ -187,6 +193,7 @@ program mksurfdat mksrf_fslp10, & mksrf_fero, & mksrf_ffert, & + mksrf_ftoprad, & nglcec, & numpft, & soil_color, & @@ -224,6 +231,7 @@ program mksurfdat map_fslp10, & map_ffert, & map_fero, & + map_ftoprad, & outnc_large_files, & outnc_double, & outnc_dims, & @@ -264,8 +272,9 @@ program mksurfdat ! mksrf_fphosphorus Soil phosphorus dataset ! mksrf_fgrvl ---- Soil gravel content dataset ! mksrf_fslp10 --- Slope percentile dataset - ! mksrf_fero ----- ELM-Erosion parameters dataset + ! mksrf_fero ----- ELM-Erosion parameters dataset ! mksrf_ffert ---- Crop Fertilizer dataset + ! mksrf_ftoprad --- TOP parameters dataset for solar radiation parameterization ! ====================================== ! Must specify mapping file for the different datafiles above ! ====================================== @@ -295,6 +304,7 @@ program mksurfdat ! map_fslp10 ------ Mapping for mksrf_fslp10 ! map_fero -------- Mapping for mksrf_fero ! map_ffert ------- Mapping for mksrf_ffert + ! map_ftoprad_______ Mapping for mksrf_ftoprad ! ====================================== ! Optionally specify setting for: ! ====================================== @@ -464,7 +474,11 @@ program mksurfdat tillage(ns_o) , & litho(ns_o) , & nfert(ns_o,0:numpft) , & - pfert(ns_o,0:numpft) ) + pfert(ns_o,0:numpft) , & + sinsl_sinas(ns_o) , & + sinsl_cosas(ns_o) , & + sky_view(ns_o) , & + terrain_config(ns_o) ) landfrac_pft(:) = spval pctlnd_pft(:) = spval @@ -511,6 +525,10 @@ program mksurfdat litho(:) = spval nfert(:,:) = 0._r8 pfert(:,:) = 0._r8 + sinsl_sinas(:) = spval + sinsl_cosas(:) = spval + sky_view(:) = spval + terrain_config(:) = spval ! ---------------------------------------------------------------------- ! Open diagnostic output log file @@ -581,6 +599,7 @@ program mksurfdat write(ndiag,*)' mapping for slope percentile ',trim(map_fslp10) write(ndiag,*)' mapping for erosion params ',trim(map_fero) write(ndiag,*)' mapping for fertilizer ',trim(map_ffert) + write(ndiag,*)' mapping for TOP params ',trim(map_ftoprad) if (mksrf_fdynuse /= ' ') then write(6,*)'mksrf_fdynuse = ',trim(mksrf_fdynuse) @@ -768,6 +787,12 @@ program mksurfdat nfert_o=nfert, pfert_o=pfert) end if + call mktoprad(ldomain, mapfname=map_ftoprad, datfname=mksrf_ftoprad, varname = 'SINSL_SINAS', ndiag=ndiag, top_o=sinsl_sinas, nodata=0.0_r8) + call mktoprad(ldomain, mapfname=map_ftoprad, datfname=mksrf_ftoprad, varname = 'SINSL_COSAS', ndiag=ndiag, top_o=sinsl_cosas, nodata=0.0_r8) + call mktoprad(ldomain, mapfname=map_ftoprad, datfname=mksrf_ftoprad, varname = 'SKY_VIEW', ndiag=ndiag, top_o=sky_view, nodata=1.0_r8) + call mktoprad(ldomain, mapfname=map_ftoprad, datfname=mksrf_ftoprad, varname = 'TERRAIN_CONFIG', ndiag=ndiag, top_o=terrain_config, nodata=0.0_r8) + + do n = 1,ns_o ! Assume wetland and/or lake when dataset landmask implies ocean @@ -797,6 +822,10 @@ program mksurfdat litho(n) = 0._r8 nfert(n,:) = 0._r8 pfert(n,:) = 0._r8 + sinsl_sinas(n) = 0._r8 + sinsl_cosas(n) = 0._r8 + sky_view(n) = 1._r8 + terrain_config(n)= 0._r8 else pftdata_mask(n) = 1 end if @@ -1103,6 +1132,18 @@ program mksurfdat call check_ret(nf_inq_varid(ncid, 'Litho', varid), subname) call check_ret(nf_put_var_double(ncid, varid, litho), subname) + call check_ret(nf_inq_varid(ncid, 'SINSL_SINAS', varid), subname) + call check_ret(nf_put_var_double(ncid, varid, sinsl_sinas), subname) + + call check_ret(nf_inq_varid(ncid, 'SINSL_COSAS', varid), subname) + call check_ret(nf_put_var_double(ncid, varid, sinsl_cosas), subname) + + call check_ret(nf_inq_varid(ncid, 'SKY_VIEW', varid), subname) + call check_ret(nf_put_var_double(ncid, varid, sky_view), subname) + + call check_ret(nf_inq_varid(ncid, 'TERRAIN_CONFIG', varid), subname) + call check_ret(nf_put_var_double(ncid, varid, terrain_config), subname) + ! Deallocate arrays NOT needed for dynamic-pft section of code deallocate ( organic ) @@ -1123,6 +1164,7 @@ program mksurfdat deallocate ( grvl, slp10 ) deallocate ( ero_c1, ero_c2, ero_c3, tillage, litho ) deallocate ( nfert, pfert ) + deallocate ( sinsl_sinas, sinsl_cosas, sky_view, terrain_config ) ! Synchronize the disk copy of a netCDF dataset with in-memory buffers call check_ret(nf_sync(ncid), subname) diff --git a/components/elm/tools/mksurfdata_map/src/mktopradMod.F90 b/components/elm/tools/mksurfdata_map/src/mktopradMod.F90 new file mode 100644 index 000000000000..e8ea66f9de49 --- /dev/null +++ b/components/elm/tools/mksurfdata_map/src/mktopradMod.F90 @@ -0,0 +1,239 @@ +module mktopradMod +!----------------------------------------------------------------------- +!BOP +! +! !MODULE: mktopradMod +! +! !DESCRIPTION: +! Make topography data for TOP solar radiation parameterization +! +! !REVISION HISTORY: +! Author: Dalei Hao +! +!----------------------------------------------------------------------- +!!USES: + use shr_kind_mod, only : r8 => shr_kind_r8 + use shr_sys_mod , only : shr_sys_flush + implicit none + + SAVE + private ! By default make data private +! +! !PUBLIC MEMBER FUNCTIONS: +! + public mktopradAtt ! Add attributes to output file + + public mktoprad ! Set topography +! +! !PUBLIC DATA MEMBERS: +! +! +! !PRIVATE DATA MEMBERS: +! +! !PRIVATE MEMBER FUNCTIONS: + +!EOP +!=============================================================== +contains +!=============================================================== + +!----------------------------------------------------------------------- +!BOP +! +! !IROUTINE: mktoprad +! +! !INTERFACE: +subroutine mktoprad(ldomain, mapfname, datfname, varname, ndiag, top_o, nodata) +! +! !DESCRIPTION: +! Make topography data for TOP solar radiation parameterization +! +! !USES: + use mkdomainMod , only : domain_type, domain_clean, domain_read, domain_checksame + use mkgridmapMod + use mkvarpar + use mkvarctl + use mkncdio +! +! !ARGUMENTS: + implicit none + type(domain_type), intent(in) :: ldomain + character(len=*) , intent(in) :: mapfname ! input mapping file name + character(len=*) , intent(in) :: datfname ! input data file name + integer , intent(in) :: ndiag ! unit number for diag out + character(len=*) , intent(in) :: varname ! topo variable name + real(r8) , intent(out):: top_o(:) ! output topography data + real(r8) , intent(in) :: nodata ! default value +! +! +! !CALLED FROM: +! subroutine mksrfdat in module mksrfdatMod +! +! !REVISION HISTORY: +! Author: Dalei Hao +! +! +! !LOCAL VARIABLES: +!EOP + type(domain_type) :: tdomain ! local domain + type(gridmap_type) :: tgridmap ! local gridmap + + real(r8), allocatable :: top_i(:) ! input top variable + real(r8), allocatable :: mask_i(:) ! input grid: mask (0, 1) + integer :: ns_i,ns_o ! indices + integer :: k,l,n,m,ni ! indices + integer :: ncidi,dimid,varid ! input netCDF id's + integer :: ier ! error status + character(len=256) :: name ! name of attribute + character(len=256) :: unit ! units of attribute + character(len= 32) :: subname = 'mktop' +!----------------------------------------------------------------------- + + write (6,*) 'Attempting to make topography .....' + call shr_sys_flush(6) + + ns_o = ldomain%ns + + ! ----------------------------------------------------------------- + ! Read input file + ! ----------------------------------------------------------------- + + ! Obtain input grid info, read local fields + + call domain_read(tdomain,datfname) + + ns_i = tdomain%ns + allocate(top_i(ns_i), stat=ier) + if (ier /= 0) then + write(6,*)'mktoprad allocation error'; call abort() + end if + + write (6,*) 'Open topography file: ', trim(datfname) + call check_ret(nf_open(datfname, 0, ncidi), subname) + call check_ret(nf_inq_varid (ncidi, trim(varname), varid), subname) + call check_ret(nf_get_var_double (ncidi, varid, top_i), subname) + call check_ret(nf_close(ncidi), subname) + + ! Read topo dataset + call gridmap_mapread(tgridmap, mapfname) + + ! Error checks for domain and map consistencies + ! Note that the topo dataset has no landmask - so a unit landmask is assumed + + call domain_checksame( tdomain, ldomain, tgridmap ) + + ! Determine top_o on output grid + + top_o(:) = nodata + + call gridmap_areaave(tgridmap, top_i, top_o, nodata=nodata) + + ! Deallocate dynamic memory + + call domain_clean(tdomain) + call gridmap_clean(tgridmap) + deallocate (top_i) + + write (6,*) 'Successfully made topography parameters' + write (6,*) + call shr_sys_flush(6) + +end subroutine mktoprad + +!----------------------------------------------------------------------- + +!----------------------------------------------------------------------- +!BOP +! +! !IROUTINE: mktopradAtt +! +! !INTERFACE: +subroutine mktopradAtt( ncid, dynlanduse, xtype ) +! +! !DESCRIPTION: +! add atttributes to output file regarding the topography module +! +! !USES: + use fileutils , only : get_filename + use mkncdio , only : check_ret, ncd_defvar + use mkvarpar + use mkvarctl + +! !ARGUMENTS: + implicit none + include 'netcdf.inc' + integer, intent(in) :: ncid ! NetCDF file ID to write out to + logical, intent(in) :: dynlanduse ! if dynamic land-use file + integer, intent(in) :: xtype ! external type to output real data as +! +! !CALLED FROM: +! subroutine mkfile in module mkfileMod +! +! !REVISION HISTORY: +! Original Author: Dalei Hao +! +! +! !LOCAL VARIABLES: +!EOP + integer :: dimid ! temporary + character(len=256) :: str ! global attribute string + character(len=32) :: subname = 'mktopAtt' +!----------------------------------------------------------------------- + + if (.not. dynlanduse) then + + ! Add global attributes to file + + str = get_filename(mksrf_fgrvl) + call check_ret(nf_put_att_text(ncid, NF_GLOBAL, & + 'top_raw_data_file_name', len_trim(str), trim(str)), subname) + + ! Define variables + + if (outnc_1d) then + call ncd_defvar(ncid=ncid, varname='SINSL_COSAS', xtype=xtype, & + dim1name='gridcell',& + long_name='sin(slope) * cos(aspect)', units='unitless') + else + call ncd_defvar(ncid=ncid, varname='SINSL_COSAS', xtype=xtype, & + dim1name='lsmlon', dim2name='lsmlat', & + long_name='sin(slope) * cos(aspect)', units='unitless') + end if + + if (outnc_1d) then + call ncd_defvar(ncid=ncid, varname='SINSL_SINAS', xtype=xtype, & + dim1name='gridcell',& + long_name='sin(slope) * sin(aspect)', units='unitless') + else + call ncd_defvar(ncid=ncid, varname='SINSL_SINAS', xtype=xtype, & + dim1name='lsmlon', dim2name='lsmlat', & + long_name='sin(slope) * sin(aspect)', units='unitless') + end if + + if (outnc_1d) then + call ncd_defvar(ncid=ncid, varname='SKY_VIEW', xtype=xtype, & + dim1name='gridcell',& + long_name='sky view factor', units='unitless') + else + call ncd_defvar(ncid=ncid, varname='SKY_VIEW', xtype=xtype, & + dim1name='lsmlon', dim2name='lsmlat', & + long_name='sky view factor', units='unitless') + end if + + if (outnc_1d) then + call ncd_defvar(ncid=ncid, varname='TERRAIN_CONFIG', xtype=xtype, & + dim1name='gridcell',& + long_name='terrain configuration factor', units='unitless') + else + call ncd_defvar(ncid=ncid, varname='TERRAIN_CONFIG', xtype=xtype, & + dim1name='lsmlon', dim2name='lsmlat', & + long_name='terrain configuration factor', units='unitless') + end if + + end if + +end subroutine mktopradAtt + +!----------------------------------------------------------------------- + +end module mktopradMod diff --git a/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 b/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 index 48e745833749..127f0e83124b 100644 --- a/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 +++ b/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 @@ -52,7 +52,8 @@ module mkvarctl character(len=256), public :: mksrf_fslp10 = ' ' ! slope percentile file name character(len=256), public :: mksrf_fero = ' ' ! ELM-Erosion parameters data file name character(len=256), public :: mksrf_ffert = ' ' ! crop fertilizer data file name - integer , public :: numpft = 16 ! number of plant types + character(len=256), public :: mksrf_ftoprad = ' ' ! topography parameters data file name for TOP solar radiation parameterization + integer , public :: numpft = 16 ! number of plant types character(len=256), public :: map_fpft = ' ' ! Mapping file for PFT character(len=256), public :: map_flakwat = ' ' ! Mapping file for lake water @@ -80,6 +81,7 @@ module mkvarctl character(len=256), public :: map_fslp10 = ' ' ! Mapping file for slope percentile character(len=256), public :: map_fero = ' ' ! Mapping file for ELM-Erosion parameters character(len=256), public :: map_ffert = ' ' ! Mapping file for crop fertilizer! + character(len=256), public :: map_ftoprad = ' ' ! Mapping file for topography parameters in TOP solar radiation parameterization ! ! Variables to override data read in with ! (all_urban is mostly for single-point mode, but could be used for sensitivity studies) From 05fe0fb235387002dd73464964f18e33aa3ec024 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 11 Mar 2024 13:40:17 -0600 Subject: [PATCH 020/904] Remove old comment as per code review. --- .../physics/mam/eamxx_mam_dry_deposition_process_interface.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index aecea66a6c4a..b12c5b12a1f3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -411,8 +411,7 @@ void fill_tracer_views( // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - // Include this but update when scream is updated to use the newer DryDeposition class: - using DryDep = mam4::DryDeposition; // mam4::DryDeposition; + using DryDep = mam4::DryDeposition; using MemberType = KT::MemberType; const auto scan_policy = ekat::ExeSpaceUtils< From 76ca0876a255864e5aa425144524aa62fed2d1ba Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 25 Mar 2024 12:08:12 -0600 Subject: [PATCH 021/904] Add p_int (pressure at interface) to DryAtmosphere structure. --- .../eamxx_mam_dry_deposition_process_interface.cpp | 6 +----- components/eamxx/src/physics/mam/mam_coupling.hpp | 11 +++++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index b12c5b12a1f3..86be78c461b5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -215,6 +215,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.p_del = get_field_in("pseudo_density").get_view(); dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.z_iface = buffer_.z_iface; dry_atm_.dz = buffer_.dz; @@ -421,8 +422,6 @@ void MAMDryDep::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - std::cout<<__FILE__<<":"<<__LINE__<<" In MAMDryDep::run_impl"< Date: Tue, 26 Mar 2024 12:38:59 -0700 Subject: [PATCH 022/904] PAM VT update --- .../eam/src/physics/crm/pam/pam_driver.cpp | 14 ++++++------- .../physics/crm/pam/pam_variance_transport.h | 20 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index 2453f2ad61b2..e6f85ed0d5ac 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -25,7 +25,7 @@ #include "p3_f90.hpp" #include "pam_debug.h" -bool constexpr enable_check_state = true; +bool constexpr enable_check_state = false; extern "C" void pam_driver() { //------------------------------------------------------------------------------------------------ @@ -180,10 +180,11 @@ extern "C" void pam_driver() { // Apply forcing tendencies if (use_MMF_VT) { pam_variance_transport_apply_forcing(coupler); } - coupler.run_module( "apply_gcm_forcing_tendencies" , modules::apply_gcm_forcing_tendencies ); if (enable_check_state) { pam_debug_check_state(coupler, 2, nstep); } - coupler.run_module( "radiation" , [&] (pam::PamCoupler &coupler) {rad .timeStep(coupler);} ); + coupler.run_module( "apply_gcm_forcing_tendencies" , modules::apply_gcm_forcing_tendencies ); if (enable_check_state) { pam_debug_check_state(coupler, 3, nstep); } + coupler.run_module( "radiation" , [&] (pam::PamCoupler &coupler) {rad .timeStep(coupler);} ); + if (enable_check_state) { pam_debug_check_state(coupler, 4, nstep); } // Dynamics if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } @@ -191,13 +192,12 @@ extern "C" void pam_driver() { coupler.run_module( "dycore", [&] (pam::PamCoupler &coupler) {dycore.timeStep(coupler);} ); if (do_density_save_recall) { pam_state_recall_dry_density(coupler); } if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"dycor"); } - if (enable_check_state) { pam_debug_check_state(coupler, 4, nstep); } + if (enable_check_state) { pam_debug_check_state(coupler, 5, nstep); } // Sponge layer damping if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } coupler.run_module( "sponge_layer", modules::sponge_layer ); if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"sponge"); } - // if (enable_check_state) { pam_debug_check_state(coupler, 4, nstep); } // Apply hyperdiffusion to account for lack of horizontal mixing in SHOC pam_hyperdiffusion(coupler); @@ -207,13 +207,13 @@ extern "C" void pam_driver() { if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } coupler.run_module( "sgs", [&] (pam::PamCoupler &coupler) {sgs .timeStep(coupler);} ); if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"sgs"); } - if (enable_check_state) { pam_debug_check_state(coupler, 5, nstep); } + if (enable_check_state) { pam_debug_check_state(coupler, 6, nstep); } // Microphysics - P3 if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } coupler.run_module( "micro", [&] (pam::PamCoupler &coupler) {micro .timeStep(coupler);} ); if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"micro"); } - if (enable_check_state) { pam_debug_check_state(coupler, 6, nstep); } + if (enable_check_state) { pam_debug_check_state(coupler, 7, nstep); } // CRM mean state acceleration if (use_crm_accel && !coupler.get_option("crm_acceleration_ceaseflag")) { diff --git a/components/eam/src/physics/crm/pam/pam_variance_transport.h b/components/eam/src/physics/crm/pam/pam_variance_transport.h index b7067631203f..1033df4280d4 100644 --- a/components/eam/src/physics/crm/pam/pam_variance_transport.h +++ b/components/eam/src/physics/crm/pam/pam_variance_transport.h @@ -2,6 +2,7 @@ #include "pam_coupler.h" + inline void pam_variance_transport_init( pam::PamCoupler &coupler ) { using yakl::c::parallel_for; using yakl::c::SimpleBounds; @@ -24,6 +25,7 @@ inline void pam_variance_transport_init( pam::PamCoupler &coupler ) { //------------------------------------------------------------------------------------------------ } + inline void pam_variance_transport_diagnose( pam::PamCoupler &coupler ) { using yakl::c::parallel_for; using yakl::c::SimpleBounds; @@ -83,6 +85,7 @@ inline void pam_variance_transport_diagnose( pam::PamCoupler &coupler ) { //------------------------------------------------------------------------------------------------ } + inline void pam_variance_transport_compute_forcing( pam::PamCoupler &coupler ) { using yakl::c::parallel_for; using yakl::c::SimpleBounds; @@ -135,9 +138,10 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { // large-scale forcing from variance transport. This is meant to // protect against creating unstable situations, although // problematic scenarios were extremely rare in testing. - // A scaling limit of +/- 10% was found to be adequate. - real constexpr pert_scale_min = 1.0 - 0.005; - real constexpr pert_scale_max = 1.0 + 0.005; + // A scaling limit of +/- 10% was found to be adequate in SAM, + // but PAM seems much more sensitive (not sure why), so we use 0.1% here + real constexpr pert_scale_min = 1.0 - 0.001; + real constexpr pert_scale_max = 1.0 + 0.001; //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp" ); auto rhov = dm_device.get("water_vapor" ); @@ -162,14 +166,15 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { //------------------------------------------------------------------------------------------------ // calculate scaling factor for local perturbations parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { + real tmp; // initialize scaling factors to 1.0 temp_pert_scale(k,n) = 1.0; rhov_pert_scale(k,n) = 1.0; uvel_pert_scale(k,n) = 1.0; - // set scaling factors as long as there are perturbations to scale - if (vt_temp(k,n)>0.0) { temp_pert_scale(k,n) = sqrt( 1.0 + crm_dt * vt_temp_forcing_tend(k,n) / vt_temp(k,n) ); } - if (vt_rhov(k,n)>0.0) { rhov_pert_scale(k,n) = sqrt( 1.0 + crm_dt * vt_rhov_forcing_tend(k,n) / vt_rhov(k,n) ); } - if (vt_uvel(k,n)>0.0) { uvel_pert_scale(k,n) = sqrt( 1.0 + crm_dt * vt_uvel_forcing_tend(k,n) / vt_uvel(k,n) ); } + // calculate variance scaling factor + tmp = 1.+crm_dt*vt_temp_forcing_tend(k,n)/vt_temp(k,n); if (tmp>0){ temp_pert_scale(k,n) = sqrt(tmp); } + tmp = 1.+crm_dt*vt_rhov_forcing_tend(k,n)/vt_rhov(k,n); if (tmp>0){ rhov_pert_scale(k,n) = sqrt(tmp); } + tmp = 1.+crm_dt*vt_uvel_forcing_tend(k,n)/vt_uvel(k,n); if (tmp>0){ uvel_pert_scale(k,n) = sqrt(tmp); } // enforce minimum scaling temp_pert_scale(k,n) = std::max( temp_pert_scale(k,n), pert_scale_min ); rhov_pert_scale(k,n) = std::max( rhov_pert_scale(k,n), pert_scale_min ); @@ -261,7 +266,6 @@ inline void pam_variance_transport_copy_to_host( pam::PamCoupler &coupler ) { vt_temp_feedback_tend.deep_copy_to(output_vt_temp_tend_host); vt_rhov_feedback_tend.deep_copy_to(output_vt_rhov_tend_host); vt_uvel_feedback_tend.deep_copy_to(output_vt_uvel_tend_host); - yakl::fence(); //------------------------------------------------------------------------------------------------ } From 6fe86f61e08038fb95f964b0cfb5907d5a53423a Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 26 Mar 2024 12:47:40 -0700 Subject: [PATCH 023/904] update PAM submodule --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 9d692dccdd67..ff652f3f2a6c 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 9d692dccdd67d9c4dc278c76290504a2af8cd187 +Subproject commit ff652f3f2a6c2d675cd957883245aa6c036ef110 From 59b9476bbc96ed060ff12e4a7ba3480795b67872 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 26 Mar 2024 12:48:49 -0700 Subject: [PATCH 024/904] update PAM hyperdiffusion timescale default --- components/eam/src/physics/crm/pam/pam_hyperdiffusion.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h b/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h index 4452c583afe3..b632b0e49093 100644 --- a/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h +++ b/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h @@ -28,7 +28,7 @@ inline void pam_hyperdiffusion( pam::PamCoupler &coupler ) { #ifdef MMF_PAM_HDT real constexpr hd_timescale = MMF_PAM_HDT; // damping time scale [sec] #else - real constexpr hd_timescale = 10.0; // damping time scale [sec] + real constexpr hd_timescale = 60.0; // damping time scale [sec] #endif //------------------------------------------------------------------------------------------------ real4d hd_temp("hd_temp",nz,ny,nx,nens); From ebdb83028c0c026a227c038ffdf9279c575bab42 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 29 Mar 2024 14:05:15 -0700 Subject: [PATCH 025/904] revert CRM timestep to 5 sec for MMF2 --- components/eam/cime_config/config_component.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 577c3256db42..784fe44195ed 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -72,8 +72,8 @@ -crm samxx -crm_dt 10 - -crm pam -pam_dycor spam -crm_dt 8 - -crm pam -pam_dycor awfl -crm_dt 8 + -crm pam -pam_dycor spam -crm_dt 5 + -crm pam -pam_dycor awfl -crm_dt 5 -use_MMF -nlev 60 -crm_nz 50 -crm_dx 2000 -crm_nx 64 -crm_ny 1 -crm_nx_rad 4 -crm_ny_rad 1 -crm_dx 2000 -crm_nx 45 -crm_ny 1 -crm_nx_rad 5 -crm_ny_rad 1 From 0194a2b1ea565e95765fd741d95def019414c3b6 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 5 Apr 2024 08:18:19 -0700 Subject: [PATCH 026/904] pam_debug update --- .../eam/src/physics/crm/pam/pam_debug.h | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_debug.h b/components/eam/src/physics/crm/pam/pam_debug.h index 937d18cf154e..2d3dc1efb894 100644 --- a/components/eam/src/physics/crm/pam/pam_debug.h +++ b/components/eam/src/physics/crm/pam/pam_debug.h @@ -27,28 +27,32 @@ inline void pam_debug_init( pam::PamCoupler &coupler ) { auto nens = coupler.get_option("ncrms"); //------------------------------------------------------------------------------------------------ dm_device.register_and_allocate("debug_save_temp", "saved temp for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); + dm_device.register_and_allocate("debug_save_rhod", "saved rhod for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); dm_device.register_and_allocate("debug_save_rhov", "saved rhov for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); dm_device.register_and_allocate("debug_save_rhoc", "saved rhoc for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); dm_device.register_and_allocate("debug_save_rhoi", "saved rhoi for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); dm_device.register_and_allocate("debug_save_uvel", "saved uvel for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); dm_device.register_and_allocate("debug_save_wvel", "saved wvel for debug", {nz,ny,nx,nens}, {"z","y","x","nens"} ); auto debug_save_temp = dm_device.get("debug_save_temp"); + auto debug_save_rhod = dm_device.get("debug_save_rhod"); auto debug_save_rhov = dm_device.get("debug_save_rhov"); auto debug_save_rhoc = dm_device.get("debug_save_rhoc"); auto debug_save_rhoi = dm_device.get("debug_save_rhoi"); auto debug_save_uvel = dm_device.get("debug_save_uvel"); auto debug_save_wvel = dm_device.get("debug_save_wvel"); //------------------------------------------------------------------------------------------------ - auto temp = dm_device.get("temp"); - auto rhov = dm_device.get("water_vapor"); - auto rhoc = dm_device.get("cloud_water"); - auto rhoi = dm_device.get("ice"); - auto uvel = dm_device.get("uvel"); - auto wvel = dm_device.get("wvel"); + auto temp = dm_device.get("temp"); + auto rhod = dm_device.get("density_dry"); + auto rhov = dm_device.get("water_vapor"); + auto rhoc = dm_device.get("cloud_water"); + auto rhoi = dm_device.get("ice"); + auto uvel = dm_device.get("uvel"); + auto wvel = dm_device.get("wvel"); //------------------------------------------------------------------------------------------------ parallel_for("copy data to saved debug variables", SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { debug_save_temp(k,j,i,iens) = temp(k,j,i,iens); + debug_save_rhod(k,j,i,iens) = rhod(k,j,i,iens); debug_save_rhov(k,j,i,iens) = rhov(k,j,i,iens); debug_save_rhoc(k,j,i,iens) = rhoc(k,j,i,iens); debug_save_rhoi(k,j,i,iens) = rhoi(k,j,i,iens); @@ -69,12 +73,14 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { auto nz = coupler.get_option("crm_nz"); auto nens = coupler.get_option("ncrms"); auto temp = dm_device.get("temp"); + auto rhod = dm_device.get("density_dry"); auto rhov = dm_device.get("water_vapor"); auto rhoc = dm_device.get("cloud_water"); auto rhoi = dm_device.get("ice"); auto uvel = dm_device.get("uvel"); auto wvel = dm_device.get("wvel"); auto debug_save_temp = dm_device.get("debug_save_temp"); + auto debug_save_rhod = dm_device.get("debug_save_rhod"); auto debug_save_rhov = dm_device.get("debug_save_rhov"); auto debug_save_rhoc = dm_device.get("debug_save_rhoc"); auto debug_save_rhoi = dm_device.get("debug_save_rhoi"); @@ -90,18 +96,21 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { auto phis = input_phis(iens)/grav; // Check for NaNs const auto is_nan_t_atm = isnan( temp(k,j,i,iens) ); + const auto is_nan_d_atm = isnan( rhod(k,j,i,iens) ); const auto is_nan_q_atm = isnan( rhov(k,j,i,iens) ); - if ( is_nan_t_atm || is_nan_q_atm ) { + if ( is_nan_t_atm || is_nan_q_atm || is_nan_d_atm ) { auto phis = input_phis(iens)/grav; - printf("PAM-DEBUG nan-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + printf("PAM-DEBUG nan-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", nstep,id,k,i,iens,lat(iens),lon(iens),phis, temp(k,j,i,iens), + rhod(k,j,i,iens), rhov(k,j,i,iens), rhoc(k,j,i,iens), rhoi(k,j,i,iens), uvel(k,j,i,iens), wvel(k,j,i,iens), debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), debug_save_rhov(k,j,i,iens), debug_save_rhoc(k,j,i,iens), debug_save_rhoi(k,j,i,iens), @@ -111,18 +120,21 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { } // Check for negative values const auto is_neg_t_atm = temp(k,j,i,iens)<0; + const auto is_neg_d_atm = rhod(k,j,i,iens)<0; const auto is_neg_q_atm = rhov(k,j,i,iens)<0; - if ( is_neg_t_atm || is_neg_q_atm ) { + if ( is_neg_t_atm || is_neg_q_atm || is_neg_d_atm ) { auto phis = input_phis(iens)/grav; - printf("PAM-DEBUG neg-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + printf("PAM-DEBUG neg-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", nstep,id,k,i,iens,lat(iens),lon(iens),phis, temp(k,j,i,iens), + rhod(k,j,i,iens), rhov(k,j,i,iens), rhoc(k,j,i,iens), rhoi(k,j,i,iens), uvel(k,j,i,iens), wvel(k,j,i,iens), debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), debug_save_rhov(k,j,i,iens), debug_save_rhoc(k,j,i,iens), debug_save_rhoi(k,j,i,iens), @@ -133,15 +145,17 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { // Check for low temperature const auto is_low_t = temp(k,j,i,iens)<100; if ( is_low_t ) { - printf("PAM-DEBUG low-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + printf("PAM-DEBUG low-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", nstep,id,k,i,iens,lat(iens),lon(iens),phis, temp(k,j,i,iens), + rhod(k,j,i,iens), rhov(k,j,i,iens), rhoc(k,j,i,iens), rhoi(k,j,i,iens), uvel(k,j,i,iens), wvel(k,j,i,iens), debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), debug_save_rhov(k,j,i,iens), debug_save_rhoc(k,j,i,iens), debug_save_rhoi(k,j,i,iens), @@ -153,15 +167,17 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { const auto is_large_pos_w = wvel(k,j,i,iens)> 40; const auto is_large_neg_w = wvel(k,j,i,iens)<-40; if ( is_large_pos_w || is_large_neg_w ) { - printf("PAM-DEBUG large-W - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + printf("PAM-DEBUG large-W - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", nstep,id,k,i,iens,lat(iens),lon(iens),phis, temp(k,j,i,iens), + rhod(k,j,i,iens), rhov(k,j,i,iens), rhoc(k,j,i,iens), rhoi(k,j,i,iens), uvel(k,j,i,iens), wvel(k,j,i,iens), debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), debug_save_rhov(k,j,i,iens), debug_save_rhoc(k,j,i,iens), debug_save_rhoi(k,j,i,iens), @@ -171,6 +187,7 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { } // update saved previous values debug_save_temp(k,j,i,iens) = temp(k,j,i,iens); + debug_save_rhod(k,j,i,iens) = rhod(k,j,i,iens); debug_save_rhov(k,j,i,iens) = rhov(k,j,i,iens); debug_save_rhoc(k,j,i,iens) = rhoc(k,j,i,iens); debug_save_rhoi(k,j,i,iens) = rhoi(k,j,i,iens); From 730931f76f3a17a9c2da0bad2407190707f377d3 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 9 Apr 2024 11:17:12 -0700 Subject: [PATCH 027/904] bug fix in pam_state --- .../eam/src/physics/crm/pam/pam_state.h | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_state.h b/components/eam/src/physics/crm/pam/pam_state.h index 3266899ba217..a34a867eb6e8 100644 --- a/components/eam/src/physics/crm/pam/pam_state.h +++ b/components/eam/src/physics/crm/pam/pam_state.h @@ -144,7 +144,7 @@ inline void pam_state_set_reference_state( pam::PamCoupler &coupler ) { real2d hmean_temp ("hmean_temp" ,nz ,nens); real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions // initialize horizontal means - parallel_for(SimpleBounds<2>(nz,nens), YAKL_LAMBDA (int k, int iens) { + parallel_for(SimpleBounds<2>(nz+1,nens), YAKL_LAMBDA (int k, int iens) { hmean_pint(k,iens) = 0; if (k < nz) { hmean_pmid (k,iens) = 0; @@ -166,24 +166,42 @@ inline void pam_state_set_reference_state( pam::PamCoupler &coupler ) { atomicAdd( hmean_temp (k,iens), temp (k,j,i,iens) * r_nx_ny ); }); // calculate interface pressure from mid-level pressure - parallel_for(SimpleBounds<4>(nz+1,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int iens) { + // parallel_for(SimpleBounds<4>(nz+1,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int iens) { + parallel_for(SimpleBounds<2>(nz+1,nens) , YAKL_LAMBDA (int k, int iens) { if (k == 0 ) { - real rho = rho_d(k ,j,i,iens)+rho_v(k ,j,i,iens); - real dz = zint(k+1,iens)-zint(k ,iens); + real rho = hmean_rho_d(k,iens)+hmean_rho_v(k,iens); + real dz = zint(k+1,iens)-zint(k,iens); hmean_pint(k,iens) = hmean_pmid(k ,iens) + grav*rho*dz/2; } else if (k == nz) { - real rho = rho_d(k-1,j,i,iens)+rho_v(k-1,j,i,iens); - real dz = zint(k ,iens)-zint(k-1,iens); + real rho = hmean_rho_d(k-1,iens)+hmean_rho_v(k-1,iens); + real dz = zint(k,iens)-zint(k-1,iens); hmean_pint(k,iens) = hmean_pmid(k-1,iens) - grav*rho*dz/2; } else { - real rhokm1 = rho_d(k-1,j,i,iens)+rho_v(k-1,j,i,iens); - real rhokm0 = rho_d(k ,j,i,iens)+rho_v(k ,j,i,iens); + real rhokm1 = hmean_rho_d(k-1,iens)+hmean_rho_v(k-1,iens);; + real rhokm0 = hmean_rho_d(k ,iens)+hmean_rho_v(k ,iens); real dzkm1 = zint(k ,iens)-zint(k-1,iens); real dzkm0 = zint(k+1,iens)-zint(k ,iens); hmean_pint(k,iens) = 0.5_fp * ( hmean_pmid(k-1,iens) - grav*rhokm1*dzkm1/2 + hmean_pmid(k ,iens) + grav*rhokm0*dzkm0/2 ); } }); + + auto &dm_host = coupler.get_data_manager_host_readonly(); + auto lat = dm_host.get("latitude" ).createDeviceCopy(); + auto lon = dm_host.get("longitude" ).createDeviceCopy(); + auto input_phis = dm_host.get("input_phis").createDeviceCopy(); + // check that interface pressure is reasonable + parallel_for(SimpleBounds<2>(nz+1,nens) , YAKL_LAMBDA (int k, int iens) { + if ( hmean_pint(k,iens) < hmean_pmid(k,iens) ) { + auto phis = input_phis(iens)/grav; + printf("PAM-DEBUG bad-pint - k:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- pint:%8.2g pmid:%8.2g \n", + k,iens,lat(iens),lon(iens),phis, + hmean_pint(k,iens), + hmean_pmid(k,iens) + ); + } + }); + // set anelastic reference state from CRM horizontal mean parallel_for(SimpleBounds<2>(nz+1,nens) , YAKL_LAMBDA (int k, int iens) { ref_presi(k,iens) = hmean_pint(k,iens); From b094fc0db0534982a3f52aadd2ba5466e09be4e7 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 9 Apr 2024 11:20:48 -0700 Subject: [PATCH 028/904] pam_debug updates --- .../eam/src/physics/crm/pam/pam_debug.h | 73 ++++++++++++------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_debug.h b/components/eam/src/physics/crm/pam/pam_debug.h index 2d3dc1efb894..43d2b5bf4f5d 100644 --- a/components/eam/src/physics/crm/pam/pam_debug.h +++ b/components/eam/src/physics/crm/pam/pam_debug.h @@ -142,32 +142,31 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { debug_save_wvel(k,j,i,iens) ); } - // Check for low temperature - const auto is_low_t = temp(k,j,i,iens)<100; - if ( is_low_t ) { - printf("PAM-DEBUG low-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", - nstep,id,k,i,iens,lat(iens),lon(iens),phis, - temp(k,j,i,iens), - rhod(k,j,i,iens), - rhov(k,j,i,iens), - rhoc(k,j,i,iens), - rhoi(k,j,i,iens), - uvel(k,j,i,iens), - wvel(k,j,i,iens), - debug_save_temp(k,j,i,iens), - debug_save_rhod(k,j,i,iens), - debug_save_rhov(k,j,i,iens), - debug_save_rhoc(k,j,i,iens), - debug_save_rhoi(k,j,i,iens), - debug_save_uvel(k,j,i,iens), - debug_save_wvel(k,j,i,iens) - ); - } - // Check for large vertical velocity - const auto is_large_pos_w = wvel(k,j,i,iens)> 40; - const auto is_large_neg_w = wvel(k,j,i,iens)<-40; - if ( is_large_pos_w || is_large_neg_w ) { - printf("PAM-DEBUG large-W - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + // // Check for low temperature + // const auto is_low_t = temp(k,j,i,iens)<100; + // if ( is_low_t ) { + // printf("PAM-DEBUG low-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + // nstep,id,k,i,iens,lat(iens),lon(iens),phis, + // temp(k,j,i,iens), + // rhod(k,j,i,iens), + // rhov(k,j,i,iens), + // rhoc(k,j,i,iens), + // rhoi(k,j,i,iens), + // uvel(k,j,i,iens), + // wvel(k,j,i,iens), + // debug_save_temp(k,j,i,iens), + // debug_save_rhod(k,j,i,iens), + // debug_save_rhov(k,j,i,iens), + // debug_save_rhoc(k,j,i,iens), + // debug_save_rhoi(k,j,i,iens), + // debug_save_uvel(k,j,i,iens), + // debug_save_wvel(k,j,i,iens) + // ); + // } + // Check for large temperature drops + const auto is_drop_t = (temp(k,j,i,iens)-debug_save_temp(k,j,i,iens))<-50; + if ( is_drop_t ) { + printf("PAM-DEBUG drop-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", nstep,id,k,i,iens,lat(iens),lon(iens),phis, temp(k,j,i,iens), rhod(k,j,i,iens), @@ -185,6 +184,28 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { debug_save_wvel(k,j,i,iens) ); } + // // Check for large vertical velocity + // const auto is_large_pos_w = wvel(k,j,i,iens)> 40; + // const auto is_large_neg_w = wvel(k,j,i,iens)<-40; + // if ( is_large_pos_w || is_large_neg_w ) { + // printf("PAM-DEBUG large-W - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + // nstep,id,k,i,iens,lat(iens),lon(iens),phis, + // temp(k,j,i,iens), + // rhod(k,j,i,iens), + // rhov(k,j,i,iens), + // rhoc(k,j,i,iens), + // rhoi(k,j,i,iens), + // uvel(k,j,i,iens), + // wvel(k,j,i,iens), + // debug_save_temp(k,j,i,iens), + // debug_save_rhod(k,j,i,iens), + // debug_save_rhov(k,j,i,iens), + // debug_save_rhoc(k,j,i,iens), + // debug_save_rhoi(k,j,i,iens), + // debug_save_uvel(k,j,i,iens), + // debug_save_wvel(k,j,i,iens) + // ); + // } // update saved previous values debug_save_temp(k,j,i,iens) = temp(k,j,i,iens); debug_save_rhod(k,j,i,iens) = rhod(k,j,i,iens); From 0720d0dca888c047638492a6c5312fb5932d88aa Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 9 Apr 2024 11:24:16 -0700 Subject: [PATCH 029/904] disable mean-state acceleration for dry density --- .../eam/src/physics/crm/pam/pam_accelerate.h | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_accelerate.h b/components/eam/src/physics/crm/pam/pam_accelerate.h index a48ca628f691..a7b179a85c9d 100644 --- a/components/eam/src/physics/crm/pam/pam_accelerate.h +++ b/components/eam/src/physics/crm/pam/pam_accelerate.h @@ -25,7 +25,7 @@ inline void pam_accelerate_init( pam::PamCoupler &coupler ) { auto crm_accel_uv = coupler.get_option("crm_accel_uv"); //------------------------------------------------------------------------------------------------ dm_device.register_and_allocate("accel_save_t", "saved temperature for MSA", {nz,nens}, {"z","nens"} ); - dm_device.register_and_allocate("accel_save_r", "saved dry density for MSA", {nz,nens}, {"z","nens"} ); + // dm_device.register_and_allocate("accel_save_r", "saved dry density for MSA", {nz,nens}, {"z","nens"} ); dm_device.register_and_allocate("accel_save_q", "saved total water for MSA", {nz,nens}, {"z","nens"} ); dm_device.register_and_allocate("accel_save_u", "saved uvel for MSA", {nz,nens}, {"z","nens"} ); dm_device.register_and_allocate("accel_save_v", "saved vvel for MSA", {nz,nens}, {"z","nens"} ); @@ -45,14 +45,14 @@ inline void pam_accelerate_diagnose( pam::PamCoupler &coupler ) { auto crm_accel_uv = coupler.get_option("crm_accel_uv"); //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp" ); - auto rhod = dm_device.get("density_dry"); + // auto rhod = dm_device.get("density_dry"); auto rhov = dm_device.get("water_vapor"); auto rhol = dm_device.get("cloud_water"); auto rhoi = dm_device.get("ice" ); auto uvel = dm_device.get("uvel" ); auto vvel = dm_device.get("vvel" ); auto accel_save_t = dm_device.get("accel_save_t"); - auto accel_save_r = dm_device.get("accel_save_r"); + // auto accel_save_r = dm_device.get("accel_save_r"); auto accel_save_q = dm_device.get("accel_save_q"); auto accel_save_u = dm_device.get("accel_save_u"); auto accel_save_v = dm_device.get("accel_save_v"); @@ -60,7 +60,7 @@ inline void pam_accelerate_diagnose( pam::PamCoupler &coupler ) { // compute horizontal means needed later for mean-state acceleration parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { accel_save_t(k,n) = 0.0; - accel_save_r(k,n) = 0.0; + // accel_save_r(k,n) = 0.0; accel_save_q(k,n) = 0.0; if (crm_accel_uv) { accel_save_u(k,n) = 0.0; @@ -70,7 +70,7 @@ inline void pam_accelerate_diagnose( pam::PamCoupler &coupler ) { real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions parallel_for( SimpleBounds<4>(nz,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int n) { yakl::atomicAdd( accel_save_t(k,n), temp(k,j,i,n) * r_nx_ny ); - yakl::atomicAdd( accel_save_r(k,n), rhod(k,j,i,n) * r_nx_ny ); + // yakl::atomicAdd( accel_save_r(k,n), rhod(k,j,i,n) * r_nx_ny ); yakl::atomicAdd( accel_save_q(k,n), ( rhov(k,j,i,n) + rhol(k,j,i,n) + rhoi(k,j,i,n) ) * r_nx_ny ); if (crm_accel_uv) { yakl::atomicAdd( accel_save_u(k,n), uvel(k,j,i,n) * r_nx_ny ); @@ -93,14 +93,14 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { auto nx = coupler.get_option("crm_nx"); //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp" ); - auto rhod = dm_device.get("density_dry"); + // auto rhod = dm_device.get("density_dry"); auto rhov = dm_device.get("water_vapor"); auto rhol = dm_device.get("cloud_water"); auto rhoi = dm_device.get("ice" ); auto uvel = dm_device.get("uvel" ); auto vvel = dm_device.get("vvel" ); auto accel_save_t = dm_device.get("accel_save_t"); - auto accel_save_r = dm_device.get("accel_save_r"); + // auto accel_save_r = dm_device.get("accel_save_r"); auto accel_save_q = dm_device.get("accel_save_q"); auto accel_save_u = dm_device.get("accel_save_u"); auto accel_save_v = dm_device.get("accel_save_v"); @@ -109,12 +109,12 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { real crm_accel_factor = coupler.get_option("crm_accel_factor"); //------------------------------------------------------------------------------------------------ real2d hmean_t ("hmean_t", nz,nens); - real2d hmean_r ("hmean_r", nz,nens); + // real2d hmean_r ("hmean_r", nz,nens); real2d hmean_q ("hmean_q", nz,nens); real2d hmean_u ("hmean_u", nz,nens); real2d hmean_v ("hmean_v", nz,nens); real2d ttend_acc("ttend_acc", nz,nens); - real2d rtend_acc("rtend_acc", nz,nens); + // real2d rtend_acc("rtend_acc", nz,nens); real2d qtend_acc("qtend_acc", nz,nens); real2d utend_acc("utend_acc", nz,nens); real2d vtend_acc("vtend_acc", nz,nens); @@ -127,7 +127,7 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { // Compute the horizontal mean for each variable parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { hmean_t(k,n) = 0.0; - hmean_r(k,n) = 0.0; + // hmean_r(k,n) = 0.0; hmean_q(k,n) = 0.0; if (crm_accel_uv) { hmean_u(k,n) = 0.0; @@ -137,7 +137,7 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions parallel_for( SimpleBounds<4>(nz,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int n) { yakl::atomicAdd( hmean_t(k,n), temp(k,j,i,n) * r_nx_ny ); - yakl::atomicAdd( hmean_r(k,n), rhod(k,j,i,n) * r_nx_ny ); + // yakl::atomicAdd( hmean_r(k,n), rhod(k,j,i,n) * r_nx_ny ); yakl::atomicAdd( hmean_q(k,n), ( rhov(k,j,i,n) + rhol(k,j,i,n) + rhoi(k,j,i,n) ) * r_nx_ny ); if (crm_accel_uv) { yakl::atomicAdd( hmean_u(k,n), uvel(k,j,i,n) * r_nx_ny ); @@ -150,7 +150,7 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { ScalarLiveOut ceaseflag_liveout(false); parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { ttend_acc(k,n) = hmean_t(k,n) - accel_save_t(k,n); - rtend_acc(k,n) = hmean_r(k,n) - accel_save_r(k,n); + // rtend_acc(k,n) = hmean_r(k,n) - accel_save_r(k,n); qtend_acc(k,n) = hmean_q(k,n) - accel_save_q(k,n); if (crm_accel_uv) { utend_acc(k,n) = hmean_u(k,n) - accel_save_u(k,n); @@ -193,7 +193,7 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { parallel_for( SimpleBounds<4>(nz,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int n) { temp(k,j,i,n) = temp(k,j,i,n) + crm_accel_factor * ttend_acc(k,n); - rhod(k,j,i,n) = rhod(k,j,i,n) + crm_accel_factor * rtend_acc(k,n); + // rhod(k,j,i,n) = rhod(k,j,i,n) + crm_accel_factor * rtend_acc(k,n); rhov(k,j,i,n) = rhov(k,j,i,n) + crm_accel_factor * qtend_acc(k,n); if (crm_accel_uv) { uvel(k,j,i,n) = uvel(k,j,i,n) + crm_accel_factor * utend_acc(k,n); From 081b18326623709d8f7019d4fcaedec551c51852 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 9 Apr 2024 11:29:11 -0700 Subject: [PATCH 030/904] update pam_driver --- components/eam/src/physics/crm/pam/pam_driver.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index e6f85ed0d5ac..69e67a90894c 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -25,7 +25,7 @@ #include "p3_f90.hpp" #include "pam_debug.h" -bool constexpr enable_check_state = false; +bool constexpr enable_check_state = true; extern "C" void pam_driver() { //------------------------------------------------------------------------------------------------ @@ -62,7 +62,7 @@ extern "C" void pam_driver() { coupler.set_option("spam_clip_vertical_velocities",true); coupler.set_option("spam_adjust_crm_per_phys_using_vert_cfl",true); coupler.set_option("spam_target_cfl",0.7); - coupler.set_option("spam_max_w",50.0); + coupler.set_option("spam_max_w",30.0); //------------------------------------------------------------------------------------------------ // Allocate the coupler state and retrieve host/device data managers coupler.allocate_coupler_state( crm_nz , crm_ny , crm_nx , nens ); @@ -91,8 +91,10 @@ extern "C" void pam_driver() { // Copy input CRM state (saved by the GCM) to coupler pam_state_copy_input_to_coupler(coupler); - // // update CRM dry density to match GCM and disable dry density forcing - // pam_state_update_dry_density(coupler); + #ifdef MMF_DISABLE_DENSITY_FORCING + // update CRM dry density to match GCM and disable dry density forcing + pam_state_update_dry_density(coupler); + #endif // if debugging - initialize saved state variables and check initial CRM state if (enable_check_state) { From fcb801bf90e7bd379b510e2c2ff3ee75e5992186 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 12:06:32 -0400 Subject: [PATCH 031/904] adjust white space --- components/eam/src/physics/crm/crm_physics.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eam/src/physics/crm/crm_physics.F90 b/components/eam/src/physics/crm/crm_physics.F90 index 2beeded06bb3..51628d5ea74e 100644 --- a/components/eam/src/physics/crm/crm_physics.F90 +++ b/components/eam/src/physics/crm/crm_physics.F90 @@ -1384,8 +1384,8 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, call pam_mirror_array_readwrite( 'output_ni_mean', crm_output%ni_mean, '' ) call pam_mirror_array_readwrite( 'output_qm_mean', crm_output%qm_mean, '' ) call pam_mirror_array_readwrite( 'output_bm_mean', crm_output%bm_mean, '' ) - call pam_mirror_array_readwrite( 'output_rho_d_mean', crm_output%rho_d_mean, '' ) - call pam_mirror_array_readwrite( 'output_rho_v_mean', crm_output%rho_v_mean, '' ) + call pam_mirror_array_readwrite( 'output_rho_d_mean', crm_output%rho_d_mean, '' ) + call pam_mirror_array_readwrite( 'output_rho_v_mean', crm_output%rho_v_mean, '' ) call pam_mirror_array_readwrite( 'output_qt_ls', crm_output%qt_ls, '' ) call pam_mirror_array_readwrite( 'output_t_ls', crm_output%t_ls, '' ) From 44ba5145df639f554be9e931740848f39a63464f Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 12:13:13 -0400 Subject: [PATCH 032/904] change how PAM reference state pressure is set --- .../eam/src/physics/crm/pam/pam_state.h | 57 +++++++++---------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_state.h b/components/eam/src/physics/crm/pam/pam_state.h index a34a867eb6e8..fb6553df07ef 100644 --- a/components/eam/src/physics/crm/pam/pam_state.h +++ b/components/eam/src/physics/crm/pam/pam_state.h @@ -108,6 +108,7 @@ inline void pam_state_set_reference_state( pam::PamCoupler &coupler ) { auto &dm_device = coupler.get_data_manager_device_readwrite(); auto nens = coupler.get_option("ncrms"); auto nz = coupler.get_option("crm_nz"); + auto gcm_nlev = coupler.get_option("gcm_nlev"); auto ref_presi = dm_device.get("ref_presi"); auto ref_pres = dm_device.get("ref_pres"); auto ref_rho_d = dm_device.get("ref_density_dry"); @@ -131,6 +132,14 @@ inline void pam_state_set_reference_state( pam::PamCoupler &coupler ) { auto rho_v = dm_device.get("water_vapor"); auto rho_c = dm_device.get("cloud_water"); auto rho_i = dm_device.get("ice"); + + auto &dm_host = coupler.get_data_manager_host_readonly(); + auto input_pmid = dm_host.get("input_pmid").createDeviceCopy(); + auto input_pint = dm_host.get("input_pint").createDeviceCopy(); + + auto lat = dm_host.get("latitude" ).createDeviceCopy(); + auto lon = dm_host.get("longitude" ).createDeviceCopy(); + auto input_phis = dm_host.get("input_phis").createDeviceCopy(); //------------------------------------------------------------------------------------------------ // Set anelastic reference state with current CRM mean state @@ -157,48 +166,34 @@ inline void pam_state_set_reference_state( pam::PamCoupler &coupler ) { }); // calculate horizontal means parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { - real pmid_tmp = rho_d(k,j,i,iens)*R_d*temp(k,j,i,iens) + rho_v(k,j,i,iens)*R_v*temp(k,j,i,iens); - atomicAdd( hmean_pmid (k,iens), pmid_tmp * r_nx_ny ); atomicAdd( hmean_rho_d(k,iens), rho_d (k,j,i,iens) * r_nx_ny ); atomicAdd( hmean_rho_v(k,iens), rho_v (k,j,i,iens) * r_nx_ny ); atomicAdd( hmean_rho_c(k,iens), rho_c (k,j,i,iens) * r_nx_ny ); atomicAdd( hmean_rho_i(k,iens), rho_i (k,j,i,iens) * r_nx_ny ); atomicAdd( hmean_temp (k,iens), temp (k,j,i,iens) * r_nx_ny ); }); - // calculate interface pressure from mid-level pressure - // parallel_for(SimpleBounds<4>(nz+1,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int iens) { - parallel_for(SimpleBounds<2>(nz+1,nens) , YAKL_LAMBDA (int k, int iens) { - if (k == 0 ) { - real rho = hmean_rho_d(k,iens)+hmean_rho_v(k,iens); - real dz = zint(k+1,iens)-zint(k,iens); - hmean_pint(k,iens) = hmean_pmid(k ,iens) + grav*rho*dz/2; - } else if (k == nz) { - real rho = hmean_rho_d(k-1,iens)+hmean_rho_v(k-1,iens); - real dz = zint(k,iens)-zint(k-1,iens); - hmean_pint(k,iens) = hmean_pmid(k-1,iens) - grav*rho*dz/2; - } else { - real rhokm1 = hmean_rho_d(k-1,iens)+hmean_rho_v(k-1,iens);; - real rhokm0 = hmean_rho_d(k ,iens)+hmean_rho_v(k ,iens); - real dzkm1 = zint(k ,iens)-zint(k-1,iens); - real dzkm0 = zint(k+1,iens)-zint(k ,iens); - hmean_pint(k,iens) = 0.5_fp * ( hmean_pmid(k-1,iens) - grav*rhokm1*dzkm1/2 + - hmean_pmid(k ,iens) + grav*rhokm0*dzkm0/2 ); - } + + // Use GCM state for reference pressure - previously, the current CRM pressure + // was used, but the way the interface pressure was calculated led to problems + // in edge cases (ex. over topography) - switching to GCM pressure works well + parallel_for( SimpleBounds<2>(nz+1,nens) , YAKL_LAMBDA (int k_crm, int iens) { + int k_gcm = (gcm_nlev+1)-1-k_crm; + hmean_pint(k_crm,iens) = input_pint(k_gcm,iens); + }); + parallel_for(SimpleBounds<2>(nz,nens), YAKL_LAMBDA (int k_crm, int iens) { + int k_gcm = gcm_nlev-1-k_crm; + hmean_pmid(k_crm,iens) = input_pmid(k_gcm,iens); }); - auto &dm_host = coupler.get_data_manager_host_readonly(); - auto lat = dm_host.get("latitude" ).createDeviceCopy(); - auto lon = dm_host.get("longitude" ).createDeviceCopy(); - auto input_phis = dm_host.get("input_phis").createDeviceCopy(); // check that interface pressure is reasonable - parallel_for(SimpleBounds<2>(nz+1,nens) , YAKL_LAMBDA (int k, int iens) { + parallel_for(SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int iens) { if ( hmean_pint(k,iens) < hmean_pmid(k,iens) ) { auto phis = input_phis(iens)/grav; - printf("PAM-DEBUG bad-pint - k:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- pint:%8.2g pmid:%8.2g \n", - k,iens,lat(iens),lon(iens),phis, - hmean_pint(k,iens), - hmean_pmid(k,iens) - ); + printf("PAM-STATE - bad interface pressure for reference state - "+\ + "k:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- "+\ + "pint:%12.4f pmid:%12.4f \n", + k,iens,lat(iens),lon(iens),phis, + hmean_pint(k,iens),hmean_pmid(k,iens)); } }); From 8c8cb1c9e358119154f9ee4f974fc2d219a15c3b Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 12:14:45 -0400 Subject: [PATCH 033/904] update PAM variance transport --- .../physics/crm/pam/pam_variance_transport.h | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_variance_transport.h b/components/eam/src/physics/crm/pam/pam_variance_transport.h index 1033df4280d4..ae9f0747ee31 100644 --- a/components/eam/src/physics/crm/pam/pam_variance_transport.h +++ b/components/eam/src/physics/crm/pam/pam_variance_transport.h @@ -38,6 +38,8 @@ inline void pam_variance_transport_diagnose( pam::PamCoupler &coupler ) { //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp" ); auto rhov = dm_device.get("water_vapor"); + auto rhoc = dm_device.get("cloud_water"); + auto rhoi = dm_device.get("ice" ); auto uvel = dm_device.get("uvel" ); auto vt_temp = dm_device.get("vt_temp" ); auto vt_rhov = dm_device.get("vt_rhov" ); @@ -64,15 +66,17 @@ inline void pam_variance_transport_diagnose( pam::PamCoupler &coupler ) { // calculate horizontal mean real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int n) { + real rhot = rhov(k,j,i,n) + rhoc(k,j,i,n) + rhoi(k,j,i,n); yakl::atomicAdd( temp_mean(k,n), temp(k,j,i,n)*r_nx_ny ); - yakl::atomicAdd( rhov_mean(k,n), rhov(k,j,i,n)*r_nx_ny ); + yakl::atomicAdd( rhov_mean(k,n), rhot *r_nx_ny ); yakl::atomicAdd( uvel_mean(k,n), uvel(k,j,i,n)*r_nx_ny ); }); //------------------------------------------------------------------------------------------------ // calculate fluctuations from horz mean parallel_for(SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int n) { + real rhot = rhov(k,j,i,n) + rhoc(k,j,i,n) + rhoi(k,j,i,n); vt_temp_pert(k,j,i,n) = temp(k,j,i,n) - temp_mean(k,n); - vt_rhov_pert(k,j,i,n) = rhov(k,j,i,n) - rhov_mean(k,n); + vt_rhov_pert(k,j,i,n) = rhot - rhov_mean(k,n); vt_uvel_pert(k,j,i,n) = uvel(k,j,i,n) - uvel_mean(k,n); }); //------------------------------------------------------------------------------------------------ @@ -101,9 +105,6 @@ inline void pam_variance_transport_compute_forcing( pam::PamCoupler &coupler ) { // update CRM variance values pam_variance_transport_diagnose(coupler); //------------------------------------------------------------------------------------------------ - auto temp = dm_device.get("temp" ); - auto rhov = dm_device.get("water_vapor" ); - auto uvel = dm_device.get("uvel" ); auto vt_temp = dm_device.get("vt_temp" ); auto vt_rhov = dm_device.get("vt_rhov" ); auto vt_uvel = dm_device.get("vt_uvel" ); @@ -134,14 +135,16 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { auto nx = coupler.get_option("crm_nx"); auto crm_dt = coupler.get_option("crm_dt"); //------------------------------------------------------------------------------------------------ + // update CRM variance values + pam_variance_transport_diagnose(coupler); + //------------------------------------------------------------------------------------------------ // min and max perturbation scaling values are used to limit the // large-scale forcing from variance transport. This is meant to // protect against creating unstable situations, although // problematic scenarios were extremely rare in testing. - // A scaling limit of +/- 10% was found to be adequate in SAM, - // but PAM seems much more sensitive (not sure why), so we use 0.1% here - real constexpr pert_scale_min = 1.0 - 0.001; - real constexpr pert_scale_max = 1.0 + 0.001; + // A scaling limit of +/- 10% was found to be adequate. + real constexpr pert_scale_min = 1.0 - 0.1; + real constexpr pert_scale_max = 1.0 + 0.1; //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp" ); auto rhov = dm_device.get("water_vapor" ); @@ -161,9 +164,6 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { real2d rhov_pert_scale("rhov_pert_scale", nz, nens); real2d uvel_pert_scale("uvel_pert_scale", nz, nens); //------------------------------------------------------------------------------------------------ - // update CRM variance values - pam_variance_transport_diagnose(coupler); - //------------------------------------------------------------------------------------------------ // calculate scaling factor for local perturbations parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { real tmp; @@ -172,9 +172,15 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { rhov_pert_scale(k,n) = 1.0; uvel_pert_scale(k,n) = 1.0; // calculate variance scaling factor - tmp = 1.+crm_dt*vt_temp_forcing_tend(k,n)/vt_temp(k,n); if (tmp>0){ temp_pert_scale(k,n) = sqrt(tmp); } - tmp = 1.+crm_dt*vt_rhov_forcing_tend(k,n)/vt_rhov(k,n); if (tmp>0){ rhov_pert_scale(k,n) = sqrt(tmp); } - tmp = 1.+crm_dt*vt_uvel_forcing_tend(k,n)/vt_uvel(k,n); if (tmp>0){ uvel_pert_scale(k,n) = sqrt(tmp); } + real tmp_t_scale = -1.0; + real tmp_q_scale = -1.0; + real tmp_u_scale = -1.0; + if (vt_temp(k,n)>0.0) { tmp_t_scale = 1. + crm_dt*vt_temp_forcing_tend(k,n) / vt_temp(k,n); } + if (vt_rhov(k,n)>0.0) { tmp_q_scale = 1. + crm_dt*vt_rhov_forcing_tend(k,n) / vt_rhov(k,n); } + if (vt_uvel(k,n)>0.0) { tmp_u_scale = 1. + crm_dt*vt_uvel_forcing_tend(k,n) / vt_uvel(k,n); } + if (tmp>0.0){ temp_pert_scale(k,n) = sqrt(tmp_t_scale); } + if (tmp>0.0){ rhov_pert_scale(k,n) = sqrt(tmp_q_scale); } + if (tmp>0.0){ uvel_pert_scale(k,n) = sqrt(tmp_u_scale); } // enforce minimum scaling temp_pert_scale(k,n) = std::max( temp_pert_scale(k,n), pert_scale_min ); rhov_pert_scale(k,n) = std::max( rhov_pert_scale(k,n), pert_scale_min ); @@ -213,9 +219,6 @@ inline void pam_variance_transport_compute_feedback( pam::PamCoupler &coupler ) // update CRM variance values pam_variance_transport_diagnose(coupler); //------------------------------------------------------------------------------------------------ - auto temp = dm_device.get("temp" ); - auto rhov = dm_device.get("water_vapor"); - auto uvel = dm_device.get("uvel" ); auto vt_temp = dm_device.get("vt_temp" ); auto vt_rhov = dm_device.get("vt_rhov" ); auto vt_uvel = dm_device.get("vt_uvel" ); From 98726502a74a6067dd2f4e96c04db227e5f3a790 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 12:16:11 -0400 Subject: [PATCH 034/904] update MMF2 default config --- components/eam/cime_config/config_component.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 784fe44195ed..056b607953f9 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -72,15 +72,15 @@ -crm samxx -crm_dt 10 - -crm pam -pam_dycor spam -crm_dt 5 - -crm pam -pam_dycor awfl -crm_dt 5 + -crm pam -pam_dycor spam -crm_dt 10 + -crm pam -pam_dycor awfl -crm_dt 10 -use_MMF -nlev 60 -crm_nz 50 -crm_dx 2000 -crm_nx 64 -crm_ny 1 -crm_nx_rad 4 -crm_ny_rad 1 -crm_dx 2000 -crm_nx 45 -crm_ny 1 -crm_nx_rad 5 -crm_ny_rad 1 -MMF_microphysics_scheme sam1mom -chem none -MMF_microphysics_scheme p3 -chem none -rad rrtmgp -rrtmgpxx - -use_MMF_VT + -use_MMF_VT -use_MMF_ESMT -aquaplanet -aquaplanet -rce From 2380d339ca04b8f6c878d2fcfdf76ecaa408663f Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 12:29:53 -0400 Subject: [PATCH 035/904] bug fix in PAM variance transport --- .../eam/src/physics/crm/pam/pam_variance_transport.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_variance_transport.h b/components/eam/src/physics/crm/pam/pam_variance_transport.h index ae9f0747ee31..e258b3b7f3ad 100644 --- a/components/eam/src/physics/crm/pam/pam_variance_transport.h +++ b/components/eam/src/physics/crm/pam/pam_variance_transport.h @@ -143,8 +143,8 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { // protect against creating unstable situations, although // problematic scenarios were extremely rare in testing. // A scaling limit of +/- 10% was found to be adequate. - real constexpr pert_scale_min = 1.0 - 0.1; - real constexpr pert_scale_max = 1.0 + 0.1; + real constexpr pert_scale_min = 1.0 - 0.05; + real constexpr pert_scale_max = 1.0 + 0.05; //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp" ); auto rhov = dm_device.get("water_vapor" ); @@ -166,7 +166,6 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { //------------------------------------------------------------------------------------------------ // calculate scaling factor for local perturbations parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { - real tmp; // initialize scaling factors to 1.0 temp_pert_scale(k,n) = 1.0; rhov_pert_scale(k,n) = 1.0; @@ -178,9 +177,9 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { if (vt_temp(k,n)>0.0) { tmp_t_scale = 1. + crm_dt*vt_temp_forcing_tend(k,n) / vt_temp(k,n); } if (vt_rhov(k,n)>0.0) { tmp_q_scale = 1. + crm_dt*vt_rhov_forcing_tend(k,n) / vt_rhov(k,n); } if (vt_uvel(k,n)>0.0) { tmp_u_scale = 1. + crm_dt*vt_uvel_forcing_tend(k,n) / vt_uvel(k,n); } - if (tmp>0.0){ temp_pert_scale(k,n) = sqrt(tmp_t_scale); } - if (tmp>0.0){ rhov_pert_scale(k,n) = sqrt(tmp_q_scale); } - if (tmp>0.0){ uvel_pert_scale(k,n) = sqrt(tmp_u_scale); } + if (tmp_t_scale>0.0){ temp_pert_scale(k,n) = sqrt(tmp_t_scale); } + if (tmp_q_scale>0.0){ rhov_pert_scale(k,n) = sqrt(tmp_q_scale); } + if (tmp_u_scale>0.0){ uvel_pert_scale(k,n) = sqrt(tmp_u_scale); } // enforce minimum scaling temp_pert_scale(k,n) = std::max( temp_pert_scale(k,n), pert_scale_min ); rhov_pert_scale(k,n) = std::max( rhov_pert_scale(k,n), pert_scale_min ); From 63df74c0bb17552d825f55cb3b04edc5f5b83014 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 12:30:50 -0400 Subject: [PATCH 036/904] implement PAM driver subcycling --- .../eam/src/physics/crm/pam/pam_driver.cpp | 173 ++++++++++++------ 1 file changed, 122 insertions(+), 51 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index 69e67a90894c..0833d2beff8d 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -1,5 +1,4 @@ #include "pam_coupler.h" -// #include "params.h" #include "Dycore.h" #include "Microphysics.h" #include "SGS.h" @@ -14,26 +13,90 @@ #include "pam_output.h" #include "pam_accelerate.h" #include "pam_variance_transport.h" +#include "pam_hyperdiffusion.h" #include "sponge_layer.h" #include "surface_friction.h" #include "scream_cxx_interface_finalize.h" -#include "pam_hyperdiffusion.h" - // Needed for p3_init #include "p3_functions.hpp" #include "p3_f90.hpp" #include "pam_debug.h" -bool constexpr enable_check_state = true; +bool constexpr enable_check_state = false; -extern "C" void pam_driver() { + +inline int pam_driver_set_subcycle_timestep( pam::PamCoupler &coupler, real crm_dt_fixed ) { + // calculate the CFL condition and adjust the PAM time loop subcylcing //------------------------------------------------------------------------------------------------ - using yakl::intrinsics::abs; - using yakl::intrinsics::maxval; - using yakl::atomicAdd; using yakl::c::parallel_for; using yakl::c::SimpleBounds; + using yakl::atomicMax; + //------------------------------------------------------------------------------------------------ + auto nens = coupler.get_option("ncrms"); + auto gcm_nlev = coupler.get_option("gcm_nlev"); + auto crm_nz = coupler.get_option("crm_nz"); + auto crm_nx = coupler.get_option("crm_nx"); + auto crm_ny = coupler.get_option("crm_ny"); + auto crm_dx = coupler.get_option("crm_dx"); + auto crm_dy = coupler.get_option("crm_dy"); + auto &dm_device = coupler.get_data_manager_device_readonly(); + auto &dm_host = coupler.get_data_manager_host_readonly(); + auto uvel = dm_device.get("uvel"); + auto wvel = dm_device.get("wvel"); + auto input_zint = dm_host.get("input_zint").createDeviceCopy(); + //------------------------------------------------------------------------------------------------ + yakl::ParallelMax pmax( crm_nz*nens ); + real cfl = 0; + int num_subcycle = 1; + int constexpr max_num_subcycle = 10; + real2d wvel_max("wvel_max",crm_nz,nens); + real2d uvel_max("uvel_max",crm_nz,nens); + real2d cfl_max("cfl_max", crm_nz,nens); + //------------------------------------------------------------------------------------------------ + // initialize max U and W arrays + parallel_for( SimpleBounds<2>(crm_nz,nens) , YAKL_LAMBDA (int k, int n) { + wvel_max(k,n) = 0.0; + uvel_max(k,n) = 0.0; + }); + // calculate max U and W + parallel_for( SimpleBounds<4>(crm_nz,crm_ny,crm_nx,nens) , YAKL_LAMBDA (int k, int j, int i, int n) { + yakl::atomicMax(uvel_max(k,n), sqrt(uvel(k,j,i,n)*uvel(k,j,i,n)) ); + yakl::atomicMax(wvel_max(k,n), fabs(wvel(k,j,i,n)) ); + }); + // find max CFL between horizontal and vertical CFL values + parallel_for( SimpleBounds<2>(crm_nz,nens) , YAKL_LAMBDA (int k, int n) { + int k_gcm = gcm_nlev-1-k; + real crm_dz = input_zint(k_gcm,n) - input_zint(k_gcm+1,n); + real cfl_u = uvel_max(k,n)*crm_dt_fixed/crm_dx; + real cfl_w = wvel_max(k,n)*crm_dt_fixed/crm_dz; + cfl_max(k,n) = max(cfl_u,cfl_w); + }); + // calculate final CFL across ensemble + real cfl_loc = pmax(cfl_max.data()); + cfl = max(cfl,cfl_loc); + // update number of subcycles and time step + num_subcycle = max(num_subcycle,max(1,static_cast(ceil(cfl/0.7)))); + real crm_dt_subcycle = crm_dt_fixed / num_subcycle; + coupler.set_option("crm_dt",crm_dt_subcycle); + // check for excessive subcylcing - don't exit, just print + if(num_subcycle > max_num_subcycle) { + real umax = pmax(uvel_max.data()); + real wmax = pmax(wvel_max.data()); + printf("PAM_DRIVER - WARNING: excessive subcycling!"+\ + " - num_subcycle: %3.3d dt: %8.4f cfl: %8.4f umax: %8.2f wmax: %8.2f \n", + num_subcycle,crm_dt_subcycle,cfl,umax,wmax); + // exit(-1); + } + + return num_subcycle; + //------------------------------------------------------------------------------------------------ +} + + +extern "C" void pam_driver() { + // This is the primary method for running the PAM CRM in E3SM-MMF. + //------------------------------------------------------------------------------------------------ auto &coupler = pam_interface::get_coupler(); //------------------------------------------------------------------------------------------------ // retreive coupler options @@ -42,8 +105,10 @@ extern "C" void pam_driver() { auto crm_nz = coupler.get_option("crm_nz"); auto crm_nx = coupler.get_option("crm_nx"); auto crm_ny = coupler.get_option("crm_ny"); + // auto crm_dx = coupler.get_option("crm_dx"); + // auto crm_dy = coupler.get_option("crm_dy"); auto gcm_dt = coupler.get_option("gcm_dt"); - auto crm_dt = coupler.get_option("crm_dt"); + auto crm_dt_fixed = coupler.get_option("crm_dt"); auto is_first_step = coupler.get_option("is_first_step"); auto is_restart = coupler.get_option("is_restart"); bool use_crm_accel = coupler.get_option("use_crm_accel"); @@ -162,7 +227,7 @@ extern "C" void pam_driver() { //------------------------------------------------------------------------------------------------ // set number of CRM steps - int nstop = int(gcm_dt/crm_dt); + int nstop = int(gcm_dt/crm_dt_fixed); // for mean-state acceleration adjust nstop and diagnose horizontal means if (use_crm_accel) { @@ -171,51 +236,58 @@ extern "C" void pam_driver() { }; // Run the CRM - real etime_crm = 0; int nstep = 0; - // while (etime_crm < gcm_dt) { while (nstep < nstop) { - if (crm_dt == 0.) { crm_dt = dycore.compute_time_step(coupler); } - if (etime_crm + crm_dt > gcm_dt) { crm_dt = gcm_dt - etime_crm; } + //-------------------------------------------------------------------------- + //-------------------------------------------------------------------------- + auto num_subcycle = pam_driver_set_subcycle_timestep(coupler,crm_dt_fixed); + #if defined(MMF_PAM_DYCOR_SPAM) + dycore.update_dt(coupler); + #endif if (enable_check_state) { pam_debug_check_state(coupler, 1, nstep); } - // Apply forcing tendencies - if (use_MMF_VT) { pam_variance_transport_apply_forcing(coupler); } - if (enable_check_state) { pam_debug_check_state(coupler, 2, nstep); } - coupler.run_module( "apply_gcm_forcing_tendencies" , modules::apply_gcm_forcing_tendencies ); - if (enable_check_state) { pam_debug_check_state(coupler, 3, nstep); } - coupler.run_module( "radiation" , [&] (pam::PamCoupler &coupler) {rad .timeStep(coupler);} ); - if (enable_check_state) { pam_debug_check_state(coupler, 4, nstep); } - - // Dynamics - if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } - if (do_density_save_recall) { pam_state_save_dry_density(coupler); } - coupler.run_module( "dycore", [&] (pam::PamCoupler &coupler) {dycore.timeStep(coupler);} ); - if (do_density_save_recall) { pam_state_recall_dry_density(coupler); } - if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"dycor"); } - if (enable_check_state) { pam_debug_check_state(coupler, 5, nstep); } - - // Sponge layer damping - if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } - coupler.run_module( "sponge_layer", modules::sponge_layer ); - if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"sponge"); } - - // Apply hyperdiffusion to account for lack of horizontal mixing in SHOC - pam_hyperdiffusion(coupler); - - // Turbulence - SHOC - coupler.run_module( "compute_surface_friction", modules::compute_surface_friction ); - if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } - coupler.run_module( "sgs", [&] (pam::PamCoupler &coupler) {sgs .timeStep(coupler);} ); - if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"sgs"); } - if (enable_check_state) { pam_debug_check_state(coupler, 6, nstep); } - - // Microphysics - P3 - if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } - coupler.run_module( "micro", [&] (pam::PamCoupler &coupler) {micro .timeStep(coupler);} ); - if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"micro"); } - if (enable_check_state) { pam_debug_check_state(coupler, 7, nstep); } + // loop for adaptive subcyling based on CFL + for(int icycle=1; icycle<=num_subcycle; icycle++) { + + // Apply forcing tendencies + if (use_MMF_VT) { pam_variance_transport_apply_forcing(coupler); } + if (enable_check_state) { pam_debug_check_state(coupler, 2, nstep); } + coupler.run_module( "apply_gcm_forcing_tendencies" , modules::apply_gcm_forcing_tendencies ); + if (enable_check_state) { pam_debug_check_state(coupler, 3, nstep); } + coupler.run_module( "radiation" , [&] (pam::PamCoupler &coupler) {rad .timeStep(coupler);} ); + if (enable_check_state) { pam_debug_check_state(coupler, 4, nstep); } + + // Dynamics + if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } + if (do_density_save_recall) { pam_state_save_dry_density(coupler); } + coupler.run_module( "dycore", [&] (pam::PamCoupler &coupler) {dycore.timeStep(coupler);} ); + if (do_density_save_recall) { pam_state_recall_dry_density(coupler); } + if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"dycor"); } + if (enable_check_state) { pam_debug_check_state(coupler, 5, nstep); } + + // Sponge layer damping + if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } + coupler.run_module( "sponge_layer", modules::sponge_layer ); + if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"sponge"); } + + // Apply hyperdiffusion to account for lack of horizontal mixing in SHOC + pam_hyperdiffusion(coupler); + + // Turbulence - SHOC + coupler.run_module( "compute_surface_friction", modules::compute_surface_friction ); + if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } + coupler.run_module( "sgs", [&] (pam::PamCoupler &coupler) {sgs .timeStep(coupler);} ); + if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"sgs"); } + if (enable_check_state) { pam_debug_check_state(coupler, 6, nstep); } + + // Microphysics - P3 + if (enable_physics_tend_stats) { pam_statistics_save_state(coupler); } + coupler.run_module( "micro", [&] (pam::PamCoupler &coupler) {micro .timeStep(coupler);} ); + if (enable_physics_tend_stats) { pam_statistics_aggregate_tendency(coupler,"micro"); } + if (enable_check_state) { pam_debug_check_state(coupler, 7, nstep); } + + } // num_subcycle // CRM mean state acceleration if (use_crm_accel && !coupler.get_option("crm_acceleration_ceaseflag")) { @@ -227,7 +299,6 @@ extern "C" void pam_driver() { pam_radiation_timestep_aggregation(coupler); pam_statistics_timestep_aggregation(coupler); - etime_crm += crm_dt; nstep += 1; } //------------------------------------------------------------------------------------------------ From 7cc042a07306b2370931de60e03cc077f70f8454 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 13:47:26 -0400 Subject: [PATCH 037/904] bug fix --- components/eam/src/physics/crm/pam/pam_driver.cpp | 2 +- components/eam/src/physics/crm/pam/pam_state.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index 0833d2beff8d..a05b5cf8b037 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -83,7 +83,7 @@ inline int pam_driver_set_subcycle_timestep( pam::PamCoupler &coupler, real crm_ if(num_subcycle > max_num_subcycle) { real umax = pmax(uvel_max.data()); real wmax = pmax(wvel_max.data()); - printf("PAM_DRIVER - WARNING: excessive subcycling!"+\ + printf("PAM_DRIVER - WARNING: excessive subcycling!" " - num_subcycle: %3.3d dt: %8.4f cfl: %8.4f umax: %8.2f wmax: %8.2f \n", num_subcycle,crm_dt_subcycle,cfl,umax,wmax); // exit(-1); diff --git a/components/eam/src/physics/crm/pam/pam_state.h b/components/eam/src/physics/crm/pam/pam_state.h index fb6553df07ef..270fef7f6c23 100644 --- a/components/eam/src/physics/crm/pam/pam_state.h +++ b/components/eam/src/physics/crm/pam/pam_state.h @@ -189,8 +189,8 @@ inline void pam_state_set_reference_state( pam::PamCoupler &coupler ) { parallel_for(SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int iens) { if ( hmean_pint(k,iens) < hmean_pmid(k,iens) ) { auto phis = input_phis(iens)/grav; - printf("PAM-STATE - bad interface pressure for reference state - "+\ - "k:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- "+\ + printf("PAM-STATE - bad interface pressure for reference state - " + "k:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- " "pint:%12.4f pmid:%12.4f \n", k,iens,lat(iens),lon(iens),phis, hmean_pint(k,iens),hmean_pmid(k,iens)); From f714cf4e2d6612210692eecdf5b2e4eda3a82933 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 13:48:07 -0400 Subject: [PATCH 038/904] bug fix in PAM variance transport --- .../physics/crm/pam/pam_variance_transport.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_variance_transport.h b/components/eam/src/physics/crm/pam/pam_variance_transport.h index e258b3b7f3ad..1eb7779f3469 100644 --- a/components/eam/src/physics/crm/pam/pam_variance_transport.h +++ b/components/eam/src/physics/crm/pam/pam_variance_transport.h @@ -118,9 +118,9 @@ inline void pam_variance_transport_compute_forcing( pam::PamCoupler &coupler ) { // calculate variance transport forcing parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k_crm, int n) { int k_gcm = gcm_nlev-1-k_crm; - vt_temp_forcing_tend(k_crm,n) = ( gcm_vt_temp(k_gcm,n) - vt_temp(k_crm,n) ) * gcm_dt ; - vt_rhov_forcing_tend(k_crm,n) = ( gcm_vt_rhov(k_gcm,n) - vt_rhov(k_crm,n) ) * gcm_dt ; - vt_uvel_forcing_tend(k_crm,n) = ( gcm_vt_uvel(k_gcm,n) - vt_uvel(k_crm,n) ) * gcm_dt ; + vt_temp_forcing_tend(k_crm,n) = ( gcm_vt_temp(k_gcm,n) - vt_temp(k_crm,n) ) / gcm_dt ; + vt_rhov_forcing_tend(k_crm,n) = ( gcm_vt_rhov(k_gcm,n) - vt_rhov(k_crm,n) ) / gcm_dt ; + vt_uvel_forcing_tend(k_crm,n) = ( gcm_vt_uvel(k_gcm,n) - vt_uvel(k_crm,n) ) / gcm_dt ; }); //------------------------------------------------------------------------------------------------ } @@ -174,9 +174,9 @@ inline void pam_variance_transport_apply_forcing( pam::PamCoupler &coupler ) { real tmp_t_scale = -1.0; real tmp_q_scale = -1.0; real tmp_u_scale = -1.0; - if (vt_temp(k,n)>0.0) { tmp_t_scale = 1. + crm_dt*vt_temp_forcing_tend(k,n) / vt_temp(k,n); } - if (vt_rhov(k,n)>0.0) { tmp_q_scale = 1. + crm_dt*vt_rhov_forcing_tend(k,n) / vt_rhov(k,n); } - if (vt_uvel(k,n)>0.0) { tmp_u_scale = 1. + crm_dt*vt_uvel_forcing_tend(k,n) / vt_uvel(k,n); } + if (vt_temp(k,n)>0.0) { tmp_t_scale = 1. + crm_dt * vt_temp_forcing_tend(k,n) / vt_temp(k,n); } + if (vt_rhov(k,n)>0.0) { tmp_q_scale = 1. + crm_dt * vt_rhov_forcing_tend(k,n) / vt_rhov(k,n); } + if (vt_uvel(k,n)>0.0) { tmp_u_scale = 1. + crm_dt * vt_uvel_forcing_tend(k,n) / vt_uvel(k,n); } if (tmp_t_scale>0.0){ temp_pert_scale(k,n) = sqrt(tmp_t_scale); } if (tmp_q_scale>0.0){ rhov_pert_scale(k,n) = sqrt(tmp_q_scale); } if (tmp_u_scale>0.0){ uvel_pert_scale(k,n) = sqrt(tmp_u_scale); } @@ -236,9 +236,9 @@ inline void pam_variance_transport_compute_feedback( pam::PamCoupler &coupler ) parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k_crm, int n) { int k_gcm = gcm_nlev-1-k_crm; if (k_crm Date: Fri, 12 Apr 2024 13:48:58 -0400 Subject: [PATCH 039/904] update default MMF_PAM_dyn_per_phys --- components/eam/bld/namelist_files/namelist_defaults_eam.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index 74f2572ea040..d7e9afecf20e 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -871,7 +871,7 @@ 0 .false. .true. - 2 + 1 90.0 0.0 From 98a6e4025e9617bfdd0ba27699fa69bbbd959772 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 15:22:03 -0400 Subject: [PATCH 040/904] update default PAM hyperdiffusion timescale --- components/eam/src/physics/crm/pam/pam_hyperdiffusion.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h b/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h index b632b0e49093..e75d80265445 100644 --- a/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h +++ b/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h @@ -28,7 +28,7 @@ inline void pam_hyperdiffusion( pam::PamCoupler &coupler ) { #ifdef MMF_PAM_HDT real constexpr hd_timescale = MMF_PAM_HDT; // damping time scale [sec] #else - real constexpr hd_timescale = 60.0; // damping time scale [sec] + real constexpr hd_timescale = 120.0; // damping time scale [sec] #endif //------------------------------------------------------------------------------------------------ real4d hd_temp("hd_temp",nz,ny,nx,nens); From 2c54c4d9aa7db1659374433517d7a2636357c690 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 12 Apr 2024 15:22:46 -0400 Subject: [PATCH 041/904] update PAM default crm_dx to 3km --- components/eam/cime_config/config_component.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 056b607953f9..ef9433ef9664 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -76,7 +76,7 @@ -crm pam -pam_dycor awfl -crm_dt 10 -use_MMF -nlev 60 -crm_nz 50 -crm_dx 2000 -crm_nx 64 -crm_ny 1 -crm_nx_rad 4 -crm_ny_rad 1 - -crm_dx 2000 -crm_nx 45 -crm_ny 1 -crm_nx_rad 5 -crm_ny_rad 1 + -crm_dx 3000 -crm_nx 45 -crm_ny 1 -crm_nx_rad 5 -crm_ny_rad 1 -MMF_microphysics_scheme sam1mom -chem none -MMF_microphysics_scheme p3 -chem none -rad rrtmgp -rrtmgpxx From 748ad0a808b404ebb3cb43b5032c7f663c3cf177 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 17 Apr 2024 18:58:56 -0400 Subject: [PATCH 042/904] reduce hd_timescale back to 60 sec fr PAM --- components/eam/src/physics/crm/pam/pam_hyperdiffusion.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h b/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h index e75d80265445..b632b0e49093 100644 --- a/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h +++ b/components/eam/src/physics/crm/pam/pam_hyperdiffusion.h @@ -28,7 +28,7 @@ inline void pam_hyperdiffusion( pam::PamCoupler &coupler ) { #ifdef MMF_PAM_HDT real constexpr hd_timescale = MMF_PAM_HDT; // damping time scale [sec] #else - real constexpr hd_timescale = 120.0; // damping time scale [sec] + real constexpr hd_timescale = 60.0; // damping time scale [sec] #endif //------------------------------------------------------------------------------------------------ real4d hd_temp("hd_temp",nz,ny,nx,nens); From 6b92cabe43740e3a36df9c7aa718d9a5a4bc0301 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 17 Apr 2024 18:59:54 -0400 Subject: [PATCH 043/904] update PAM submodule --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index ff652f3f2a6c..8f9538db7138 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit ff652f3f2a6c2d675cd957883245aa6c036ef110 +Subproject commit 8f9538db7138baf26fa2b0f163fae3050d3fe29d From f750a1d9caca3e853a215fbf012888d07bba3176 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 18 Apr 2024 16:23:30 -0400 Subject: [PATCH 044/904] make PAM dry density acceleration optional --- .../eam/src/physics/crm/pam/pam_accelerate.h | 89 +++++++++---------- 1 file changed, 41 insertions(+), 48 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_accelerate.h b/components/eam/src/physics/crm/pam/pam_accelerate.h index a7b179a85c9d..5b277fdca966 100644 --- a/components/eam/src/physics/crm/pam/pam_accelerate.h +++ b/components/eam/src/physics/crm/pam/pam_accelerate.h @@ -17,16 +17,22 @@ inline void pam_accelerate_init( pam::PamCoupler &coupler ) { using yakl::c::parallel_for; using yakl::c::SimpleBounds; using yakl::atomicAdd; + //------------------------------------------------------------------------------------------------ + // The ability to accelerate dry density is implemented, but tests suggested + // that this was contributing to instabilities in PAM. Combined with the + // uncertainty of how to handle dry density in the anelastic case, we will + // disable dry density mean-state acceleration by default. + coupler.set_option("crm_accel_rd",false); + //------------------------------------------------------------------------------------------------ auto &dm_device = coupler.get_data_manager_device_readwrite(); auto nens = coupler.get_option("ncrms"); auto nz = coupler.get_option("crm_nz"); auto ny = coupler.get_option("crm_ny"); auto nx = coupler.get_option("crm_nx"); - auto crm_accel_uv = coupler.get_option("crm_accel_uv"); //------------------------------------------------------------------------------------------------ dm_device.register_and_allocate("accel_save_t", "saved temperature for MSA", {nz,nens}, {"z","nens"} ); - // dm_device.register_and_allocate("accel_save_r", "saved dry density for MSA", {nz,nens}, {"z","nens"} ); dm_device.register_and_allocate("accel_save_q", "saved total water for MSA", {nz,nens}, {"z","nens"} ); + dm_device.register_and_allocate("accel_save_r", "saved dry density for MSA", {nz,nens}, {"z","nens"} ); dm_device.register_and_allocate("accel_save_u", "saved uvel for MSA", {nz,nens}, {"z","nens"} ); dm_device.register_and_allocate("accel_save_v", "saved vvel for MSA", {nz,nens}, {"z","nens"} ); //------------------------------------------------------------------------------------------------ @@ -42,40 +48,37 @@ inline void pam_accelerate_diagnose( pam::PamCoupler &coupler ) { auto nz = coupler.get_option("crm_nz"); auto ny = coupler.get_option("crm_ny"); auto nx = coupler.get_option("crm_nx"); + auto crm_accel_rd = coupler.get_option("crm_accel_rd"); auto crm_accel_uv = coupler.get_option("crm_accel_uv"); //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp" ); - // auto rhod = dm_device.get("density_dry"); auto rhov = dm_device.get("water_vapor"); auto rhol = dm_device.get("cloud_water"); auto rhoi = dm_device.get("ice" ); + auto rhod = dm_device.get("density_dry"); auto uvel = dm_device.get("uvel" ); auto vvel = dm_device.get("vvel" ); auto accel_save_t = dm_device.get("accel_save_t"); - // auto accel_save_r = dm_device.get("accel_save_r"); auto accel_save_q = dm_device.get("accel_save_q"); + auto accel_save_r = dm_device.get("accel_save_r"); auto accel_save_u = dm_device.get("accel_save_u"); auto accel_save_v = dm_device.get("accel_save_v"); //------------------------------------------------------------------------------------------------ // compute horizontal means needed later for mean-state acceleration parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { accel_save_t(k,n) = 0.0; - // accel_save_r(k,n) = 0.0; accel_save_q(k,n) = 0.0; - if (crm_accel_uv) { - accel_save_u(k,n) = 0.0; - accel_save_v(k,n) = 0.0; - } + if (crm_accel_rd) { accel_save_r(k,n) = 0.0; } + if (crm_accel_uv) { accel_save_u(k,n) = 0.0; } + if (crm_accel_uv) { accel_save_v(k,n) = 0.0; } }); real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions parallel_for( SimpleBounds<4>(nz,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int n) { yakl::atomicAdd( accel_save_t(k,n), temp(k,j,i,n) * r_nx_ny ); - // yakl::atomicAdd( accel_save_r(k,n), rhod(k,j,i,n) * r_nx_ny ); yakl::atomicAdd( accel_save_q(k,n), ( rhov(k,j,i,n) + rhol(k,j,i,n) + rhoi(k,j,i,n) ) * r_nx_ny ); - if (crm_accel_uv) { - yakl::atomicAdd( accel_save_u(k,n), uvel(k,j,i,n) * r_nx_ny ); - yakl::atomicAdd( accel_save_v(k,n), vvel(k,j,i,n) * r_nx_ny ); - } + if (crm_accel_rd) { yakl::atomicAdd( accel_save_r(k,n), rhod(k,j,i,n) * r_nx_ny ); } + if (crm_accel_uv) { yakl::atomicAdd( accel_save_u(k,n), uvel(k,j,i,n) * r_nx_ny ); } + if (crm_accel_uv) { yakl::atomicAdd( accel_save_v(k,n), vvel(k,j,i,n) * r_nx_ny ); } }); //------------------------------------------------------------------------------------------------ } @@ -86,35 +89,35 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { using yakl::c::SimpleBounds; using yakl::atomicAdd; using yakl::ScalarLiveOut; - auto &dm_device = coupler.get_data_manager_device_readwrite(); - auto nens = coupler.get_option("ncrms"); - auto nz = coupler.get_option("crm_nz"); - auto ny = coupler.get_option("crm_ny"); - auto nx = coupler.get_option("crm_nx"); + auto &dm_device = coupler.get_data_manager_device_readwrite(); + auto nens = coupler.get_option("ncrms"); + auto nz = coupler.get_option("crm_nz"); + auto ny = coupler.get_option("crm_ny"); + auto nx = coupler.get_option("crm_nx"); + auto crm_accel_rd = coupler.get_option("crm_accel_rd"); + auto crm_accel_uv = coupler.get_option("crm_accel_uv"); + real crm_accel_factor = coupler.get_option("crm_accel_factor"); //------------------------------------------------------------------------------------------------ auto temp = dm_device.get("temp" ); - // auto rhod = dm_device.get("density_dry"); auto rhov = dm_device.get("water_vapor"); auto rhol = dm_device.get("cloud_water"); auto rhoi = dm_device.get("ice" ); + auto rhod = dm_device.get("density_dry"); auto uvel = dm_device.get("uvel" ); auto vvel = dm_device.get("vvel" ); auto accel_save_t = dm_device.get("accel_save_t"); - // auto accel_save_r = dm_device.get("accel_save_r"); auto accel_save_q = dm_device.get("accel_save_q"); + auto accel_save_r = dm_device.get("accel_save_r"); auto accel_save_u = dm_device.get("accel_save_u"); auto accel_save_v = dm_device.get("accel_save_v"); //------------------------------------------------------------------------------------------------ - bool crm_accel_uv = coupler.get_option("crm_accel_uv"); - real crm_accel_factor = coupler.get_option("crm_accel_factor"); - //------------------------------------------------------------------------------------------------ real2d hmean_t ("hmean_t", nz,nens); - // real2d hmean_r ("hmean_r", nz,nens); + real2d hmean_r ("hmean_r", nz,nens); real2d hmean_q ("hmean_q", nz,nens); real2d hmean_u ("hmean_u", nz,nens); real2d hmean_v ("hmean_v", nz,nens); real2d ttend_acc("ttend_acc", nz,nens); - // real2d rtend_acc("rtend_acc", nz,nens); + real2d rtend_acc("rtend_acc", nz,nens); real2d qtend_acc("qtend_acc", nz,nens); real2d utend_acc("utend_acc", nz,nens); real2d vtend_acc("vtend_acc", nz,nens); @@ -127,22 +130,18 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { // Compute the horizontal mean for each variable parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { hmean_t(k,n) = 0.0; - // hmean_r(k,n) = 0.0; hmean_q(k,n) = 0.0; - if (crm_accel_uv) { - hmean_u(k,n) = 0.0; - hmean_v(k,n) = 0.0; - } + if (crm_accel_rd) { hmean_r(k,n) = 0.0; } + if (crm_accel_uv) { hmean_u(k,n) = 0.0; } + if (crm_accel_uv) { hmean_v(k,n) = 0.0; } }); real r_nx_ny = 1._fp/(nx*ny); // precompute reciprocal to avoid costly divisions parallel_for( SimpleBounds<4>(nz,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int n) { yakl::atomicAdd( hmean_t(k,n), temp(k,j,i,n) * r_nx_ny ); - // yakl::atomicAdd( hmean_r(k,n), rhod(k,j,i,n) * r_nx_ny ); yakl::atomicAdd( hmean_q(k,n), ( rhov(k,j,i,n) + rhol(k,j,i,n) + rhoi(k,j,i,n) ) * r_nx_ny ); - if (crm_accel_uv) { - yakl::atomicAdd( hmean_u(k,n), uvel(k,j,i,n) * r_nx_ny ); - yakl::atomicAdd( hmean_v(k,n), vvel(k,j,i,n) * r_nx_ny ); - } + if (crm_accel_rd) { yakl::atomicAdd( hmean_r(k,n), rhod(k,j,i,n) * r_nx_ny ); } + if (crm_accel_uv) { yakl::atomicAdd( hmean_u(k,n), uvel(k,j,i,n) * r_nx_ny ); } + if (crm_accel_uv) { yakl::atomicAdd( hmean_v(k,n), vvel(k,j,i,n) * r_nx_ny ); } }); //------------------------------------------------------------------------------------------------ // Compute the accelerated tendencies @@ -150,12 +149,10 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { ScalarLiveOut ceaseflag_liveout(false); parallel_for( SimpleBounds<2>(nz,nens) , YAKL_LAMBDA (int k, int n) { ttend_acc(k,n) = hmean_t(k,n) - accel_save_t(k,n); - // rtend_acc(k,n) = hmean_r(k,n) - accel_save_r(k,n); qtend_acc(k,n) = hmean_q(k,n) - accel_save_q(k,n); - if (crm_accel_uv) { - utend_acc(k,n) = hmean_u(k,n) - accel_save_u(k,n); - vtend_acc(k,n) = hmean_v(k,n) - accel_save_v(k,n); - } + if (crm_accel_rd) { rtend_acc(k,n) = hmean_r(k,n) - accel_save_r(k,n); } + if (crm_accel_uv) { utend_acc(k,n) = hmean_u(k,n) - accel_save_u(k,n); } + if (crm_accel_uv) { vtend_acc(k,n) = hmean_v(k,n) - accel_save_v(k,n); } if (abs(ttend_acc(k,n)) > dtemp_max) { ceaseflag_liveout = true; } @@ -191,14 +188,11 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { //------------------------------------------------------------------------------------------------ // Apply the accelerated tendencies parallel_for( SimpleBounds<4>(nz,ny,nx,nens) , YAKL_LAMBDA (int k, int j, int i, int n) { - temp(k,j,i,n) = temp(k,j,i,n) + crm_accel_factor * ttend_acc(k,n); - // rhod(k,j,i,n) = rhod(k,j,i,n) + crm_accel_factor * rtend_acc(k,n); rhov(k,j,i,n) = rhov(k,j,i,n) + crm_accel_factor * qtend_acc(k,n); - if (crm_accel_uv) { - uvel(k,j,i,n) = uvel(k,j,i,n) + crm_accel_factor * utend_acc(k,n); - vvel(k,j,i,n) = vvel(k,j,i,n) + crm_accel_factor * vtend_acc(k,n); - } + if (crm_accel_rd) { rhod(k,j,i,n) = rhod(k,j,i,n) + crm_accel_factor * rtend_acc(k,n); } + if (crm_accel_uv) { uvel(k,j,i,n) = uvel(k,j,i,n) + crm_accel_factor * utend_acc(k,n); } + if (crm_accel_uv) { vvel(k,j,i,n) = vvel(k,j,i,n) + crm_accel_factor * vtend_acc(k,n); } // apply limiter on temperature temp(k,j,i,n) = std::max( temp_min, temp(k,j,i,n) ); }); @@ -231,4 +225,3 @@ inline void pam_accelerate( pam::PamCoupler &coupler, int nstep, int &nstop ) { }); //------------------------------------------------------------------------------------------------ } - From 8e65b050739206ac2490374af86e0721562194a8 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 18 Apr 2024 16:24:38 -0400 Subject: [PATCH 045/904] remove MMF_PAM_dyn_per_phys namelist option --- components/eam/bld/build-namelist | 3 --- .../eam/bld/namelist_files/namelist_defaults_eam.xml | 1 - components/eam/bld/namelist_files/namelist_definition.xml | 6 ------ components/eam/src/physics/cam/phys_control.F90 | 8 ++------ components/eam/src/physics/crm/crm_physics.F90 | 7 ------- components/eam/src/physics/crm/pam/pam_driver.cpp | 1 + 6 files changed, 3 insertions(+), 23 deletions(-) diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index ac9001c3f7b1..8dc532b6a3df 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -3941,9 +3941,6 @@ if ($cfg->get('use_MMF')) { # MMF CRM domain orientation add_default($nl, 'MMF_orientation_angle'); - - # PAM dynamics sub-stepping parameter - add_default($nl, 'MMF_PAM_dyn_per_phys'); } add_default($nl, 'do_aerocom_ind3'); diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index d7e9afecf20e..fd7a91f19a1b 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -871,7 +871,6 @@ 0 .false. .true. - 1 90.0 0.0 diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index cc59697d5bbf..19e356ca6ccb 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -4824,12 +4824,6 @@ Defaults: 3D CRM: 0 - -Number of PAM dycor calls per physics time step. -Default: 2 - - Wavenumber cutoff for filtered MMF variance transport. diff --git a/components/eam/src/physics/cam/phys_control.F90 b/components/eam/src/physics/cam/phys_control.F90 index b87ea442c4e7..b7c9b37fa817 100644 --- a/components/eam/src/physics/cam/phys_control.F90 +++ b/components/eam/src/physics/cam/phys_control.F90 @@ -72,7 +72,6 @@ module phys_control logical :: use_crm_accel = .false. ! true => use MMF CRM mean-state acceleration (MSA) real(r8) :: crm_accel_factor = 2.D0 ! CRM acceleration factor logical :: crm_accel_uv = .true. ! true => apply MMF CRM MSA to momentum fields -integer :: MMF_PAM_dyn_per_phys = 2 ! PAM CRM dynamics steps per CRM physics steps logical :: use_subcol_microp = .false. ! if .true. then use sub-columns in microphysics @@ -229,7 +228,7 @@ subroutine phys_ctl_readnl(nlfile) eddy_scheme, microp_scheme, macrop_scheme, radiation_scheme, srf_flux_avg, & MMF_microphysics_scheme, MMF_orientation_angle, use_MMF, use_ECPP, & use_MMF_VT, MMF_VT_wn_max, use_MMF_ESMT, & - use_crm_accel, crm_accel_factor, crm_accel_uv, MMF_PAM_dyn_per_phys, & + use_crm_accel, crm_accel_factor, crm_accel_uv, & use_subcol_microp, atm_dep_flux, history_amwg, history_verbose, history_vdiag, & get_presc_aero_data,history_aerosol, history_aero_optics, & is_output_interactive_volc, & @@ -314,7 +313,6 @@ subroutine phys_ctl_readnl(nlfile) call mpibcast(use_crm_accel, 1 , mpilog, 0, mpicom) call mpibcast(crm_accel_factor, 1 , mpir8, 0, mpicom) call mpibcast(crm_accel_uv, 1 , mpilog, 0, mpicom) - call mpibcast(MMF_PAM_dyn_per_phys, 1 , mpiint, 0, mpicom) call mpibcast(use_subcol_microp, 1 , mpilog, 0, mpicom) call mpibcast(atm_dep_flux, 1 , mpilog, 0, mpicom) call mpibcast(history_amwg, 1 , mpilog, 0, mpicom) @@ -598,7 +596,7 @@ subroutine phys_getopts(deep_scheme_out, shallow_scheme_out, eddy_scheme_out, & prog_modal_aero_out, macrop_scheme_out, ideal_phys_option_out, & use_MMF_out, use_ECPP_out, MMF_microphysics_scheme_out, & MMF_orientation_angle_out, use_MMF_VT_out, MMF_VT_wn_max_out, use_MMF_ESMT_out, & - use_crm_accel_out, crm_accel_factor_out, crm_accel_uv_out, MMF_PAM_dyn_per_phys_out, & + use_crm_accel_out, crm_accel_factor_out, crm_accel_uv_out, & do_clubb_sgs_out, do_shoc_sgs_out, do_tms_out, state_debug_checks_out, & linearize_pbl_winds_out, & do_aerocom_ind3_out, & @@ -641,7 +639,6 @@ subroutine phys_getopts(deep_scheme_out, shallow_scheme_out, eddy_scheme_out, & logical, intent(out), optional :: use_crm_accel_out real(r8), intent(out), optional :: crm_accel_factor_out logical, intent(out), optional :: crm_accel_uv_out - integer, intent(out), optional :: MMF_PAM_dyn_per_phys_out logical, intent(out), optional :: use_subcol_microp_out logical, intent(out), optional :: atm_dep_flux_out logical, intent(out), optional :: history_amwg_out @@ -745,7 +742,6 @@ subroutine phys_getopts(deep_scheme_out, shallow_scheme_out, eddy_scheme_out, & if ( present(use_crm_accel_out ) ) use_crm_accel_out = use_crm_accel if ( present(crm_accel_factor_out ) ) crm_accel_factor_out = crm_accel_factor if ( present(crm_accel_uv_out ) ) crm_accel_uv_out = crm_accel_uv - if ( present(MMF_PAM_dyn_per_phys_out) ) MMF_PAM_dyn_per_phys_out = MMF_PAM_dyn_per_phys if ( present(use_subcol_microp_out ) ) use_subcol_microp_out = use_subcol_microp if ( present(macrop_scheme_out ) ) macrop_scheme_out = macrop_scheme diff --git a/components/eam/src/physics/crm/crm_physics.F90 b/components/eam/src/physics/crm/crm_physics.F90 index 51628d5ea74e..39ed6a0876c7 100644 --- a/components/eam/src/physics/crm/crm_physics.F90 +++ b/components/eam/src/physics/crm/crm_physics.F90 @@ -620,8 +620,6 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, logical :: use_MMF_VT_tmp ! flag for MMF variance transport (for Fortran CRM) logical :: use_MMF_ESMT_tmp ! flag for MMF scalar momentum transport (for Fortran CRM) integer :: MMF_VT_wn_max ! wavenumber cutoff for filtered variance transport - - integer :: MMF_PAM_dyn_per_phys ! PAM CRM dynamics steps per CRM physics steps real(r8) :: tmp_e_sat ! temporary saturation vapor pressure real(r8) :: tmp_q_sat ! temporary saturation specific humidity @@ -734,9 +732,6 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, use_crm_accel = use_crm_accel_tmp crm_accel_uv = crm_accel_uv_tmp - ! PAM namelist options - call phys_getopts(MMF_PAM_dyn_per_phys_out = MMF_PAM_dyn_per_phys) - nstep = get_nstep() itim = pbuf_old_tim_idx() ! "Old" pbuf time index (what does all this mean?) @@ -1466,8 +1461,6 @@ subroutine crm_physics_tend(ztodt, state, tend, ptend, pbuf2d, cam_in, cam_out, call pam_set_option('enable_physics_tend_stats', .false. ) - call pam_set_option('crm_dyn_per_phys', MMF_PAM_dyn_per_phys ) - call pam_set_option('is_first_step', (nstep<=1) ) call pam_set_option('is_restart', (nsrest>0) ) call pam_set_option('am_i_root', masterproc ) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index a05b5cf8b037..f8f575ec8a1d 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -122,6 +122,7 @@ extern "C" void pam_driver() { coupler.set_option("crm_acceleration_ceaseflag",false); //------------------------------------------------------------------------------------------------ // coupler options for SPAM dycor + coupler.set_option("crm_dyn_per_phys",1); coupler.set_option("spam_couple_wind_exact_inverse",true); coupler.set_option("spam_clip_negative_densities",true); coupler.set_option("spam_clip_vertical_velocities",true); From 743b8ff2bdcc8a50672e5135ea3fa5c2e6a4ea4d Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 18 Apr 2024 16:26:14 -0400 Subject: [PATCH 046/904] fix white space --- components/eam/bld/namelist_files/namelist_defaults_eam.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index fd7a91f19a1b..56c816b1753c 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -871,6 +871,7 @@ 0 .false. .true. + 90.0 0.0 From 56c11f70c5137f887f204d00cd6bd7de9ed9e29a Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 18 Apr 2024 16:37:56 -0400 Subject: [PATCH 047/904] pam_debug.h clean-up --- .../eam/src/physics/crm/pam/pam_debug.h | 235 ++++++++++-------- 1 file changed, 129 insertions(+), 106 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_debug.h b/components/eam/src/physics/crm/pam/pam_debug.h index 43d2b5bf4f5d..7e54f437f691 100644 --- a/components/eam/src/physics/crm/pam/pam_debug.h +++ b/components/eam/src/physics/crm/pam/pam_debug.h @@ -91,121 +91,144 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { auto input_phis = dm_host.get("input_phis").createDeviceCopy(); real grav = 9.80616; //------------------------------------------------------------------------------------------------ + // logical switches to set which checks are active + bool chk_nan = true; // check for NaN values in select fields + bool chk_neg = true; // check for negative values in select fields + bool chk_low_t = false; // check for low temperatures + bool chk_drop_t = true; // check for large drops in temperature + bool chk_wvel = false; // check for large vertical velocity + //------------------------------------------------------------------------------------------------ // Check for invalid values parallel_for("", SimpleBounds<4>(nz,ny,nx,nens), YAKL_LAMBDA (int k, int j, int i, int iens) { auto phis = input_phis(iens)/grav; + //-------------------------------------------------------------------------- // Check for NaNs - const auto is_nan_t_atm = isnan( temp(k,j,i,iens) ); - const auto is_nan_d_atm = isnan( rhod(k,j,i,iens) ); - const auto is_nan_q_atm = isnan( rhov(k,j,i,iens) ); - if ( is_nan_t_atm || is_nan_q_atm || is_nan_d_atm ) { - auto phis = input_phis(iens)/grav; - printf("PAM-DEBUG nan-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", - nstep,id,k,i,iens,lat(iens),lon(iens),phis, - temp(k,j,i,iens), - rhod(k,j,i,iens), - rhov(k,j,i,iens), - rhoc(k,j,i,iens), - rhoi(k,j,i,iens), - uvel(k,j,i,iens), - wvel(k,j,i,iens), - debug_save_temp(k,j,i,iens), - debug_save_rhod(k,j,i,iens), - debug_save_rhov(k,j,i,iens), - debug_save_rhoc(k,j,i,iens), - debug_save_rhoi(k,j,i,iens), - debug_save_uvel(k,j,i,iens), - debug_save_wvel(k,j,i,iens) - ); + if (chk_nan) { + const auto is_nan_t_atm = isnan( temp(k,j,i,iens) ); + const auto is_nan_d_atm = isnan( rhod(k,j,i,iens) ); + const auto is_nan_q_atm = isnan( rhov(k,j,i,iens) ); + if ( is_nan_t_atm || is_nan_q_atm || is_nan_d_atm ) { + auto phis = input_phis(iens)/grav; + printf("PAM-DEBUG nan-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + nstep,id,k,i,iens,lat(iens),lon(iens),phis, + temp(k,j,i,iens), + rhod(k,j,i,iens), + rhov(k,j,i,iens), + rhoc(k,j,i,iens), + rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), + debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), + debug_save_rhov(k,j,i,iens), + debug_save_rhoc(k,j,i,iens), + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) + ); + } } + //-------------------------------------------------------------------------- // Check for negative values - const auto is_neg_t_atm = temp(k,j,i,iens)<0; - const auto is_neg_d_atm = rhod(k,j,i,iens)<0; - const auto is_neg_q_atm = rhov(k,j,i,iens)<0; - if ( is_neg_t_atm || is_neg_q_atm || is_neg_d_atm ) { - auto phis = input_phis(iens)/grav; - printf("PAM-DEBUG neg-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", - nstep,id,k,i,iens,lat(iens),lon(iens),phis, - temp(k,j,i,iens), - rhod(k,j,i,iens), - rhov(k,j,i,iens), - rhoc(k,j,i,iens), - rhoi(k,j,i,iens), - uvel(k,j,i,iens), - wvel(k,j,i,iens), - debug_save_temp(k,j,i,iens), - debug_save_rhod(k,j,i,iens), - debug_save_rhov(k,j,i,iens), - debug_save_rhoc(k,j,i,iens), - debug_save_rhoi(k,j,i,iens), - debug_save_uvel(k,j,i,iens), - debug_save_wvel(k,j,i,iens) - ); + if (chk_neg) { + const auto is_neg_t_atm = temp(k,j,i,iens)<0; + const auto is_neg_d_atm = rhod(k,j,i,iens)<0; + const auto is_neg_q_atm = rhov(k,j,i,iens)<0; + if ( is_neg_t_atm || is_neg_q_atm || is_neg_d_atm ) { + auto phis = input_phis(iens)/grav; + printf("PAM-DEBUG neg-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + nstep,id,k,i,iens,lat(iens),lon(iens),phis, + temp(k,j,i,iens), + rhod(k,j,i,iens), + rhov(k,j,i,iens), + rhoc(k,j,i,iens), + rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), + debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), + debug_save_rhov(k,j,i,iens), + debug_save_rhoc(k,j,i,iens), + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) + ); + } + } + //-------------------------------------------------------------------------- + // Check for low temperature + if (chk_low_t) { + const auto is_low_t = temp(k,j,i,iens)<100; + if ( is_low_t ) { + printf("PAM-DEBUG low-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + nstep,id,k,i,iens,lat(iens),lon(iens),phis, + temp(k,j,i,iens), + rhod(k,j,i,iens), + rhov(k,j,i,iens), + rhoc(k,j,i,iens), + rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), + debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), + debug_save_rhov(k,j,i,iens), + debug_save_rhoc(k,j,i,iens), + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) + ); + } } - // // Check for low temperature - // const auto is_low_t = temp(k,j,i,iens)<100; - // if ( is_low_t ) { - // printf("PAM-DEBUG low-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", - // nstep,id,k,i,iens,lat(iens),lon(iens),phis, - // temp(k,j,i,iens), - // rhod(k,j,i,iens), - // rhov(k,j,i,iens), - // rhoc(k,j,i,iens), - // rhoi(k,j,i,iens), - // uvel(k,j,i,iens), - // wvel(k,j,i,iens), - // debug_save_temp(k,j,i,iens), - // debug_save_rhod(k,j,i,iens), - // debug_save_rhov(k,j,i,iens), - // debug_save_rhoc(k,j,i,iens), - // debug_save_rhoi(k,j,i,iens), - // debug_save_uvel(k,j,i,iens), - // debug_save_wvel(k,j,i,iens) - // ); - // } + //-------------------------------------------------------------------------- // Check for large temperature drops - const auto is_drop_t = (temp(k,j,i,iens)-debug_save_temp(k,j,i,iens))<-50; - if ( is_drop_t ) { - printf("PAM-DEBUG drop-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", - nstep,id,k,i,iens,lat(iens),lon(iens),phis, - temp(k,j,i,iens), - rhod(k,j,i,iens), - rhov(k,j,i,iens), - rhoc(k,j,i,iens), - rhoi(k,j,i,iens), - uvel(k,j,i,iens), - wvel(k,j,i,iens), - debug_save_temp(k,j,i,iens), - debug_save_rhod(k,j,i,iens), - debug_save_rhov(k,j,i,iens), - debug_save_rhoc(k,j,i,iens), - debug_save_rhoi(k,j,i,iens), - debug_save_uvel(k,j,i,iens), - debug_save_wvel(k,j,i,iens) - ); + if (chk_drop_t) { + const auto is_drop_t = (temp(k,j,i,iens)-debug_save_temp(k,j,i,iens))<-50; + if ( is_drop_t ) { + printf("PAM-DEBUG drop-T - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + nstep,id,k,i,iens,lat(iens),lon(iens),phis, + temp(k,j,i,iens), + rhod(k,j,i,iens), + rhov(k,j,i,iens), + rhoc(k,j,i,iens), + rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), + debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), + debug_save_rhov(k,j,i,iens), + debug_save_rhoc(k,j,i,iens), + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) + ); + } + } + //-------------------------------------------------------------------------- + // Check for large vertical velocity + if (chk_wvel) { + const auto is_large_pos_w = wvel(k,j,i,iens)> 40; + const auto is_large_neg_w = wvel(k,j,i,iens)<-40; + if ( is_large_pos_w || is_large_neg_w ) { + printf("PAM-DEBUG large-W - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", + nstep,id,k,i,iens,lat(iens),lon(iens),phis, + temp(k,j,i,iens), + rhod(k,j,i,iens), + rhov(k,j,i,iens), + rhoc(k,j,i,iens), + rhoi(k,j,i,iens), + uvel(k,j,i,iens), + wvel(k,j,i,iens), + debug_save_temp(k,j,i,iens), + debug_save_rhod(k,j,i,iens), + debug_save_rhov(k,j,i,iens), + debug_save_rhoc(k,j,i,iens), + debug_save_rhoi(k,j,i,iens), + debug_save_uvel(k,j,i,iens), + debug_save_wvel(k,j,i,iens) + ); + } } - // // Check for large vertical velocity - // const auto is_large_pos_w = wvel(k,j,i,iens)> 40; - // const auto is_large_neg_w = wvel(k,j,i,iens)<-40; - // if ( is_large_pos_w || is_large_neg_w ) { - // printf("PAM-DEBUG large-W - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", - // nstep,id,k,i,iens,lat(iens),lon(iens),phis, - // temp(k,j,i,iens), - // rhod(k,j,i,iens), - // rhov(k,j,i,iens), - // rhoc(k,j,i,iens), - // rhoi(k,j,i,iens), - // uvel(k,j,i,iens), - // wvel(k,j,i,iens), - // debug_save_temp(k,j,i,iens), - // debug_save_rhod(k,j,i,iens), - // debug_save_rhov(k,j,i,iens), - // debug_save_rhoc(k,j,i,iens), - // debug_save_rhoi(k,j,i,iens), - // debug_save_uvel(k,j,i,iens), - // debug_save_wvel(k,j,i,iens) - // ); - // } + //-------------------------------------------------------------------------- // update saved previous values debug_save_temp(k,j,i,iens) = temp(k,j,i,iens); debug_save_rhod(k,j,i,iens) = rhod(k,j,i,iens); From 8eae43e1a3df9f9ddde9a52d37ecad06ec77b199 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 25 Apr 2024 15:13:25 -0400 Subject: [PATCH 048/904] update PAM submodule --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 8f9538db7138..5931666c95a3 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 8f9538db7138baf26fa2b0f163fae3050d3fe29d +Subproject commit 5931666c95a3b75ae63fbe5bff6e743e74cb2c0a From 567fa1d665a2909266a3ee774e725e97b30a7f05 Mon Sep 17 00:00:00 2001 From: daleihao Date: Wed, 1 May 2024 11:56:37 -0700 Subject: [PATCH 049/904] change top_rad rawdate to 1km --- .../bld/namelist_files/namelist_defaults.xml | 9 ++++++++ .../namelist_defaults_tools.xml | 6 +++--- .../tools/mksurfdata_map/src/mktopradMod.F90 | 21 ++++++++++++++----- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 8d7ad4fba900..0132320a2153 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -1657,6 +1657,13 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/2.5x3.33/map_1km-merge-10min_HYDRO1K-merge-nomask_to_2.5x3.33_nomask_aave_da_c130405.nc + + +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240424.nc + + + @@ -1704,6 +1711,8 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/0.5x0.5/map_360x720cru_cruncep_to_0.5x0.5_nomask_aave_da_c190417.nc lnd/clm2/mappingdata/maps/0.5x0.5/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.5x0.5_nomask_aave_da_c190417.nc +lnd/clm2/mappingdata/maps/0.5x0.5/map_0.01x0.01_nomask_to_0.5x0.5_nomask_aave_da_c240424.nc diff --git a/components/elm/bld/namelist_files/namelist_defaults_tools.xml b/components/elm/bld/namelist_files/namelist_defaults_tools.xml index d948b16c95a5..f97a41314be0 100644 --- a/components/elm/bld/namelist_files/namelist_defaults_tools.xml +++ b/components/elm/bld/namelist_files/namelist_defaults_tools.xml @@ -177,7 +177,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 0.5x0.5 0.5x0.5 5x5min -0.1x0.1 +0.01x0.01 @@ -317,8 +317,8 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/rawdata/mksrf_fert.c220309.nc -lnd/clm2/rawdata/mksrf_toprad_0.1x0.1.c231218.nc +lnd/clm2/rawdata/mksrf_toprad_0.01x0.01.c240422.nc diff --git a/components/elm/tools/mksurfdata_map/src/mktopradMod.F90 b/components/elm/tools/mksurfdata_map/src/mktopradMod.F90 index e8ea66f9de49..0325019706f7 100644 --- a/components/elm/tools/mksurfdata_map/src/mktopradMod.F90 +++ b/components/elm/tools/mksurfdata_map/src/mktopradMod.F90 @@ -114,25 +114,36 @@ subroutine mktoprad(ldomain, mapfname, datfname, varname, ndiag, top_o, nodata) call check_ret(nf_get_var_double (ncidi, varid, top_i), subname) call check_ret(nf_close(ncidi), subname) - ! Read topo dataset + ! set mask as 0 when topo data is filled value: -9999 + allocate(mask_i(ns_i), stat=ier) + if (ier /= 0) then + write(6,*)'mktoprad allocation error'; call abort() + end if + + mask_i(:) = 1._r8 + do ni = 1,ns_i + if (top_i(ni) < -1000._r8) then + mask_i(ni) = 0._r8 + end if + enddo + + ! Read mapping file call gridmap_mapread(tgridmap, mapfname) ! Error checks for domain and map consistencies - ! Note that the topo dataset has no landmask - so a unit landmask is assumed - call domain_checksame( tdomain, ldomain, tgridmap ) ! Determine top_o on output grid - top_o(:) = nodata - call gridmap_areaave(tgridmap, top_i, top_o, nodata=nodata) + call gridmap_areaave(tgridmap, top_i, top_o, nodata=nodata, mask_src=mask_i) ! Deallocate dynamic memory call domain_clean(tdomain) call gridmap_clean(tgridmap) deallocate (top_i) + deallocate (mask_i) write (6,*) 'Successfully made topography parameters' write (6,*) From cb43f07642f97625c9c3a66afda40b0322449326 Mon Sep 17 00:00:00 2001 From: daleihao Date: Thu, 2 May 2024 06:21:00 -0700 Subject: [PATCH 050/904] update mapping files --- components/elm/bld/namelist_files/namelist_defaults.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 0132320a2153..9a5294055647 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -1660,7 +1660,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240424.nc +>lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc @@ -1712,7 +1712,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/0.5x0.5/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.5x0.5_nomask_aave_da_c190417.nc lnd/clm2/mappingdata/maps/0.5x0.5/map_0.01x0.01_nomask_to_0.5x0.5_nomask_aave_da_c240424.nc +>lnd/clm2/mappingdata/maps/0.5x0.5/map_0.01x0.01_nomask_to_0.5x0.5_nomask_aave_da_c240501.nc From b5d1e12a94ccbf153f15b2a7ebdd249a3d4f7c9b Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 7 May 2024 16:07:39 -0700 Subject: [PATCH 051/904] update RCEMIP zenith angle --- components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl index 3ecd465f7a14..e3d4433d4a8d 100644 --- a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl +++ b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -31,4 +31,4 @@ seq_flux_mct_albdif = 0.07 seq_flux_mct_albdir = 0.07 seq_flux_atmocn_minwind = 1 -constant_zenith_deg = 42.05 \ No newline at end of file +constant_zenith_deg = 42.04 \ No newline at end of file From b33fa7dacce0a31f9552649dc14d0f774ead8780 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 7 May 2024 16:08:06 -0700 Subject: [PATCH 052/904] update RCE use case comment --- components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml b/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml index c1c8e9e027ad..146a9483b36f 100644 --- a/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml +++ b/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml @@ -35,7 +35,7 @@ 0.0 - + From f3278f20b89fe1ed80681e0c8d2935c911e70adb Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:43:37 -0700 Subject: [PATCH 053/904] add RCEMIP2 compsets --- .../eam/cime_config/config_compsets.xml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 39971fbb0945..01e99a9a1489 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -229,6 +229,50 @@ 2000_EAM%RCE-MMF2_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV + + + FRCE-MW_295dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW_300dT0p625 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW_300dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW_300dT2p5 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW_305dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + + FRCE-MW-MMF1_295dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT0p625 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT2p5 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_305dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + From b63cbcadde51a50160d1221a740d142d20ea5c0c Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:44:11 -0700 Subject: [PATCH 054/904] add new data ocean SST patterns or RCEMIP2 --- .../docn/cime_config/config_component.xml | 10 +++- .../cime_config/namelist_definition_docn.xml | 7 ++- .../data_comps/docn/src/docn_comp_mod.F90 | 54 ++++++++++++++++++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/components/data_comps/docn/cime_config/config_component.xml b/components/data_comps/docn/cime_config/config_component.xml index 5a30c69df6cf..431d358f9951 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/components/data_comps/docn/cime_config/config_component.xml @@ -13,7 +13,7 @@ This file may have ocn desc entries. --> - DOCN + DOCN null mode prescribed ocean mode slab ocean mode @@ -45,7 +45,7 @@ char - prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null + prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquap11,sst_aquap12,sst_aquap13,sst_aquap14,sst_aquap15,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null prescribed null @@ -63,6 +63,12 @@ sst_aquap8 sst_aquap9 sst_aquap10 + + sst_aquap11 + sst_aquap12 + sst_aquap13 + sst_aquap14 + sst_aquap15 sst_aquapfile sst_aquap_constant diff --git a/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 948902e37324..a191d088d7f5 100644 --- a/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -257,7 +257,7 @@ char streams shr_strdata_nml - SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL + SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAP11,SST_AQUAP12,SST_AQUAP13,SST_AQUAP14,SST_AQUAP15,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL General method that operates on the data. This is generally implemented in the data models but is set in the strdata method for @@ -323,6 +323,11 @@ SST_AQUAP8 SST_AQUAP9 SST_AQUAP10 + SST_AQUAP11 + SST_AQUAP12 + SST_AQUAP13 + SST_AQUAP14 + SST_AQUAP15 SST_AQUAPFILE SST_AQUAP_CONSTANT SOM diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index e692882c9db1..002ed655b404 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -984,6 +984,7 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) integer :: i real(r8) :: tmp, tmp1, pi real(r8) :: rlon(lsize), rlat(lsize) + real(r8) :: mean_SST, delta_SST real(r8), parameter :: pio180 = SHR_CONST_PI/180._r8 @@ -1013,8 +1014,8 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) ! Control - if (sst_option < 1 .or. sst_option > 10) then - call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 10') + if (sst_option < 1 .or. sst_option > 15) then + call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 15') end if if (sst_option == 1 .or. sst_option == 6 .or. sst_option == 7 .or. sst_option == 8) then @@ -1174,6 +1175,55 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) end do end if + !------------------------------------------------------------------------------- + ! RCEMIP phase 2 - Mock-Walker + + ! MW_295dT1p25 - mean SST = 295 / dSST = 1.25 K + + if (sst_option == 11) then + mean_SST = 295 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT0p625 - mean SST = 300 / dSST = 0.625 K + if (sst_option == 12) then + mean_SST = 300 + delta_SST = 0.625 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT1p25 - mean SST = 300 / dSST = 1.25 K + if (sst_option == 13) then + mean_SST = 300 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT2p5 - mean SST = 300 / dSST = 2.5 K + if (sst_option == 14) then + mean_SST = 300 + delta_SST = 2.5 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_305dT1p25 - mean SST = 305 / dSST = 1.25 K + if (sst_option == 15) then + mean_SST = 305 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + end subroutine prescribed_sst end module docn_comp_mod From 3803486652580fc8f983ec5af650c4399fa21870 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:45:03 -0700 Subject: [PATCH 055/904] add TMQS diagnostic for RCEMIP2 --- components/eam/src/physics/cam/cam_diagnostics.F90 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/eam/src/physics/cam/cam_diagnostics.F90 b/components/eam/src/physics/cam/cam_diagnostics.F90 index ead4f558f05b..0b4abed44720 100644 --- a/components/eam/src/physics/cam/cam_diagnostics.F90 +++ b/components/eam/src/physics/cam/cam_diagnostics.F90 @@ -306,6 +306,7 @@ subroutine diag_init() call addfld ('MQ',(/ 'lev' /), 'A','kg/m2','Water vapor mass in layer') call addfld ('TMQ',horiz_only, 'A','kg/m2','Total (vertically integrated) precipitable water', & standard_name='atmosphere_mass_content_of_water_vapor') + call addfld ('TMQS',horiz_only, 'A','kg/m2','Total (vertically integrated) saturated precipitable water') call addfld ('TTQ',horiz_only, 'A', 'kg/m/s','Total (vertically integrated) vapor transport') call addfld ('TUQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) zonal water flux') call addfld ('TVQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) meridional water flux') @@ -1335,6 +1336,15 @@ subroutine diag_phys_writeout(state, psl) ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) end do call outfld ('TMQ ',ftem, pcols ,lchnk ) +! +! Mass of saturated q vertically integrated +! + call qsat(state%t(:ncol,:), state%pmid(:ncol,:), tem2(:ncol,:), ftem(:ncol,:)) + ftem(:ncol,:) = ftem(:ncol,:) * state%pdel(:ncol,:) * rga + do k=2,pver + ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) + end do + call outfld ('TMQS ',ftem, pcols ,lchnk ) ! ! Mass of vertically integrated water vapor flux ! From f835a2c3c157223b1299e93e04503dfa876e99f4 Mon Sep 17 00:00:00 2001 From: daleihao Date: Tue, 28 May 2024 12:31:54 -0700 Subject: [PATCH 056/904] use std_elev when stdev_elev is not available in fsurdata & remove latitude limit fix a small bug --- components/elm/src/biogeophys/SurfaceAlbedoMod.F90 | 2 +- components/elm/src/main/surfrdMod.F90 | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 b/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 index 073c836c19bc..49c6d061d3d8 100644 --- a/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 +++ b/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 @@ -1811,7 +1811,7 @@ subroutine Albedo_TOP_Adjustment(bounds, num_pft, filter_pft, & lon_180 = lon(g) if (lon_180 > pi) lon_180 = lon_180-2._r8*pi - if (cosz > 0._r8 .and. abs(lat(g)) < 1.047_r8 .and. stdev_elev(g) > 0._r8) then + if (cosz > 0._r8 .and. stdev_elev(g) > 0._r8) then local_timeofday = next_tod + lon_180 / pi * 180._r8 * 240._r8 if (local_timeofday >= 86400._r8) then diff --git a/components/elm/src/main/surfrdMod.F90 b/components/elm/src/main/surfrdMod.F90 index 9cbcd4b2b8e0..31ce8cd5b5f6 100755 --- a/components/elm/src/main/surfrdMod.F90 +++ b/components/elm/src/main/surfrdMod.F90 @@ -1702,7 +1702,12 @@ subroutine surfrd_get_topo_for_solar_rad(domain,filename) call ncd_io(ncid=ncid, varname='STDEV_ELEV', flag='read', data=domain%stdev_elev, & dim1name=grlnd, readvar=readvar) - if (.not. readvar) call endrun( trim(subname)//' ERROR: STDEV_ELEV NOT on fsurdat file' ) + if (.not. readvar) then + write(iulog,*) trim(subname),' ERROR: STDEV_ELEV NOT on fsurdat file' + call ncd_io(ncid=ncid, varname='STD_ELEV', flag='read', data=domain%stdev_elev, & + dim1name=grlnd, readvar=readvar) + if (.not. readvar) call endrun( trim(subname)//' ERROR: BOTH STD_ELEV and STDEV_ELEV NOT on fsurdat file' ) + endif call ncd_io(ncid=ncid, varname='SKY_VIEW', flag='read', data=domain%sky_view, & dim1name=grlnd, readvar=readvar) if (.not. readvar) call endrun( trim(subname)//' ERROR: SKY_VIEW NOT on fsurdat file' ) From e4b362d0269854dd5537e935cf2d94fdf1ac8a1b Mon Sep 17 00:00:00 2001 From: Dalei Hao Date: Tue, 28 May 2024 14:48:55 -0700 Subject: [PATCH 057/904] modidy error message --- components/elm/src/main/surfrdMod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/main/surfrdMod.F90 b/components/elm/src/main/surfrdMod.F90 index 31ce8cd5b5f6..43bb7965314a 100755 --- a/components/elm/src/main/surfrdMod.F90 +++ b/components/elm/src/main/surfrdMod.F90 @@ -1703,7 +1703,7 @@ subroutine surfrd_get_topo_for_solar_rad(domain,filename) call ncd_io(ncid=ncid, varname='STDEV_ELEV', flag='read', data=domain%stdev_elev, & dim1name=grlnd, readvar=readvar) if (.not. readvar) then - write(iulog,*) trim(subname),' ERROR: STDEV_ELEV NOT on fsurdat file' + write(iulog,*) trim(subname),' WARNING: STDEV_ELEV NOT on fsurdat file. Try to use STD_ELEV instead.' call ncd_io(ncid=ncid, varname='STD_ELEV', flag='read', data=domain%stdev_elev, & dim1name=grlnd, readvar=readvar) if (.not. readvar) call endrun( trim(subname)//' ERROR: BOTH STD_ELEV and STDEV_ELEV NOT on fsurdat file' ) From 630a785b904339cd9f93eb4a0220869c4740bc4b Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 29 May 2024 17:03:52 -0700 Subject: [PATCH 058/904] work in progress mods to update definition of vtheta_dp to be consistent with v1 def in IOP interface --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 9e04b4a00920..2cd2ef95be66 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -18,6 +18,9 @@ #include "SimulationParams.hpp" #include "Types.hpp" +// SCREAM includes +#include "share/util/scream_common_physics_functions.hpp" + // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -227,6 +230,8 @@ apply_iop_forcing(const Real dt) using ElementOps = Homme::ElementOps; using KV = Homme::KernelVariables; + using PF = PhysicsFunctions; + using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -436,7 +441,7 @@ apply_iop_forcing(const Real dt) } // Update T and qv according to large scale forcing as specified in IOP file. - advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); +// advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); if (iop_coriolis) { // Apply coriolis forcing to u and v winds @@ -457,6 +462,13 @@ apply_iop_forcing(const Real dt) const int igp = idx/NGP; const int jgp = idx%NGP; + // Get temp views from workspace + auto ws = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); + + auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); @@ -466,6 +478,17 @@ apply_iop_forcing(const Real dt) auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); @@ -481,11 +504,13 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - // Recompute rstar with updated qv, and convert updated temperature back to potential temperature - elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); + // Convert updated temperature back to density weight virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + const auto th = PF::calculate_theta_from_T(temperature_i(k),s_pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); + // Release WS views + ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); }); From d04dc45f77cb514abe3890627c1918db3dd77ac7 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 30 May 2024 08:25:26 -0700 Subject: [PATCH 059/904] uncomment out advance_iop_forcing --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 2cd2ef95be66..cc3f2e6f3027 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -441,7 +441,7 @@ apply_iop_forcing(const Real dt) } // Update T and qv according to large scale forcing as specified in IOP file. -// advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); + advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); if (iop_coriolis) { // Apply coriolis forcing to u and v winds From 7a7333d268b432497137794ae6de99d8454e23c4 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sun, 2 Jun 2024 20:50:20 -0400 Subject: [PATCH 060/904] set explicit out and expand type --- components/eamxx/src/physics/p3/p3_iso_c.f90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index d75af1cb2edb..625ee48bc506 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,8 +16,8 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=256), intent(inout) :: string - character(*), intent(in) :: prefix + character(kind=c_char, len=256), intent(out) :: string + character(kind=c_char, len=*), intent(in) :: prefix real(kind=c_real) :: s write (string, '(a,i1,a1)') prefix, sizeof(s), C_NULL_CHAR From b2bd480441774296c4bc12803ae6b1d0dec1e48e Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sun, 2 Jun 2024 21:18:22 -0400 Subject: [PATCH 061/904] avoid specifying prefix type Co-authored-by: Andrew M. Bradley --- components/eamxx/src/physics/p3/p3_iso_c.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index 625ee48bc506..35c1154325f7 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -17,7 +17,7 @@ module p3_iso_c subroutine append_precision(string, prefix) character(kind=c_char, len=256), intent(out) :: string - character(kind=c_char, len=*), intent(in) :: prefix + character(*), intent(in) :: prefix real(kind=c_real) :: s write (string, '(a,i1,a1)') prefix, sizeof(s), C_NULL_CHAR From d0a7290ff1bf7f8f4fc09bfe2125ce927d7909ba Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 3 Jun 2024 14:47:24 -0400 Subject: [PATCH 062/904] use char concat instead of write routine --- components/eamxx/src/physics/p3/p3_iso_c.f90 | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index 35c1154325f7..5f887607c182 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=256), intent(out) :: string + character(kind=c_char, len=512), intent(out) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,10 +69,17 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. - call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") - call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") - call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") - call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") +#ifdef SCREAM_DOUBLE_PRECISION + mu_r_filename = lookup_file_dir//'/mu_r_table_vals.dat8'//C_NULL_CHAR + revap_filename = lookup_file_dir//'/revap_table_vals.dat8'//C_NULL_CHAR + vn_filename = lookup_file_dir//'/vn_table_vals.dat8'//C_NULL_CHAR + vm_filename = lookup_file_dir//'/vm_table_vals.dat8'//C_NULL_CHAR +#else + mu_r_filename = lookup_file_dir//'/mu_r_table_vals.dat4'//C_NULL_CHAR + revap_filename = lookup_file_dir//'/revap_table_vals.dat4'//C_NULL_CHAR + vn_filename = lookup_file_dir//'/vn_table_vals.dat4'//C_NULL_CHAR + vm_filename = lookup_file_dir//'/vm_table_vals.dat4'//C_NULL_CHAR +#endif if (write_tables) then call p3_init_b() From d2ed08cf79276ae8e9fc9bab3bb16bc5aa0eb53d Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 3 Jun 2024 15:21:27 -0400 Subject: [PATCH 063/904] null char offset --- components/eamxx/src/physics/p3/p3_iso_c.f90 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index 5f887607c182..ea0a18411c10 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -70,15 +70,15 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) ok = .false. #ifdef SCREAM_DOUBLE_PRECISION - mu_r_filename = lookup_file_dir//'/mu_r_table_vals.dat8'//C_NULL_CHAR - revap_filename = lookup_file_dir//'/revap_table_vals.dat8'//C_NULL_CHAR - vn_filename = lookup_file_dir//'/vn_table_vals.dat8'//C_NULL_CHAR - vm_filename = lookup_file_dir//'/vm_table_vals.dat8'//C_NULL_CHAR + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR #else - mu_r_filename = lookup_file_dir//'/mu_r_table_vals.dat4'//C_NULL_CHAR - revap_filename = lookup_file_dir//'/revap_table_vals.dat4'//C_NULL_CHAR - vn_filename = lookup_file_dir//'/vn_table_vals.dat4'//C_NULL_CHAR - vm_filename = lookup_file_dir//'/vm_table_vals.dat4'//C_NULL_CHAR + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR #endif if (write_tables) then From bfc54464aa92a62159127aae5cab9c4a3599fb0f Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 4 Jun 2024 10:12:06 -0700 Subject: [PATCH 064/904] update temperature consistent with defintions of vtheta_dp that homme uses, two way --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index cc3f2e6f3027..6c7f41cf5972 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -355,10 +355,17 @@ apply_iop_forcing(const Real dt) auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); + // Get temp views from workspace + auto ws2 = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws2.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; const int jgp = idx%NGP; + auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); @@ -368,6 +375,18 @@ apply_iop_forcing(const Real dt) auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + team.team_barrier(); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); @@ -389,12 +408,19 @@ apply_iop_forcing(const Real dt) eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); } - // Get the temperature from dynamics states - elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + // Compute temperature + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + auto& T_val = vtheta_dp_i(k); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); + }); + }); // Release WS views ws.release_macro_block(pnh_slot, NGP*NGP); + ws2.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); }; @@ -489,6 +515,8 @@ apply_iop_forcing(const Real dt) } }); + team.team_barrier(); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); @@ -504,11 +532,12 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - // Convert updated temperature back to density weight virtual potential temperature + // Recompute rstar with updated qv, and convert updated temperature back to potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),s_pmid(k)); + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); + // Release WS views ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); From 0161bd7a45a1c879794a1995a5b571f867685c04 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 5 Jun 2024 17:08:33 -0600 Subject: [PATCH 065/904] Port scream_rrtmgp_interface to new pattern --- .../eamxx/src/physics/rrtmgp/CMakeLists.txt | 1 - .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 6 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 1167 +------------- .../rrtmgp/scream_rrtmgp_interface.hpp | 1432 ++++++++++++++--- 4 files changed, 1243 insertions(+), 1363 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index c098a0829d52..c24c9d9fd1aa 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -109,7 +109,6 @@ set(EAM_RRTMGP_DIR ${SCREAM_BASE_DIR}/../eam/src/physics/rrtmgp) # NOTE: The external RRTMGP build needs some fixes to work with CUDA in a library build, so for now we will build these ourselves set(EXTERNAL_SRC ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels/mo_gas_optics_kernels.cpp - ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_constants.cpp ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_util_reorder.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/expand_and_transpose.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_fluxes_broadband_kernels.cpp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 0dfe4945e0c8..119093d5f9d3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -109,14 +109,14 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, RealT xmin, RealT xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - bool1dk bad_mask("bad_mask", x.size()); + Kokkos::View bad_mask("bad_mask", x.size()); Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { if (x.data()[i] < xmin or x.data()[i] > xmax) { bad_mask.data()[i] = true; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 450d7ad216be..1696e7933685 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,5 +1,4 @@ #include "scream_rrtmgp_interface.hpp" -#include "rrtmgp_utils.hpp" #include "cpp/examples/mo_load_coefficients.h" #include "examples/all-sky/mo_load_cloud_coefficients.h" #include "cpp/rrtmgp/mo_gas_concentrations.h" @@ -8,10 +7,6 @@ #include "cpp/rte/mo_rte_sw.h" #include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" -#include "ekat/util/ekat_math_utils.hpp" -#ifdef RRTMGP_ENABLE_KOKKOS -#include "Kokkos_Random.hpp" -#endif namespace scream { @@ -38,41 +33,27 @@ void finalize_kls() #endif } +#ifdef RRTMGP_ENABLE_YAKL namespace rrtmgp { -#ifdef RRTMGP_ENABLE_YAKL using yakl::fortran::parallel_for; using yakl::fortran::SimpleBounds; using yakl::intrinsics::merge; -#endif - /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -#ifdef RRTMGP_ENABLE_YAKL GasOpticsRRTMGP k_dist_sw; GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -GasOpticsRRTMGPK k_dist_sw_k; -GasOpticsRRTMGPK k_dist_lw_k; -#endif /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -#ifdef RRTMGP_ENABLE_YAKL CloudOptics cloud_optics_sw; CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -CloudOpticsK cloud_optics_sw_k; -CloudOpticsK cloud_optics_lw_k; -#endif bool initialized = false; bool initialized_k = false; @@ -80,7 +61,6 @@ bool initialized_k = false; // local functions namespace { -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_cloud_optics_sw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -106,36 +86,7 @@ OpticalProps2str get_cloud_optics_sw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_cloud_optics_sw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps2strK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_2str(ncol, nlay); - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_cloud_optics_lw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -161,36 +112,7 @@ OpticalProps1scl get_cloud_optics_lw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_cloud_optics_lw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps1sclK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2str &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -242,60 +164,7 @@ OpticalProps2str get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps2strK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_2str(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, - // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), - // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should - // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very - // high resolution. - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); - subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - subsampled_optics.ssa(icol,ilay,igpt) = 0; - subsampled_optics.g (icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1scl &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -339,51 +208,6 @@ OpticalProps1scl get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_1scl(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM; use different - // seed values for longwave and shortwave - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif } @@ -392,7 +216,6 @@ OpticalProps1sclK get_subsampled_clouds( * can be used as-is, but are intended to be wrapped by the SCREAM AD * interface to radiation. */ -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_initialize(GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, @@ -419,65 +242,15 @@ void rrtmgp_initialize(GasConcs &gas_concs, // We are now initialized! initialized = true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger) -{ - // If we've already initialized, just exit - if (initialized_k) { - if (logger) - logger->info("RRTMGP is already initialized; skipping\n"); - return; - } - - // Initialize Kokkos - if (!Kokkos::is_initialized()) { Kokkos::initialize(); } - - // Load and initialize absorption coefficient data - load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); - load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); - - // Load and initialize cloud optical property look-up table information - load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); - load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); - - // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 18000; - const size_t ncol = gas_concs.ncol; - const size_t nlay = gas_concs.nlay; - const size_t nlev = SCREAM_NUM_VERTICAL_LEV; - const size_t my_size_ref = ncol * nlay * nlev; - conv::MemPoolSingleton::init(2e6 * (float(my_size_ref) / base_ref)); - - // We are now initialized! - initialized_k = true; -} -#endif - void rrtmgp_finalize() { -#ifdef RRTMGP_ENABLE_YAKL initialized = false; k_dist_sw.finalize(); k_dist_lw.finalize(); cloud_optics_sw.finalize(); //~CloudOptics(); cloud_optics_lw.finalize(); //~CloudOptics(); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - initialized_k = false; - k_dist_sw_k.finalize(); - k_dist_lw_k.finalize(); - cloud_optics_sw_k.finalize(); //~CloudOptics(); - cloud_optics_lw_k.finalize(); //~CloudOptics(); - conv::MemPoolSingleton::finalize(); -#endif } -#ifdef RRTMGP_ENABLE_YAKL void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, @@ -527,56 +300,7 @@ void compute_band_by_band_surface_albedos( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif) { - - EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - - EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, - "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); - EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), - "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); - - // Loop over bands, and determine for each band whether it is broadly in the - // visible or infrared part of the spectrum (visible or "not visible") - Kokkos::parallel_for(conv::get_mdrp<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); - } - else { - // Band straddles the visible to near-infrared transition, so we take - // the albedo to be the average of the visible and near-infrared - // broadband albedos - sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); - sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , @@ -631,62 +355,7 @@ void compute_broadband_surface_fluxes( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir) { - // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums - // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when - // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This - // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. - //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - - // Initialize sums over bands - Kokkos::deep_copy(sfc_flux_dir_nir, 0); - Kokkos::deep_copy(sfc_flux_dir_vis, 0); - Kokkos::deep_copy(sfc_flux_dif_nir, 0); - Kokkos::deep_copy(sfc_flux_dif_vis, 0); - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { - for (int ibnd = 0; ibnd < nswbands; ++ibnd) { - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else { - // Band straddles the visible to near-infrared transition, so put half - // the flux in visible and half in near-infrared fluxes - sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - } - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -855,191 +524,19 @@ void rrtmgp_main( ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, - real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - -#ifdef SCREAM_RRTMGP_DEBUG - // Sanity check inputs, and possibly repair - check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); - check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); - check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); - check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); - check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); - check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); - check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); - check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); - check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); - check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); - check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); -#endif - - // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; - fluxes_sw.flux_up = sw_flux_up; - fluxes_sw.flux_dn = sw_flux_dn; - fluxes_sw.flux_dn_dir = sw_flux_dn_dir; - fluxes_sw.bnd_flux_up = sw_bnd_flux_up; - fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; - fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; - clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; - clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; - clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; - clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; - clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; - clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; - clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; - clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; - clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; - // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; - fluxes_lw.flux_up = lw_flux_up; - fluxes_lw.flux_dn = lw_flux_dn; - fluxes_lw.bnd_flux_up = lw_bnd_flux_up; - fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; - clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; - clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; - clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; - clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; - clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; - clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; +int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - auto nswbands = k_dist_sw_k.get_nband(); - auto nlwbands = k_dist_lw_k.get_nband(); + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; - aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); - aerosol_sw.alloc_2str(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); - aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); - aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); - }); - aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); - aerosol_lw.alloc_1scl(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Check aerosol optical properties - // NOTE: these should already have been checked by precondition checks, but someday we might have - // non-trivial aerosol optics, so this is still good to do here. - check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); - check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); - check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); - check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); -#endif - - // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); - Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); - Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); - - // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; - // This implements the Monte Carlo Independing Column Approximation by mapping only a single - // subcolumn (cloud state) to each gpoint. - auto nswgpts = k_dist_sw_k.get_ngpt(); - auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); - // Longwave - auto nlwgpts = k_dist_lw_k.get_ngpt(); - auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); - - // Copy cloud properties to outputs (is this needed, or can we just use pointers?) - // Alternatively, just compute and output a subcolumn cloud mask - Kokkos::parallel_for(conv::get_mdrp<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, - // but we might want to provide additional debug info here. NOTE: we may actually want to move this - // up higher in the code, I think optical props should go up higher since optical props are kind of - // a parameterization of their own, and we might want to swap different choices. These checks go here - // only because we need to run them on computed optical props, so if the optical props themselves get - // computed up higher, then perform these checks higher as well - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); - check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); - check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); -#endif - - // Do shortwave - rrtmgp_sw( - ncol, nlay, - k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, - fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, - tsi_scaling, logger, - extra_clnclrsky_diag, extra_clnsky_diag - ); - - // Do longwave - rrtmgp_lw( - ncol, nlay, - k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - aerosol_lw, clouds_lw_gpt, - fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, - extra_clnclrsky_diag, extra_clnsky_diag - ); - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL -int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3d("cldx", ncol, nlay, ngpt); + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = real3d("cldx", ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1090,84 +587,7 @@ int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d }); return subcolumn_mask; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3dk("cldx", ncol, nlay, ngpt); - - // Apply overlap assumption to set cldx - if (overlap_option == 0) { // Dummy mask, always cloudy - Kokkos::deep_copy(cldx, 1); - } else { // Default case, maximum-random overlap - // Maximum-random overlap: - // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, - // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same - // algorithm used in RRTMG implementation of maximum-random overlap (see - // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) - // - // First, fill cldx with random numbers. Need to use a unique seed for each column! - // auto seeds_host = Kokkos::create_mirror_view(seeds); - // Kokkos::deep_copy(seeds_host, seeds); - // for (int icol = 0; icol < ncol; ++icol) { - // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { - // auto generator = random_pool.get_state(); - // cldx(icol,ilay,igpt) = generator.drand(0., 1.); - // random_pool.free_state(generator); - // }); - // } - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - conv::Random rand(seeds(icol)); - for (int igpt = 0; igpt < ngpt; igpt++) { - for (int ilay = 0; ilay < nlay; ilay++) { - cldx(icol,ilay,igpt) = rand.genFP(); - } - } - }); - - // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { - for (int ilay = 1; ilay < nlay; ilay++) { - // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn - if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { - // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent - // layers are maximimally overlapped - cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); - } else { - // Cloud-less above, use new random number so that clouds are distributed - // randomly in this layer. Need to scale new random number to range - // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution - // of random numbers in this layer with the above branch of the conditional, - // which would otherwise inflate cloud fraction in this layer. - cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); - } - } - }); - } - - // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { - subcolumn_mask(icol,ilay,igpt) = 1; - } else { - subcolumn_mask(icol,ilay,igpt) = 0; - } - }); - return subcolumn_mask; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1423,267 +843,7 @@ void rrtmgp_sw( }); } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &flux_dn_dir = fluxes.flux_dn_dir; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; - - // Reset fluxes to zero - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up (icol,ilev) = 0; - flux_dn (icol,ilev) = 0; - flux_dn_dir(icol,ilev) = 0; - clnclrsky_flux_up (icol,ilev) = 0; - clnclrsky_flux_dn (icol,ilev) = 0; - clnclrsky_flux_dn_dir(icol,ilev) = 0; - clrsky_flux_up (icol,ilev) = 0; - clrsky_flux_dn (icol,ilev) = 0; - clrsky_flux_dn_dir(icol,ilev) = 0; - clnsky_flux_up (icol,ilev) = 0; - clnsky_flux_dn (icol,ilev) = 0; - clnsky_flux_dn_dir(icol,ilev) = 0; - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up (icol,ilev,ibnd) = 0; - bnd_flux_dn (icol,ilev,ibnd) = 0; - bnd_flux_dn_dir(icol,ilev,ibnd) = 0; - }); - - // Get daytime indices - auto dayIndices = int1dk("dayIndices", ncol); - Kokkos::deep_copy(dayIndices, -1); - - int nday = 0; - // Serialized for now. - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { - for (int icol = 0; icol < ncol; ++icol) { - if (mu0(icol) > 0) { - dayIndices(nday_inner++) = icol; - } - } - }, Kokkos::Sum(nday)); - - if (nday == 0) { - // No daytime columns in this chunk, skip the rest of this routine - return; - } - - // Subset mu0 - auto mu0_day = real1dk("mu0_day", nday); - Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { - mu0_day(iday) = mu0(dayIndices(iday)); - }); - - // subset state variables - auto p_lay_day = real2dk("p_lay_day", nday, nlay); - auto t_lay_day = real2dk("t_lay_day", nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); - t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); - }); - auto p_lev_day = real2dk("p_lev_day", nday, nlay+1); - auto t_lev_day = real2dk("t_lev_day", nday, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); - t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); - }); - - // Subset gases - auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; - gas_concs_day.init(gas_names, nday, nlay); - for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = real2dk("vmr_day", nday, nlay); - auto vmr = real2dk("vmr" , ncol, nlay); - gas_concs.get_vmr(gas_names[igas], vmr); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); - }); - gas_concs_day.set_vmr(gas_names[igas], vmr_day); - } - - // Subset aerosol optics - OpticalProps2strK aerosol_day; - aerosol_day.init(k_dist.get_band_lims_wavenumber()); - aerosol_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { - aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); - aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); - aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); - }); - - // Subset cloud optics - // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; - clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); - clouds_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { - clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); - clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); - clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); - }); - - // RRTMGP assumes surface albedos have a screwy dimension ordering - // for some strange reason, so we need to transpose these; also do - // daytime subsetting in the same kernel - real2dk sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - real2dk sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); - Kokkos::parallel_for(conv::get_mdrp<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { - sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); - sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); - }); - - // Temporaries we need for daytime-only fluxes - auto flux_up_day = real2dk("flux_up_day", nday, nlay+1); - auto flux_dn_day = real2dk("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = real2dk("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = real3dk("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = real3dk("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = real3dk("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; - fluxes_day.flux_up = flux_up_day; - fluxes_day.flux_dn = flux_dn_day; - fluxes_day.flux_dn_dir = flux_dn_dir_day; - fluxes_day.bnd_flux_up = bnd_flux_up_day; - fluxes_day.bnd_flux_dn = bnd_flux_dn_day; - fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; - - // Allocate space for optical properties - OpticalProps2strK optics; - optics.alloc_2str(nday, nlay, k_dist); - - OpticalProps2strK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_2str(nday, nlay, k_dist); - } - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", nday, nlay); - limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); - - // Do gas optics - real2dk toa_flux("toa_flux", nday, ngpt); - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); - if (extra_clnsky_diag) { - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); - check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); - check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); -#endif - - // Apply tsi_scaling - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { - toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); - }); - - if (extra_clnclrsky_diag) { - // Compute clear-clean-sky (just gas) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } - - // Combine gas and aerosol optics - aerosol_day.delta_scale(); - aerosol_day.increment(optics); - - // Compute clearsky (gas + aerosol) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - - // Now merge in cloud optics and do allsky calculations - - // Combine gas and cloud optics - clouds_day.delta_scale(); - clouds_day.increment(optics); - // Compute fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - flux_up (icol,ilev) = flux_up_day (iday,ilev); - flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { - const int icol = dayIndices(iday); - bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); - bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); - bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); - }); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds_day.increment(optics_no_aerosols); - // Compute cleansky (gas + clouds) fluxes on daytime columns - rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1816,147 +976,7 @@ void rrtmgp_lw( } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, - OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Problem size - int nbnd = k_dist.get_nband(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - - // Reset fluxes to zero - Kokkos::parallel_for( - conv::get_mdrp<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up(icol, ilev) = 0; - flux_dn(icol, ilev) = 0; - clnclrsky_flux_up(icol, ilev) = 0; - clnclrsky_flux_dn(icol, ilev) = 0; - clrsky_flux_up(icol, ilev) = 0; - clrsky_flux_dn(icol, ilev) = 0; - clnsky_flux_up(icol, ilev) = 0; - clnsky_flux_dn(icol, ilev) = 0; - }); - Kokkos::parallel_for( - conv::get_mdrp<3>({nbnd, nlay + 1, ncol}), - KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up(icol, ilev, ibnd) = 0; - bnd_flux_dn(icol, ilev, ibnd) = 0; - }); - // Allocate space for optical properties - OpticalProps1sclK optics; - optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); - } - - // Boundary conditions - SourceFuncLWK lw_sources; - lw_sources.alloc(ncol, nlay, k_dist); - real1dk t_sfc ("t_sfc" ,ncol); - real2dk emis_sfc("emis_sfc",nbnd,ncol); - - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - // Surface temperature - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); - }); - Kokkos::deep_copy(emis_sfc , 0.98); - - // Get Gaussian quadrature weights - // TODO: move this crap out of userland! - // Weights and angle secants for first order (k=1) Gaussian quadrature. - // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 - // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - realHost2dk gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - realHost2dk gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - real2dk gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - real2dk gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); - Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); - Kokkos::deep_copy(gauss_wts, gauss_wts_host); - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", ncol, nlay); - auto t_lev_limited = real2dk("t_lev_limited", ncol, nlay+1); - limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); - limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); - - // Do gas optics - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, real2dk(), t_lev_limited); - if (extra_clnsky_diag) { - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, real2dk(), t_lev_limited); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); -#endif - - if (extra_clnclrsky_diag) { - // Compute clean-clear-sky fluxes before we add in aerosols and clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); - } - - // Combine gas and aerosol optics - aerosol.increment(optics); - - // Compute clear-sky fluxes before we add in clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); - - // Combine gas and cloud optics - clouds.increment(optics); - - // Compute allsky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds.increment(optics_no_aerosols); - // Compute clean-sky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); - } - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area) { @@ -1981,46 +1001,11 @@ void compute_cloud_area( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area) { - // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy - // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = real2dk("subcol_mask", ncol, ngpt); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but - // using play/pmid does not - if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { - subcol_mask(icol,igpt) = 1; - } - }); - // Compute average over subcols to get cloud area - auto ngpt_inv = 1.0 / ngpt; - Kokkos::deep_copy(cld_area, 0); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // This loop needs to be serial because of the atomic reduction - for (int igpt = 0; igpt < ngpt; ++igpt) { - cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index_sw_k(double wavelength) { return get_wavelength_index(k_dist_sw_k, wavelength); } - -int get_wavelength_index_lw_k(double wavelength) { return get_wavelength_index(k_dist_lw_k, wavelength); } -#endif - -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index(OpticalProps &kdist, double wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -2043,33 +1028,7 @@ int get_wavelength_index(OpticalProps &kdist, double wavelength) { }); return band_index.hostRead(); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength) { - // Get wavelength bounds for all wavelength bands - auto wavelength_bounds = kdist.get_band_lims_wavelength(); - - // Find the band index for the specified wavelength - // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength - // in units of meters, we need a conversion factor of 10^2 - const int nbnds = kdist.get_nband(); - int band_index = -1; - Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { - if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { - if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } else { - if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } - }, Kokkos::Max(band_index)); - return band_index; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -2166,109 +1125,7 @@ void compute_aerocom_cloudtop( cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop) { - /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCom recommendation. See reference for routine - * get_subcolumn_mask above, where equation 14 is used for the - * maximum-random overlap assumption for subcolumn generation. We use - * equation 13, the column counterpart. - */ - // Set outputs to zero - Kokkos::deep_copy(T_mid_at_cldtop, 0.0); - Kokkos::deep_copy(p_mid_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); - Kokkos::deep_copy(cdnc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); - - // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = real1dk("aerocom_clr", ncol); - Kokkos::deep_copy(aerocom_clr, 1.0); - - // Get gravity acceleration constant from constants - using physconst = scream::physics::Constants; - - // TODO: move tunable constant to namelist - constexpr real q_threshold = 0.0; // BAD_CONSTANT! - - // TODO: move tunable constant to namelist - constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! - - // Loop over all columns in parallel - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // Loop over all layers in serial (due to accumulative - // product), starting at 2 (second highest) layer because the - // highest is assumed to hav no clouds - for(int ilay = 1; ilay < nlay; ++ilay) { - // Only do the calculation if certain conditions are met - if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && - (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { - /* PART I: Probabilistically determining cloud top */ - // Populate aerocom_tmp as the clear-sky fraction - // probability of this level, where aerocom_clr is that of - // the previous level - auto aerocom_tmp = - aerocom_clr(icol) * - (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), - cldfrac_tot(icol, ilay))) / - (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), - 1.0 - cldfrac_tot_threshold)); - // Temporary variable for probability "weights" - auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; - // Temporary variable for liquid "phase" - auto aerocom_phi = - qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); - /* PART II: The inferred properties */ - /* In general, converting a 3D property X to a 2D cloud-top - * counterpart x follows: x(i) += X(i,k) * weights * Phase - * but X and Phase are not always needed */ - // T_mid_at_cldtop - T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; - // p_mid_at_cldtop - p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; - // cldfrac_ice_at_cldtop - cldfrac_ice_at_cldtop(icol) += - (1.0 - aerocom_phi) * aerocom_wts; - // cldfrac_liq_at_cldtop - cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; - // cdnc_at_cldtop - /* We need to convert nc from 1/mass to 1/volume first, and - * from grid-mean to in-cloud, but after that, the - * calculation follows the general logic */ - auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / - z_del(icol, ilay) / physconst::gravit / - cldfrac_tot(icol, ilay); - cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; - // eff_radius_qc_at_cldtop - eff_radius_qc_at_cldtop(icol) += - rel(icol, ilay) * aerocom_phi * aerocom_wts; - // eff_radius_qi_at_cldtop - eff_radius_qi_at_cldtop(icol) += - rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset aerocom_clr to aerocom_tmp to accumulate - aerocom_clr(icol) = aerocom_tmp; - } - } - // After the serial loop over levels, the cloudy fraction is - // defined as (1 - aerocom_clr). This is true because - // aerocom_clr is the result of accumulative probabilities - // (their products) - cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); - }); -} -#endif } // namespace rrtmgp +#endif } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 49fbfeb4486b..c40be4904302 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -6,10 +6,17 @@ #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" #include "cpp/rrtmgp_const.h" +#include "rrtmgp_utils.hpp" + #include "physics/share/physics_constants.hpp" #include "ekat/mpi/ekat_comm.hpp" #include "ekat/logging/ekat_logger.hpp" +#include "ekat/util/ekat_math_utils.hpp" + +#ifdef RRTMGP_ENABLE_KOKKOS +#include "Kokkos_Random.hpp" +#endif namespace scream { @@ -18,105 +25,34 @@ void finalize_kls(); namespace rrtmgp { -/* - * Objects containing k-distribution information need to be initialized - * once and then persist throughout the life of the program, so we - * declare them here within the rrtmgp namespace. - */ #ifdef RRTMGP_ENABLE_YAKL extern GasOpticsRRTMGP k_dist_sw; extern GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern GasOpticsRRTMGPK k_dist_sw_k; -extern GasOpticsRRTMGPK k_dist_lw_k; -#endif -/* - * Objects containing cloud optical property look-up table information. - * We want to initialize these once and use throughout the life of the - * program, so declare here and read data in during rrtmgp_initialize(). - */ -#ifdef RRTMGP_ENABLE_YAKL extern CloudOptics cloud_optics_sw; extern CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern CloudOpticsK cloud_optics_sw_k; -extern CloudOpticsK cloud_optics_lw_k; -#endif -/* - * Flag to indicate whether or not we have initialized RRTMGP - */ -#ifdef RRTMGP_ENABLE_YAKL extern bool initialized; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern bool initialized_k; -#endif -/* - * Initialize data for RRTMGP driver - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_initialize( +void rrtmgp_initialize( GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger); -#endif -/* - * Compute band-by-band surface albedos from broadband albedos. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_band_by_band_surface_albedos( +void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, real2d &sfc_alb_dir, real2d &sfc_alb_dif); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif); -#endif -/* - * Compute broadband visible/UV and near-infrared surface fluxes. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_broadband_surface_fluxes( +void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir); -#endif -/* - * Main driver code to run RRTMGP. - * The input logger is in charge of outputing info to - * screen and/or to file (or neither), depending on how it was set up. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_main( +void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, GasConcs &gas_concs, @@ -138,42 +74,10 @@ extern void rrtmgp_main( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -/* - * Perform any clean-up tasks - */ -extern void rrtmgp_finalize(); +void rrtmgp_finalize(); -/* - * Shortwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_sw( +void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -184,26 +88,8 @@ extern void rrtmgp_sw( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Longwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_lw( +void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -211,46 +97,13 @@ extern void rrtmgp_lw( OpticalProps1scl &aerosol, OpticalProps1scl &clouds, FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Return a subcolumn mask consistent with a specified overlap assumption - */ -#ifdef RRTMGP_ENABLE_YAKL int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds); -#endif -/* - * Compute cloud area from 3d subcol cloud property - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, Real pmin, Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, Real pmin, Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area); -#endif -/* - * Return select cloud-top diagnostics following AeroCom recommendation - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -260,26 +113,7 @@ void compute_aerocom_cloudtop( real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop); -#endif -/* - * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area - * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion - * with the standard practice of using "water path" to refer to the total column-integrated - * quantities). - */ -#ifdef RRTMGP_ENABLE_YAKL template void mixing_ratio_to_cloud_mass( yakl::Array const &mixing_ratio, @@ -303,10 +137,1035 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif + +template +void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { + yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { + arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + }); +} + +int get_wavelength_index(OpticalProps &kdist, double wavelength); +int get_wavelength_index_sw(double wavelength); +int get_wavelength_index_lw(double wavelength); +#endif // RRTMGP_ENABLE_YAKL + +// New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS +template +struct rrtmgp_interface { + +using MDRP = typename conv::MDRP; + +template +using view_t = Kokkos::View; + +template +using oview_t = Kokkos::Experimental::OffsetView; + +template +using hview_t = Kokkos::View; + +using pool_t = conv::MemPoolSingleton; + +/* + * Objects containing k-distribution information need to be initialized + * once and then persist throughout the life of the program, so we + * declare them here within the rrtmgp namespace. + */ +static inline GasOpticsRRTMGPK k_dist_sw_k; +static inline GasOpticsRRTMGPK k_dist_lw_k; + +/* + * Objects containing cloud optical property look-up table information. + * We want to initialize these once and use throughout the life of the + * program, so declare here and read data in during rrtmgp_initialize(). + */ +static inline CloudOpticsK cloud_optics_sw_k; +static inline CloudOpticsK cloud_optics_lw_k; + +/* + * Flag to indicate whether or not we have initialized RRTMGP + */ +static inline bool initialized_k = false; + +/* + * Initialize data for RRTMGP driver + */ +static void rrtmgp_initialize( + GasConcsK &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger) +{ + // If we've already initialized, just exit + if (initialized_k) { + if (logger) + logger->info("RRTMGP is already initialized; skipping\n"); + return; + } + + // Initialize Kokkos + if (!Kokkos::is_initialized()) { Kokkos::initialize(); } + + // Load and initialize absorption coefficient data + load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); + load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); + + // Load and initialize cloud optical property look-up table information + load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); + load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); + + // initialize kokkos rrtmgp pool allocator + const size_t base_ref = 18000; + const size_t ncol = gas_concs.ncol; + const size_t nlay = gas_concs.nlay; + const size_t nlev = SCREAM_NUM_VERTICAL_LEV; + const size_t my_size_ref = ncol * nlay * nlev; + pool_t::init(2e6 * (float(my_size_ref) / base_ref)); + + // We are now initialized! + initialized_k = true; +} + +/* + * Compute band-by-band surface albedos from broadband albedos. + */ +template +static void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + SfcAlbDirVisT &sfc_alb_dir_vis, SfcAlbDirNirT &sfc_alb_dir_nir, + SfcAlbDifVisT &sfc_alb_dif_vis, SfcAlbDifNirT &sfc_alb_dif_nir, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif) +{ + EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + + EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, + "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); + EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), + "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); + + // Loop over bands, and determine for each band whether it is broadly in the + // visible or infrared part of the spectrum (visible or "not visible") + Kokkos::parallel_for(MDRP::template get<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); + } + else { + // Band straddles the visible to near-infrared transition, so we take + // the albedo to be the average of the visible and near-infrared + // broadband albedos + sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); + sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); + } + }); +} + +/* + * Compute broadband visible/UV and near-infrared surface fluxes. + */ +template +static void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + SwBndFluxDirT &sw_bnd_flux_dir , SwBndFluxDifT &sw_bnd_flux_dif , + SfcFluxDirVisT &sfc_flux_dir_vis, SfcFluxDirNirT &sfc_flux_dir_nir, + SfcFluxDifVisT &sfc_flux_dif_vis, SfcFluxDifNirT &sfc_flux_dif_nir) +{ + // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums + // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when + // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This + // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. + //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + + // Initialize sums over bands + Kokkos::deep_copy(sfc_flux_dir_nir, 0); + Kokkos::deep_copy(sfc_flux_dir_vis, 0); + Kokkos::deep_copy(sfc_flux_dif_nir, 0); + Kokkos::deep_copy(sfc_flux_dif_vis, 0); + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { + for (int ibnd = 0; ibnd < nswbands; ++ibnd) { + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else { + // Band straddles the visible to near-infrared transition, so put half + // the flux in visible and half in near-infrared fluxes + sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + } + } + }); +} + +/* + * Main driver code to run RRTMGP. + * The input logger is in charge of outputing info to + * screen and/or to file (or neither), depending on how it was set up. + */ +template +static void rrtmgp_main( + const int ncol, const int nlay, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei, CldfracT &cldfrac, + AerTauSwT &aer_tau_sw, AerSsaSwT &aer_ssa_sw, AerAsmSwT &aer_asm_sw, AerTauLwT &aer_tau_lw, + CldTauSwBndT &cld_tau_sw_bnd, CldTauLwBndT &cld_tau_lw_bnd, + CldTauSwGptT &cld_tau_sw_gpt, CldTauLwGptT &cld_tau_lw_gpt, + SwFluxUpT &sw_flux_up, SwFluxDnT &sw_flux_dn, SwFluxDnDirT &sw_flux_dn_dir, + LwFluxUpT &lw_flux_up, LwFluxDnT &lw_flux_dn, + SwClnclrskyFluxUpT &sw_clnclrsky_flux_up, SwClnclrskyFluxDnT &sw_clnclrsky_flux_dn, SwClnclrskyFluxDnDirT &sw_clnclrsky_flux_dn_dir, + SwClrskyFluxUpT &sw_clrsky_flux_up, SwClrskyFluxDnT &sw_clrsky_flux_dn, SwClrskyFluxDnDirT &sw_clrsky_flux_dn_dir, + SwClnskyFluxUpT &sw_clnsky_flux_up, SwClnskyFluxDnT &sw_clnsky_flux_dn, SwClnskyFluxDnDirT &sw_clnsky_flux_dn_dir, + LwClnclrskyFluxUpT &lw_clnclrsky_flux_up, LwClnclrskyFluxDnT &lw_clnclrsky_flux_dn, + LwClrskyFluxUpT &lw_clrsky_flux_up, LwClrskyFluxDnT &lw_clrsky_flux_dn, + LwClnskyFluxUpT &lw_clnsky_flux_up, LwClnskyFluxDnT &lw_clnsky_flux_dn, + SwBndFluxUpT &sw_bnd_flux_up, SwBndFluxDnT &sw_bnd_flux_dn, SwBndFluxDnDirT &sw_bnd_flux_dn_dir, + LwBndFluxUpT &lw_bnd_flux_up, LwBndFluxDnT &lw_bnd_flux_dn, + const RealT tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) +{ +#ifdef SCREAM_RRTMGP_DEBUG + // Sanity check inputs, and possibly repair + check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); + check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); + check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); + check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); + check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); + check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); + check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); + check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); + check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); + check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); + check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); +#endif + + // Setup pointers to RRTMGP SW fluxes + FluxesBybandK fluxes_sw; + fluxes_sw.flux_up = sw_flux_up; + fluxes_sw.flux_dn = sw_flux_dn; + fluxes_sw.flux_dn_dir = sw_flux_dn_dir; + fluxes_sw.bnd_flux_up = sw_bnd_flux_up; + fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; + fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_sw; + clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; + clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; + clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_sw; + clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; + clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; + clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_sw; + clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; + clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; + clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; + + // Setup pointers to RRTMGP LW fluxes + FluxesBybandK fluxes_lw; + fluxes_lw.flux_up = lw_flux_up; + fluxes_lw.flux_dn = lw_flux_dn; + fluxes_lw.bnd_flux_up = lw_bnd_flux_up; + fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_lw; + clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; + clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_lw; + clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; + clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_lw; + clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; + clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; + + auto nswbands = k_dist_sw_k.get_nband(); + auto nlwbands = k_dist_lw_k.get_nband(); + + // Setup aerosol optical properties + OpticalProps2strK aerosol_sw; + OpticalProps1sclK aerosol_lw; + aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); + aerosol_sw.alloc_2str(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); + aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); + aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); + }); + aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); + aerosol_lw.alloc_1scl(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Check aerosol optical properties + // NOTE: these should already have been checked by precondition checks, but someday we might have + // non-trivial aerosol optics, so this is still good to do here. + check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); + check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); + check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); + check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); +#endif + + // Convert cloud physical properties to optical properties for input to RRTMGP + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); + Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); + + // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; + // This implements the Monte Carlo Independing Column Approximation by mapping only a single + // subcolumn (cloud state) to each gpoint. + auto nswgpts = k_dist_sw_k.get_ngpt(); + auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); + // Longwave + auto nlwgpts = k_dist_lw_k.get_ngpt(); + auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); + + // Copy cloud properties to outputs (is this needed, or can we just use pointers?) + // Alternatively, just compute and output a subcolumn cloud mask + Kokkos::parallel_for(MDRP::template get<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); + }); + Kokkos::parallel_for(MDRP::template get<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, + // but we might want to provide additional debug info here. NOTE: we may actually want to move this + // up higher in the code, I think optical props should go up higher since optical props are kind of + // a parameterization of their own, and we might want to swap different choices. These checks go here + // only because we need to run them on computed optical props, so if the optical props themselves get + // computed up higher, then perform these checks higher as well + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); + check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); +#endif + + // Do shortwave + rrtmgp_sw( + ncol, nlay, + k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, + fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, + tsi_scaling, logger, + extra_clnclrsky_diag, extra_clnsky_diag + ); + + // Do longwave + rrtmgp_lw( + ncol, nlay, + k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + aerosol_lw, clouds_lw_gpt, + fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, + extra_clnclrsky_diag, extra_clnsky_diag + ); + +} + +/* + * Perform any clean-up tasks + */ +static void rrtmgp_finalize() +{ + initialized_k = false; + k_dist_sw_k.finalize(); + k_dist_lw_k.finalize(); + cloud_optics_sw_k.finalize(); //~CloudOptics(); + cloud_optics_lw_k.finalize(); //~CloudOptics(); + pool_t::finalize(); +} + +/* + * Shortwave driver (called by rrtmgp_main) + */ +template +static void rrtmgp_sw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, + OpticalProps2strK &aerosol, OpticalProps2strK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const RealT tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Get problem sizes + int nbnd = k_dist.get_nband(); + int ngpt = k_dist.get_ngpt(); + int ngas = gas_concs.get_num_gases(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &flux_dn_dir = fluxes.flux_dn_dir; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; + + // Reset fluxes to zero + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up (icol,ilev) = 0; + flux_dn (icol,ilev) = 0; + flux_dn_dir(icol,ilev) = 0; + clnclrsky_flux_up (icol,ilev) = 0; + clnclrsky_flux_dn (icol,ilev) = 0; + clnclrsky_flux_dn_dir(icol,ilev) = 0; + clrsky_flux_up (icol,ilev) = 0; + clrsky_flux_dn (icol,ilev) = 0; + clrsky_flux_dn_dir(icol,ilev) = 0; + clnsky_flux_up (icol,ilev) = 0; + clnsky_flux_dn (icol,ilev) = 0; + clnsky_flux_dn_dir(icol,ilev) = 0; + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up (icol,ilev,ibnd) = 0; + bnd_flux_dn (icol,ilev,ibnd) = 0; + bnd_flux_dn_dir(icol,ilev,ibnd) = 0; + }); + + // Get daytime indices + auto dayIndices = view_t("dayIndices", ncol); + Kokkos::deep_copy(dayIndices, -1); + + int nday = 0; + // Serialized for now. + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { + for (int icol = 0; icol < ncol; ++icol) { + if (mu0(icol) > 0) { + dayIndices(nday_inner++) = icol; + } + } + }, Kokkos::Sum(nday)); + + if (nday == 0) { + // No daytime columns in this chunk, skip the rest of this routine + return; + } + + // Subset mu0 + auto mu0_day = view_t("mu0_day", nday); + Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { + mu0_day(iday) = mu0(dayIndices(iday)); + }); + + // subset state variables + auto p_lay_day = view_t("p_lay_day", nday, nlay); + auto t_lay_day = view_t("t_lay_day", nday, nlay); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); + t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); + }); + auto p_lev_day = view_t("p_lev_day", nday, nlay+1); + auto t_lev_day = view_t("t_lev_day", nday, nlay+1); + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); + t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); + }); + + // Subset gases + auto gas_names = gas_concs.get_gas_names(); + GasConcsK gas_concs_day; + gas_concs_day.init(gas_names, nday, nlay); + for (int igas = 0; igas < ngas; igas++) { + auto vmr_day = view_t("vmr_day", nday, nlay); + auto vmr = view_t("vmr" , ncol, nlay); + gas_concs.get_vmr(gas_names[igas], vmr); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); + }); + gas_concs_day.set_vmr(gas_names[igas], vmr_day); + } + + // Subset aerosol optics + OpticalProps2strK aerosol_day; + aerosol_day.init(k_dist.get_band_lims_wavenumber()); + aerosol_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { + aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); + aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); + aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); + }); + + // Subset cloud optics + // TODO: nbnd -> ngpt once we pass sub-sampled cloud state + OpticalProps2strK clouds_day; + clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); + clouds_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { + clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); + clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); + clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); + }); + + // RRTMGP assumes surface albedos have a screwy dimension ordering + // for some strange reason, so we need to transpose these; also do + // daytime subsetting in the same kernel + view_t sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); + view_t sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); + Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { + sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); + sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); + }); + + // Temporaries we need for daytime-only fluxes + auto flux_up_day = view_t("flux_up_day", nday, nlay+1); + auto flux_dn_day = view_t("flux_dn_day", nday, nlay+1); + auto flux_dn_dir_day = view_t("flux_dn_dir_day", nday, nlay+1); + auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); + FluxesBybandK fluxes_day; + fluxes_day.flux_up = flux_up_day; + fluxes_day.flux_dn = flux_dn_day; + fluxes_day.flux_dn_dir = flux_dn_dir_day; + fluxes_day.bnd_flux_up = bnd_flux_up_day; + fluxes_day.bnd_flux_dn = bnd_flux_dn_day; + fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; + + // Allocate space for optical properties + OpticalProps2strK optics; + optics.alloc_2str(nday, nlay, k_dist); + + OpticalProps2strK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_2str(nday, nlay, k_dist); + } + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = view_t("t_lay_limited", nday, nlay); + limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); + + // Do gas optics + view_t toa_flux("toa_flux", nday, ngpt); + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); + if (extra_clnsky_diag) { + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); + check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); + check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); +#endif + + // Apply tsi_scaling + Kokkos::parallel_for(MDRP::template get<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { + toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); + }); + + if (extra_clnclrsky_diag) { + // Compute clear-clean-sky (just gas) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + + // Combine gas and aerosol optics + aerosol_day.delta_scale(); + aerosol_day.increment(optics); + + // Compute clearsky (gas + aerosol) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + + // Now merge in cloud optics and do allsky calculations + + // Combine gas and cloud optics + clouds_day.delta_scale(); + clouds_day.increment(optics); + // Compute fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + flux_up (icol,ilev) = flux_up_day (iday,ilev); + flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { + const int icol = dayIndices(iday); + bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); + bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); + bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); + }); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds_day.increment(optics_no_aerosols); + // Compute cleansky (gas + clouds) fluxes on daytime columns + rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } +} + +/* + * Longwave driver (called by rrtmgp_main) + */ +template +static void rrtmgp_lw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Problem size + int nbnd = k_dist.get_nband(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + + // Reset fluxes to zero + Kokkos::parallel_for( + MDRP::template get<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up(icol, ilev) = 0; + flux_dn(icol, ilev) = 0; + clnclrsky_flux_up(icol, ilev) = 0; + clnclrsky_flux_dn(icol, ilev) = 0; + clrsky_flux_up(icol, ilev) = 0; + clrsky_flux_dn(icol, ilev) = 0; + clnsky_flux_up(icol, ilev) = 0; + clnsky_flux_dn(icol, ilev) = 0; + }); + Kokkos::parallel_for( + MDRP::template get<3>({nbnd, nlay + 1, ncol}), + KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up(icol, ilev, ibnd) = 0; + bnd_flux_dn(icol, ilev, ibnd) = 0; + }); + + // Allocate space for optical properties + OpticalProps1sclK optics; + optics.alloc_1scl(ncol, nlay, k_dist); + OpticalProps1sclK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); + } + + // Boundary conditions + SourceFuncLWK lw_sources; + lw_sources.alloc(ncol, nlay, k_dist); + view_t t_sfc ("t_sfc" ,ncol); + view_t emis_sfc("emis_sfc",nbnd,ncol); + + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + // Surface temperature + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); + }); + Kokkos::deep_copy(emis_sfc , 0.98); + + // Get Gaussian quadrature weights + // TODO: move this crap out of userland! + // Weights and angle secants for first order (k=1) Gaussian quadrature. + // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 + // after Abramowitz & Stegun 1972, page 921 + int constexpr max_gauss_pts = 4; + hview_t gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; + gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; + gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; + gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; + + hview_t gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); + gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; + gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; + gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; + gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; + + view_t gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + view_t gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); + Kokkos::deep_copy(gauss_wts, gauss_wts_host); + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = view_t("t_lay_limited", ncol, nlay); + auto t_lev_limited = view_t("t_lev_limited", ncol, nlay+1); + limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); + limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); + + // Do gas optics + oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); + if (extra_clnsky_diag) { + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); +#endif + + if (extra_clnclrsky_diag) { + // Compute clean-clear-sky fluxes before we add in aerosols and clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); + } + + // Combine gas and aerosol optics + aerosol.increment(optics); + + // Compute clear-sky fluxes before we add in clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); + + // Combine gas and cloud optics + clouds.increment(optics); + + // Compute allsky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds.increment(optics_no_aerosols); + // Compute clean-sky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); + } +} + +/* + * Return a subcolumn mask consistent with a specified overlap assumption + */ +template +static view_t get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, + CldfT &cldf, const int overlap_option, SeedsT &seeds) +{ + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = view_t("subcolumn_mask", ncol, nlay, ngpt); + + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = view_t("cldx", ncol, nlay, ngpt); + + // Apply overlap assumption to set cldx + if (overlap_option == 0) { // Dummy mask, always cloudy + Kokkos::deep_copy(cldx, 1); + } else { // Default case, maximum-random overlap + // Maximum-random overlap: + // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, + // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same + // algorithm used in RRTMG implementation of maximum-random overlap (see + // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) + // + // First, fill cldx with random numbers. Need to use a unique seed for each column! + // auto seeds_host = Kokkos::create_mirror_view(seeds); + // Kokkos::deep_copy(seeds_host, seeds); + // for (int icol = 0; icol < ncol; ++icol) { + // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); + // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // auto generator = random_pool.get_state(); + // cldx(icol,ilay,igpt) = generator.drand(0., 1.); + // random_pool.free_state(generator); + // }); + // } + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + conv::Random rand(seeds(icol)); + for (int igpt = 0; igpt < ngpt; igpt++) { + for (int ilay = 0; ilay < nlay; ilay++) { + cldx(icol,ilay,igpt) = rand.genFP(); + } + } + }); + + // Step down columns and apply algorithm from eq (14) + Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + for (int ilay = 1; ilay < nlay; ilay++) { + // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn + if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { + // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent + // layers are maximimally overlapped + cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); + } else { + // Cloud-less above, use new random number so that clouds are distributed + // randomly in this layer. Need to scale new random number to range + // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution + // of random numbers in this layer with the above branch of the conditional, + // which would otherwise inflate cloud fraction in this layer. + cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); + } + } + }); + } + + // Use cldx array to create subcolumn mask + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { + subcolumn_mask(icol,ilay,igpt) = 1; + } else { + subcolumn_mask(icol,ilay,igpt) = 0; + } + }); + return subcolumn_mask; +} + +/* + * Compute cloud area from 3d subcol cloud property + */ +template +static void compute_cloud_area( + int ncol, int nlay, int ngpt, RealT pmin, RealT pmax, + const PmidT& pmid, const CldTauGptT& cld_tau_gpt, CldAreaT& cld_area) +{ + // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy + // then 2d subcol mask is 1, otherwise it is 0 + auto subcol_mask = view_t("subcol_mask", ncol, ngpt); + Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but + // using play/pmid does not + if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { + subcol_mask(icol,igpt) = 1; + } + }); + // Compute average over subcols to get cloud area + auto ngpt_inv = 1.0 / ngpt; + Kokkos::deep_copy(cld_area, 0); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // This loop needs to be serial because of the atomic reduction + for (int igpt = 0; igpt < ngpt; ++igpt) { + cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; + } + }); +} + +/* + * Return select cloud-top diagnostics following AeroCom recommendation + */ +template +static void compute_aerocom_cloudtop( + int ncol, int nlay, const TmidT &tmid, const PmidT &pmid, + const PdelT &p_del, const ZdelT &z_del, const QcT &qc, + const QiT &qi, const RelT &rel, const ReiT &rei, + const CldfracTotT &cldfrac_tot, const NcT &nc, + TmidAtCldtopT &T_mid_at_cldtop, PmidAtCldtopT &p_mid_at_cldtop, + CldfracIceAtCldtopT &cldfrac_ice_at_cldtop, CldfracLiqAtCldtopT &cldfrac_liq_at_cldtop, + CldfracTotAtCldtopT &cldfrac_tot_at_cldtop, CdncAtCldtopT &cdnc_at_cldtop, + EffRadiusQcAtCldtopT &eff_radius_qc_at_cldtop, EffRadiusQiAtCldTopT &eff_radius_qi_at_cldtop) +{ + /* The goal of this routine is to calculate properties at cloud top + * based on the AeroCom recommendation. See reference for routine + * get_subcolumn_mask above, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. + */ + // Set outputs to zero + Kokkos::deep_copy(T_mid_at_cldtop, 0.0); + Kokkos::deep_copy(p_mid_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); + Kokkos::deep_copy(cdnc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); + + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto aerocom_clr = view_t("aerocom_clr", ncol); + Kokkos::deep_copy(aerocom_clr, 1.0); + + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + + // TODO: move tunable constant to namelist + constexpr RealT q_threshold = 0.0; // BAD_CONSTANT! + + // TODO: move tunable constant to namelist + constexpr RealT cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! + + // Loop over all columns in parallel + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds + for(int ilay = 1; ilay < nlay; ++ilay) { + // Only do the calculation if certain conditions are met + if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && + (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + aerocom_clr(icol) * + (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), + cldfrac_tot(icol, ilay))) / + (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = + qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; + // p_mid_at_cldtop + p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + cldfrac_ice_at_cldtop(icol) += + (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / + z_del(icol, ilay) / physconst::gravit / + cldfrac_tot(icol, ilay); + cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtop + eff_radius_qc_at_cldtop(icol) += + rel(icol, ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtop + eff_radius_qi_at_cldtop(icol) += + rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + aerocom_clr(icol) = aerocom_tmp; + } + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); + }); +} + +/* + * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area + * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion + * with the standard practice of using "water path" to refer to the total column-integrated + * quantities). + */ template -void mixing_ratio_to_cloud_mass( +static void mixing_ratio_to_cloud_mass( View1 const& mixing_ratio, View2 const& cloud_fraction, View3 const& dp, @@ -315,7 +1174,7 @@ void mixing_ratio_to_cloud_mass( int ncol = mixing_ratio.extent(0); int nlay = mixing_ratio.extent(1); using physconst = scream::physics::Constants; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { // Compute in-cloud mixing ratio (mixing ratio of the cloudy part of the layer) // NOTE: these thresholds (from E3SM) seem arbitrary, but included here for consistency // This limits in-cloud mixing ratio to 0.005 kg/kg. According to note in cloud_diagnostics @@ -329,7 +1188,6 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif /* * Routine to limit a quantity to set bounds. Used to make sure @@ -337,33 +1195,199 @@ void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -#ifdef RRTMGP_ENABLE_YAKL template -void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { - yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { +static void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { + Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -template -void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + +static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) +{ + // Get wavelength bounds for all wavelength bands + auto wavelength_bounds = kdist.get_band_lims_wavelength(); + + // Find the band index for the specified wavelength + // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength + // in units of meters, we need a conversion factor of 10^2 + const int nbnds = kdist.get_nband(); + int band_index = -1; + Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { + if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { + if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } else { + if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } + }, Kokkos::Max(band_index)); + return band_index; +} + +static inline int get_wavelength_index_sw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_sw_k, wavelength); +} + +static inline int get_wavelength_index_lw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_lw_k, wavelength); +} + +template +static OpticalProps2strK get_cloud_optics_sw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + + // Initialize optics + OpticalProps2strK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_2str(ncol, nlay); + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = view_t("rel_limited", ncol, nlay); + auto rei_limited = view_t("rei_limited", ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} + +template +static OpticalProps1sclK get_cloud_optics_lw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + + // Initialize optics + OpticalProps1sclK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = view_t("rel_limited", ncol, nlay); + auto rei_limited = view_t("rei_limited", ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} + +template +OpticalProps2strK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { + // Initialized subsampled optics + OpticalProps2strK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_2str(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, + // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), + // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should + // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very + // high resolution. + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM + auto seeds = view_t("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); + subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + subsampled_optics.ssa(icol,ilay,igpt) = 0; + subsampled_optics.g (icol,ilay,igpt) = 0; + } }); + return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_YAKL -int get_wavelength_index(OpticalProps &kdist, double wavelength); -int get_wavelength_index_sw(double wavelength); -int get_wavelength_index_lw(double wavelength); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength); -int get_wavelength_index_sw_k(double wavelength); -int get_wavelength_index_lw_k(double wavelength); -#endif +template +OpticalProps1sclK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, + CldT &cld, PlayT &p_lay) { + // Initialized subsampled optics + OpticalProps1sclK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_1scl(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM; use different + // seed values for longwave and shortwave + auto seeds = view_t("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + } + }); + return subsampled_optics; +} + +}; // struct rrtmgp_interface +#endif // RRTMGP_ENABLE_KOKKOS } // namespace rrtmgp } // namespace scream From 75892307589e0f6734aa0501d2a439d95879731c Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 5 Jun 2024 16:32:50 -0700 Subject: [PATCH 066/904] first round of code clean up changes --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 61 ++++--------------- 1 file changed, 13 insertions(+), 48 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 6c7f41cf5972..da628df92e0f 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -338,28 +338,22 @@ apply_iop_forcing(const Real dt) // TODO: Create a memory buffer for this class // and add the below WSM and views WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); - WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); view_Nd rstar ("rstar", nelem, NGP, NGP, NLEV), exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing rstar, exner, and temperature from Hommexx + // Lambda for computing temperature from Hommexx auto compute_homme_states = [&] () { - Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("compute_temperature_for_iop", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { KV kv(team); const int ie = team.league_rank(); // Get temp views from workspace - auto ws = homme_wsm.get_workspace(team); - auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); - uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); - - // Get temp views from workspace - auto ws2 = eamxx_wsm.get_workspace(team); + auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid, pint, pdel; - ws2.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); + ws.take_many_contiguous_unsafe<1>({"pmid"}, + {&pmid}); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; @@ -368,49 +362,26 @@ apply_iop_forcing(const Real dt) auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto pnh_i = ekat::subview(pnh, idx); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); // Compute reference pressures and layer thickness. // TODO: Allow geometry data to allocate packsize auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; }); team.team_barrier(); // Reinterperate into views of Homme::Scalar for calling Hommexx function. Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); - Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - // Compute exner from EOS - if (theta_hydrostatic_mode) { - auto hydro_p_int = ws.take("hydro_p_int"); - Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); - elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); - eos.compute_exner(kv, pnh_scalar, exner_scalar); - ws.release(hydro_p_int); - } else { - eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); - } - - // Compute temperature + // Compute temperature from virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - auto& T_val = vtheta_dp_i(k); + auto T_val = vtheta_dp_i(k); T_val /= dp3d_i(k); T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); @@ -419,12 +390,11 @@ apply_iop_forcing(const Real dt) }); // Release WS views - ws.release_macro_block(pnh_slot, NGP*NGP); - ws2.release_many_contiguous<3>({&pmid, &pint, &pdel}); + ws.release_many_contiguous<1>({&pmid}); }); }; - // Preprocess some homme states to get temperature and exner + // Preprocess some homme states to get temperature compute_homme_states(); Kokkos::fence(); @@ -500,8 +470,6 @@ apply_iop_forcing(const Real dt) auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); // Compute reference pressures and layer thickness. @@ -517,10 +485,6 @@ apply_iop_forcing(const Real dt) team.team_barrier(); - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - // Compute Qdp from updated Q Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { const int ilev = k/qsize; @@ -532,7 +496,7 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - // Recompute rstar with updated qv, and convert updated temperature back to potential temperature + // Convert updated temperature back to psuedo density virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); @@ -661,6 +625,7 @@ apply_iop_forcing(const Real dt) temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); // Convert updated temperature back to potential temperature + // NEED TO REPLACE THIS DEFINITION! vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); } if (iop_nudge_uv) { From d3284e41b6bd0381f31f3ff1d9b94534712a4cfc Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 5 Jun 2024 17:10:47 -0700 Subject: [PATCH 067/904] update definition of vtheta_dp in IOP nudging --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index da628df92e0f..eff94346895e 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -352,8 +352,7 @@ apply_iop_forcing(const Real dt) // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<1>({"pmid"}, - {&pmid}); + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; @@ -594,10 +593,16 @@ apply_iop_forcing(const Real dt) KV kv(team); const int ie = team.league_rank(); + // Get temp views from workspace + auto ws = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; const int jgp = idx%NGP; + auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); auto rstar_i = ekat::subview(rstar, ie, igp, jgp); @@ -607,6 +612,14 @@ apply_iop_forcing(const Real dt) auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user @@ -624,9 +637,9 @@ apply_iop_forcing(const Real dt) qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to potential temperature - // NEED TO REPLACE THIS DEFINITION! - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); @@ -634,6 +647,8 @@ apply_iop_forcing(const Real dt) } }); }); + // Release WS views + ws.release_many_contiguous<1>({&pmid}); }); } } From e6e4fb7bb4d1bccfd65e096c31edc3a4b28500b1 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 6 Jun 2024 15:39:18 -0400 Subject: [PATCH 068/904] remove line --- components/eam/src/physics/crm/pam/pam_state.h | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/pam_state.h b/components/eam/src/physics/crm/pam/pam_state.h index 270fef7f6c23..980fb7f4eb0a 100644 --- a/components/eam/src/physics/crm/pam/pam_state.h +++ b/components/eam/src/physics/crm/pam/pam_state.h @@ -156,7 +156,6 @@ inline void pam_state_set_reference_state( pam::PamCoupler &coupler ) { parallel_for(SimpleBounds<2>(nz+1,nens), YAKL_LAMBDA (int k, int iens) { hmean_pint(k,iens) = 0; if (k < nz) { - hmean_pmid (k,iens) = 0; hmean_rho_d(k,iens) = 0; hmean_rho_v(k,iens) = 0; hmean_rho_c(k,iens) = 0; From 30faaba50681d0821a80ccb9b5592c9d81b6074e Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 6 Jun 2024 15:39:41 -0400 Subject: [PATCH 069/904] make variables const --- .../eam/src/physics/crm/pam/pam_variance_transport.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_variance_transport.h b/components/eam/src/physics/crm/pam/pam_variance_transport.h index 1eb7779f3469..70b985cc2b7b 100644 --- a/components/eam/src/physics/crm/pam/pam_variance_transport.h +++ b/components/eam/src/physics/crm/pam/pam_variance_transport.h @@ -36,11 +36,11 @@ inline void pam_variance_transport_diagnose( pam::PamCoupler &coupler ) { auto ny = coupler.get_option("crm_ny"); auto nx = coupler.get_option("crm_nx"); //------------------------------------------------------------------------------------------------ - auto temp = dm_device.get("temp" ); - auto rhov = dm_device.get("water_vapor"); - auto rhoc = dm_device.get("cloud_water"); - auto rhoi = dm_device.get("ice" ); - auto uvel = dm_device.get("uvel" ); + auto temp = dm_device.get("temp" ); + auto rhov = dm_device.get("water_vapor"); + auto rhoc = dm_device.get("cloud_water"); + auto rhoi = dm_device.get("ice" ); + auto uvel = dm_device.get("uvel" ); auto vt_temp = dm_device.get("vt_temp" ); auto vt_rhov = dm_device.get("vt_rhov" ); auto vt_uvel = dm_device.get("vt_uvel" ); From f9e6cc20fbe98699adcbae004932871bf198c943 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 6 Jun 2024 15:40:13 -0400 Subject: [PATCH 070/904] switch to std::abs --- components/eam/src/physics/crm/pam/pam_driver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index f8f575ec8a1d..bed8980ea67d 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -61,8 +61,8 @@ inline int pam_driver_set_subcycle_timestep( pam::PamCoupler &coupler, real crm_ }); // calculate max U and W parallel_for( SimpleBounds<4>(crm_nz,crm_ny,crm_nx,nens) , YAKL_LAMBDA (int k, int j, int i, int n) { - yakl::atomicMax(uvel_max(k,n), sqrt(uvel(k,j,i,n)*uvel(k,j,i,n)) ); - yakl::atomicMax(wvel_max(k,n), fabs(wvel(k,j,i,n)) ); + yakl::atomicMax(uvel_max(k,n), std::abs(uvel(k,j,i,n)) ); + yakl::atomicMax(wvel_max(k,n), std::abs(wvel(k,j,i,n)) ); }); // find max CFL between horizontal and vertical CFL values parallel_for( SimpleBounds<2>(crm_nz,nens) , YAKL_LAMBDA (int k, int n) { From 2590f226f43c316ecb224649c00e597a7dce8b43 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 6 Jun 2024 15:42:58 -0400 Subject: [PATCH 071/904] remove pmax --- components/eam/src/physics/crm/pam/pam_driver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index bed8980ea67d..5fd1c5071d16 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -73,7 +73,7 @@ inline int pam_driver_set_subcycle_timestep( pam::PamCoupler &coupler, real crm_ cfl_max(k,n) = max(cfl_u,cfl_w); }); // calculate final CFL across ensemble - real cfl_loc = pmax(cfl_max.data()); + real cfl_loc = yakl::intrinsics::maxval(cfl_max); cfl = max(cfl,cfl_loc); // update number of subcycles and time step num_subcycle = max(num_subcycle,max(1,static_cast(ceil(cfl/0.7)))); From 6b80ad2b9aded0b5b71b94df626a8066de3329e7 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 6 Jun 2024 16:55:23 -0600 Subject: [PATCH 072/904] Builds and runs --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 109 +++++----- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 130 ++++++------ .../src/physics/rrtmgp/rrtmgp_test_utils.cpp | 200 +----------------- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 128 ++++++++++- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 5 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 7 - .../rrtmgp/scream_rrtmgp_interface.hpp | 33 +-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 68 +++--- 9 files changed, 306 insertions(+), 376 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 59e6f72cc922..3d7fba5ac72d 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 59e6f72cc92268c69d1bd6ede570a959c1ebc28a +Subproject commit 3d7fba5ac72dab7c8448411e7452e2b409f22d93 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1ff3f27b2fce..a24f4d5ddc32 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -624,17 +624,17 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.init(gas_names_yakl_offset,m_col_chunk_size,m_nlay); - rrtmgp::rrtmgp_initialize( + interface_t::rrtmgp_initialize( m_gas_concs_k, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, m_atm_logger ); VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_sw, rrtmgp::k_dist_sw_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_lw, rrtmgp::k_dist_lw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, rrtmgp::cloud_optics_sw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, rrtmgp::cloud_optics_lw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_sw, interface_t::k_dist_sw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_lw, interface_t::k_dist_lw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, interface_t::cloud_optics_sw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, interface_t::cloud_optics_lw_k); #endif // Set property checks for fields in this process @@ -924,30 +924,30 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS // If YAKL is on, we don't want aliased memory in both the yakl and kokos // subviews. - auto subview_1dk = [&](const real1dk v) -> real1dk { - real1dk subv(v, std::make_pair(0, ncol)); + auto subview_1dk = [&](const view_1d_real v) -> view_1d_real { + view_1d_real subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - real1dk rv(v.label(), ncol); + view_1d_real rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_2dk = [&](const real2dk v) -> real2dk { - real2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_2dk = [&](const view_2d_real v) -> view_2d_real { + view_2d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); + view_2d_real rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_3dk = [&](const real3dk v) -> real3dk { - real3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + auto subview_3dk = [&](const view_3d_real v) -> view_3d_real { + view_3d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); + view_3d_real rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); return rv; #else @@ -1194,7 +1194,7 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::fence(); #ifdef RRTMGP_ENABLE_KOKKOS COMPARE_ALL_WRAP(std::vector({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1206,7 +1206,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real2d tmp2d = subview_2d(m_buffer.tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); + view_2d_real tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1308,14 +1308,14 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + interface_t::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + interface_t::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); + interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); + interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), - std::vector({lwp_k, iwp_k})); + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1350,13 +1350,13 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dir, sfc_alb_dif); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis_k, sfc_alb_dir_nir_k, sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), - std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1384,7 +1384,7 @@ void RRTMGPRadiation::run_impl (const double dt) { ); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, m_nlay, p_lay_k, t_lay_k, p_lev_k, t_lev_k, m_gas_concs_k, @@ -1412,7 +1412,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, lw_clnsky_flux_up, lw_clnsky_flux_dn}), - std::vector({ + std::vector({ sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, @@ -1422,7 +1422,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), - std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1472,7 +1472,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), - std::vector({sw_heating_k, lw_heating_k})); + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1509,14 +1509,14 @@ void RRTMGPRadiation::run_impl (const double dt) { sw_bnd_flux_dif_k(icol,ilev,ibnd) = sw_bnd_flux_dn_k(icol,ilev,ibnd) - sw_bnd_flux_dir_k(icol,ilev,ibnd); }); // Compute surface fluxes - rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot_k, nswbands, sw_bnd_flux_dir_k, sw_bnd_flux_dif_k, sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), - std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1537,22 +1537,22 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); - real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); - real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); - real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably // does not matter in practice, as clouds probably should not be produced above 50 hPa and we // should not be encountering surface pressure above 1200 hPa, but in the event that things go off // the rails we might want to look at these still. - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), - std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Compute cloud-top diagnostics following AeroCOM recommendation @@ -1580,20 +1580,20 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // Get visible 0.67 micron band for COSP - auto idx_067_k = rrtmgp::get_wavelength_index_sw_k(0.67e-6); + auto idx_067_k = interface_t::get_wavelength_index_sw_k(0.67e-6); // Get IR 10.5 micron band for COSP - auto idx_105_k = rrtmgp::get_wavelength_index_lw_k(10.5e-6); - - real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - - rrtmgp::compute_aerocom_cloudtop( + auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); + + view_1d_real T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + + interface_t::compute_aerocom_cloudtop( ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, @@ -1602,7 +1602,7 @@ void RRTMGPRadiation::run_impl (const double dt) { T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), - std::vector({ + std::vector({ T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); @@ -1699,7 +1699,7 @@ void RRTMGPRadiation::run_impl (const double dt) { }); #ifdef RRTMGP_ENABLE_YAKL // Sync back to gas_concs_k - real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + view_3d_real temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); Kokkos::deep_copy(temp, m_gas_concs_k.concs); #endif #endif @@ -1769,11 +1769,12 @@ void RRTMGPRadiation::run_impl (const double dt) { void RRTMGPRadiation::finalize_impl () { #ifdef RRTMGP_ENABLE_YAKL m_gas_concs.reset(); + rrtmgp::rrtmgp_finalize(); #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.reset(); + interface_t::rrtmgp_finalize(); #endif - rrtmgp::rrtmgp_finalize(); finalize_kls(); } diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 66c89476028b..28c5033c46d4 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -28,6 +28,10 @@ class RRTMGPRadiation : public AtmosphereProcess { template using uview_2d = Unmanaged>; + using layout_t = typename ekat::KokkosTypes::Layout; + + using interface_t = rrtmgp::rrtmgp_interface; + // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -92,7 +96,7 @@ class RRTMGPRadiation : public AtmosphereProcess { GasConcs m_gas_concs; #endif #ifdef RRTMGP_ENABLE_KOKKOS - GasConcsK m_gas_concs_k; + GasConcsK m_gas_concs_k; #endif // Prescribed greenhouse gas surface concentrations in moles / moles air @@ -137,15 +141,15 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + view_1d_real mu0_k; + view_1d_real sfc_alb_dir_vis_k; + view_1d_real sfc_alb_dir_nir_k; + view_1d_real sfc_alb_dif_vis_k; + view_1d_real sfc_alb_dif_nir_k; + view_1d_real sfc_flux_dir_vis_k; + view_1d_real sfc_flux_dir_nir_k; + view_1d_real sfc_flux_dif_vis_k; + view_1d_real sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) @@ -168,21 +172,21 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + view_2d_real p_lay_k; + view_2d_real t_lay_k; + view_2d_real z_del_k; + view_2d_real p_del_k; + view_2d_real qc_k; + view_2d_real nc_k; + view_2d_real qi_k; + view_2d_real cldfrac_tot_k; + view_2d_real eff_radius_qc_k; + view_2d_real eff_radius_qi_k; + view_2d_real tmp2d_k; + view_2d_real lwp_k; + view_2d_real iwp_k; + view_2d_real sw_heating_k; + view_2d_real lw_heating_k; #endif // 2d size (ncol, nlay+1) @@ -212,28 +216,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + view_2d_real p_lev_k; + view_2d_real t_lev_k; + view_2d_real sw_flux_up_k; + view_2d_real sw_flux_dn_k; + view_2d_real sw_flux_dn_dir_k; + view_2d_real lw_flux_up_k; + view_2d_real lw_flux_dn_k; + view_2d_real sw_clnclrsky_flux_up_k; + view_2d_real sw_clnclrsky_flux_dn_k; + view_2d_real sw_clnclrsky_flux_dn_dir_k; + view_2d_real sw_clrsky_flux_up_k; + view_2d_real sw_clrsky_flux_dn_k; + view_2d_real sw_clrsky_flux_dn_dir_k; + view_2d_real sw_clnsky_flux_up_k; + view_2d_real sw_clnsky_flux_dn_k; + view_2d_real sw_clnsky_flux_dn_dir_k; + view_2d_real lw_clnclrsky_flux_up_k; + view_2d_real lw_clnclrsky_flux_dn_k; + view_2d_real lw_clrsky_flux_up_k; + view_2d_real lw_clrsky_flux_dn_k; + view_2d_real lw_clnsky_flux_up_k; + view_2d_real lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -244,10 +248,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + view_3d_real sw_bnd_flux_up_k; + view_3d_real sw_bnd_flux_dn_k; + view_3d_real sw_bnd_flux_dir_k; + view_3d_real sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -256,8 +260,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + view_3d_real lw_bnd_flux_up_k; + view_3d_real lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -266,8 +270,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + view_2d_real sfc_alb_dir_k; + view_2d_real sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -278,10 +282,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + view_3d_real aero_tau_sw_k; + view_3d_real aero_ssa_sw_k; + view_3d_real aero_g_sw_k; + view_3d_real aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -290,8 +294,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + view_3d_real cld_tau_sw_bnd_k; + view_3d_real cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -300,8 +304,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + view_3d_real cld_tau_sw_gpt_k; + view_3d_real cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp index 54a32133da92..65cc99e09cc3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp @@ -1,25 +1,13 @@ -#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" #include "physics/rrtmgp/rrtmgp_test_utils.hpp" #ifdef RRTMGP_ENABLE_YAKL #include "YAKL_netcdf.h" #endif -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rte/mo_fluxes.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" - #include #include namespace rrtmgpTest { -#ifdef RRTMGP_ENABLE_YAKL -using yakl::fortran::parallel_for; -using yakl::fortran::SimpleBounds; -using yakl::intrinsics::mod; -using yakl::intrinsics::merge; -#endif - bool file_exists(const char *filename) { if (auto file = fopen(filename, "r")) { fclose(file); @@ -29,8 +17,12 @@ bool file_exists(const char *filename) { } } -// TODO: use YAKL intrinsics for this to avoid needing to make host copies #ifdef RRTMGP_ENABLE_YAKL +using yakl::fortran::parallel_for; +using yakl::fortran::SimpleBounds; +using yakl::intrinsics::mod; +using yakl::intrinsics::merge; + bool all_close(real2d &arr1, real2d &arr2, double tolerance) { int nx = arr1.dimension[0]; int ny = arr2.dimension[1]; @@ -46,28 +38,7 @@ bool all_close(real2d &arr1, real2d &arr2, double tolerance) { } return true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { - int nx = arr1.extent(0); - int ny = arr2.extent(1); - auto arr1_h = Kokkos::create_mirror_view(arr1); - auto arr2_h = Kokkos::create_mirror_view(arr2); - Kokkos::deep_copy(arr1_h, arr1); - Kokkos::deep_copy(arr2_h, arr2); - for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { - printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); - return false; - } - } - } - return true; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void dummy_atmos( std::string inputfile, int ncol, real2d &p_lay, real2d &t_lay, @@ -94,37 +65,7 @@ void dummy_atmos( // create a dummy atmosphere. dummy_clouds(scream::rrtmgp::cloud_optics_sw, p_lay, t_lay, lwp, iwp, rel, rei, cld); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_atmos( - std::string inputfile, - int ncol, real2dk &p_lay, real2dk &t_lay, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld) { - - // Setup boundary conditions, solar zenith angle, etc - // NOTE: this stuff would come from the model in a real run - // Ocean-ish values for surface albedos, just for example - Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); - - // Pick a solar zenith angle; this should come from the model - Kokkos::deep_copy(mu0, 0.86 ); - - // Get dummy cloud PHYSICAL properties. Note that this function call - // needs the CloudOptics object only because it uses the min and max - // valid values from the lookup tables for liquid and ice water path to - // create a dummy atmosphere. - dummy_clouds(scream::rrtmgp::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void dummy_clouds( CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cloud_mask) { @@ -151,38 +92,7 @@ void dummy_clouds( rei(icol,ilay) = merge(rei_val, 0._wp, iwp(icol,ilay) > 0._wp); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask) { - - // Problem sizes - int ncol = t_lay.extent(0); - int nlay = t_lay.extent(1); - - // Generate some fake liquid and ice water data. We pick values to be midway between - // the min and max of the valid lookup table values for effective radii - real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); - real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); - // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and - // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. - // Set sane values for liquid and ice water path. - // NOTE: these "sane" values are in g/m2! - Kokkos::parallel_for( conv::get_mdrp<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { - cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; - // Ice and liquid will overlap in a few layers - lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); - iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); - rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); - rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); - }); -} -#endif - -// Function to read fluxes from input file so we can compare our answers against the reference -#ifdef RRTMGP_ENABLE_YAKL void read_fluxes( std::string inputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -208,36 +118,7 @@ void read_fluxes( io.read(lw_flux_up, "lw_flux_up" ); io.read(lw_flux_dn, "lw_flux_dn" ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void read_fluxes( - std::string inputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - // Initialize netcdf reader - conv::SimpleNetCDF io; - io.open(inputfile, NC_NOWRITE); - - // Initialize arrays to hold fluxes - int nlev = io.getDimSize("lev"); - int ncol = io.getDimSize("col_flx"); - sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); - sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); - sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); - lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); - lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); - - // Read data - io.read(sw_flux_up, "sw_flux_up" ); - io.read(sw_flux_dn, "sw_flux_dn" ); - io.read(sw_flux_dir, "sw_flux_dir"); - io.read(lw_flux_up, "lw_flux_up" ); - io.read(lw_flux_dn, "lw_flux_dn" ); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void write_fluxes( std::string outputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -253,76 +134,5 @@ void write_fluxes( io.close(); } #endif -#ifdef RRTMGP_ENABLE_KOKKOS -void write_fluxes( - std::string outputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - conv::SimpleNetCDF io; - io.create(outputfile); - io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); - io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); - io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); - io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); - io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); - io.close(); -} -#endif - -// TODO: This function should instead take values, not file names, -// because for the test we do not want to write to file -#ifdef RRTMGP_ENABLE_YAKL -int compare_y(std::string file1, std::string file2) { - // Read data from baseline and test file - real2d sw_flux_up_1; - real2d sw_flux_dn_1; - real2d sw_flux_dir_1; - real2d lw_flux_up_1; - real2d lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2d sw_flux_up_2; - real2d sw_flux_dn_2; - real2d sw_flux_dir_2; - real2d lw_flux_up_2; - real2d lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int compare_k(std::string file1, std::string file2) { - // Read data from baseline and test file - real2dk sw_flux_up_1; - real2dk sw_flux_dn_1; - real2dk sw_flux_dir_1; - real2dk lw_flux_up_1; - real2dk lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2dk sw_flux_up_2; - real2dk sw_flux_dn_2; - real2dk sw_flux_dir_2; - real2dk lw_flux_up_2; - real2dk lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index b9498a9a34f9..79f6e570950d 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -1,5 +1,10 @@ #ifndef RRTMGP_TEST_UTILS_HPP #define RRTMGP_TEST_UTILS_HPP + +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rte/mo_fluxes.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" namespace rrtmgpTest { @@ -37,33 +42,134 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance); +template +struct rrtmgp_test_utils { -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +using real1dk = typename ekat::KokkosTypes::template view_1d; +using real2dk = typename ekat::KokkosTypes::template view_2d; +using real3dk = typename ekat::KokkosTypes::template view_3d; +using MDRP = typename conv::MDRP; +using interface_t = scream::rrtmgp::rrtmgp_interface; -void dummy_atmos( +static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) +{ + int nx = arr1.extent(0); + int ny = arr2.extent(1); + auto arr1_h = Kokkos::create_mirror_view(arr1); + auto arr2_h = Kokkos::create_mirror_view(arr2); + Kokkos::deep_copy(arr1_h, arr1); + Kokkos::deep_copy(arr2_h, arr2); + for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { + printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); + return false; + } + } + } + return true; +} + +static void dummy_clouds( + CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask +) +{ + // Problem sizes + int ncol = t_lay.extent(0); + int nlay = t_lay.extent(1); + + // Generate some fake liquid and ice water data. We pick values to be midway between + // the min and max of the valid lookup table values for effective radii + real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); + real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); + + // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and + // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. + // Set sane values for liquid and ice water path. + // NOTE: these "sane" values are in g/m2! + Kokkos::parallel_for( MDRP::template get<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { + cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; + // Ice and liquid will overlap in a few layers + lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); + iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); + rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); + rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); + }); +} + +static void dummy_atmos( std::string inputfile, int ncol, real2dk &p_lay, real2dk &t_lay, real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, real1dk &mu0, real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +) +{ + // Setup boundary conditions, solar zenith angle, etc + // NOTE: this stuff would come from the model in a real run -void read_fluxes( + // Ocean-ish values for surface albedos, just for example + Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); + + // Pick a solar zenith angle; this should come from the model + Kokkos::deep_copy(mu0, 0.86 ); + + // Get dummy cloud PHYSICAL properties. Note that this function call + // needs the CloudOptics object only because it uses the min and max + // valid values from the lookup tables for liquid and ice water path to + // create a dummy atmosphere. + dummy_clouds(interface_t::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); +} + +static void read_fluxes( std::string inputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + // Initialize netcdf reader + conv::SimpleNetCDF io; + io.open(inputfile, NC_NOWRITE); -void write_fluxes( + // Initialize arrays to hold fluxes + int nlev = io.getDimSize("lev"); + int ncol = io.getDimSize("col_flx"); + sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); + sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); + sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); + lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); + lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); + + // Read data + io.read(sw_flux_up, "sw_flux_up" ); + io.read(sw_flux_dn, "sw_flux_dn" ); + io.read(sw_flux_dir, "sw_flux_dir"); + io.read(lw_flux_up, "lw_flux_up" ); + io.read(lw_flux_dn, "lw_flux_dn" ); +} + +static void write_fluxes( std::string outputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + conv::SimpleNetCDF io; + io.create(outputfile); + io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); + io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); + io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); + io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); + io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); + io.close(); +} + +}; #endif } diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 119093d5f9d3..6e924103229f 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -109,8 +109,9 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, RealT xmin, RealT xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 1696e7933685..10281df8078c 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,11 +1,4 @@ #include "scream_rrtmgp_interface.hpp" -#include "cpp/examples/mo_load_coefficients.h" -#include "examples/all-sky/mo_load_cloud_coefficients.h" -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" -#include "cpp/rte/mo_rte_sw.h" -#include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" namespace scream { diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index c40be4904302..2038cb62ff79 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -4,7 +4,14 @@ #include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" +#include "cpp/examples/mo_load_coefficients.h" #include "cpp/rrtmgp_const.h" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "cpp/rte/mo_rte_sw.h" +#include "cpp/rte/mo_rte_lw.h" +#include "examples/all-sky/mo_load_cloud_coefficients.h" #include "rrtmgp_utils.hpp" @@ -152,7 +159,7 @@ int get_wavelength_index_lw(double wavelength); // New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_interface { using MDRP = typename conv::MDRP; @@ -469,8 +476,8 @@ static void rrtmgp_main( #endif // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); @@ -640,7 +647,7 @@ static void rrtmgp_sw( // Subset gases auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; + GasConcsK gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { auto vmr_day = view_t("vmr_day", nday, nlay); @@ -653,7 +660,7 @@ static void rrtmgp_sw( } // Subset aerosol optics - OpticalProps2strK aerosol_day; + OpticalProps2strK aerosol_day; aerosol_day.init(k_dist.get_band_lims_wavenumber()); aerosol_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { @@ -664,7 +671,7 @@ static void rrtmgp_sw( // Subset cloud optics // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; + OpticalProps2strK clouds_day; clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); clouds_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { @@ -690,7 +697,7 @@ static void rrtmgp_sw( auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; + FluxesBybandK fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; fluxes_day.flux_dn_dir = flux_dn_dir_day; @@ -699,10 +706,10 @@ static void rrtmgp_sw( fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; // Allocate space for optical properties - OpticalProps2strK optics; + OpticalProps2strK optics; optics.alloc_2str(nday, nlay, k_dist); - OpticalProps2strK optics_no_aerosols; + OpticalProps2strK optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_2str(nday, nlay, k_dist); @@ -1195,8 +1202,8 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template -static void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { +template +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); @@ -1291,7 +1298,7 @@ static OpticalProps1sclK get_cloud_optics_lw( } template -OpticalProps2strK get_subsampled_clouds( +static OpticalProps2strK get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { // Initialized subsampled optics @@ -1342,7 +1349,7 @@ OpticalProps2strK get_subsampled_clouds( } template -OpticalProps1sclK get_subsampled_clouds( +static OpticalProps1sclK get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index ddef61808fe0..36e2a59e02ff 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -849,6 +849,14 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { #endif #ifdef RRTMGP_ENABLE_KOKKOS +using interface_t = scream::rrtmgp::rrtmgp_interface<>; +using real1dk = interface_t::view_t; +using real2dk = interface_t::view_t; +using real3dk = interface_t::view_t; +using int1dk = interface_t::view_t; +using int2dk = interface_t::view_t; +using int3dk = interface_t::view_t; + TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos scream::init_kls(); @@ -921,7 +929,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 1.0; }); auto cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with no cloud @@ -931,7 +939,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.0; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with empty clouds (cloud fraction but with no associated mixing ratio) @@ -943,7 +951,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.1; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with cell half filled with cloud @@ -953,7 +961,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.5; }); cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up @@ -977,14 +985,14 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { }); // Limit to bounds that contain the data; should be no change in values - scream::rrtmgp::limit_to_bounds(arr, 0.0, 5.0, arr_limited); + interface_t::limit_to_bounds_k(arr, 0.0, 5.0, arr_limited); REQUIRE(chc(arr)(0,0) == chc(arr_limited)(0,0)); REQUIRE(chc(arr)(0,1) == chc(arr_limited)(0,1)); REQUIRE(chc(arr)(1,0) == chc(arr_limited)(1,0)); REQUIRE(chc(arr)(1,1) == chc(arr_limited)(1,1)); // Limit to bounds that do not completely contain the data; should be a change in values! - scream::rrtmgp::limit_to_bounds(arr, 1.5, 3.5, arr_limited); + interface_t::limit_to_bounds_k(arr, 1.5, 3.5, arr_limited); REQUIRE(chc(arr_limited)(0,0) == 1.5); REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); @@ -1078,7 +1086,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Create simple test cases; We expect, given the input data, that band 10 // will straddle the NIR and VIS, bands 1-9 will be purely NIR, and bands 11-14 @@ -1106,7 +1114,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1138,7 +1146,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1169,7 +1177,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1200,7 +1208,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1216,7 +1224,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Finalize YAKL logger->info("Free memory...\n"); - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); gas_concs.reset(); scream::finalize_kls(); } @@ -1282,7 +1290,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { @@ -1316,7 +1324,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); auto cldmask_h = chc(cldmask); for (int igpt = 0; igpt < ngpt; igpt++) { if (cldmask_h(0,0,igpt) == 1) { @@ -1359,7 +1367,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); // Case: @@ -1376,7 +1384,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 1; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); // Case: @@ -1393,11 +1401,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1.0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0 / 3.0); // Case: @@ -1414,11 +1422,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); scream::finalize_kls(); } @@ -1463,7 +1471,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { Kokkos::deep_copy(rel, 10.0); Kokkos::deep_copy(rei, 10.0); // Call the function - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1481,7 +1489,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { // Case 2: if all clouds, everything goes to 1 * its value Kokkos::deep_copy(cldfrac_tot, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1504,7 +1512,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 3) = 0.3; cldfrac_tot(0, 4) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1520,7 +1528,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 5) = 0.4; cldfrac_tot(0, 6) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1534,7 +1542,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 4) = 0.0; cldfrac_tot(0, 5) = 0.1; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1550,7 +1558,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 1.0); Kokkos::deep_copy(qi, 0.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1568,7 +1576,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 0.0); Kokkos::deep_copy(qi, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1604,7 +1612,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { qc(0, 6) = 50; qc(0, 7) = 10; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, From df2d2da92a5fdb59baa02088fe5a04dbd3f42cf1 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 6 Jun 2024 16:10:58 -0700 Subject: [PATCH 073/904] remove exner and rstar in homme iop as they are no longer needed and fix formatting issues --- .../eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index eff94346895e..81765e2c6a7e 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -339,8 +339,6 @@ apply_iop_forcing(const Real dt) // and add the below WSM and views WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); view_Nd - rstar ("rstar", nelem, NGP, NGP, NLEV), - exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); // Lambda for computing temperature from Hommexx @@ -381,9 +379,9 @@ apply_iop_forcing(const Real dt) // Compute temperature from virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); }); }); @@ -498,7 +496,7 @@ apply_iop_forcing(const Real dt) // Convert updated temperature back to psuedo density virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); // Release WS views @@ -511,7 +509,7 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute rstar, exner and temperature from Hommexx + // Compute temperature from Hommexx compute_homme_states(); Kokkos::fence(); } @@ -605,8 +603,6 @@ apply_iop_forcing(const Real dt) auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); From 0819dd38967299ec94e4ff11656998fc02a7a41b Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 10 Jun 2024 09:30:25 -0700 Subject: [PATCH 074/904] address a minor formatting issue --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 81765e2c6a7e..ee76f7229bbc 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -356,7 +356,7 @@ apply_iop_forcing(const Real dt) const int igp = idx/NGP; const int jgp = idx%NGP; - auto ps_i = ps_dyn(ie, igp, jgp); + auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); From 771fa4ee77919b0a102e0e92e6e45ffe1f254a00 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 09:10:32 -0700 Subject: [PATCH 075/904] clean up of unnecessary code --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 65b51b4c11be..22e0add1fe99 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -341,15 +341,15 @@ apply_iop_forcing(const Real dt) view_Nd temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing temperature from Hommexx - auto compute_homme_states = [&] () { + // Lambda for computing temperature + auto compute_temperature = [&] () { Kokkos::parallel_for("compute_temperature_for_iop", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { KV kv(team); const int ie = team.league_rank(); // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; + uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { @@ -370,12 +370,6 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); - Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - // Compute temperature from virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { auto T_val = vtheta_dp_i(k); @@ -392,7 +386,7 @@ apply_iop_forcing(const Real dt) }; // Preprocess some homme states to get temperature - compute_homme_states(); + compute_temperature(); Kokkos::fence(); // Apply IOP forcing @@ -509,8 +503,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute temperature from Hommexx - compute_homme_states(); + // Compute temperature + compute_temperature(); Kokkos::fence(); } @@ -593,7 +587,7 @@ apply_iop_forcing(const Real dt) // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; + uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { From 5145ccf2f4fd5f14d717cb100a1f3e8b21887424 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 10:28:02 -0700 Subject: [PATCH 076/904] change two instances of policy_homme to policy_eamxx --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 133 ++++++++---------- 1 file changed, 62 insertions(+), 71 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 22e0add1fe99..4bff0c0fa1ee 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -343,41 +343,36 @@ apply_iop_forcing(const Real dt) // Lambda for computing temperature auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + Kokkos::parallel_for("compute_temperature_for_iop", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); - }); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + // Compute temperature from virtual potential temperature + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + auto T_val = vtheta_dp_i(k); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); }); // Release WS views @@ -441,61 +436,57 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; - // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); + // Get temp views from workspace + auto ws = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); - team.team_barrier(); + team.team_barrier(); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - }); + // Convert updated temperature back to psuedo density virtual potential temperature + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + }); // Release WS views ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); - }); }); if (iop_nudge_tq or iop_nudge_uv) { From f9e7595585ce0825f0eb3270657b77a05134e138 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 11:10:13 -0700 Subject: [PATCH 077/904] last change from policy_homme to policy_eamxx in the nudging routine --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 98 +++++++++---------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 4bff0c0fa1ee..93cddb99877d 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -571,65 +571,63 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_homme, + policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { - if (iop_nudge_tq) { - // Restrict nudging of T and qv to certain levels if requested by user - // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high - // is in units of [hPa], thus convert iop_nudge_tq_low/high - Mask nudge_level(false); - int max_size = hyam.size(); - for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { - const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; - nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); - } - - qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); - temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - } - if (iop_nudge_uv) { - u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); - v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { + if (iop_nudge_tq) { + // Restrict nudging of T and qv to certain levels if requested by user + // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high + // is in units of [hPa], thus convert iop_nudge_tq_low/high + Mask nudge_level(false); + int max_size = hyam.size(); + for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { + const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; + nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 + and + pressure_from_iop >= iop_nudge_tq_high*100); } - }); + + qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); + temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); + + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + } + if (iop_nudge_uv) { + u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); + v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); + } }); - // Release WS views - ws.release_many_contiguous<1>({&pmid}); + + // Release WS views + ws.release_many_contiguous<1>({&pmid}); }); } } From 8731c480f22bb64a18737d3e43cf364dd6d2bf87 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 11:38:11 -0700 Subject: [PATCH 078/904] remove unnecessary includes statements --- .../eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 93cddb99877d..b37985608f73 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,11 +10,8 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" -#include "ElementOps.hpp" -#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" -#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" @@ -225,13 +222,7 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - - using EOS = Homme::EquationOfState; - using ElementOps = Homme::ElementOps; - using KV = Homme::KernelVariables; - using PF = PhysicsFunctions; - using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -268,7 +259,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states and EOS/EO classes + // Homme element states auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -277,11 +268,6 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - EOS eos; - eos.init(params.theta_hydrostatic_mode, hvcoord); - - ElementOps elem_ops; - elem_ops.init(hvcoord); const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); // Load data from IOP files, if necessary From 09cfa6a9d896e0e3cf574daa75e71b9f7cc0f0b9 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 11:43:14 -0700 Subject: [PATCH 079/904] remove lingering definition of policy_homme and update comment --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index b37985608f73..77330878f21a 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -310,15 +310,10 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for both homme and scream - // related loops. We need separate policies since hommexx functions used here - // assume they are called inside nested loops for elements and Gaussian points, - // whereas EAMxx function we use expects a single level of parallelism - // for elements and Guassian points. + // Team policy and workspace manager for eamxx // TODO: scream::ColumnOps functions could take an arbitary loop boundary // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that // all 3 kernel launches here could be combined. - const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class From 2edf9b3c5649c798f7552e045416fd68b283b96c Mon Sep 17 00:00:00 2001 From: Dalei Hao Date: Thu, 13 Jun 2024 15:44:32 -0700 Subject: [PATCH 080/904] Undo the change related to latitude limits --- components/elm/src/biogeophys/SurfaceAlbedoMod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 b/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 index 49c6d061d3d8..073c836c19bc 100644 --- a/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 +++ b/components/elm/src/biogeophys/SurfaceAlbedoMod.F90 @@ -1811,7 +1811,7 @@ subroutine Albedo_TOP_Adjustment(bounds, num_pft, filter_pft, & lon_180 = lon(g) if (lon_180 > pi) lon_180 = lon_180-2._r8*pi - if (cosz > 0._r8 .and. stdev_elev(g) > 0._r8) then + if (cosz > 0._r8 .and. abs(lat(g)) < 1.047_r8 .and. stdev_elev(g) > 0._r8) then local_timeofday = next_tod + lon_180 / pi * 180._r8 * 240._r8 if (local_timeofday >= 86400._r8) then From f8211b93bf9de0f6544a523ba7cd7d7e2cc394d4 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 17 Jun 2024 09:44:20 -0700 Subject: [PATCH 081/904] rename policy and workspaces and change a few instances of improper ThreadVectorRange to TeamVectorRange --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 77330878f21a..37e6c9de8adc 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -311,26 +311,23 @@ apply_iop_forcing(const Real dt) } // Team policy and workspace manager for eamxx - // TODO: scream::ColumnOps functions could take an arbitary loop boundary - // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that - // all 3 kernel launches here could be combined. - const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); + WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); view_Nd temperature("temperature", nelem, NGP, NGP, NLEV); // Lambda for computing temperature auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); @@ -349,7 +346,7 @@ apply_iop_forcing(const Real dt) team.team_barrier(); // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { auto T_val = vtheta_dp_i(k); T_val /= dp3d_i(k); T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); @@ -366,13 +363,13 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -417,13 +414,13 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -450,7 +447,7 @@ apply_iop_forcing(const Real dt) team.team_barrier(); // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { const int ilev = k/qsize; const int q = k%qsize; @@ -461,7 +458,7 @@ apply_iop_forcing(const Real dt) team.team_barrier(); // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); @@ -552,7 +549,7 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_eamxx, + policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); @@ -560,7 +557,7 @@ apply_iop_forcing(const Real dt) const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); From eac5b2f2ea2d4694a5bd13c353e28ba9b4d3b9aa Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 14:15:42 -0600 Subject: [PATCH 082/904] Everything working --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 327 +++++++++--------- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 158 ++++----- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 10 +- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 89 ++++- .../rrtmgp/scream_rrtmgp_interface.hpp | 24 +- .../src/physics/rrtmgp/tests/rrtmgp_tests.cpp | 64 ++-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 15 +- .../rrtmgp/rrtmgp_standalone_unit.cpp | 37 +- 9 files changed, 405 insertions(+), 321 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 3d7fba5ac72d..4c542032b3ac 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 3d7fba5ac72dab7c8448411e7452e2b409f22d93 +Subproject commit 4c542032b3ac3f842a9619c9eadc005c507a0366 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index a24f4d5ddc32..1556280eab4b 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -19,6 +19,28 @@ namespace scream { +template +void check_views2(const View1& view1, const View2& view2) +{ + constexpr auto krank1 = View1::rank; + constexpr auto krank2 = View2::rank; + + RRT_REQUIRE(krank1 == krank2, "Rank mismatch for: " << view1.label()); + RRT_REQUIRE(krank1 == 2, "Rank mismatch for: " << view1.label()); + + for (auto r = 0; r < krank1; ++r) { + RRT_REQUIRE(view1.extent(r) == view2.extent(r), "Dim mismatch for: " << view1.label() << ", rank: " << r << ", " << view1.extent(r) << " != " << view2.extent(r)); + } + + for (auto i = 0; i < view1.extent(0); ++i) { + for (auto j = 0; j < view1.extent(1); ++j) { + const auto data1 = view1(i, j); + const auto data2 = view2(i, j); + RRT_REQUIRE(data1 == data2, "Data mismatch for: " << view1.label() << ", i: " << i << ", j: " << j << ", " << data1 << " != " << data2); + } + } +} + using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; @@ -596,7 +618,7 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // Names of active gases auto gas_names_yakl_offset = string1dv(m_ngas); - m_gas_mol_weights = view_1d_real("gas_mol_weights",m_ngas); + m_gas_mol_weights = real1dk("gas_mol_weights",m_ngas); // the lookup function for getting the gas mol weights doesn't work on device auto gas_mol_w_host = Kokkos::create_mirror_view(m_gas_mol_weights); for (int igas = 0; igas < m_ngas; igas++) { @@ -848,11 +870,9 @@ void RRTMGPRadiation::run_impl (const double dt) { "[RRTMGP::run_impl] Col chunk beg,end: " + std::to_string(beg) + ", " + std::to_string(beg+ncol) + "\n"); +#ifdef RRTMGP_ENABLE_YAKL // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. - // JGF: this doesn't appear to be copying the data, just returning a new array - // pointing to the same memory. -#ifdef RRTMGP_ENABLE_YAKL auto subview_1d = [&](const real1d v) -> real1d { return real1d(v.label(),v.myData,ncol); }; @@ -923,31 +943,64 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // If YAKL is on, we don't want aliased memory in both the yakl and kokos - // subviews. - auto subview_1dk = [&](const view_1d_real v) -> view_1d_real { - view_1d_real subv(v, std::make_pair(0, ncol)); + // subviews, so make new views and deep_copy. Also, be sure to trim the excess + // items from the field manager views due to simd packs. If we don't trim, then + // check_range_k will fail due to looking at unused values. Once rrtmgp can handle + // packs, this won't be necessary. + auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { + ureal1dk subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - view_1d_real rv(v.label(), ncol); + real1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_2dk = [&](const view_2d_real v) -> view_2d_real { - view_2d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { + cureal1dk subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - view_2d_real rv(v.label(), ncol, v.extent(1)); + creal1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_3dk = [&](const view_3d_real v) -> view_3d_real { - view_3d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + auto subview_2dk = [&](const ureal2dk& v) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - view_3d_real rv(v.label(), ncol, v.extent(1), v.extent(2)); + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_2dkc = [&](const cureal2dk& v, const int inner_dim) -> cureal2dk { + cureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); +#ifdef RRTMGP_ENABLE_YAKL + creal2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_2dk_new = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); +#ifdef RRTMGP_ENABLE_YAKL + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { + ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); +#ifdef RRTMGP_ENABLE_YAKL + real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); return rv; #else @@ -955,59 +1008,59 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; - auto p_lay_k = subview_2dk(m_buffer.p_lay_k); - auto t_lay_k = subview_2dk(m_buffer.t_lay_k); - auto p_lev_k = subview_2dk(m_buffer.p_lev_k); - auto z_del_k = subview_2dk(m_buffer.z_del_k); - auto p_del_k = subview_2dk(m_buffer.p_del_k); - auto t_lev_k = subview_2dk(m_buffer.t_lev_k); - auto mu0_k = subview_1dk(m_buffer.mu0_k); - auto sfc_alb_dir_k = subview_2dk(m_buffer.sfc_alb_dir_k); - auto sfc_alb_dif_k = subview_2dk(m_buffer.sfc_alb_dif_k); - auto sfc_alb_dir_vis_k = subview_1dk(m_buffer.sfc_alb_dir_vis_k); - auto sfc_alb_dir_nir_k = subview_1dk(m_buffer.sfc_alb_dir_nir_k); - auto sfc_alb_dif_vis_k = subview_1dk(m_buffer.sfc_alb_dif_vis_k); - auto sfc_alb_dif_nir_k = subview_1dk(m_buffer.sfc_alb_dif_nir_k); - auto qc_k = subview_2dk(m_buffer.qc_k); - auto nc_k = subview_2dk(m_buffer.nc_k); - auto qi_k = subview_2dk(m_buffer.qi_k); - auto cldfrac_tot_k = subview_2dk(m_buffer.cldfrac_tot_k); - auto rel_k = subview_2dk(m_buffer.eff_radius_qc_k); - auto rei_k = subview_2dk(m_buffer.eff_radius_qi_k); - auto sw_flux_up_k = subview_2dk(m_buffer.sw_flux_up_k); - auto sw_flux_dn_k = subview_2dk(m_buffer.sw_flux_dn_k); - auto sw_flux_dn_dir_k = subview_2dk(m_buffer.sw_flux_dn_dir_k); - auto lw_flux_up_k = subview_2dk(m_buffer.lw_flux_up_k); - auto lw_flux_dn_k = subview_2dk(m_buffer.lw_flux_dn_k); - auto sw_clnclrsky_flux_up_k = subview_2dk(m_buffer.sw_clnclrsky_flux_up_k); - auto sw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_k); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_dir_k); - auto sw_clrsky_flux_up_k = subview_2dk(m_buffer.sw_clrsky_flux_up_k); - auto sw_clrsky_flux_dn_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_k); - auto sw_clrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_dir_k); - auto sw_clnsky_flux_up_k = subview_2dk(m_buffer.sw_clnsky_flux_up_k); - auto sw_clnsky_flux_dn_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_k); - auto sw_clnsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_dir_k); - auto lw_clnclrsky_flux_up_k = subview_2dk(m_buffer.lw_clnclrsky_flux_up_k); - auto lw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.lw_clnclrsky_flux_dn_k); - auto lw_clrsky_flux_up_k = subview_2dk(m_buffer.lw_clrsky_flux_up_k); - auto lw_clrsky_flux_dn_k = subview_2dk(m_buffer.lw_clrsky_flux_dn_k); - auto lw_clnsky_flux_up_k = subview_2dk(m_buffer.lw_clnsky_flux_up_k); - auto lw_clnsky_flux_dn_k = subview_2dk(m_buffer.lw_clnsky_flux_dn_k); - auto sw_bnd_flux_up_k = subview_3dk(m_buffer.sw_bnd_flux_up_k); - auto sw_bnd_flux_dn_k = subview_3dk(m_buffer.sw_bnd_flux_dn_k); - auto sw_bnd_flux_dir_k = subview_3dk(m_buffer.sw_bnd_flux_dir_k); - auto sw_bnd_flux_dif_k = subview_3dk(m_buffer.sw_bnd_flux_dif_k); - auto lw_bnd_flux_up_k = subview_3dk(m_buffer.lw_bnd_flux_up_k); - auto lw_bnd_flux_dn_k = subview_3dk(m_buffer.lw_bnd_flux_dn_k); - auto sfc_flux_dir_vis_k = subview_1dk(m_buffer.sfc_flux_dir_vis_k); - auto sfc_flux_dir_nir_k = subview_1dk(m_buffer.sfc_flux_dir_nir_k); - auto sfc_flux_dif_vis_k = subview_1dk(m_buffer.sfc_flux_dif_vis_k); - auto sfc_flux_dif_nir_k = subview_1dk(m_buffer.sfc_flux_dif_nir_k); - auto aero_tau_sw_k = subview_3dk(m_buffer.aero_tau_sw_k); - auto aero_ssa_sw_k = subview_3dk(m_buffer.aero_ssa_sw_k); - auto aero_g_sw_k = subview_3dk(m_buffer.aero_g_sw_k); - auto aero_tau_lw_k = subview_3dk(m_buffer.aero_tau_lw_k); + // Note, ncol will not necessary be m_col_chunk_size because the number of cols + // will not always be evenly divided by m_col_chunk_size. In most cases, the + // extra space will not cause any problems, but it does sometimes. + auto p_lay_k = subview_2dkc(d_pmid, m_nlay); + auto t_lay_k = subview_2dkc(d_tmid, m_nlay); + auto p_lev_k = subview_2dkc(d_pint, m_nlay+1); + auto p_del_k = subview_2dkc(d_pdel, m_nlay); + auto sfc_alb_dir_k = m_buffer.sfc_alb_dir_k; + auto sfc_alb_dif_k = m_buffer.sfc_alb_dif_k; + auto sfc_alb_dir_vis_k = subview_1dkc(d_sfc_alb_dir_vis); + auto sfc_alb_dir_nir_k = subview_1dkc(d_sfc_alb_dir_nir); + auto sfc_alb_dif_vis_k = subview_1dkc(d_sfc_alb_dif_vis); + auto sfc_alb_dif_nir_k = subview_1dkc(d_sfc_alb_dif_nir); + auto qc_k = subview_2dkc(d_qc, m_nlay); + auto nc_k = subview_2dkc(d_nc, m_nlay); + auto qi_k = subview_2dkc(d_qi, m_nlay); + auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; + auto rel_k = subview_2dkc(d_rel, m_nlay); + auto rei_k = subview_2dkc(d_rei, m_nlay); + auto sw_flux_up_k = subview_2dk_new(d_sw_flux_up, m_nlay+1); + auto sw_flux_dn_k = subview_2dk_new(d_sw_flux_dn, m_nlay+1); + auto sw_flux_dn_dir_k = subview_2dk_new(d_sw_flux_dn_dir, m_nlay+1); + auto lw_flux_up_k = subview_2dk_new(d_lw_flux_up, m_nlay+1); + auto lw_flux_dn_k = subview_2dk_new(d_lw_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_up_k = subview_2dk_new(d_sw_clnclrsky_flux_up, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = subview_2dk_new(d_sw_clnclrsky_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); + auto sw_clrsky_flux_up_k = subview_2dk_new(d_sw_clrsky_flux_up, m_nlay+1); + auto sw_clrsky_flux_dn_k = subview_2dk_new(d_sw_clrsky_flux_dn, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clrsky_flux_dn_dir, m_nlay+1); + auto sw_clnsky_flux_up_k = subview_2dk_new(d_sw_clnsky_flux_up, m_nlay+1); + auto sw_clnsky_flux_dn_k = subview_2dk_new(d_sw_clnsky_flux_dn, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnsky_flux_dn_dir, m_nlay+1); + auto lw_clnclrsky_flux_up_k = subview_2dk_new(d_lw_clnclrsky_flux_up, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = subview_2dk_new(d_lw_clnclrsky_flux_dn, m_nlay+1); + auto lw_clrsky_flux_up_k = subview_2dk_new(d_lw_clrsky_flux_up, m_nlay+1); + auto lw_clrsky_flux_dn_k = subview_2dk_new(d_lw_clrsky_flux_dn, m_nlay+1); + auto lw_clnsky_flux_up_k = subview_2dk_new(d_lw_clnsky_flux_up, m_nlay+1); + auto lw_clnsky_flux_dn_k = subview_2dk_new(d_lw_clnsky_flux_dn, m_nlay+1); + auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; + auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; + auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; + auto sw_bnd_flux_dif_k = m_buffer.sw_bnd_flux_dif_k; + auto lw_bnd_flux_up_k = m_buffer.lw_bnd_flux_up_k; + auto lw_bnd_flux_dn_k = m_buffer.lw_bnd_flux_dn_k; + auto sfc_flux_dir_vis_k = subview_1dk(d_sfc_flux_dir_vis); + auto sfc_flux_dir_nir_k = subview_1dk(d_sfc_flux_dir_nir); + auto sfc_flux_dif_vis_k = subview_1dk(d_sfc_flux_dif_vis); + auto sfc_flux_dif_nir_k = subview_1dk(d_sfc_flux_dif_nir); + auto aero_tau_sw_k = m_buffer.aero_tau_sw_k; + auto aero_ssa_sw_k = m_buffer.aero_ssa_sw_k; + auto aero_g_sw_k = m_buffer.aero_g_sw_k; + auto aero_tau_lw_k = m_buffer.aero_tau_lw_k; auto cld_tau_sw_bnd_k = subview_3dk(m_buffer.cld_tau_sw_bnd_k); auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); @@ -1015,7 +1068,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif auto d_tint = m_buffer.d_tint; auto d_dz = m_buffer.d_dz; - + auto d_mu0 = m_buffer.cosine_zenith; // Set gas concs to "view" only the first ncol columns #ifdef RRTMGP_ENABLE_YAKL @@ -1032,7 +1085,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // Determine the cosine zenith angle // NOTE: Since we are bridging to F90 arrays this must be done on HOST and then // deep copied to a device view. - auto d_mu0 = m_buffer.cosine_zenith; auto h_mu0 = Kokkos::create_mirror_view(d_mu0); if (m_fixed_solar_zenith_angle > 0) { for (int i=0; i({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1206,7 +1226,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real2d tmp2d = subview_2d(m_buffer.tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real tmp2d_k = subview_2dk(m_buffer.tmp2d_k); + real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1315,7 +1335,7 @@ void RRTMGPRadiation::run_impl (const double dt) { interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), - std::vector({lwp_k, iwp_k})); + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1356,7 +1376,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), - std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1386,9 +1406,9 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS interface_t::rrtmgp_main( ncol, m_nlay, - p_lay_k, t_lay_k, p_lev_k, t_lev_k, + p_lay_k, t_lay_k, p_lev_k, d_tint, m_gas_concs_k, - sfc_alb_dir_k, sfc_alb_dif_k, mu0_k, + sfc_alb_dir_k, sfc_alb_dif_k, d_mu0, lwp_k, iwp_k, rel_k, rei_k, cldfrac_tot_k, aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k, cld_tau_sw_bnd_k, cld_tau_lw_bnd_k, @@ -1412,7 +1432,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, lw_clnsky_flux_up, lw_clnsky_flux_dn}), - std::vector({ + std::vector({ sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, @@ -1420,9 +1440,8 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up_k, lw_clnclrsky_flux_dn_k, lw_clrsky_flux_up_k, lw_clrsky_flux_dn_k, lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); - COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), - std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1472,7 +1491,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), - std::vector({sw_heating_k, lw_heating_k})); + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1516,7 +1535,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), - std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1537,10 +1556,10 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_1d_real cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably @@ -1552,7 +1571,7 @@ void RRTMGPRadiation::run_impl (const double dt) { interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), - std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Compute cloud-top diagnostics following AeroCOM recommendation @@ -1584,17 +1603,17 @@ void RRTMGPRadiation::run_impl (const double dt) { // Get IR 10.5 micron band for COSP auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); - view_1d_real T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); interface_t::compute_aerocom_cloudtop( - ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, + ncol, nlay, t_lay_k, p_lay_k, p_del_k, d_dz, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k); @@ -1602,7 +1621,7 @@ void RRTMGPRadiation::run_impl (const double dt) { T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), - std::vector({ + std::vector({ T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); @@ -1658,34 +1677,8 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; - d_sfc_flux_dir_nir(icol) = sfc_flux_dir_nir_k(i); - d_sfc_flux_dir_vis(icol) = sfc_flux_dir_vis_k(i); - d_sfc_flux_dif_nir(icol) = sfc_flux_dif_nir_k(i); - d_sfc_flux_dif_vis(icol) = sfc_flux_dif_vis_k(i); d_sfc_flux_sw_net(icol) = sw_flux_dn_k(i,kbot_k) - sw_flux_up_k(i,kbot_k); d_sfc_flux_lw_dn(icol) = lw_flux_dn_k(i,kbot_k); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - d_sw_flux_up(icol,k) = sw_flux_up_k(i,k); - d_sw_flux_dn(icol,k) = sw_flux_dn_k(i,k); - d_sw_flux_dn_dir(icol,k) = sw_flux_dn_dir_k(i,k); - d_lw_flux_up(icol,k) = lw_flux_up_k(i,k); - d_lw_flux_dn(icol,k) = lw_flux_dn_k(i,k); - d_sw_clnclrsky_flux_up(icol,k) = sw_clnclrsky_flux_up_k(i,k); - d_sw_clnclrsky_flux_dn(icol,k) = sw_clnclrsky_flux_dn_k(i,k); - d_sw_clnclrsky_flux_dn_dir(icol,k) = sw_clnclrsky_flux_dn_dir_k(i,k); - d_sw_clrsky_flux_up(icol,k) = sw_clrsky_flux_up_k(i,k); - d_sw_clrsky_flux_dn(icol,k) = sw_clrsky_flux_dn_k(i,k); - d_sw_clrsky_flux_dn_dir(icol,k) = sw_clrsky_flux_dn_dir_k(i,k); - d_sw_clnsky_flux_up(icol,k) = sw_clnsky_flux_up_k(i,k); - d_sw_clnsky_flux_dn(icol,k) = sw_clnsky_flux_dn_k(i,k); - d_sw_clnsky_flux_dn_dir(icol,k) = sw_clnsky_flux_dn_dir_k(i,k); - d_lw_clnclrsky_flux_up(icol,k) = lw_clnclrsky_flux_up_k(i,k); - d_lw_clnclrsky_flux_dn(icol,k) = lw_clnclrsky_flux_dn_k(i,k); - d_lw_clrsky_flux_up(icol,k) = lw_clrsky_flux_up_k(i,k); - d_lw_clrsky_flux_dn(icol,k) = lw_clrsky_flux_dn_k(i,k); - d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up_k(i,k); - d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn_k(i,k); - }); // Extract optical properties for COSP Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067_k); @@ -1699,7 +1692,7 @@ void RRTMGPRadiation::run_impl (const double dt) { }); #ifdef RRTMGP_ENABLE_YAKL // Sync back to gas_concs_k - view_3d_real temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); Kokkos::deep_copy(temp, m_gas_concs_k.concs); #endif #endif diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 28c5033c46d4..c60a1cb3181a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -16,17 +16,21 @@ namespace scream { class RRTMGPRadiation : public AtmosphereProcess { public: - using view_1d_real = typename ekat::KokkosTypes::template view_1d; - using view_2d_real = typename ekat::KokkosTypes::template view_2d; - using view_3d_real = typename ekat::KokkosTypes::template view_3d; - using view_2d_real_const = typename ekat::KokkosTypes::template view_2d; - using ci_string = ekat::CaseInsensitiveString; - - using KT = ekat::KokkosTypes; - template - using uview_1d = Unmanaged>; - template - using uview_2d = Unmanaged>; + using KT = ekat::KokkosTypes; + using real1dk = typename KT::template view_1d; + using real2dk = typename KT::template view_2d; + using real3dk = typename KT::template view_3d; + using creal1dk = typename KT::template view_1d; + using creal2dk = typename KT::template view_2d; + using creal3dk = typename KT::template view_3d; + using ureal1dk = Unmanaged; + using ureal2dk = Unmanaged; + using ureal3dk = Unmanaged; + using cureal1dk = Unmanaged; + using cureal2dk = Unmanaged; + using cureal3dk = Unmanaged; + + using ci_string = ekat::CaseInsensitiveString; using layout_t = typename ekat::KokkosTypes::Layout; @@ -91,7 +95,7 @@ class RRTMGPRadiation : public AtmosphereProcess { // These are the gases that we keep track of int m_ngas; std::vector m_gas_names; - view_1d_real m_gas_mol_weights; + real1dk m_gas_mol_weights; #ifdef RRTMGP_ENABLE_YAKL GasConcs m_gas_concs; #endif @@ -128,7 +132,7 @@ class RRTMGPRadiation : public AtmosphereProcess { static constexpr int num_3d_nlay_nlwgpts = 1; // 1d size (ncol) - uview_1d cosine_zenith; + ureal1dk cosine_zenith; #ifdef RRTMGP_ENABLE_YAKL real1d mu0; real1d sfc_alb_dir_vis; @@ -141,19 +145,19 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_1d_real mu0_k; - view_1d_real sfc_alb_dir_vis_k; - view_1d_real sfc_alb_dir_nir_k; - view_1d_real sfc_alb_dif_vis_k; - view_1d_real sfc_alb_dif_nir_k; - view_1d_real sfc_flux_dir_vis_k; - view_1d_real sfc_flux_dir_nir_k; - view_1d_real sfc_flux_dif_vis_k; - view_1d_real sfc_flux_dif_nir_k; + real1dk mu0_k; + real1dk sfc_alb_dir_vis_k; + real1dk sfc_alb_dir_nir_k; + real1dk sfc_alb_dif_vis_k; + real1dk sfc_alb_dif_nir_k; + real1dk sfc_flux_dir_vis_k; + real1dk sfc_flux_dir_nir_k; + real1dk sfc_flux_dif_vis_k; + real1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) - uview_2d d_dz; + ureal2dk d_dz; #ifdef RRTMGP_ENABLE_YAKL real2d p_lay; real2d t_lay; @@ -172,25 +176,25 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real p_lay_k; - view_2d_real t_lay_k; - view_2d_real z_del_k; - view_2d_real p_del_k; - view_2d_real qc_k; - view_2d_real nc_k; - view_2d_real qi_k; - view_2d_real cldfrac_tot_k; - view_2d_real eff_radius_qc_k; - view_2d_real eff_radius_qi_k; - view_2d_real tmp2d_k; - view_2d_real lwp_k; - view_2d_real iwp_k; - view_2d_real sw_heating_k; - view_2d_real lw_heating_k; + real2dk p_lay_k; + real2dk t_lay_k; + real2dk z_del_k; + real2dk p_del_k; + real2dk qc_k; + real2dk nc_k; + real2dk qi_k; + real2dk cldfrac_tot_k; + real2dk eff_radius_qc_k; + real2dk eff_radius_qi_k; + real2dk tmp2d_k; + real2dk lwp_k; + real2dk iwp_k; + real2dk sw_heating_k; + real2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) - uview_2d d_tint; + ureal2dk d_tint; #ifdef RRTMGP_ENABLE_YAKL real2d p_lev; real2d t_lev; @@ -216,28 +220,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real p_lev_k; - view_2d_real t_lev_k; - view_2d_real sw_flux_up_k; - view_2d_real sw_flux_dn_k; - view_2d_real sw_flux_dn_dir_k; - view_2d_real lw_flux_up_k; - view_2d_real lw_flux_dn_k; - view_2d_real sw_clnclrsky_flux_up_k; - view_2d_real sw_clnclrsky_flux_dn_k; - view_2d_real sw_clnclrsky_flux_dn_dir_k; - view_2d_real sw_clrsky_flux_up_k; - view_2d_real sw_clrsky_flux_dn_k; - view_2d_real sw_clrsky_flux_dn_dir_k; - view_2d_real sw_clnsky_flux_up_k; - view_2d_real sw_clnsky_flux_dn_k; - view_2d_real sw_clnsky_flux_dn_dir_k; - view_2d_real lw_clnclrsky_flux_up_k; - view_2d_real lw_clnclrsky_flux_dn_k; - view_2d_real lw_clrsky_flux_up_k; - view_2d_real lw_clrsky_flux_dn_k; - view_2d_real lw_clnsky_flux_up_k; - view_2d_real lw_clnsky_flux_dn_k; + real2dk p_lev_k; + real2dk t_lev_k; + real2dk sw_flux_up_k; + real2dk sw_flux_dn_k; + real2dk sw_flux_dn_dir_k; + real2dk lw_flux_up_k; + real2dk lw_flux_dn_k; + real2dk sw_clnclrsky_flux_up_k; + real2dk sw_clnclrsky_flux_dn_k; + real2dk sw_clnclrsky_flux_dn_dir_k; + real2dk sw_clrsky_flux_up_k; + real2dk sw_clrsky_flux_dn_k; + real2dk sw_clrsky_flux_dn_dir_k; + real2dk sw_clnsky_flux_up_k; + real2dk sw_clnsky_flux_dn_k; + real2dk sw_clnsky_flux_dn_dir_k; + real2dk lw_clnclrsky_flux_up_k; + real2dk lw_clnclrsky_flux_dn_k; + real2dk lw_clrsky_flux_up_k; + real2dk lw_clrsky_flux_dn_k; + real2dk lw_clnsky_flux_up_k; + real2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -248,10 +252,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real sw_bnd_flux_up_k; - view_3d_real sw_bnd_flux_dn_k; - view_3d_real sw_bnd_flux_dir_k; - view_3d_real sw_bnd_flux_dif_k; + real3dk sw_bnd_flux_up_k; + real3dk sw_bnd_flux_dn_k; + real3dk sw_bnd_flux_dir_k; + real3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -260,8 +264,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real lw_bnd_flux_up_k; - view_3d_real lw_bnd_flux_dn_k; + real3dk lw_bnd_flux_up_k; + real3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -270,8 +274,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real sfc_alb_dir_k; - view_2d_real sfc_alb_dif_k; + real2dk sfc_alb_dir_k; + real2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -282,10 +286,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real aero_tau_sw_k; - view_3d_real aero_ssa_sw_k; - view_3d_real aero_g_sw_k; - view_3d_real aero_tau_lw_k; + real3dk aero_tau_sw_k; + real3dk aero_ssa_sw_k; + real3dk aero_g_sw_k; + real3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -294,8 +298,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real cld_tau_sw_bnd_k; - view_3d_real cld_tau_lw_bnd_k; + real3dk cld_tau_sw_bnd_k; + real3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -304,8 +308,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real cld_tau_sw_gpt_k; - view_3d_real cld_tau_lw_gpt_k; + real3dk cld_tau_sw_gpt_k; + real3dk cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index 79f6e570950d..466467919fcf 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -42,14 +42,14 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_test_utils { -using real1dk = typename ekat::KokkosTypes::template view_1d; -using real2dk = typename ekat::KokkosTypes::template view_2d; -using real3dk = typename ekat::KokkosTypes::template view_3d; -using MDRP = typename conv::MDRP; using interface_t = scream::rrtmgp::rrtmgp_interface; +using real1dk = typename interface_t::view_t; +using real2dk = typename interface_t::view_t; +using real3dk = typename interface_t::view_t; +using MDRP = typename conv::MDRP; static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 6e924103229f..8c236e64e811 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -57,9 +57,10 @@ void compute_heating_rate ( View4 const &heating_rate) { using physconst = scream::physics::Constants; + using MDRP = typename conv::MDRP; auto ncol = flux_up.extent(0); auto nlay = flux_up.extent(1)-1; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { heating_rate(icol,ilay) = ( flux_up(icol,ilay+1) - flux_up(icol,ilay) - flux_dn(icol,ilay+1) + flux_dn(icol,ilay) @@ -98,18 +99,20 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } }); auto num_bad = sum(bad_mask); - pass = false; - out << msg << ": " - << num_bad << " values outside range " - << "[" << xmin << "," << xmax << "]" - << "; minval = " << _xmin - << "; maxval = " << _xmax << "\n"; + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } } return pass; } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template ::type* = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; @@ -117,10 +120,10 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - Kokkos::View bad_mask("bad_mask", x.size()); - Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { - if (x.data()[i] < xmin or x.data()[i] > xmax) { - bad_mask.data()[i] = true; + Kokkos::View bad_mask("bad_mask", x.extent(0)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + if (x(i) < xmin or x(i) > xmax) { + bad_mask(i) = true; } }); auto num_bad = conv::sum(bad_mask); @@ -133,6 +136,68 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val } return pass; } + +template ::type* = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + if (x(i, j) < xmin or x(i, j) > xmax) { + bad_mask(i, j) = true; + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + +template ::type* = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1), x.extent(2)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + for (size_t k = 0; k < x.extent(2); ++k) { + if (x(i, j, k) < xmin or x(i, j, k) > xmax) { + bad_mask(i, j, k) = true; + } + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + + #endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 2038cb62ff79..c82d9e38a3b6 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -159,7 +159,7 @@ int get_wavelength_index_lw(double wavelength); // New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_interface { using MDRP = typename conv::MDRP; @@ -980,7 +980,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i // Kokkos::deep_copy(seeds_host, seeds); // for (int icol = 0; icol < ncol; ++icol) { // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // Kokkos::parallel_for(MDRP::template get<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { // auto generator = random_pool.get_state(); // cldx(icol,ilay,igpt) = generator.drand(0., 1.); // random_pool.free_state(generator); @@ -996,7 +996,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i }); // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { for (int ilay = 1; ilay < nlay; ilay++) { // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { @@ -1016,7 +1016,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i } // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { subcolumn_mask(icol,ilay,igpt) = 1; } else { @@ -1202,13 +1202,21 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template +template::type* = nullptr> static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + Kokkos::parallel_for(arr_out.size(), KOKKOS_LAMBDA(int i) { + arr_out(i) = std::min(std::max(arr_in(i), lower), upper); + }); +} + +template::type* = nullptr> +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { + Kokkos::parallel_for(MDRP::template get<2>({arr_out.extent(0), arr_out.extent(1)}), KOKKOS_LAMBDA(int i, int j) { + arr_out(i, j) = std::min(std::max(arr_in(i, j), lower), upper); }); } + static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) { // Get wavelength bounds for all wavelength bands @@ -1382,7 +1390,7 @@ static OpticalProps1sclK get_subsampled_clouds( auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { auto ibnd = gpoint_bands(igpt); if (cldmask(icol,ilay,igpt) == 1) { subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 65d1da9f1d68..0d3f18e7d841 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -320,6 +320,12 @@ int run_yakl(int argc, char** argv) { int run_kokkos(int argc, char** argv) { using namespace ekat::logger; using logger_t = Logger; + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using MDRP = utils_t::MDRP; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; ekat::Comm comm(MPI_COMM_WORLD); auto logger = std::make_shared("",LogLevel::info,comm); @@ -374,7 +380,7 @@ int run_kokkos(int argc, char** argv) { real2dk sw_flux_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Get dimension sizes int ncol = sw_flux_up_ref.extent(0); @@ -394,12 +400,12 @@ int run_kokkos(int argc, char** argv) { real2dk p_lev("p_lev", ncol, nlay+1); real2dk t_lev("t_lev", ncol, nlay+1); real2dk col_dry; - GasConcsK gas_concs; + GasConcsK gas_concs; read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry, ncol); // Initialize absorption coefficients logger->info("Initialize RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Setup our dummy atmosphere based on the input data we read in logger->info("Setup dummy atmos...\n"); @@ -413,7 +419,7 @@ int run_kokkos(int argc, char** argv) { real2dk rel("rel", ncol, nlay); real2dk rei("rei", ncol, nlay); real2dk cld("cld", ncol, nlay); - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol, p_lay, t_lay, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -426,8 +432,8 @@ int run_kokkos(int argc, char** argv) { // we would just have to setup the pointers to them in the // FluxesBroadband object logger->info("Setup fluxes...\n"); - const auto nswbands = scream::rrtmgp::k_dist_sw_k.get_nband(); - const auto nlwbands = scream::rrtmgp::k_dist_lw_k.get_nband(); + const auto nswbands = interface_t::k_dist_sw_k.get_nband(); + const auto nlwbands = interface_t::k_dist_lw_k.get_nband(); real2dk sw_flux_up ("sw_flux_up" , ncol, nlay+1); real2dk sw_flux_dn ("sw_flux_dn" , ncol, nlay+1); real2dk sw_flux_dir("sw_flux_dir", ncol, nlay+1); @@ -457,7 +463,7 @@ int run_kokkos(int argc, char** argv) { // Compute band-by-band surface_albedos. real2dk sfc_alb_dir("sfc_alb_dir", ncol, nswbands); real2dk sfc_alb_dif("sfc_alb_dif", ncol, nswbands); - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -468,19 +474,19 @@ int run_kokkos(int argc, char** argv) { auto aer_ssa_sw = real3dk("aer_ssa_sw", ncol, nlay, nswbands); auto aer_asm_sw = real3dk("aer_asm_sw", ncol, nlay, nswbands); auto aer_tau_lw = real3dk("aer_tau_lw", ncol, nlay, nlwbands); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_sw(icol,ilay,ibnd) = 0; aer_ssa_sw(icol,ilay,ibnd) = 0; aer_asm_sw(icol,ilay,ibnd) = 0; }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_lw(icol,ilay,ibnd) = 0; }); // These are returned as outputs now from rrtmgp_main // TODO: provide as inputs consistent with how aerosol is treated? - const auto nswgpts = scream::rrtmgp::k_dist_sw_k.get_ngpt(); - const auto nlwgpts = scream::rrtmgp::k_dist_lw_k.get_ngpt(); + const auto nswgpts = interface_t::k_dist_sw_k.get_ngpt(); + const auto nlwgpts = interface_t::k_dist_lw_k.get_ngpt(); auto cld_tau_sw_bnd = real3dk("cld_tau_sw_bnd", ncol, nlay, nswbands); auto cld_tau_lw_bnd = real3dk("cld_tau_lw_bnd", ncol, nlay, nlwbands); auto cld_tau_sw = real3dk("cld_tau_sw", ncol, nlay, nswgpts); @@ -489,7 +495,7 @@ int run_kokkos(int argc, char** argv) { // Run RRTMGP code on dummy atmosphere logger->info("Run RRTMGP...\n"); const Real tsi_scaling = 1; - scream::rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, nlay, p_lay, t_lay, p_lev, t_lev, gas_concs, sfc_alb_dir, sfc_alb_dif, mu0, @@ -513,33 +519,33 @@ int run_kokkos(int argc, char** argv) { // Check values against baseline logger->info("Check values...\n"); - rrtmgpTest::read_fluxes( + utils_t::read_fluxes( baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; // Because the aerosol optical properties are all set to zero, these fluxes must be equal - if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; logger->info("Cleaning up...\n"); // Clean up or else YAKL will throw errors - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); scream::finalize_kls(); return nerr != 0 ? 1 : 0; diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 36e2a59e02ff..99a582445616 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -856,6 +856,7 @@ using real3dk = interface_t::view_t; using int1dk = interface_t::view_t; using int2dk = interface_t::view_t; using int3dk = interface_t::view_t; +using MDRP = interface_t::MDRP; TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos @@ -1082,7 +1083,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Need to initialize RRTMGP with dummy gases logger->info("Init gases...\n"); - GasConcsK gas_concs; + GasConcsK gas_concs; string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); @@ -1100,7 +1101,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { auto sw_bnd_flux_dir = real3dk("sw_bnd_flux_dir", ncol, nlay+1, nbnd); auto sw_bnd_flux_dif = real3dk("sw_bnd_flux_dif", ncol, nlay+1, nbnd); logger->info("Populate band-resolved 3d fluxes for test case with only transition band flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1132,7 +1133,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in NIR bands logger->info("Populate band-resolved 3d fluxes for test case with only NIR flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1; sw_bnd_flux_dif(icol,ilay,ibnd) = 1; @@ -1163,7 +1164,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in VIS bands logger->info("Populate band-resolved 3d fluxes for test case with only VIS/UV flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1194,7 +1195,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in all bands logger->info("Populate band-resolved 3d fluxes for test with non-zero flux in all bands...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1.0; sw_bnd_flux_dif(icol,ilay,ibnd) = 2.0; @@ -1293,13 +1294,13 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { for (int igpt = 0; igpt < ngpt; ++igpt) { real cldmask_real = cldmask(icol,ilay,igpt); cldfrac_from_mask(icol,ilay) += cldmask_real; } }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { cldfrac_from_mask(icol,ilay) = cldfrac_from_mask(icol,ilay) / ngpt; }); // For cldfrac 1 we should get 1, for cldfrac 0 we should get 0, but in between we cannot be sure diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 9e6e94a2164f..75feeb1b8dd4 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -422,6 +422,13 @@ TEST_CASE("rrtmgp_scream_standalone", "") { #endif #ifdef RRTMGP_ENABLE_KOKKOS TEST_CASE("rrtmgp_scream_standalone_k", "") { + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using MDRP = interface_t::MDRP; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; + // Get baseline name (needs to be passed as an arg) std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); @@ -439,7 +446,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { real2dk sw_flux_dn_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Load ad parameter list std::string fname = "input_unit.yaml"; @@ -526,11 +533,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { // this will copy the first column of the input data (the first profile) ncol // times. We will then fill some fraction of these columns with clouds for // the test problem. - GasConcsK gas_concs; + GasConcsK gas_concs; real2dk col_dry; read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, col_dry, ncol_all); // Setup dummy problem; need to use tmp arrays with ncol_all size - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol_all, p_lay_all, t_lay_all, sfc_alb_dir_vis_all, sfc_alb_dir_nir_all, sfc_alb_dif_vis_all, sfc_alb_dif_nir_all, @@ -547,7 +554,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { sfc_alb_dif_nir(icol) = sfc_alb_dif_nir_all(icol_all); mu0(icol) = mu0_all(icol_all); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lay(icol,ilay) = p_lay_all(icol_all,ilay); t_lay(icol,ilay) = t_lay_all(icol_all,ilay); @@ -557,14 +564,14 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { rei(icol,ilay) = rei_all(icol_all,ilay); cld(icol,ilay) = cld_all(icol_all,ilay); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lev(icol,ilay) = p_lev_all(icol_all,ilay); t_lev(icol,ilay) = t_lev_all(icol_all,ilay); }); // Need to calculate a dummy pseudo_density for our test problem - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { p_del(icol,ilay) = abs(p_lev(icol,ilay+1) - p_lev(icol,ilay)); }); @@ -573,13 +580,13 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto qc = real2dk("qc", ncol, nlay); auto nc = real2dk("nc", ncol, nlay); auto qi = real2dk("qi", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); qi(icol,ilay) = 1e-3 * iwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); }); // Copy gases from gas_concs to gas_vmr array - Kokkos::parallel_for(conv::get_mdrp<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { + Kokkos::parallel_for(MDRP::template get<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { auto icol_all = ncol * irank + icol; gas_vmr(icol,ilay,igas) = gas_concs.concs(icol_all,ilay,igas); }); @@ -691,7 +698,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { const int i = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - if (k < nlay) t_lay(i+1,k+1) = d_tmid(i,k); + if (k < nlay) t_lay(i,k) = d_tmid(i,k); sw_flux_up_test(i,k) = d_sw_flux_up(i,k); sw_flux_dn_test(i,k) = d_sw_flux_dn(i,k); @@ -722,7 +729,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto sw_flux_dn_dir_loc = real2dk("sw_flux_dn_dir_loc", ncol, nlay+1); auto lw_flux_up_loc = real2dk("lw_flux_up_loc" , ncol, nlay+1); auto lw_flux_dn_loc = real2dk("lw_flux_dn_loc" , ncol, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; sw_flux_up_loc(icol,ilay) = sw_flux_up_ref(icol_all,ilay); sw_flux_dn_loc(icol,ilay) = sw_flux_dn_ref(icol_all,ilay); @@ -730,11 +737,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { lw_flux_up_loc(icol,ilay) = lw_flux_up_ref(icol_all,ilay); lw_flux_dn_loc(icol,ilay) = lw_flux_dn_ref(icol_all,ilay); }); - REQUIRE(rrtmgpTest::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); + REQUIRE(utils_t::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); // Finalize the driver. YAKL will be finalized inside // RRTMGPRadiation::finalize_impl after RRTMGP has had the From 4badc7bbad04be736fd7f046cfc526f275d46f21 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 14:50:19 -0600 Subject: [PATCH 083/904] cleanup --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 89 ++++++------------- 1 file changed, 27 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1556280eab4b..cde298cc8cd9 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -19,28 +19,6 @@ namespace scream { -template -void check_views2(const View1& view1, const View2& view2) -{ - constexpr auto krank1 = View1::rank; - constexpr auto krank2 = View2::rank; - - RRT_REQUIRE(krank1 == krank2, "Rank mismatch for: " << view1.label()); - RRT_REQUIRE(krank1 == 2, "Rank mismatch for: " << view1.label()); - - for (auto r = 0; r < krank1; ++r) { - RRT_REQUIRE(view1.extent(r) == view2.extent(r), "Dim mismatch for: " << view1.label() << ", rank: " << r << ", " << view1.extent(r) << " != " << view2.extent(r)); - } - - for (auto i = 0; i < view1.extent(0); ++i) { - for (auto j = 0; j < view1.extent(1); ++j) { - const auto data1 = view1(i, j); - const auto data2 = view2(i, j); - RRT_REQUIRE(data1 == data2, "Data mismatch for: " << view1.label() << ", i: " << i << ", j: " << j << ", " << data1 << " != " << data2); - } - } -} - using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; @@ -967,8 +945,8 @@ void RRTMGPRadiation::run_impl (const double dt) { return subv; #endif }; - auto subview_2dk = [&](const ureal2dk& v) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); #ifdef RRTMGP_ENABLE_YAKL real2dk rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); @@ -985,16 +963,6 @@ void RRTMGPRadiation::run_impl (const double dt) { return rv; #else return subv; -#endif - }; - auto subview_2dk_new = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); -#ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; #endif }; auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { @@ -1027,26 +995,26 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; auto rel_k = subview_2dkc(d_rel, m_nlay); auto rei_k = subview_2dkc(d_rei, m_nlay); - auto sw_flux_up_k = subview_2dk_new(d_sw_flux_up, m_nlay+1); - auto sw_flux_dn_k = subview_2dk_new(d_sw_flux_dn, m_nlay+1); - auto sw_flux_dn_dir_k = subview_2dk_new(d_sw_flux_dn_dir, m_nlay+1); - auto lw_flux_up_k = subview_2dk_new(d_lw_flux_up, m_nlay+1); - auto lw_flux_dn_k = subview_2dk_new(d_lw_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_up_k = subview_2dk_new(d_sw_clnclrsky_flux_up, m_nlay+1); - auto sw_clnclrsky_flux_dn_k = subview_2dk_new(d_sw_clnclrsky_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); - auto sw_clrsky_flux_up_k = subview_2dk_new(d_sw_clrsky_flux_up, m_nlay+1); - auto sw_clrsky_flux_dn_k = subview_2dk_new(d_sw_clrsky_flux_dn, m_nlay+1); - auto sw_clrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clrsky_flux_dn_dir, m_nlay+1); - auto sw_clnsky_flux_up_k = subview_2dk_new(d_sw_clnsky_flux_up, m_nlay+1); - auto sw_clnsky_flux_dn_k = subview_2dk_new(d_sw_clnsky_flux_dn, m_nlay+1); - auto sw_clnsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnsky_flux_dn_dir, m_nlay+1); - auto lw_clnclrsky_flux_up_k = subview_2dk_new(d_lw_clnclrsky_flux_up, m_nlay+1); - auto lw_clnclrsky_flux_dn_k = subview_2dk_new(d_lw_clnclrsky_flux_dn, m_nlay+1); - auto lw_clrsky_flux_up_k = subview_2dk_new(d_lw_clrsky_flux_up, m_nlay+1); - auto lw_clrsky_flux_dn_k = subview_2dk_new(d_lw_clrsky_flux_dn, m_nlay+1); - auto lw_clnsky_flux_up_k = subview_2dk_new(d_lw_clnsky_flux_up, m_nlay+1); - auto lw_clnsky_flux_dn_k = subview_2dk_new(d_lw_clnsky_flux_dn, m_nlay+1); + auto sw_flux_up_k = subview_2dk(d_sw_flux_up, m_nlay+1); + auto sw_flux_dn_k = subview_2dk(d_sw_flux_dn, m_nlay+1); + auto sw_flux_dn_dir_k = subview_2dk(d_sw_flux_dn_dir, m_nlay+1); + auto lw_flux_up_k = subview_2dk(d_lw_flux_up, m_nlay+1); + auto lw_flux_dn_k = subview_2dk(d_lw_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_up_k = subview_2dk(d_sw_clnclrsky_flux_up, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = subview_2dk(d_sw_clnclrsky_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); + auto sw_clrsky_flux_up_k = subview_2dk(d_sw_clrsky_flux_up, m_nlay+1); + auto sw_clrsky_flux_dn_k = subview_2dk(d_sw_clrsky_flux_dn, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = subview_2dk(d_sw_clrsky_flux_dn_dir, m_nlay+1); + auto sw_clnsky_flux_up_k = subview_2dk(d_sw_clnsky_flux_up, m_nlay+1); + auto sw_clnsky_flux_dn_k = subview_2dk(d_sw_clnsky_flux_dn, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = subview_2dk(d_sw_clnsky_flux_dn_dir, m_nlay+1); + auto lw_clnclrsky_flux_up_k = subview_2dk(d_lw_clnclrsky_flux_up, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = subview_2dk(d_lw_clnclrsky_flux_dn, m_nlay+1); + auto lw_clrsky_flux_up_k = subview_2dk(d_lw_clrsky_flux_up, m_nlay+1); + auto lw_clrsky_flux_dn_k = subview_2dk(d_lw_clrsky_flux_dn, m_nlay+1); + auto lw_clnsky_flux_up_k = subview_2dk(d_lw_clnsky_flux_up, m_nlay+1); + auto lw_clnsky_flux_dn_k = subview_2dk(d_lw_clnsky_flux_dn, m_nlay+1); auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; @@ -1224,9 +1192,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // correct size that uses m_buffer.tmp2d's pointer #ifdef RRTMGP_ENABLE_YAKL real2d tmp2d = subview_2d(m_buffer.tmp2d); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1235,22 +1200,22 @@ void RRTMGPRadiation::run_impl (const double dt) { // 'o3' is marked as 'Required' rather than 'Computed', so we need to get the proper field auto f = name=="o3" ? get_field_in(full_name) : get_field_out(full_name); auto d_vmr = f.get_view(); +#ifdef RRTMGP_ENABLE_KOKKOS + auto tmp2d_k = subview_2dkc(d_vmr, m_nlay); +#endif +#ifdef RRTMGP_ENABLE_YAKL // Copy to YAKL const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { -#ifdef RRTMGP_ENABLE_YAKL tmp2d(i+1,k+1) = d_vmr(icol,k); // Note that for YAKL arrays i and k start with index 1 -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - tmp2d_k(i,k) = d_vmr(icol,k); -#endif }); }); Kokkos::fence(); +#endif // Populate GasConcs object #ifdef RRTMGP_ENABLE_YAKL From 46595e4404340126f43f126df89776a563ad140a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 15:48:32 -0600 Subject: [PATCH 084/904] subview_2dk bug fix --- .../eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index cde298cc8cd9..9fab85444fcc 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -946,7 +946,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); + ureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); #ifdef RRTMGP_ENABLE_YAKL real2dk rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); From a11d4e51b372ba2b27bd8130a99d8af021e1ee86 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 17:44:50 -0600 Subject: [PATCH 085/904] More fixes --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 9fab85444fcc..7a5af5843d19 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -926,7 +926,7 @@ void RRTMGPRadiation::run_impl (const double dt) { // check_range_k will fail due to looking at unused values. Once rrtmgp can handle // packs, this won't be necessary. auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { - ureal1dk subv(v, std::make_pair(0, ncol)); + ureal1dk subv(v, std::make_pair(beg, beg+ncol)); #ifdef RRTMGP_ENABLE_YAKL real1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); @@ -936,7 +936,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { - cureal1dk subv(v, std::make_pair(0, ncol)); + cureal1dk subv(v, std::make_pair(beg, beg+ncol)); #ifdef RRTMGP_ENABLE_YAKL creal1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); @@ -966,7 +966,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { - ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); // The range assumes these are buffer views, not fields #ifdef RRTMGP_ENABLE_YAKL real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); From 325c3a3f8f29b97e6246d94fe2c9b88308571813 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 13:04:44 -0600 Subject: [PATCH 086/904] Set to correct rrtmgp branch --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 4c542032b3ac..02c2e03df33e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 4c542032b3ac3f842a9619c9eadc005c507a0366 +Subproject commit 02c2e03df33edca6101670168ca187cb94c0768e From aa300be9f1ae3fc732b2228464d07ab390311e86 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 14:01:18 -0600 Subject: [PATCH 087/904] Add buffer views should be unmanaged --- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index c60a1cb3181a..5dece435fb0a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -145,15 +145,15 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + ureal1dk mu0_k; + ureal1dk sfc_alb_dir_vis_k; + ureal1dk sfc_alb_dir_nir_k; + ureal1dk sfc_alb_dif_vis_k; + ureal1dk sfc_alb_dif_nir_k; + ureal1dk sfc_flux_dir_vis_k; + ureal1dk sfc_flux_dir_nir_k; + ureal1dk sfc_flux_dif_vis_k; + ureal1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) @@ -176,21 +176,21 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + ureal2dk p_lay_k; + ureal2dk t_lay_k; + ureal2dk z_del_k; + ureal2dk p_del_k; + ureal2dk qc_k; + ureal2dk nc_k; + ureal2dk qi_k; + ureal2dk cldfrac_tot_k; + ureal2dk eff_radius_qc_k; + ureal2dk eff_radius_qi_k; + ureal2dk tmp2d_k; + ureal2dk lwp_k; + ureal2dk iwp_k; + ureal2dk sw_heating_k; + ureal2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) @@ -220,28 +220,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + ureal2dk p_lev_k; + ureal2dk t_lev_k; + ureal2dk sw_flux_up_k; + ureal2dk sw_flux_dn_k; + ureal2dk sw_flux_dn_dir_k; + ureal2dk lw_flux_up_k; + ureal2dk lw_flux_dn_k; + ureal2dk sw_clnclrsky_flux_up_k; + ureal2dk sw_clnclrsky_flux_dn_k; + ureal2dk sw_clnclrsky_flux_dn_dir_k; + ureal2dk sw_clrsky_flux_up_k; + ureal2dk sw_clrsky_flux_dn_k; + ureal2dk sw_clrsky_flux_dn_dir_k; + ureal2dk sw_clnsky_flux_up_k; + ureal2dk sw_clnsky_flux_dn_k; + ureal2dk sw_clnsky_flux_dn_dir_k; + ureal2dk lw_clnclrsky_flux_up_k; + ureal2dk lw_clnclrsky_flux_dn_k; + ureal2dk lw_clrsky_flux_up_k; + ureal2dk lw_clrsky_flux_dn_k; + ureal2dk lw_clnsky_flux_up_k; + ureal2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -252,10 +252,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + ureal3dk sw_bnd_flux_up_k; + ureal3dk sw_bnd_flux_dn_k; + ureal3dk sw_bnd_flux_dir_k; + ureal3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -264,8 +264,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + ureal3dk lw_bnd_flux_up_k; + ureal3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -274,8 +274,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + ureal2dk sfc_alb_dir_k; + ureal2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -286,10 +286,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + ureal3dk aero_tau_sw_k; + ureal3dk aero_ssa_sw_k; + ureal3dk aero_g_sw_k; + ureal3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -298,8 +298,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + ureal3dk cld_tau_sw_bnd_k; + ureal3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -308,8 +308,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + ureal3dk cld_tau_sw_gpt_k; + ureal3dk cld_tau_lw_gpt_k; #endif }; From 75ae15d16933b68f9343c0d694fc884622cbc7c0 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 15:55:26 -0600 Subject: [PATCH 088/904] Build fixes for yakl mode --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 4 ++-- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 7a5af5843d19..62de7f8416cd 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -1293,8 +1293,8 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - interface_t::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - interface_t::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 5dece435fb0a..e036e7ac08c0 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -34,7 +34,9 @@ class RRTMGPRadiation : public AtmosphereProcess { using layout_t = typename ekat::KokkosTypes::Layout; +#ifdef RRTMGP_ENABLE_KOKKOS using interface_t = rrtmgp::rrtmgp_interface; +#endif // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); From 047111c9358244b176cce8543119d727d7c3b7ef Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 21 Jun 2024 16:01:17 -0600 Subject: [PATCH 089/904] add new mkatmsrffile tool --- .../eam/tools/mkatmsrffile/mkatmsrffile.py | 295 ++++++++++++++++++ .../{ => old_mkatmsrffile}/Makefile | 0 .../{ => old_mkatmsrffile}/README | 0 .../{ => old_mkatmsrffile}/mkatmsrffile.F90 | 0 .../{ => old_mkatmsrffile}/nml_atmsrf | 0 .../test_mkatmsrffile.sh | 0 .../generate-dry-deposition.md | 38 ++- 7 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 components/eam/tools/mkatmsrffile/mkatmsrffile.py rename components/eam/tools/mkatmsrffile/{ => old_mkatmsrffile}/Makefile (100%) rename components/eam/tools/mkatmsrffile/{ => old_mkatmsrffile}/README (100%) rename components/eam/tools/mkatmsrffile/{ => old_mkatmsrffile}/mkatmsrffile.F90 (100%) rename components/eam/tools/mkatmsrffile/{ => old_mkatmsrffile}/nml_atmsrf (100%) rename components/eam/tools/mkatmsrffile/{ => old_mkatmsrffile}/test_mkatmsrffile.sh (100%) diff --git a/components/eam/tools/mkatmsrffile/mkatmsrffile.py b/components/eam/tools/mkatmsrffile/mkatmsrffile.py new file mode 100644 index 000000000000..50999b6e945f --- /dev/null +++ b/components/eam/tools/mkatmsrffile/mkatmsrffile.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +#--------------------------------------------------------------------------------------------------- +''' +This is a replacement for the legacy gen_domain tool created for CESM. +Most legacy functionality is reproduced, with the notable exception of +the pole point latitude adjustment needed for the CESM FV grid. + +Created April, 2024 by Walter Hannah (LLNL) +''' +#--------------------------------------------------------------------------------------------------- +''' +The map file used to generate the atmsrf files can be created a few different ways. +For a typical E3SM configuration we recommend using a conservative, monotone map. +Here is an example command that can be used to generate one as of NCO version 5.2.2 + + SRC_GRID=${DIN_LOC_ROOT}/../mapping/grids/1x1d.nc + DST_GRID=${GRID_ROOT}/ne${NE}pg2_scrip.nc + MAP_FILE=${MAP_ROOT}/map_1x1_to_ne${NE}pg2_traave.nc + ncremap -a traave --src_grd=${SRC_GRID} --dst_grd=${DST_GRID} --map_file=${MAP_FILE} + +''' +#--------------------------------------------------------------------------------------------------- +import datetime, os, numpy as np, xarray as xr, numba, itertools, subprocess as sp +user, host = os.getenv('USER'), os.getenv('HOST') +source_code_meta = 'generate_domain_E3SM.py' +#--------------------------------------------------------------------------------------------------- +class clr:END,RED,GREEN,MAGENTA,CYAN = '\033[0m','\033[31m','\033[32m','\033[35m','\033[36m' +#--------------------------------------------------------------------------------------------------- +usage = ''' +python mkatmsrffile.py --map_file + --dst_grid + [--output-root ] + [--date-stamp ] + +Purpose: + This tool generates a file needed to prescribe atmospheric dry deposition of aerosols at the surface. + +Environment + + This tool requires a few special packages, such as xarray, numba, and itertools. + These are all included in the E3SM unified environment: + https://e3sm.org/resources/tools/other-tools/e3sm-unified-environment/ + + Otherwise a simple conda environment can be created: + conda create --name example_env --channel conda-forge xarray numpy numba scikit-learn netcdf4 + +The following output file is created: + + atmsrf__.nc + ???? + +''' +from optparse import OptionParser +parser = OptionParser(usage=usage) +parser.add_option('--map_file', + dest='map_file', + default=None, + help='Mapping file from a 1x1 degree grid to the target atmosphere grid') +parser.add_option('--vegetation_file', + dest='vegetation_file', + default=None, + help='Path to file containing area fractions of surface types and plant functional type (PFT) data (regrid_vegetation.nc)') +parser.add_option('--soil_water_file', + dest='soil_water_file', + default=None, + help='Path to file containing soil water data (clim_soilw.nc)') +parser.add_option('--dst_grid', + dest='dst_grid', + default=None, + help='destination atmosphere grid name') +parser.add_option('--output_root', + dest='output_root', + default='./', + help='Output path for domain files') +parser.add_option('--date-stamp', + dest='date_stamp', + default=None, + help='Creation date stamp for domain files') +(opts, args) = parser.parse_args() +#--------------------------------------------------------------------------------------------------- +def main(): + #----------------------------------------------------------------------------- + # check for valid input arguments + if opts.map_file is None: + raise ValueError(f'{clr.RED}map_file was not specified{clr.END}') + if opts.vegetation_file is None: + raise ValueError(f'{clr.RED}vegetation_file was not specified{clr.END}') + if opts.soil_water_file is None: + raise ValueError(f'{clr.RED}soil_water_file was not specified{clr.END}') + if opts.dst_grid is None: + raise ValueError(f'{clr.RED}dst_grid was not specified{clr.END}') + #----------------------------------------------------------------------------- + # check existence of input files and output path + if not os.path.exists(opts.vegetation_file) : + raise ValueError(f'{clr.RED}Vegetaion file does not exist:{clr.END} {opts.vegetation_file}') + if not os.path.exists(opts.soil_water_file) : + raise ValueError(f'{clr.RED}Soil water file does not exist:{clr.END} {opts.soil_water_file}') + if not os.path.exists(opts.output_root) : + raise ValueError(f'{clr.RED}Output root path does not exist:{clr.END} {opts.output_root}') + + #----------------------------------------------------------------------------- + # Set date stamp for file name + + if opts.date_stamp is None: + cdate = datetime.datetime.now().strftime('%Y%m%d') + else: + cdate = opts.date_stamp + + #----------------------------------------------------------------------------- + # specify output file name + + output_file = f'{opts.output_root}/atm_srf_{opts.dst_grid}_{cdate}.nc' + + #----------------------------------------------------------------------------- + # open input files as datasets + + ds_map = xr.open_dataset(opts.map_file) + ds_veg = xr.open_dataset(opts.vegetation_file) + ds_slw = xr.open_dataset(opts.soil_water_file) + + #----------------------------------------------------------------------------- + # check that dimension sizes are consistent + + nxy_veg = len(ds_veg.lat) * len(ds_veg.lon) + nxy_slw = len(ds_slw.lat) * len(ds_slw.lon) + n_a = len(ds_map.n_a) + n_b = len(ds_map.n_b) + + if nxy_veg!=nxy_slw or n_a!=nxy_veg or n_a!=nxy_slw: + err_msg = f'{clr.RED}Input file dimension sizes are inconsistent:{clr.END}' + err_msg += f' n_a: {n_a} nxy_veg: {nxy_veg} nxy_slw: {nxy_slw}' + raise ValueError(err_msg) + + ntime = 12 # expected length of time dimension + + if len(ds_veg.time)!=ntime: + err_msg = f'vegetation_file time record error:' + err_msg += f' expected length = {ntime}, actual length = {len(ds_veg.time)}' + raise ValueError(err_msg) + + if len(ds_slw.time)!=ntime: + err_msg = f'soil_water_file time record error:' + err_msg += f' expected length = {ntime}, actual length = {len(ds_slw.time)}' + raise ValueError(err_msg) + + if len(ds_veg.lat)!=len(ds_slw.lat): + err_msg = f'inconsistent latitude coordinate lengths:' + err_msg += f' ds_veg.lat = {len(ds_veg.lat)}' + err_msg += f' ds_slw.lat = {len(ds_slw.lat)}' + raise ValueError(err_msg) + + if len(ds_veg.lon)!=len(ds_slw.lon): + err_msg = f'inconsistent latitude coordinate lengths:' + err_msg += f' ds_veg.lon = {len(ds_veg.lon)}' + err_msg += f' ds_slw.lon = {len(ds_slw.lon)}' + raise ValueError(err_msg) + + #----------------------------------------------------------------------------- + # print some informative stuff + + print(f''' + File and parameter values: + {clr.GREEN}map_file {clr.END}: {opts.map_file} + {clr.GREEN}vegetation_file {clr.END}: {opts.vegetation_file} + {clr.GREEN}soil_water_file {clr.END}: {opts.soil_water_file} + {clr.GREEN}output_root {clr.END}: {opts.output_root} + {clr.GREEN}output_file {clr.END}: {output_file} + {clr.GREEN}dst_grid {clr.END}: {opts.dst_grid} + {clr.GREEN}map_file n_a {clr.END}: {n_a} + {clr.GREEN}map_file n_b {clr.END}: {n_b} + ''') + + #----------------------------------------------------------------------------- + # Load input data + # (also convert percentage to fraction [0,1]) + LANDMASK_in = ds_veg['LANDMASK' ].values + PCT_LAKE_in = ds_veg['PCT_LAKE' ].values/1e2 + PCT_URBAN_in = ds_veg['PCT_URBAN' ].values/1e2 + PCT_WETLAND_in = ds_veg['PCT_WETLAND'].values/1e2 + PCT_PFT_in = ds_veg['PCT_PFT' ].values/1e2 + SOILW_in = ds_slw['SOILW' ].values + #----------------------------------------------------------------------------- + # Adjust input data based on land fraction (for consistency with fortran tool) + nlat = len(ds_veg.lat) + nlon = len(ds_veg.lon) + adjust_inputs(ntime, nlat, nlon, LANDMASK_in, PCT_LAKE_in, PCT_URBAN_in, PCT_WETLAND_in, SOILW_in) + #----------------------------------------------------------------------------- + # Remap data to the atmosphere grid + + n_s = len(ds_map.n_s) + wgt = ds_map.S.values + row = ds_map.row.values-1 + col = ds_map.col.values-1 + + shp_1D = (n_b) + shp_2D = (ntime,n_b) + + SOILW = apply_map_2D( np.zeros(shp_2D), ntime, n_s, wgt, row, col, np.reshape(SOILW_in,(ntime,-1)) ) + + PCT_LAKE = apply_map_1D( np.zeros(shp_1D), n_s, wgt, row, col, np.reshape(PCT_LAKE_in ,-1) ) + PCT_URBAN = apply_map_1D( np.zeros(shp_1D), n_s, wgt, row, col, np.reshape(PCT_URBAN_in ,-1) ) + PCT_WETLAND = apply_map_1D( np.zeros(shp_1D), n_s, wgt, row, col, np.reshape(PCT_WETLAND_in,-1) ) + + npft = len(ds_veg.PCT_PFT[:,0,0]) + PCT_PFT = np.zeros((npft,n_b)) + for p in range(npft): + PCT_PFT[p,:] = apply_map_1D( np.zeros(shp_1D), n_s, wgt, row, col, np.reshape(PCT_PFT_in[p,:,:],-1) ) + + #----------------------------------------------------------------------------- + # Compute fields to output + + nclass_landuse = 11 + fraction_landuse = np.zeros([nclass_landuse,n_b]) + total_soilw = np.zeros([ntime,n_b]) + + for i in range(n_b): + # Calculate total_land as sum as all surface type fractions + total_land = PCT_LAKE[i] + PCT_URBAN[i] + PCT_WETLAND[i] + for p in range(npft): total_land += PCT_PFT[p,i] + # Adjust lake area fraction + if total_land<1.0: + PCT_LAKE[i] = PCT_LAKE[i] + (1.0 - total_land) + # Calculate total_soilw + fraction_soilw = total_land - ( PCT_LAKE[i] + PCT_WETLAND[i] ) + for t in range(ntime): + total_soilw[t,i] = total_soilw[t,i] + SOILW[t,i] * fraction_soilw + + # Calculate fraction_landuse - n-1 indexing is used to correspond to fortran indexing + fraction_landuse[ 1-1,i] = PCT_URBAN[i] + fraction_landuse[ 2-1,i] = np.sum( PCT_PFT[[n-1 for n in [16,17]] ,i], axis=0) + fraction_landuse[ 3-1,i] = np.sum( PCT_PFT[[n-1 for n in [13,14,15]] ,i], axis=0) + fraction_landuse[ 4-1,i] = np.sum( PCT_PFT[[n-1 for n in [5,6,7,8,9]],i], axis=0) + fraction_landuse[ 5-1,i] = np.sum( PCT_PFT[[n-1 for n in [2,3,4]] ,i], axis=0) + fraction_landuse[ 6-1,i] = PCT_WETLAND[i] + fraction_landuse[ 7-1,i] = PCT_LAKE[i] + fraction_landuse[ 8-1,i] = PCT_PFT[1-1,i] + fraction_landuse[11-1,i] = np.sum( PCT_PFT[[n-1 for n in [10,11,12]] ,i], axis=0) + + # Normalize fraction_landuse if it does not sum to 1 +/- tolerance + fraction_landuse_tolerance = 0.001 + fraction_landuse_sum = np.sum(fraction_landuse[:,i]) + fraction_landuse_error = fraction_landuse_sum - 1.0 + if np.absolute(fraction_landuse_error) > fraction_landuse_tolerance: + for c in range(nclass_landuse): + fraction_landuse[c,i] = fraction_landuse[c,i] / fraction_landuse_sum + + #----------------------------------------------------------------------------- + # Create output file and add fields + + ds_out = xr.Dataset() + ds_out['fraction_landuse'] = xr.DataArray(fraction_landuse,dims=['class','ncol']) + ds_out['soilw'] = xr.DataArray(SOILW ,dims=['month','ncol']) + ds_out['lon'] = ds_map.xc_b.rename({'n_b':'ncol'}) + ds_out['lat'] = ds_map.yc_b.rename({'n_b':'ncol'}) + ds_out.to_netcdf(path=output_file,mode='w') + + #----------------------------------------------------------------------------- + # convert to 64bit_data + + cmd = f'ncks -O --fl_fmt=64bit_data {output_file} {output_file} ' + sp.check_call(cmd,shell=True) + + #----------------------------------------------------------------------------- + # print status message + + print(f'\nsuccessfully created atmsrf file: {clr.MAGENTA}{output_file}{clr.END}') + print() + +#--------------------------------------------------------------------------------------------------- +@numba.njit() +def adjust_inputs(ntime, nlat, nlon, LANDMASK, PCT_LAKE, PCT_URBAN, PCT_WETLAND, SOILW): + for j in range(nlat): + for i in range(nlon): + if LANDMASK[j,i]==0: + PCT_LAKE[j,i] = 1.0 + PCT_URBAN[j,i] = 0.0 + PCT_WETLAND[j,i] = 0.0 + for t in range(ntime): SOILW[t,j,i] = 0.0 + +@numba.njit() +def apply_map_1D( data_out, n_s, wgt, row, col, data_in ): + for k in range(n_s): + data_out[row[k]] = data_out[row[k]] + data_in[col[k]] * wgt[k] + return data_out + +@numba.njit() +def apply_map_2D( data_out, ntime, n_s, wgt, row, col, data_in ): + for t in range(ntime): + for k in range(n_s): + data_out[t,row[k]] = data_out[t,row[k]] + data_in[t,col[k]] * wgt[k] + return data_out +#--------------------------------------------------------------------------------------------------- +if __name__ == '__main__': + main() +#--------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/components/eam/tools/mkatmsrffile/Makefile b/components/eam/tools/mkatmsrffile/old_mkatmsrffile/Makefile similarity index 100% rename from components/eam/tools/mkatmsrffile/Makefile rename to components/eam/tools/mkatmsrffile/old_mkatmsrffile/Makefile diff --git a/components/eam/tools/mkatmsrffile/README b/components/eam/tools/mkatmsrffile/old_mkatmsrffile/README similarity index 100% rename from components/eam/tools/mkatmsrffile/README rename to components/eam/tools/mkatmsrffile/old_mkatmsrffile/README diff --git a/components/eam/tools/mkatmsrffile/mkatmsrffile.F90 b/components/eam/tools/mkatmsrffile/old_mkatmsrffile/mkatmsrffile.F90 similarity index 100% rename from components/eam/tools/mkatmsrffile/mkatmsrffile.F90 rename to components/eam/tools/mkatmsrffile/old_mkatmsrffile/mkatmsrffile.F90 diff --git a/components/eam/tools/mkatmsrffile/nml_atmsrf b/components/eam/tools/mkatmsrffile/old_mkatmsrffile/nml_atmsrf similarity index 100% rename from components/eam/tools/mkatmsrffile/nml_atmsrf rename to components/eam/tools/mkatmsrffile/old_mkatmsrffile/nml_atmsrf diff --git a/components/eam/tools/mkatmsrffile/test_mkatmsrffile.sh b/components/eam/tools/mkatmsrffile/old_mkatmsrffile/test_mkatmsrffile.sh similarity index 100% rename from components/eam/tools/mkatmsrffile/test_mkatmsrffile.sh rename to components/eam/tools/mkatmsrffile/old_mkatmsrffile/test_mkatmsrffile.sh diff --git a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/generate-dry-deposition.md b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/generate-dry-deposition.md index f9dc7ff9eda4..cf777c418a2b 100644 --- a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/generate-dry-deposition.md +++ b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/generate-dry-deposition.md @@ -1,6 +1,40 @@ # Generate a Dry Deposition File -!!!WARNING - This page is still under construction +Atmospheric dry deposition of aerosols at the surface depends on certain surface properties, such as soil type. In some cases these calculations can be handled in the land model and passed to the atmosphere through the coupler. However, with modal areosols this method is not adequate and we must recalculate these fields in the atmosphere (see subroutine `interp_map` in `components/eam/src/chemistry/mozart/mo_drydep.F90`). + +For unstructured grids it was determined to create this offline interpolation tool rather than generalize the subroutine interp_map. + +Be sure to activate the E3SM unified environment when performing the steps below. + +## Map File Generation + +The destination atmosphere grid file should be on the "pg2" grid. These grid files are easily generated with three TempestRemap commands as follows: + +```shell +NE=30 +GenerateCSMesh --alt --res ${NE} --file ${GRID_ROOT}/ne${NE}.g +GenerateVolumetricMesh --in ${GRID_ROOT}/ne${NE}.g --out ${GRID_ROOT}/ne${NE}pg2.g --np 2 --uniform +ConvertMeshToSCRIP --in ${GRID_ROOT}/ne${NE}pg2.g --out ${GRID_ROOT}/ne${NE}pg2_scrip.nc +``` + +The map file used to generate atmsrf files can be created a few different ways. For a typical E3SM configuration we recommend using a conservative, monotone map. Here is an example command that can be used to generate one (as of NCO version 5.2.2) + +```shell +SRC_GRID=${DIN_LOC_ROOT}/../mapping/grids/1x1d.nc +DST_GRID=${GRID_ROOT}/ne${NE}pg2_scrip.nc +MAP_FILE=${MAP_ROOT}/map_1x1_to_ne${NE}pg2_traave.nc +ncremap -5 -a traave --src_grd=${SRC_GRID} --dst_grd=${DST_GRID} --map_file=${MAP_FILE} +``` + +For RRM grids the last two commands would be used on the exodus file produced by [SQuadGen](https://github.com/ClimateGlobalChange/squadgen) (See the [Adding Support for New Grids](https://docs.e3sm.org/user-guides/adding-grid-support-overview.md) tutorial for more information.). + +## Generating a New Dry Desposition File (atmsrf) + +```shell +VEGE_FILE=${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart/dvel/regrid_vegetation.nc +SOIL_FILE=${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart/dvel/clim_soilw.nc + +python ~/E3SM/mkatmsrffile.py --map_file=${MAP_FILE} --vegetation_file=${VEGE_FILE} --soil_water_file=${SOIL_FILE} --output_root=${atmsrf_root} --dst_grid=ne${NE}pg2 --date-stamp=20240613 +``` Back to step-by-step guide for [Adding Support for New Grids](../adding-grid-support-step-by-step-guide.md) From 51b8482f3ab1a7754471ef713e7bf0629a54cde3 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 24 Jun 2024 21:38:42 -0400 Subject: [PATCH 090/904] workaround to point to md snippets outside of the docs dir --- components/eam/tools/mkatmsrffile/test.md | 3 +++ docs/dev-guide/adding-grid-support/index.md | 1 + docs/dev-guide/some-tools/mkatmsrffile.md | 2 ++ mkdocs.yaml | 2 ++ 4 files changed, 8 insertions(+) create mode 100644 components/eam/tools/mkatmsrffile/test.md create mode 100644 docs/dev-guide/some-tools/mkatmsrffile.md diff --git a/components/eam/tools/mkatmsrffile/test.md b/components/eam/tools/mkatmsrffile/test.md new file mode 100644 index 000000000000..bc0ef89536e0 --- /dev/null +++ b/components/eam/tools/mkatmsrffile/test.md @@ -0,0 +1,3 @@ +# test + +Some text goes here. diff --git a/docs/dev-guide/adding-grid-support/index.md b/docs/dev-guide/adding-grid-support/index.md index f7c25043747a..4decc3976d33 100644 --- a/docs/dev-guide/adding-grid-support/index.md +++ b/docs/dev-guide/adding-grid-support/index.md @@ -6,3 +6,4 @@ - [Atmosphere Grid Overview ](../../EAM/tech-guide/atmosphere-grid-overview.md) - [Grid Description File Overview ](adding-grid-support-grid-types.md) - [Step-by-Step Guide to Support a New Grid ](adding-grid-support-step-by-step-guide.md) +- [Some test ](../some-tools/mkatmsrffile.md) diff --git a/docs/dev-guide/some-tools/mkatmsrffile.md b/docs/dev-guide/some-tools/mkatmsrffile.md new file mode 100644 index 000000000000..1c0fec1cf2bd --- /dev/null +++ b/docs/dev-guide/some-tools/mkatmsrffile.md @@ -0,0 +1,2 @@ + +--8<-- 'components/eam/tools/mkatmsrffile/test.md' diff --git a/mkdocs.yaml b/mkdocs.yaml index 5c57a22993c3..66a694115b9c 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -9,6 +9,7 @@ nav: - Development: - Introduction: 'dev-guide/index.md' - Adding Support for New Grids: 'dev-guide/adding-grid-support/index.md' + - mkatmsrffile: 'dev-guide/some-tools/mkatmsrffile.md' - Components: '*include ./components/*/mkdocs.yml' - Tools: '*include ./tools/*/mkdocs.yml' - More Information: @@ -57,6 +58,7 @@ markdown_extensions: - pymdownx.details - pymdownx.highlight - pymdownx.superfences + - pymdownx.snippets - pymdownx.tabbed: alternate_style: true - pymdownx.arithmatex: From 8603487991499be6db1ce149a496333084c28dd9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 10:45:31 -0600 Subject: [PATCH 091/904] EAMxx: fix how we pass IOType to scorpio The enum underlying values are not to be trusted. Do a manual conversion to PIO_IOTYPE instead. --- .../src/share/io/scream_scorpio_interface.cpp | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index f354b44efc1d..cb2b501f2b5c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -207,6 +207,21 @@ size_t dtype_size (const std::string& dtype) { return 0; } +int pio_iotype (IOType iotype) { + int iotype_int; + auto& s = ScorpioSession::instance(); + switch(iotype){ + case IOType::DefaultIOType: iotype_int = s.pio_type_default; break; + case IOType::NetCDF: iotype_int = static_cast(PIO_IOTYPE_NETCDF); break; + case IOType::PnetCDF: iotype_int = static_cast(PIO_IOTYPE_PNETCDF); break; + case IOType::Adios: iotype_int = static_cast(PIO_IOTYPE_ADIOS); break; + case IOType::Hdf5: iotype_int = static_cast(PIO_IOTYPE_HDF5); break; + default: + EKAT_ERROR_MSG ("Unrecognized/unsupported iotype.\n"); + } + return iotype_int; +} + // ====================== Local utilities ========================== // namespace impl { @@ -313,9 +328,9 @@ void init_subsystem(const ekat::Comm& comm, const int atm_id) s.pio_rearranger = PIO_REARR_SUBSET; s.pio_format = PIO_64BIT_DATA; #if PIO_USE_PNETCDF - s.pio_type_default = PIO_IOTYPE_PNETCDF; + s.pio_type_default = pio_iotype(IOType::PnetCDF); #elif PIO_USE_NETCDF - s.pio_type_default = PIO_IOTYPE_NETCDF; + s.pio_type_default = pio_iotype(IOType::NetCDF); #else #error "Standalone EAMxx requires either PNETCDF or NETCDF iotype to be available in Scorpio" #endif @@ -395,9 +410,7 @@ void register_file (const std::string& filename, if (f.mode == Unset) { // First time we ask for this file. Call PIO open routine(s) int err; - int iotype_int = iotype==IOType::DefaultIOType - ? s.pio_type_default - : static_cast(iotype); + int iotype_int = pio_iotype(iotype); if (mode & Read) { auto write = mode & Write ? PIO_WRITE : PIO_NOWRITE; err = PIOc_openfile(s.pio_sysid,&f.ncid,&iotype_int,filename.c_str(),write); From 7ecf8709c508467914314b5a85af737420e44dbd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 14:03:06 -0600 Subject: [PATCH 092/904] PyEAMxx: changed how user creates grids - Make user create a GridsManager rather than a single grid, to prepare for more complex use cases (e.g., dyn-phys) - Create a PySession singleton to store some data - Misc fixes to p3_standalone_py test --- components/eamxx/src/python/CMakeLists.txt | 9 ++- components/eamxx/src/python/pyatmproc.hpp | 41 ++++++----- components/eamxx/src/python/pyeamxx.cpp | 8 +++ components/eamxx/src/python/pyeamxx.hpp | 30 ++++++++ components/eamxx/src/python/pygrid.hpp | 72 +++++++------------ .../eamxx/tests/python/pyp3/p3_standalone_py | 22 +++--- 6 files changed, 106 insertions(+), 76 deletions(-) create mode 100644 components/eamxx/src/python/pyeamxx.hpp diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 8c7cf2aaba9c..43e2db73e533 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -2,4 +2,11 @@ find_package(pybind11 REQUIRED) find_package(mpi4py REQUIRED) pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC mpi4py scream_share scream_io diagnostics eamxx_physics) +target_link_libraries(pyeamxx PUBLIC + mpi4py + scream_share + scream_io + diagnostics + eamxx_physics + scream_test_support +) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index 4e7a88b21e67..f4daf87233ff 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -10,6 +10,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" +#include "pyeamxx.hpp" #include @@ -20,7 +21,6 @@ namespace scream { struct PyAtmProc { std::shared_ptr ap; - PyGrid phys_grid; std::map fields; util::TimeStamp t0; util::TimeStamp time; @@ -28,14 +28,10 @@ struct PyAtmProc { std::shared_ptr output_mgr; - PyAtmProc (const PyParamList& params, const PyGrid& phys_grid_in) - : phys_grid(phys_grid_in) + PyAtmProc (const PyParamList& params) { // Get the comm - const auto& comm = phys_grid.grid->get_comm(); - - // Create a grids manager on the fly - auto gm = std::make_shared(phys_grid.grid); + const auto& comm = PySession::get().comm; // Create the atm proc register_physics(); @@ -46,6 +42,7 @@ struct PyAtmProc { ap = apf.create(ap_type,comm,params.pl); // Create the fields + auto gm = PySession::get().gm; ap->set_grids(gm); create_fields(); } @@ -118,33 +115,41 @@ struct PyAtmProc { } } } - AtmosphereInput reader (ic_filename,phys_grid.grid,ic_fields,true); - reader.read_variables(); + if (ic_fields.size()>0) { + const auto& gn = ic_fields[0].get_header().get_identifier().get_grid_name(); + auto gm = PySession::get().gm; + auto grid = gm->get_grid(gn); + AtmosphereInput reader (ic_filename,grid,ic_fields,true); + reader.read_variables(); + } scorpio::release_file(ic_filename); return pybind11::cast(missing); } void setup_output (const std::string& yaml_file) { + auto comm = PySession::get().comm; // Load output params auto params = ekat::parse_yaml_file(yaml_file); // Stuff all fields in a field manager - auto fm = std::make_shared(phys_grid.grid); - fm->registration_begins(); - fm->registration_ends(); + auto gm = PySession::get().gm; + std::map> fms; + for (auto it : gm->get_repo()) { + fms[it.first] = std::make_shared(it.second); + fms[it.first]->registration_begins(); + fms[it.first]->registration_ends(); + } for (auto it : fields) { - fm->add_field(it.second.f); + const auto& gn = it.second.f.get_header().get_identifier().get_grid_name(); + fms.at(gn)->add_field(it.second.f); } - // Create a grids manager on the fly - auto gm = std::make_shared(phys_grid.grid); - // Make sure diagnostics are available, then create the output mgr register_diagnostics(); output_mgr = std::make_shared(); - output_mgr->setup(phys_grid.grid->get_comm(),params,fm,gm,t0,t0,false); + output_mgr->setup(comm,params,fms,gm,t0,t0,false); output_mgr->set_logger(ap->get_logger()); } @@ -161,7 +166,7 @@ struct PyAtmProc { inline void pybind_pyatmproc(pybind11::module& m) { pybind11::class_(m,"AtmProc") - .def(pybind11::init()) + .def(pybind11::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) .def("setup_output",&PyAtmProc::setup_output) diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index 5834ca10f74f..93a1e0ab2dfa 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -19,6 +19,10 @@ void initialize (MPI_Comm mpi_comm) { ekat::Comm comm(mpi_comm); initialize_scream_session(comm.am_i_root()); scorpio::init_subsystem(comm); + + auto& s = PySession::get(); + s.comm = comm; + s.inited = true; } void initialize () { @@ -28,6 +32,10 @@ void initialize (pybind11::object py_comm) { initialize(get_c_comm(py_comm)); } void finalize () { + auto& s = PySession::get(); + s.gm = nullptr; + s.inited = false; + scorpio::finalize_subsystem(); finalize_scream_session(); } diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/pyeamxx.hpp new file mode 100644 index 000000000000..59371844a763 --- /dev/null +++ b/components/eamxx/src/python/pyeamxx.hpp @@ -0,0 +1,30 @@ +#ifndef PYEAMXX_HPP +#define PYEAMXX_HPP + +#include "physics/register_physics.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "dynamics/register_dynamics.hpp" + +#include "share/grid/grids_manager.hpp" + +#include + +namespace scream { + +struct PySession { + static PySession& get () { + static PySession s; + return s; + } + + ekat::Comm comm; + std::shared_ptr gm; + bool inited = false; +private: + + PySession () = default; +}; + +} // namespace scream + +#endif // PYEAMXX_HPP diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp index 874b1aef17a0..102cf6037c62 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/pygrid.hpp @@ -1,10 +1,9 @@ #ifndef PYGRID_HPP #define PYGRID_HPP -#include "share/grid/grids_manager.hpp" -#include "share/grid/point_grid.hpp" -#include "pyfield.hpp" -#include "pyutils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "pyeamxx.hpp" #include @@ -12,53 +11,34 @@ namespace scream { -// Small grids manager class, to hold a pre-built grid -// We will use this to build a GM on the fly from a single grid -class SingleGridGM : public GridsManager +inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) { -public: - SingleGridGM (const std::shared_ptr& grid) - { - add_grid(grid); + EKAT_REQUIRE_MSG (PySession::get().inited, + "Error! You did not initialize pyeamxx, or you already finalized it!\n"); + auto& comm = PySession::get().comm; + ekat::ParameterList gm_params; + std::vector grids_names = {"Physics"}; + auto& pl = gm_params.sublist("Physics"); + pl.set("type",std::string("point_grid")); + pl.set("number_of_global_columns",ncols); + pl.set("number_of_vertical_levels",nlevs); + gm_params.set("grids_names",grids_names); + + if (latlon_nc_file!="") { + gm_params.set("ic_filename",latlon_nc_file); } - std::string name () const override { return "SingleGridGM"; } - - void build_grids () override {} - -protected: - remapper_ptr_type - do_create_remapper (const grid_ptr_type /* from_grid */, - const grid_ptr_type /* to_grid */) const override - { - EKAT_ERROR_MSG ("Error! do_create_remapper not implemented for SingleGridGM.\n"); - } -}; - -struct PyGrid { - std::shared_ptr grid; - - PyGrid () = default; - - PyGrid(const std::string& name, int ncols, int nlevs) - : PyGrid (name,ncols,nlevs,MPI_COMM_WORLD) - {} - - PyGrid(const std::string& name, int ncols, int nlevs, pybind11::object py_comm) - : PyGrid (name,ncols,nlevs,get_c_comm(py_comm)) - {} - - PyGrid(const std::string& name, int ncols, int nlevs, MPI_Comm mpi_comm) - { - ekat::Comm comm(mpi_comm); - grid = create_point_grid(name,ncols,nlevs,comm); - } -}; + PySession::get().gm = create_mesh_free_grids_manager (comm, gm_params); + PySession::get().gm->build_grids(); +} +inline void create_grids_manager (int ncols, int nlevs) +{ + create_grids_manager(ncols,nlevs,""); +} inline void pybind_pygrid (pybind11::module& m) { - pybind11::class_(m,"Grid") - .def(pybind11::init()) - .def(pybind11::init()); + m.def("create_grids_manager",pybind11::overload_cast(&create_grids_manager)); + m.def("create_grids_manager",pybind11::overload_cast(&create_grids_manager)); } } // namespace scream diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index 9d2b648060bf..def3c7f01b6d 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -6,7 +6,7 @@ import sys sys.path.append('@SCREAM_BASE_DIR@/scripts') # Add path to pyeamxx libs -sys.path.append('@CMAKE_BINARY_DIR@/src/python/libpyeamxx') +sys.path.append('@CMAKE_BINARY_DIR@/src/python') # Without these, and manual init/finalize, on my laptop I get # obscure MPI errors at exit. @@ -15,8 +15,7 @@ mpi4py.rc.initialize = False # do not initialize MPI automatically mpi4py.rc.finalize = False # do not finalize MPI automatically from mpi4py import MPI -import pyscream as ps -import pyp3 +import pyeamxx from pathlib import Path from utils import ensure_yaml @@ -34,17 +33,18 @@ def main (): dt = yaml_input['time_stepping']['time_step'] t0_str = yaml_input['time_stepping']['run_t0'] + ic_file = Path('@SCREAM_DATA_DIR@/init/screami_unit_tests_ne2np4L72_20220822.nc') + # Create the grid ncols = 218 nlevs = 72 - grid = ps.Grid("Physics",ncols,nlevs) - - ic_file = Path('@SCREAM_DATA_DIR@/init/screami_unit_tests_ne2np4L72_20220822.nc') + pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) - p3params = ps.ParameterList(yaml_input['atmosphere_processes']['p3']) - p3 = pyp3.P3(grid,p3params) + p3params = pyeamxx.ParameterList(yaml_input['atmosphere_processes']['p3'],'p3') + p3 = pyeamxx.AtmProc(p3params) missing = p3.read_ic(str(ic_file)) - print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") + if len(missing)>0: + print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") p3.initialize(t0_str) p3.setup_output("output_py.yaml") @@ -57,7 +57,7 @@ if __name__ == "__main__": # This level of indirection ensures all pybind structs are destroyed # before we finalize eamxx (and hence kokkos) MPI.Init() - ps.init() + pyeamxx.init() main () - ps.finalize() + pyeamxx.finalize() MPI.Finalize() From 1b69dd67b323c05cea912ca3daf5a10cd83f11bd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 14:03:27 -0600 Subject: [PATCH 093/904] PyEAMxx: register all procs/diags during pyeamxx init --- components/eamxx/src/python/pyatmproc.hpp | 6 +----- components/eamxx/src/python/pyeamxx.cpp | 5 +++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index f4daf87233ff..cbf4a1706f8d 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -1,8 +1,6 @@ #ifndef PYATMPROC_HPP #define PYATMPROC_HPP -#include "physics/register_physics.hpp" -#include "diagnostics/register_diagnostics.hpp" #include "share/atm_process/atmosphere_process.hpp" #include "share/io/scorpio_input.hpp" #include "share/io/scream_output_manager.hpp" @@ -34,7 +32,6 @@ struct PyAtmProc { const auto& comm = PySession::get().comm; // Create the atm proc - register_physics(); auto& apf = AtmosphereProcessFactory::instance(); const auto& ap_type = params.pl.isParameter("Type") ? params.pl.get("Type") @@ -146,8 +143,7 @@ struct PyAtmProc { fms.at(gn)->add_field(it.second.f); } - // Make sure diagnostics are available, then create the output mgr - register_diagnostics(); + // Create/setup the output mgr output_mgr = std::make_shared(); output_mgr->setup(comm,params,fms,gm,t0,t0,false); output_mgr->set_logger(ap->get_logger()); diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index 93a1e0ab2dfa..c4ec360d80a1 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -20,6 +20,11 @@ void initialize (MPI_Comm mpi_comm) { initialize_scream_session(comm.am_i_root()); scorpio::init_subsystem(comm); + // Register everything in the eamxx factories + register_physics(); + register_dynamics(); + register_diagnostics(); + auto& s = PySession::get(); s.comm = comm; s.inited = true; From b0f4ab07863c3bcab6a753e5c5eba349aa3159df Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 14:13:07 -0600 Subject: [PATCH 094/904] PyEAMxx: prune unused tests directory --- .../eamxx/src/python/tests/basic_test.py | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 components/eamxx/src/python/tests/basic_test.py diff --git a/components/eamxx/src/python/tests/basic_test.py b/components/eamxx/src/python/tests/basic_test.py deleted file mode 100644 index 26e43cad0d2b..000000000000 --- a/components/eamxx/src/python/tests/basic_test.py +++ /dev/null @@ -1,20 +0,0 @@ -# luca's verbatim example works from any dir now! -from pyeamxx import pyscream as ps -import numpy as np - -ps.init() - -v = np.zeros(shape=(10,2)) - -for i in range(0,10): - for j in range(0,2): - v[i,j] = i*2 + j + 1 - -print(f"v:{v}") - -f = ps.Field("T_mid",v) - -f.print() - -f.cleanup() -ps.finalize() From 52301555cba7c65f0c04d67011c1f73098105d55 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 25 Jun 2024 16:18:02 -0600 Subject: [PATCH 095/904] cosmetic fix for consistency --- .../adding-grid-support-step-by-step-guide/add-grid-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md index b872658db45e..f46a84afa767 100644 --- a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md +++ b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md @@ -1,3 +1,3 @@ # Add New Grid Configuration to E3SM -[Back to Adding Support for New Grids](../adding-grid-support-step-by-step-guide.md) +Back to step-by-step guide for [Adding Support for New Grids](../adding-grid-support-step-by-step-guide.md) From 7cd80e3d82d9748b9ef0a5df8c3760a4aa027976 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 25 Jun 2024 16:18:33 -0600 Subject: [PATCH 096/904] add link to unified page --- .../generate-dry-deposition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/generate-dry-deposition.md b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/generate-dry-deposition.md index cf777c418a2b..79e6fd017a40 100644 --- a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/generate-dry-deposition.md +++ b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/generate-dry-deposition.md @@ -4,7 +4,7 @@ Atmospheric dry deposition of aerosols at the surface depends on certain surface For unstructured grids it was determined to create this offline interpolation tool rather than generalize the subroutine interp_map. -Be sure to activate the E3SM unified environment when performing the steps below. +Be sure to activate the [E3SM unified environment](https://e3sm.org/resources/tools/other-tools/e3sm-unified-environment/) when performing the steps below. ## Map File Generation From 19ad5048f55f3a77d62757e6f54446937162466d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 16:35:20 -0600 Subject: [PATCH 097/904] PyEAMxx: add view semantic to PyParamList Allows to store reference_wrapper to existing parameter list --- components/eamxx/src/python/pyparamlist.hpp | 53 +++++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/pyparamlist.hpp index 00ee1c894f84..5d31a2c64274 100644 --- a/components/eamxx/src/python/pyparamlist.hpp +++ b/components/eamxx/src/python/pyparamlist.hpp @@ -7,19 +7,54 @@ #include #include +#include + namespace scream { struct PyParamList { ekat::ParameterList pl; + std::reference_wrapper pl_ref; + + PyParamList (ekat::ParameterList& src) + : pl_ref(src) + {} - PyParamList(const pybind11::dict& d) { + PyParamList(const pybind11::dict& d) + : PyParamList(d,"") + {} + + PyParamList(const pybind11::dict& d, const std::string& name) + : pl(name) + , pl_ref(pl) + { parse_dict(d,pl); } - PyParamList(const pybind11::dict& d, const std::string& name) { - pl.rename(name); - parse_dict(d,pl); + PyParamList sublist (const std::string& name) { + PyParamList spl(pl.sublist(name)); + return spl; + } + + int get_int (const std::string& name) const { + return pl_ref.get().get(name); + } + double get_dbl (const std::string& name) const { + return pl_ref.get().get(name); } + std::string get_str (const std::string& name) const { + return pl_ref.get().get(name); + } + + template + void set (const std::string& name, T val) { + pl_ref.get().set(name,val); + } + + void print () { + pl_ref.get().print(); + } + +private: void parse_dict(const pybind11::dict& d, ekat::ParameterList& p) { for (auto item : d) { @@ -84,7 +119,15 @@ inline void pybind_pyparamlist (pybind11::module& m) // Param list pybind11::class_(m,"ParameterList") .def(pybind11::init()) - .def(pybind11::init()); + .def(pybind11::init()) + .def("sublist",&PyParamList::sublist) + .def("print",&PyParamList::print) + .def("set",&PyParamList::set) + .def("set",&PyParamList::set) + .def("set",&PyParamList::set) + .def("get_int",&PyParamList::get_int) + .def("get_dbl",&PyParamList::get_dbl) + .def("get_str",&PyParamList::get_str); } } // namespace scream From 9c327e8c72d5e88f4fa64608954560c80613e064 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 16:36:09 -0600 Subject: [PATCH 098/904] PyEAMxx: changed AtmProc constructor to build straight from a dict --- components/eamxx/src/python/pyatmproc.hpp | 6 ++++-- components/eamxx/tests/python/pyp3/p3_standalone_py | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index cbf4a1706f8d..cc697c0e6c0b 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -26,8 +26,10 @@ struct PyAtmProc { std::shared_ptr output_mgr; - PyAtmProc (const PyParamList& params) + PyAtmProc (const pybind11::dict& d, const std::string& name) { + PyParamList params(d,name); + // Get the comm const auto& comm = PySession::get().comm; @@ -162,7 +164,7 @@ struct PyAtmProc { inline void pybind_pyatmproc(pybind11::module& m) { pybind11::class_(m,"AtmProc") - .def(pybind11::init()) + .def(pybind11::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) .def("setup_output",&PyAtmProc::setup_output) diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index def3c7f01b6d..c6a02d27010a 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -40,8 +40,7 @@ def main (): nlevs = 72 pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) - p3params = pyeamxx.ParameterList(yaml_input['atmosphere_processes']['p3'],'p3') - p3 = pyeamxx.AtmProc(p3params) + p3 = pyeamxx.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') missing = p3.read_ic(str(ic_file)) if len(missing)>0: print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") From cab6fa3d69acead6b50f824e9e7aa87bf54bf9dc Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 16:37:52 -0600 Subject: [PATCH 099/904] PyEAMxx: allow to set/get parameters into/from a py AtmProc --- components/eamxx/src/python/pyatmproc.hpp | 6 ++++++ components/eamxx/tests/python/pyp3/p3_standalone_py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index cc697c0e6c0b..98d7b9a525ed 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -49,6 +49,11 @@ struct PyAtmProc { // I don't think virtual is needed, but just in case virtual ~PyAtmProc () = default; + PyParamList get_params () const { + PyParamList pypl(ap->get_params()); + return pypl; + } + void create_fields () { // Create fields that are input/output to the atm proc for (const auto& req : ap->get_required_field_requests()) { @@ -167,6 +172,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def(pybind11::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) + .def("get_params",&PyAtmProc::get_params) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) .def("read_ic",&PyAtmProc::read_ic); diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index c6a02d27010a..0db6bcfcef61 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -41,6 +41,14 @@ def main (): pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) p3 = pyeamxx.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') + params = p3.get_params() + old = params.get_dbl('max_total_ni') + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") + params.set("max_total_ni",1000000.0) + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") + params.set("max_total_ni",old) + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") + missing = p3.read_ic(str(ic_file)) if len(missing)>0: print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") From 0de9b423a8066e0e4c7a03c6ee84f3be7d6abfc5 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 11:57:53 -0600 Subject: [PATCH 100/904] PyEAMxx: add support for vector parameters in PyParamList --- components/eamxx/src/python/pyparamlist.hpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/pyparamlist.hpp index 5d31a2c64274..3afc0a1cf1af 100644 --- a/components/eamxx/src/python/pyparamlist.hpp +++ b/components/eamxx/src/python/pyparamlist.hpp @@ -45,6 +45,16 @@ struct PyParamList { return pl_ref.get().get(name); } + std::vector get_int_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + std::vector get_dbl_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + std::vector get_str_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + template void set (const std::string& name, T val) { pl_ref.get().set(name,val); @@ -125,9 +135,15 @@ inline void pybind_pyparamlist (pybind11::module& m) .def("set",&PyParamList::set) .def("set",&PyParamList::set) .def("set",&PyParamList::set) + .def("set",&PyParamList::set>) + .def("set",&PyParamList::set>) + .def("set",&PyParamList::set>) .def("get_int",&PyParamList::get_int) .def("get_dbl",&PyParamList::get_dbl) - .def("get_str",&PyParamList::get_str); + .def("get_str",&PyParamList::get_str) + .def("get_int_vec",&PyParamList::get_int_vec) + .def("get_dbl_vec",&PyParamList::get_dbl_vec) + .def("get_str_vec",&PyParamList::get_str_vec); } } // namespace scream From 0cbf490490aa4357e719553c622f015d5468f10b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 11:59:21 -0600 Subject: [PATCH 101/904] PyEAMxx: add support for bool params in PyParamList --- components/eamxx/src/python/pyparamlist.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/pyparamlist.hpp index 3afc0a1cf1af..a631b6622e90 100644 --- a/components/eamxx/src/python/pyparamlist.hpp +++ b/components/eamxx/src/python/pyparamlist.hpp @@ -35,6 +35,9 @@ struct PyParamList { return spl; } + bool get_bool (const std::string& name) const { + return pl_ref.get().get(name); + } int get_int (const std::string& name) const { return pl_ref.get().get(name); } @@ -132,12 +135,14 @@ inline void pybind_pyparamlist (pybind11::module& m) .def(pybind11::init()) .def("sublist",&PyParamList::sublist) .def("print",&PyParamList::print) + .def("set",&PyParamList::set) .def("set",&PyParamList::set) .def("set",&PyParamList::set) .def("set",&PyParamList::set) .def("set",&PyParamList::set>) .def("set",&PyParamList::set>) .def("set",&PyParamList::set>) + .def("get_bool",&PyParamList::get_bool) .def("get_int",&PyParamList::get_int) .def("get_dbl",&PyParamList::get_dbl) .def("get_str",&PyParamList::get_str) From 9498dcc9a6ccb1784a3ea67d9b55e71061312010 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 14:10:41 -0600 Subject: [PATCH 102/904] EAMxx: remove pointless stuff from scorpio_input.hpp --- .../share/grid/mesh_free_grids_manager.cpp | 1 + .../share/grid/remap/vertical_remapper.cpp | 3 ++- .../eamxx/src/share/io/scorpio_input.hpp | 19 ++----------------- .../iop/intensive_observation_period.cpp | 1 + .../share/util/eamxx_time_interpolation.cpp | 1 + 5 files changed, 7 insertions(+), 18 deletions(-) diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index e7e11ef20e2a..c930013f8398 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -4,6 +4,7 @@ #include "share/grid/remap/do_nothing_remapper.hpp" #include "share/property_checks/field_nan_check.hpp" #include "share/property_checks/field_within_interval_check.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/io/scorpio_input.hpp" #include "physics/share/physics_constants.hpp" diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 5e714c48aa0d..9b524cec5e49 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -5,8 +5,9 @@ #include "share/field/field_tag.hpp" #include "share/field/field_identifier.hpp" #include "share/util/scream_universal_constants.hpp" +#include "share/io/scream_scorpio_interface.hpp" -#include "ekat/util/ekat_units.hpp" +#include #include #include #include diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 801d224be831..2c0a7e76c3eb 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -1,10 +1,8 @@ #ifndef SCREAM_SCORPIO_INPUT_HPP #define SCREAM_SCORPIO_INPUT_HPP -#include "share/io/scream_scorpio_interface.hpp" #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" -#include "share/grid/grids_manager.hpp" #include "ekat/ekat_parameter_list.hpp" #include "ekat/logging/ekat_logger.hpp" @@ -23,22 +21,11 @@ * ----- * Input Parameters * Filename: STRING - * Fields: ARRAY OF STRINGS + * Field Names: ARRAY OF STRINGS * ----- * The meaning of these parameters is the following: * - Filename: the name of the input file to be read. - * - Fields: list of names of fields to load from file. Should match the name in the file and the name in the field manager. - * Note: you can specify lists (such as the 'Fields' list above) with either of the two syntaxes - * Fields: [field_name1, field_name2, ... , field_name_N] - * Fields: - * - field_name_1 - * - field_name_2 - * ... - * - field_name_N - * Note: an alternative way of specifying Fields names is to have - * Grid: STRING - * Fields: - * $GRID: [field_name1,...,field_name_N] + * - Field Names: list of names of fields to load from file. Should match the name in the file and the name in the field manager. * * TODO: add a rename option if variable names differ in file and field manager. * @@ -55,8 +42,6 @@ class AtmosphereInput public: using fm_type = FieldManager; using grid_type = AbstractGrid; - using gm_type = GridsManager; - using remapper_type = AbstractRemapper; using KT = KokkosTypes; template diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index bbe3306a415e..657f7ad742a0 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -1,5 +1,6 @@ #include "share/grid/point_grid.hpp" #include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/iop/intensive_observation_period.hpp" #include "ekat/ekat_assert.hpp" diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index e0dba0cf8382..aa46328acac3 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -1,4 +1,5 @@ #include "share/util/eamxx_time_interpolation.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/io/scream_io_utils.hpp" namespace scream{ From 07955c6851987ee59620598316e819b4bd466462 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 14:11:38 -0600 Subject: [PATCH 103/904] EAMxx: add input file reader for a single-column model --- components/eamxx/src/share/io/CMakeLists.txt | 1 + .../eamxx/src/share/io/scorpio_scm_input.cpp | 246 ++++++++++++++++++ .../eamxx/src/share/io/scorpio_scm_input.hpp | 68 +++++ 3 files changed, 315 insertions(+) create mode 100644 components/eamxx/src/share/io/scorpio_scm_input.cpp create mode 100644 components/eamxx/src/share/io/scorpio_scm_input.hpp diff --git a/components/eamxx/src/share/io/CMakeLists.txt b/components/eamxx/src/share/io/CMakeLists.txt index afb0ef888db7..6c24d3bc51da 100644 --- a/components/eamxx/src/share/io/CMakeLists.txt +++ b/components/eamxx/src/share/io/CMakeLists.txt @@ -54,6 +54,7 @@ endif() add_library(scream_io scream_output_manager.cpp scorpio_input.cpp + scorpio_scm_input.cpp scorpio_output.cpp scream_io_utils.cpp ) diff --git a/components/eamxx/src/share/io/scorpio_scm_input.cpp b/components/eamxx/src/share/io/scorpio_scm_input.cpp new file mode 100644 index 000000000000..c5a4b9149ae9 --- /dev/null +++ b/components/eamxx/src/share/io/scorpio_scm_input.cpp @@ -0,0 +1,246 @@ +#include "share/io/scorpio_scm_input.hpp" +#include "share/io/scorpio_input.hpp" + +#include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/point_grid.hpp" + +#include + +#include + +namespace scream +{ + +SCMInput:: +SCMInput (const std::string& filename, + const double lat, const double lon, + const std::vector& fields, + const ekat::Comm& comm) + : m_comm(comm) + , m_filename (filename) +{ + auto iotype = scorpio::str2iotype("default"); + scorpio::register_file(m_filename,scorpio::Read,iotype); + + // Some input files have the "time" dimension as non-unlimited. This messes up our + // scorpio interface. To avoid trouble, if a dim called 'time' is present we + // treat it as unlimited, even though it isn't. + if (scorpio::has_dim(m_filename,"time") and not scorpio::is_dim_unlimited(m_filename,"time")) { + scorpio::pretend_dim_is_unlimited(m_filename,"time"); + } + + create_io_grid (); + auto ncols = m_io_grid->get_num_local_dofs(); + + create_closest_col_info (lat, lon); + + // Init fields specs + for (const auto& f : fields) { + const auto& fh = f.get_header(); + const auto& fid = fh.get_identifier(); + const auto& fl = fid.get_layout(); + + EKAT_REQUIRE_MSG (fl.tags()[0]==FieldTag::Column, + "Error! SCMInput only works for physics-type layouts.\n" + " - field name: " + f.name() + "\n" + " - field layout: " + fl.to_string() + "\n"); + + m_fields.push_back(f.subfield(0,0)); + FieldIdentifier fid_io(f.name(),fl.clone().reset_dim(0,ncols),fid.get_units(),m_io_grid->name()); + auto& f_io = m_io_fields.emplace_back(fid_io); + f_io.allocate_view(); + } + + // Init scorpio internal structures + init_scorpio_structures (); +} + +SCMInput:: +~SCMInput () +{ + scorpio::release_file(m_filename); +} + +void SCMInput::create_io_grid () +{ + EKAT_REQUIRE_MSG (scorpio::has_dim(m_filename,"ncol"), + "Error! Dimension 'ncol' not found in input file.\n" + " - filename: " + m_filename + "\n"); + const int ncols = scorpio::get_dimlen(m_filename,"ncol"); + const int nlevs = scorpio::has_dim(m_filename,"lev") ? scorpio::get_dimlen(m_filename,"lev") : 1; + + m_io_grid = create_point_grid("scm_io_grid",ncols,nlevs,m_comm); +} + +void SCMInput::create_closest_col_info (double target_lat, double target_lon) +{ + // Read lat/lon fields + const auto ncols = m_io_grid->get_num_local_dofs(); + + auto nondim = ekat::units::Units::nondimensional(); + auto lat = m_io_grid->create_geometry_data("lat",m_io_grid->get_2d_scalar_layout(),nondim); + auto lon = m_io_grid->create_geometry_data("lon",m_io_grid->get_2d_scalar_layout(),nondim); + + // Read from file + AtmosphereInput file_reader(m_filename, m_io_grid, {lat,lon}); + file_reader.read_variables(); + file_reader.finalize(); + + // Find column index of closest lat/lon to target_lat/lon params + auto lat_d = lat.get_view(); + auto lon_d = lon.get_view(); + using minloc_t = Kokkos::MinLoc; + using minloc_value_t = typename minloc_t::value_type; + minloc_value_t minloc; + Kokkos::parallel_reduce(ncols, KOKKOS_LAMBDA (int icol, minloc_value_t& result) { + auto dist = std::abs(lat_d(icol)-target_lat)+std::abs(lon_d(icol)-target_lon); + if(dist min_dist_and_rank = {minloc.val, my_rank}; + m_comm.all_reduce>(&min_dist_and_rank, 1, MPI_MINLOC); + + // Set local col idx to -1 for mpi ranks not containing minimum lat/lon distance + m_closest_col_info.mpi_rank = min_dist_and_rank.second; + m_closest_col_info.col_lid = my_rank==min_dist_and_rank.second ? minloc.loc : -1; +} + +void SCMInput::read_variables (const int time_index) +{ + auto func_start = std::chrono::steady_clock::now(); + auto fname = [](const Field& f) { return f.name(); }; + if (m_atm_logger) { + m_atm_logger->info("[EAMxx::scorpio_scm_input] Reading variables from file"); + m_atm_logger->info(" file name: " + m_filename); + m_atm_logger->info(" var names: " + ekat::join(m_fields,fname,", ")); + if (time_index!=-1) { + m_atm_logger->info(" time idx : " + std::to_string(time_index)); + } + } + + // MPI rank with closest column index store column data + const int n = m_fields.size(); + for (int i=0; i(),time_index); + + auto& f = m_fields[i]; + if (m_comm.rank() == m_closest_col_info.mpi_rank) { + // This rank read the column we need, so copy it in the output field + f.deep_copy(f_io.subfield(0,m_closest_col_info.col_lid)); + } + + // Broadcast column data to all other ranks + const auto col_size = f.get_header().get_identifier().get_layout().size(); + m_comm.broadcast(f.get_internal_view_data(), col_size, m_closest_col_info.mpi_rank); + + // Sync fields to device + f.sync_to_dev(); + } + + auto func_finish = std::chrono::steady_clock::now(); + if (m_atm_logger) { + auto duration = std::chrono::duration_cast(func_finish - func_start)/1000.0; + m_atm_logger->info(" Done! Elapsed time: " + std::to_string(duration.count()) +" seconds"); + } +} + +void SCMInput::init_scorpio_structures() +{ + using namespace ShortFieldTagsNames; + + // Check variables are in the input file + for (const auto& f : m_io_fields) { + const auto& layout = f.get_header().get_identifier().get_layout(); + auto dim_names = layout.names(); + for (int i=0; iget_partitioned_dim_global_size() : layout.dim(i); + EKAT_REQUIRE_MSG (eamxx_len==file_len, + "Error! Dimension mismatch for input file variable.\n" + " - filename : " + m_filename + "\n" + " - varname : " + f.name() + "\n" + " - var dims : " + ekat::join(dim_names,",") + "\n" + " - dim name : " + dim_names[i] + "\n" + " - expected extent : " + std::to_string(eamxx_len) + "\n" + " - extent from file: " + std::to_string(file_len) + "\n"); + } + + // Ensure that we can read the var using Real data type + scorpio::change_var_dtype (m_filename,f.name(),"real"); + } + + // Set decompositions for the variables + set_decompositions(); +} + +/* ---------------------------------------------------------- */ +void SCMInput::set_decompositions() +{ + using namespace ShortFieldTagsNames; + + // First, check if any of the vars is indeed partitioned + const auto decomp_tag = m_io_grid->get_partitioned_dim_tag(); + + bool has_decomposed_layouts = false; + for (const auto& f : m_io_fields) { + const auto& layout = f.get_header().get_identifier().get_layout(); + if (layout.has_tag(decomp_tag)) { + has_decomposed_layouts = true; + break; + } + } + if (not has_decomposed_layouts) { + // If none of the input vars are decomposed on this grid, + // then there's nothing to do here + return; + } + + // Set the decomposition for the partitioned dimension + const int local_dim = m_io_grid->get_partitioned_dim_local_size(); + std::string decomp_dim = m_io_grid->has_special_tag_name(decomp_tag) + ? m_io_grid->get_special_tag_name(decomp_tag) + : e2str(decomp_tag); + + auto gids_f = m_io_grid->get_partitioned_dim_gids(); + auto gids_h = gids_f.get_view(); + auto min_gid = m_io_grid->get_global_min_partitioned_dim_gid(); + std::vector offsets(local_dim); + for (int idof=0; idof + +namespace scream +{ + +// Similar to AtmosphereInput, but reads in a single column from +// a file with N columns. A few assumptions: +// - lat and lon variables are present in the file +// - fields have layout +class SCMInput +{ +public: + // --- Constructor(s) & Destructor --- // + SCMInput (const std::string& filename, + const double lat, const double lon, + const std::vector& fields, + const ekat::Comm& comm); + + ~SCMInput (); + + // Due to resource acquisition (in scorpio), avoid copies + SCMInput (const SCMInput&) = delete; + SCMInput& operator= (const SCMInput&) = delete; + + // Read fields that were required via parameter list. + void read_variables (const int time_index = -1); + + // Option to add a logger + void set_logger(const std::shared_ptr& atm_logger) { + m_atm_logger = atm_logger; + } +protected: + + struct ClosestColInfo { + // MPI rank which owns the columns whose lat/lon pair is the closest to target lat/lon + int mpi_rank; + // Local column index of on rank=mpi_rank (-1 on all other ranks) + int col_lid; + }; + + void create_io_grid (); + void create_closest_col_info (double target_lat, double target_lon); + void init_scorpio_structures (); + void set_decompositions(); + + // Internal variables + ekat::Comm m_comm; + + std::shared_ptr m_io_grid; + + std::string m_filename; + std::vector m_fields; + std::vector m_io_fields; + + ClosestColInfo m_closest_col_info; + + // The logger to be used throughout the ATM to log message + std::shared_ptr m_atm_logger; +}; + +} //namespace scream + +#endif // SCREAM_SCORPIO_SCM_INPUT_HPP From 49ad7b36a397de7b63d831f6751abc4bec32e8cc Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 15:41:55 -0600 Subject: [PATCH 104/904] EAMxx: add unit test for SCMInput --- .../eamxx/src/share/io/tests/CMakeLists.txt | 6 + .../src/share/io/tests/io_scm_reader.cpp | 130 ++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 components/eamxx/src/share/io/tests/io_scm_reader.cpp diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index 16ce2efd4376..8c819f7896cc 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -81,3 +81,9 @@ CreateUnitTest(io_remap_test "io_remap_test.cpp" LIBS scream_io diagnostics LABELS io remap MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) + +## Test single-column reader +CreateUnitTest(io_scm_reader "io_scm_reader.cpp" + LIBS scream_io LABELS io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} +) diff --git a/components/eamxx/src/share/io/tests/io_scm_reader.cpp b/components/eamxx/src/share/io/tests/io_scm_reader.cpp new file mode 100644 index 000000000000..403d557582a9 --- /dev/null +++ b/components/eamxx/src/share/io/tests/io_scm_reader.cpp @@ -0,0 +1,130 @@ +#include + +#include "share/io/scorpio_scm_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/field/field.hpp" +#include "share/field/field_utils.hpp" + +#include "share/util/scream_setup_random_test.hpp" + +namespace scream { + +// Returns fields after initialization +void write (const int seed, const int ncols, const int nlevs, const ekat::Comm& comm) +{ + using ekat::units::Units; + using RPDF = std::uniform_real_distribution; + using Engine = std::mt19937_64; + + Engine engine(seed); + RPDF lat_pdf(-90.0,90.0); + RPDF lon_pdf(-180.0,180.0); + + // Create grid + auto grid = create_point_grid("test",ncols,nlevs,comm); + + // Create lat/lon grid + Units deg (Units::nondimensional(),"deg"); + auto lat = grid->create_geometry_data("lat",grid->get_2d_scalar_layout(),deg); + auto lon = grid->create_geometry_data("lon",grid->get_2d_scalar_layout(),deg); + randomize(lat,engine,lat_pdf); + randomize(lon,engine,lon_pdf); + + // Create variable data + FieldIdentifier fid("var",grid->get_3d_scalar_layout(true),Units::nondimensional(),""); + Field var(fid); + var.allocate_view(); + randomize(var,engine,RPDF(-1.0,1.0)); + + // Create file + auto filename = "io_scm_np" + std::to_string(comm.size()) + ".nc"; + scorpio::register_file(filename,scorpio::Write,scorpio::DefaultIOType); + + scorpio::define_dim(filename,"ncol",ncols); + scorpio::define_dim(filename,"lev",nlevs); + + scorpio::define_var(filename,"lat",{"ncol"},"real"); + scorpio::define_var(filename,"lon",{"ncol"},"real"); + + scorpio::define_var(filename,"var",{"ncol","lev"},"real"); + + auto my_col_gids = grid->get_partitioned_dim_gids().get_view(); + std::vector my_offsets(my_col_gids.size()); + for (size_t i=0; iget_global_min_partitioned_dim_gid (); + } + scorpio::set_dim_decomp(filename,"ncol",my_offsets); + scorpio::enddef(filename); + + // Write to file + scorpio::write_var(filename,"lat",lat.get_internal_view_data()); + scorpio::write_var(filename,"lon",lon.get_internal_view_data()); + scorpio::write_var(filename,"var",var.get_internal_view_data()); + + scorpio::release_file(filename); +} + +void read (const int seed, const int nlevs, const ekat::Comm& comm) +{ + using ekat::units::Units; + using IPDF = std::uniform_int_distribution; + using Engine = std::mt19937_64; + + Engine engine(seed); + + auto grid = create_point_grid("scm_grid",1,nlevs,comm); + + // Pick a col index, and find its lat/lon from file + auto filename = "io_scm_np" + std::to_string(comm.size()) + ".nc"; + scorpio::register_file(filename,scorpio::Read,scorpio::DefaultIOType); + int ncols = scorpio::get_dimlen(filename,"ncol"); + + std::vector lat(ncols), lon(ncols), var(ncols*nlevs); + scorpio::read_var(filename,"lat",lat.data()); + scorpio::read_var(filename,"lon",lon.data()); + scorpio::read_var(filename,"var",var.data()); + + scorpio::release_file(filename); + + auto tgt_col = IPDF(0,ncols-1)(engine); + + auto tgt_lat = lat[tgt_col]; + auto tgt_lon = lon[tgt_col]; + + // Create field to read + FieldIdentifier fid("var",grid->get_3d_scalar_layout(true),Units::nondimensional(),""); + Field var_f(fid); + var_f.allocate_view(); + + // Read field + SCMInput reader(filename,tgt_lat,tgt_lon,{var_f},comm); + reader.read_variables(); + + // Check + auto var_h = var_f.get_view(); + for (int ilev=0; ilev; + using Engine = std::mt19937_64; + + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::init_subsystem(comm); + + auto seed = get_random_test_seed(&comm); + Engine engine(seed); + const int nlevs = IPDF(5,50)(engine); + const int ncols = IPDF(20,50)(engine); + + write(seed,ncols,nlevs,comm); + read(seed,nlevs,comm); + + scorpio::finalize_subsystem(); +} + +} // anonymous namespace From d39248d423fb02ab7e678b3aa464f8dbc8879b51 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 27 Jun 2024 09:52:56 -0600 Subject: [PATCH 105/904] EAMxx: add missing include in nudging interface --- .../src/physics/nudging/eamxx_nudging_process_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 3af33eaea164..8ebfc9c14c01 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -4,6 +4,7 @@ #include "share/grid/remap/refining_remapper_p2p.hpp" #include "share/grid/remap/do_nothing_remapper.hpp" #include "share/util/scream_utils.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include #include From 5ffbba0f7817813a37f797a95348d8c7e223e74b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 27 Jun 2024 14:35:15 -0600 Subject: [PATCH 106/904] EAMxx: fix scm reader unit test --- .../eamxx/src/share/io/tests/io_scm_reader.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/io/tests/io_scm_reader.cpp b/components/eamxx/src/share/io/tests/io_scm_reader.cpp index 403d557582a9..111f82947504 100644 --- a/components/eamxx/src/share/io/tests/io_scm_reader.cpp +++ b/components/eamxx/src/share/io/tests/io_scm_reader.cpp @@ -74,7 +74,9 @@ void read (const int seed, const int nlevs, const ekat::Comm& comm) Engine engine(seed); - auto grid = create_point_grid("scm_grid",1,nlevs,comm); + auto grid = std::make_shared("scm_grid",1,nlevs,comm); + auto dofs_gids = grid->get_dofs_gids(); + dofs_gids.deep_copy(0); // Pick a col index, and find its lat/lon from file auto filename = "io_scm_np" + std::to_string(comm.size()) + ".nc"; @@ -89,6 +91,7 @@ void read (const int seed, const int nlevs, const ekat::Comm& comm) scorpio::release_file(filename); auto tgt_col = IPDF(0,ncols-1)(engine); + comm.broadcast(&tgt_col,1,comm.root_rank()); auto tgt_lat = lat[tgt_col]; auto tgt_lon = lon[tgt_col]; @@ -118,8 +121,11 @@ TEST_CASE ("scm_io") { auto seed = get_random_test_seed(&comm); Engine engine(seed); - const int nlevs = IPDF(5,50)(engine); - const int ncols = IPDF(20,50)(engine); + int nlevs = IPDF(5,8)(engine); + int ncols = IPDF(4,5)(engine); + + comm.broadcast(&ncols,1,comm.root_rank()); + comm.broadcast(&nlevs,1,comm.root_rank()); write(seed,ncols,nlevs,comm); read(seed,nlevs,comm); From 241899b1d07b0677f85f7b19254a623f3b817003 Mon Sep 17 00:00:00 2001 From: noel Date: Thu, 27 Jun 2024 17:46:54 -0700 Subject: [PATCH 107/904] revert back to hex hash prints --- .../src/share/atm_process/atmosphere_process_hash.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 6b2f2fd080bb..37cb251d7796 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,9 +129,9 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lld (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), - i, (long long int)gaccum[i], label.c_str()); + i, gaccum[i], label.c_str()); } void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) const { @@ -140,8 +140,8 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lld (%s)\n", - timestamp().get_num_steps(), (long long int) gaccum, label.c_str()); + fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + timestamp().get_num_steps(), gaccum, label.c_str()); } } // namespace scream From c195b3c01f60922f5ff36fab65ec02560807a37f Mon Sep 17 00:00:00 2001 From: noel Date: Fri, 28 Jun 2024 11:51:26 -0700 Subject: [PATCH 108/904] use llx instead of lx for print format --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 37cb251d7796..9fb5726084f4 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } From f9a89377f327ba15cf20c637f237bd26f9cafe3a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 1 Jul 2024 09:52:35 -0600 Subject: [PATCH 109/904] EAMxx: fix CUDA issue in scm reader --- components/eamxx/src/share/io/scorpio_scm_input.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scorpio_scm_input.hpp b/components/eamxx/src/share/io/scorpio_scm_input.hpp index 55b4c637cb1b..44b54043dcad 100644 --- a/components/eamxx/src/share/io/scorpio_scm_input.hpp +++ b/components/eamxx/src/share/io/scorpio_scm_input.hpp @@ -34,6 +34,12 @@ class SCMInput void set_logger(const std::shared_ptr& atm_logger) { m_atm_logger = atm_logger; } + +#ifndef KOKKOS_ENABLE_CUDA + // Cuda requires methods enclosing __device__ lambda's to be public +protected: +#endif + void create_closest_col_info (double target_lat, double target_lon); protected: struct ClosestColInfo { @@ -44,7 +50,6 @@ class SCMInput }; void create_io_grid (); - void create_closest_col_info (double target_lat, double target_lon); void init_scorpio_structures (); void set_decompositions(); From 6160a60eef2c2fee930e2df751ed39b7591b158e Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:06:33 -0500 Subject: [PATCH 110/904] Add shr_const to analysis mode --- components/mpas-ocean/src/analysis_members/Makefile | 5 ++++- components/mpas-ocean/src/analysis_members/shr_const_mod.F | 1 + components/mpas-ocean/src/analysis_members/shr_kind_mod.F | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 components/mpas-ocean/src/analysis_members/shr_const_mod.F create mode 100644 components/mpas-ocean/src/analysis_members/shr_kind_mod.F diff --git a/components/mpas-ocean/src/analysis_members/Makefile b/components/mpas-ocean/src/analysis_members/Makefile index c07186ad9adc..9afb438b77e3 100644 --- a/components/mpas-ocean/src/analysis_members/Makefile +++ b/components/mpas-ocean/src/analysis_members/Makefile @@ -2,6 +2,9 @@ OBJS = mpas_ocn_analysis_driver.o +UTILS = shr_kind_mod.o \ + shr_const_mod.o + MEMBERS = mpas_ocn_global_stats.o \ mpas_ocn_okubo_weiss.o \ mpas_ocn_layer_volume_weighted_averages.o \ @@ -35,7 +38,7 @@ MEMBERS = mpas_ocn_global_stats.o \ all: $(OBJS) -mpas_ocn_analysis_driver.o: $(MEMBERS) +mpas_ocn_analysis_driver.o: $(UTILS) $(MEMBERS) mpas_ocn_okubo_weiss.o: mpas_ocn_okubo_weiss_eigenvalues.o diff --git a/components/mpas-ocean/src/analysis_members/shr_const_mod.F b/components/mpas-ocean/src/analysis_members/shr_const_mod.F new file mode 100644 index 000000000000..41538d8af2f1 --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_const_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_const_mod.F90 diff --git a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F new file mode 100644 index 000000000000..44b22288cce1 --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_kind_mod.F90 From 2da96ef7e2350afec441612a4e1a3b7e97594420 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:15:21 -0500 Subject: [PATCH 111/904] Use pi for debug tracer --- .../src/mode_forward/mpas_ocn_time_integration_si.F | 4 ++-- .../src/mode_forward/mpas_ocn_time_integration_split.F | 4 ++-- .../src/mode_forward/mpas_ocn_time_integration_split_ab2.F | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F index 9529fbdc6ea1..8e542d5ac61b 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F @@ -2587,7 +2587,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2604,7 +2604,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 653d88198131..2b35a4efae4f 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -2225,7 +2225,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2242,7 +2242,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F index c6040aeba1d6..0693d00a72fd 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F @@ -2405,7 +2405,7 @@ subroutine ocn_time_integrator_split_ab2(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2422,7 +2422,7 @@ subroutine ocn_time_integrator_split_ab2(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & From 561ce771eed0e886437236d1682e47170fd32ce0 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:15:30 -0500 Subject: [PATCH 112/904] Use pi in cvmix --- components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F index fcede65e420c..87fb36f2145f 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F @@ -345,8 +345,8 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim ! specify geometry/location cvmix_variables % SeaSurfaceHeight = ssh(iCell) cvmix_variables % Coriolis = fCell(iCell) - cvmix_variables % lat = latCell(iCell) * 180.0_RKIND / 3.14_RKIND - cvmix_variables % lon = lonCell(iCell) * 180.0_RKIND / 3.14_RKIND + cvmix_variables % lat = latCell(iCell) * 180.0_RKIND / pi + cvmix_variables % lon = lonCell(iCell) * 180.0_RKIND / pi ! fill vertical position of column ! CVMix assume top of ocean is at z=0, so building all z-coordinate data based on layerThickness From 6223ff4f520f7917120e313adf74a49cfae7cd56 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:15:45 -0500 Subject: [PATCH 113/904] Use pi in gm --- components/mpas-ocean/src/shared/mpas_ocn_gm.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_gm.F b/components/mpas-ocean/src/shared/mpas_ocn_gm.F index 6f4442103fa7..3db04c4d3e8f 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_gm.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_gm.F @@ -484,7 +484,7 @@ subroutine ocn_GM_compute_Bolus_velocity(statePool, & ! Compute the speed of the first baroclinic mode from the Brunt-Vaisala frequency. cGMphaseSpeed(iEdge) = max(c_min, & - sum_hN/(config_GM_spatially_variable_baroclinic_mode*3.141592_RKIND)) + sum_hN/(config_GM_spatially_variable_baroclinic_mode*pi)) end do !$omp end do @@ -615,7 +615,7 @@ subroutine ocn_GM_compute_Bolus_velocity(statePool, & ! for the first-mode (m=1) baroclinic gravity wave speed in m/s ! c_m = 1/pi integral(N dz ) - c_mode1 = max(1.0E-5_RKIND, sum_hN/3.141592_RKIND) + c_mode1 = max(1.0E-5_RKIND, sum_hN/pi) ! See Hallberg (2013) https://doi.org/10.1016/j.ocemod.2013.08.007 ! just after eqn 4 the defines the deformation From 1b92637325d779f78fceda7fb25fd3b29e596e6c Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:16:07 -0500 Subject: [PATCH 114/904] Set pi in mpas_ocn_constants --- components/mpas-ocean/src/shared/mpas_ocn_constants.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_constants.F b/components/mpas-ocean/src/shared/mpas_ocn_constants.F index 35ada1306f8e..6c7023748205 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_constants.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_constants.F @@ -117,6 +117,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ !----------------------------------------------------------------------- T0_Kelvin = 273.16_RKIND ! zero point for Celsius + pi = 3.141592_RKIND ! pi rho_air = 1.2_RKIND ! ambient air density (kg/m^3) rho_sw = config_density0 ! density of salt water (kg/m^3) rho_fw = 1.0e3_RKIND ! avg. water density (kg/m^3) @@ -159,6 +160,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ #ifdef MPAS_ESM_SHR_CONST T0_Kelvin = SHR_CONST_TKFRZ ! zero point for Celsius + pi = SHR_CONST_PI ! zero point for Celsius cp_sw = SHR_CONST_CPSW ! erg/g/K cp_air = SHR_CONST_CPDAIR ! J/kg/K rho_air = SHR_CONST_RHODAIR ! kg/m^3 From 5e7322ed4f39d51429d744117be258c25f6c4006 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:16:30 -0500 Subject: [PATCH 115/904] Use pi in conservation earth area --- .../src/analysis_members/mpas_ocn_conservation_check.F | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index bdd07e799b5d..fdfc219789ea 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -19,6 +19,9 @@ module ocn_conservation_check + use shr_kind_mod, only: SHR_KIND_R8 + use shr_const_mod + use mpas_derived_types use mpas_pool_routines use mpas_dmpar @@ -157,10 +160,9 @@ subroutine ocn_init_conservation_check(domain, err)!{{{ !----------------------------------------------------------------- ! taking PI from SHR_CONST_PI in share/util/shr_const_mod.F90 to match coupler - !real (kind=RKIND), parameter :: piE3SM = 3.14159265358979323846_RKIND ! pi - real (kind=RKIND), parameter :: piE3SM = 3.141592653589793_RKIND + real (kind=RKIND), parameter :: piE3SM = SHR_CONST_PI ! taking earth radius from SHR_CONST_REARTH in share/util/shr_const_mod.F90 to match coupler - real (kind=RKIND), parameter :: earthRadiusE3SM = 6.371229e6_RKIND ! radius of earth, m + real (kind=RKIND), parameter :: earthRadiusE3SM = SHR_CONST_REARTH ! radius of earth, m err = 0 From ba7b48e8b2ded2735203aad3ee899c6a7aecb963 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:40:17 -0500 Subject: [PATCH 116/904] Set pi in mpas_ocn_constants --- components/mpas-ocean/src/shared/mpas_ocn_constants.F | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_constants.F b/components/mpas-ocean/src/shared/mpas_ocn_constants.F index 6c7023748205..08b70dce3777 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_constants.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_constants.F @@ -65,6 +65,7 @@ module ocn_constants real (kind=RKIND), public :: & T0_Kelvin ,&! zero point for Celsius + pi ,&! pi mpercm ,&! meters per m cmperm ,&! m per meter days_per_second ,&! days per second From 58a8d60ce428d6e493b596456f677ec2f694bc0e Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 21 Jun 2024 16:47:52 -0500 Subject: [PATCH 117/904] Add shr_const to analysis mode --- components/mpas-ocean/src/analysis_members/shr_const_mod.F | 2 +- components/mpas-ocean/src/analysis_members/shr_kind_mod.F | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 120000 components/mpas-ocean/src/analysis_members/shr_const_mod.F mode change 100644 => 120000 components/mpas-ocean/src/analysis_members/shr_kind_mod.F diff --git a/components/mpas-ocean/src/analysis_members/shr_const_mod.F b/components/mpas-ocean/src/analysis_members/shr_const_mod.F deleted file mode 100644 index 41538d8af2f1..000000000000 --- a/components/mpas-ocean/src/analysis_members/shr_const_mod.F +++ /dev/null @@ -1 +0,0 @@ -../../../../share/util/shr_const_mod.F90 diff --git a/components/mpas-ocean/src/analysis_members/shr_const_mod.F b/components/mpas-ocean/src/analysis_members/shr_const_mod.F new file mode 120000 index 000000000000..c471e79113fd --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_const_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_const_mod.F90 \ No newline at end of file diff --git a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F deleted file mode 100644 index 44b22288cce1..000000000000 --- a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F +++ /dev/null @@ -1 +0,0 @@ -../../../../share/util/shr_kind_mod.F90 diff --git a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F new file mode 120000 index 000000000000..77a61f967b6a --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_kind_mod.F90 \ No newline at end of file From 17aed5bb359eb04727c4bcc5e016fbda05422599 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 1 Jul 2024 14:25:51 -0500 Subject: [PATCH 118/904] Use MPASO's pi instead of MPAS-framework's pi in AM --- .../src/analysis_members/mpas_ocn_harmonic_analysis.F | 2 +- .../analysis_members/mpas_ocn_lagrangian_particle_tracking.F | 2 +- .../mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F | 4 ++-- .../mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F index ee1495a3e59a..c60ac179ccf4 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F @@ -681,7 +681,7 @@ SUBROUTINE harmonic_analysis_solve(MNP,nfreq,hmat,GLOELV,haff,haface,emagt,phase REAL(kind=RKIND),ALLOCATABLE :: hap(:),hax(:) REAL(kind=RKIND),ALLOCATABLE :: ha(:,:) - convrd=180.0_RKIND/pii + convrd=180.0_RKIND/pi mm = 2*nfreq ALLOCATE ( PHASEE(nfreq),EMAG(nfreq) ) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F index ba02542eb45d..8a8a2665bcb9 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F @@ -3833,7 +3833,7 @@ subroutine convert_latlon_from_xyz(lat, lon, x, y, z) !{{{ ! ensure range in 0, 2*pi if (lon < 0.0_RKIND) then - lon = 2*pii + lon + lon = 2*pi + lon end if end subroutine convert_latlon_from_xyz !}}} diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F index 88c9960092b7..64ab02e05733 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F @@ -946,8 +946,8 @@ subroutine ocn_compute_eddy_stats(dminfo, block, nVertLevels, nCells, nCellsSolv ! for lat/lon coordinates, convert from radians to degrees for output if (config_AM_okuboWeiss_use_lat_lon_coords) then - wsPosX = wsPosX *180.0_RKIND/pii - wsPosY = wsPosY *180.0_RKIND/pii + wsPosX = wsPosX *180.0_RKIND/pi + wsPosY = wsPosY *180.0_RKIND/pi end if call mpas_timer_stop("stats per proc") diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F index 78adb9a0583f..498f3f09c7ac 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F @@ -62,7 +62,7 @@ module ocn_time_filters !-------------------------------------------------------------------- #ifdef TIME_FILTERS_DEBUG integer :: iEdgeOutput = 0, iBlockOutput = 0, iklevel = 1 - real (kind=RKIND) :: lonEdgePoint = 10.0_RKIND*pii/180.0_RKIND, latEdgePoint = 30.0_RKIND*pii/180.0_RKIND + real (kind=RKIND) :: lonEdgePoint = 10.0_RKIND*pi/180.0_RKIND, latEdgePoint = 30.0_RKIND*pi/180.0_RKIND #endif !*********************************************************************** @@ -182,7 +182,7 @@ subroutine ocn_init_time_filters(domain, err)!{{{ call mpas_pool_get_subpool(block % structs, 'mesh', statePool) call mpas_pool_get_array(statePool, 'latEdge', latEdge) call mpas_pool_get_array(statePool, 'lonEdge', lonEdge) - print *, 'lon = ', 180.0_RKIND/pii*lonEdge(iEdgeOutput), ' lat = ', 180.0_RKIND/pii*latEdge(iEdgeOutput), & + print *, 'lon = ', 180.0_RKIND/pi*lonEdge(iEdgeOutput), ' lat = ', 180.0_RKIND/pi*latEdge(iEdgeOutput), & ' iklevel=',iklevel, ' iEdgeOutput=',iEdgeOutput, ' iBlockOutput = ', iBlockOutput #endif From 5ce53c730807bfb626fd811640fb501ceed310f1 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 1 Jul 2024 14:26:44 -0500 Subject: [PATCH 119/904] Use MPASO's pi instead of MPAS-framework's pi in shared --- components/mpas-ocean/src/shared/Makefile | 2 +- .../src/shared/mpas_ocn_manufactured_solution.F | 4 ++-- components/mpas-ocean/src/shared/mpas_ocn_mesh.F | 3 ++- components/mpas-ocean/src/shared/mpas_ocn_tendency.F | 4 ++-- .../mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F | 4 ++-- .../src/shared/mpas_ocn_tracer_advection_shared.F | 10 +++++----- .../mpas_ocn_vel_forcing_topographic_wave_drag.F | 6 +++--- .../mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F | 2 +- .../src/shared/mpas_ocn_vel_self_attraction_loading.F | 6 +++--- .../src/shared/mpas_ocn_vel_tidal_potential.F | 10 +++++----- 10 files changed, 26 insertions(+), 25 deletions(-) diff --git a/components/mpas-ocean/src/shared/Makefile b/components/mpas-ocean/src/shared/Makefile index 35017551c754..d378b68624bb 100644 --- a/components/mpas-ocean/src/shared/Makefile +++ b/components/mpas-ocean/src/shared/Makefile @@ -143,7 +143,7 @@ mpas_ocn_tracer_advection_std.o: mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_trac mpas_ocn_tracer_advection_vert.o: mpas_ocn_mesh.o mpas_ocn_config.o -mpas_ocn_tracer_advection_shared.o: mpas_ocn_mesh.o mpas_ocn_config.o +mpas_ocn_tracer_advection_shared.o: mpas_ocn_constants.o mpas_ocn_mesh.o mpas_ocn_config.o mpas_ocn_tracer_hmix_redi.o: mpas_ocn_constants.o mpas_ocn_config.o diff --git a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F index e9ad6786103c..2631ba1737ba 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F @@ -217,8 +217,8 @@ subroutine ocn_manufactured_solution_init(domain, err)!{{{ if (.not. config_use_manufactured_solution) return - kx = 2.0_RKIND*pii / config_manufactured_solution_wavelength_x - ky = 2.0_RKIND*pii / config_manufactured_solution_wavelength_y + kx = 2.0_RKIND*pi / config_manufactured_solution_wavelength_x + ky = 2.0_RKIND*pi / config_manufactured_solution_wavelength_y eta0 = config_manufactured_solution_amplitude diff --git a/components/mpas-ocean/src/shared/mpas_ocn_mesh.F b/components/mpas-ocean/src/shared/mpas_ocn_mesh.F index 5e0690704e40..9eadb339b660 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_mesh.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_mesh.F @@ -33,6 +33,7 @@ module ocn_mesh use mpas_log use ocn_config + use ocn_constants implicit none private @@ -1792,7 +1793,7 @@ subroutine ocn_meshScaling() !{{{ ! neighboring cells are circles for this calculation. cellWidth = 2.0_RKIND* & sqrt((areaCell(cell1) + areaCell(cell2))/ & - 2.0_RKIND/pii) + 2.0_RKIND/pi) meshScalingDel2(iEdge) = cellWidth/ & config_hmix_ref_cell_width meshScalingDel4(iEdge) = (cellWidth/ & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index a16e52f14499..a17c48150744 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -1453,7 +1453,7 @@ subroutine ocn_tend_freq_filtered_thickness(tendPool, statePool, & do k = 1, maxLevelCell(iCell) tend_lowFreqDivergence(k,iCell) = & - -2.0*pii/thickness_filter_timescale_sec & + -2.0*pi/thickness_filter_timescale_sec & *(lowFreqDivergence(k,iCell) - div_hu(k) & + div_hu_btr * layerThickness(k,iCell)/totalThickness) @@ -1461,7 +1461,7 @@ subroutine ocn_tend_freq_filtered_thickness(tendPool, statePool, & div_hu_btr*layerThickness(k,iCell)/totalThickness + & lowFreqDivergence(k,iCell) + & use_highFreqThick_restore* & - (-2.0*pii/highFreqThick_restore_time_sec* & + (-2.0*pi/highFreqThick_restore_time_sec* & highFreqThickness(k,iCell) ) end do ! vert (k) loop diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F index 2a9b400dd90b..48901fbf5d51 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F @@ -243,8 +243,8 @@ subroutine ocn_tidal_forcing_build_array(domain, meshPool, forcingPool, statePoo ! compute the tidalHeight if (trim(config_tidal_forcing_model) == 'monochromatic') then tidalHeight = config_tidal_forcing_monochromatic_amp * & - SIN(2.0_RKIND*pii/config_tidal_forcing_monochromatic_period * daysSinceStartOfSim - & - pii*config_tidal_forcing_monochromatic_phaseLag/180.0_RKIND) - & + SIN(2.0_RKIND*pi/config_tidal_forcing_monochromatic_period * daysSinceStartOfSim - & + pi*config_tidal_forcing_monochromatic_phaseLag/180.0_RKIND) - & config_tidal_forcing_monochromatic_baseline elseif (trim(config_tidal_forcing_model) == 'linear') then tidalHeight = max(config_tidal_forcing_linear_min, & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F index 2d14c961fba1..3439776094de 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F @@ -24,10 +24,12 @@ module ocn_tracer_advection_shared use mpas_derived_types use mpas_hash use mpas_sort + use mpas_constants use mpas_geometry_utils use mpas_log use ocn_config + use ocn_constants use ocn_mesh implicit none @@ -370,7 +372,6 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ xec, yec, zec, &! arc bisection coords thetae_tmp, &! angle xv1, xv2, yv1, yv2, zv1, zv2, &! vertex cart coords - pii, &! pi math constant length_scale, &! length scale cos2t, costsint, sin2t ! trig function temps @@ -413,7 +414,6 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ ! Initialize derivTwo and pi - pii = 2.*asin(1.0) derivTwo(:,:,:) = 0.0_RKIND do iCell = 1, nCellsAll @@ -461,9 +461,9 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ end do if ( zc(1) == 1.0_RKIND) then - theta_abs(iCell) = pii/2.0_RKIND + theta_abs(iCell) = pi/2.0_RKIND else - theta_abs(iCell) = pii/2.0_RKIND & + theta_abs(iCell) = pi/2.0_RKIND & - mpas_sphere_angle( xc(1), yc(1), zc(1), & xc(2), yc(2), zc(2), & 0.0_RKIND, 0.0_RKIND, 1.0_RKIND) @@ -509,7 +509,7 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ angle_2d(i) = angleEdge(edgesOnCell(i,iCell)) iEdge = edgesOnCell(i,iCell) if ( iCell .ne. cellsOnEdge(1,iEdge)) & - angle_2d(i) = angle_2d(i) - pii + angle_2d(i) = angle_2d(i) - pi xp(i) = dcEdge(edgesOnCell(i,iCell)) * cos(angle_2d(i)) yp(i) = dcEdge(edgesOnCell(i,iCell)) * sin(angle_2d(i)) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F index 8f641ca4f31c..a024ec97e51b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F @@ -199,7 +199,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ call mpas_log_write(" Topographic wave drag scheme is: Jayne and St. Laurent") call mpas_log_write("") tensorScheme = .false. - kappa = pii/10000.0_RKIND + kappa = pi/10000.0_RKIND do iEdge = 1, nEdgesAll kmax = maxLevelEdgeTop(iEdge) @@ -241,7 +241,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ topographicWaveDrag(iEdge) = topographicWaveDragCoeff * gam & * bed_slope_edges(iEdge)**2 * Nbar * Nb & - / (8.0_RKIND * omegaM2 * pii**2) + / (8.0_RKIND * omegaM2 * pi**2) endif enddo else if (config_topographic_wave_drag_scheme.EQ."LGF") then @@ -256,7 +256,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ topographicWaveDrag(iEdge) = topographicWaveDragCoeff & * (sqrt((topo_buoyancy_N1B(iEdge)**2 - omegaM2**2) & * (topo_buoyancy_N1V(iEdge)**2 - omegaM2**2))) & - / (4.0_RKIND*pii*omegaM2) + / (4.0_RKIND*pi*omegaM2) normalCoeffTWD(iEdge) = (lonGradEdge(iEdge)**2) & * (cos(angleEdge(iEdge))**2) + (latGradEdge(iEdge)**2)*(sin(angleEdge(iEdge))**2) & - 2.0_RKIND*latGradEdge(iEdge)*lonGradEdge(iEdge)*sin(angleEdge(iEdge))*(cos(angleEdge(iEdge))) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F index a6f9ec5c073c..db3e78e9b881 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F @@ -159,7 +159,7 @@ subroutine ocn_vel_hmix_leith_tend(div, relVort, tend, err)!{{{ dcEdgeInv = 1.0_RKIND / dcEdge(iEdge) dvEdgeInv = 1.0_RKIND / dvEdge(iEdge) - visc2tmp = (leithParam*dxLeith*meshScalingDel2(iEdge)/pii)**3 + visc2tmp = (leithParam*dxLeith*meshScalingDel2(iEdge)/pi)**3 do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F index 90d1a4e4630d..a1c7a65452f8 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F @@ -863,8 +863,8 @@ subroutine ocn_vel_self_attraction_loading_init(domain,err)!{{{ ! Pre-compute sin and cos of latCell (co-latitude) values allocate(sinLatCell(nCellsAll), cosLatCell(nCellsAll)) do iCell = 1,nCellsAll - sinLatCell(iCell) = sin(0.5_RKIND*pii-latCell(iCell)) - cosLatCell(iCell) = cos(0.5_RKIND*pii-latCell(iCell)) + sinLatCell(iCell) = sin(0.5_RKIND*pi-latCell(iCell)) + cosLatCell(iCell) = cos(0.5_RKIND*pi-latCell(iCell)) enddo ! Calculate blocking indices @@ -2526,7 +2526,7 @@ subroutine associatedLegendrePolynomials(n, m, startIdx, endIdx, l, pmnm2, pmnm1 if (n == m) then do iCell = startIdx,endIdx - pmnm2(iCell) = sqrt(1.0_RKIND/(4.0_RKIND*pii))*sinLatCell(iCell)**m + pmnm2(iCell) = sqrt(1.0_RKIND/(4.0_RKIND*pi))*sinLatCell(iCell)**m do i = 1,m pmnm2(iCell) = pmnm2(iCell)*sqrt(real(2*i+1,RKIND)/real(2*i,RKIND)) enddo diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F index 211538ac0a73..daafa816fab6 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F @@ -282,7 +282,7 @@ subroutine ocn_compute_tidal_potential_forcing(err)!{{{ !*** Compute eta by summing all constituent contributions do jCon = 1, nTidalConstituents - period = 2.0_RKIND*pii/tidalConstituentFrequency(jCon) + period = 2.0_RKIND*pi/tidalConstituentFrequency(jCon) nCycles = real(int(t/period),RKIND) targ = tidalConstituentFrequency(jCon)*(t - nCycles*period) + & tidalConstituentNodalPhase(jCon) + & @@ -583,7 +583,7 @@ subroutine tidal_constituent_factors(constituentList,nTidalConstituents,refTime, !h = 280.4661_RKIND + 0.98564736_RKIND*T; !p = 83.3535_RKIND + 0.11140353_RKIND*T; !N = 125.0445_RKIND - 0.05295377_RKIND*T; - !N = N*pii/180.0_RKIND + !N = N*pi/180.0_RKIND !! M2 !tidalConstituentAstronomical(j) = 2.0_RKIND*h - 2.0_RKIND*s @@ -611,7 +611,7 @@ subroutine tidal_constituent_factors(constituentList,nTidalConstituents,refTime, !tidalConstituentNodalAmplitude(j) = 1.009_RKIND + 0.187_RKIND*cos(N) - deg2rad = pii/180.0_RKIND + deg2rad = pi/180.0_RKIND T = adjust_angle(180.0_RKIND + real(refHour,RKIND)*(360.0_RKIND/24.0_RKIND)) do j = 1,nTidalConstituents @@ -742,8 +742,8 @@ subroutine orbit(year,julianDay,hour, & real (kind=RKIND) :: deg2rad,rad2deg real (kind=RKIND) :: NRad,pRad,IRad,nuRad,xiRad,nupRad,nup2Rad - deg2rad = pii/180.0_RKIND - rad2deg = 180.0_RKIND/pii + deg2rad = pi/180.0_RKIND + rad2deg = 180.0_RKIND/pi x = int((real(year,RKIND)-1901.0_RKIND)/4.0_RKIND) yr = real(year,RKIND) - 1900.0_RKIND From b3c8342d89983b960c3f7f8d1116c2fe451c3efd Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 1 Jul 2024 13:50:46 -0600 Subject: [PATCH 120/904] Increase precision of MPAS-O standalone pi Co-authored-by: Xylar Asay-Davis --- components/mpas-ocean/src/shared/mpas_ocn_constants.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_constants.F b/components/mpas-ocean/src/shared/mpas_ocn_constants.F index 08b70dce3777..dd3214d8d4fc 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_constants.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_constants.F @@ -118,7 +118,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ !----------------------------------------------------------------------- T0_Kelvin = 273.16_RKIND ! zero point for Celsius - pi = 3.141592_RKIND ! pi + pi = 3.14159265358979323846_RKIND ! pi rho_air = 1.2_RKIND ! ambient air density (kg/m^3) rho_sw = config_density0 ! density of salt water (kg/m^3) rho_fw = 1.0e3_RKIND ! avg. water density (kg/m^3) From e43b59f1b378f096d83682aebacc3e10dad361f0 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 3 May 2024 18:39:16 -0400 Subject: [PATCH 121/904] add hacky atm tend diag --- .../eamxx/src/diagnostics/CMakeLists.txt | 1 + components/eamxx/src/diagnostics/atm_tend.cpp | 76 ++++++++++++ components/eamxx/src/diagnostics/atm_tend.hpp | 50 ++++++++ .../src/diagnostics/register_diagnostics.hpp | 2 + .../src/diagnostics/tests/CMakeLists.txt | 3 + .../src/diagnostics/tests/atm_tend_test.cpp | 111 ++++++++++++++++++ .../eamxx/src/share/io/scorpio_output.cpp | 7 ++ 7 files changed, 250 insertions(+) create mode 100644 components/eamxx/src/diagnostics/atm_tend.cpp create mode 100644 components/eamxx/src/diagnostics/atm_tend.hpp create mode 100644 components/eamxx/src/diagnostics/tests/atm_tend_test.cpp diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index a7a1ee9eaeaf..75c6f420fc61 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -20,6 +20,7 @@ set(DIAGNOSTIC_SRCS aodvis.cpp number_path.cpp aerocom_cld.cpp + atm_tend.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/atm_tend.cpp b/components/eamxx/src/diagnostics/atm_tend.cpp new file mode 100644 index 000000000000..e4903e6d5573 --- /dev/null +++ b/components/eamxx/src/diagnostics/atm_tend.cpp @@ -0,0 +1,76 @@ +#include "diagnostics/atm_tend.hpp" + +#include + +#include "share/util/scream_universal_constants.hpp" + +namespace scream { + +AtmTendDiag::AtmTendDiag(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereDiagnostic(comm, params) { + EKAT_REQUIRE_MSG(params.isParameter("Tend Name"), + "Error! AtmTendDiag requires 'Tend Name' in its " + "input parameters.\n"); + + m_name = m_params.get("Tend Name"); +} + +std::string AtmTendDiag::name() const { return m_name + "_atm_tend"; } + +void AtmTendDiag::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + const auto &gname = m_params.get("grid_name"); + add_field(m_name, gname); +} + +void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { + const auto &f = get_field_in(m_name); + const auto &fid = f.get_header().get_identifier(); + + // Sanity checks + using namespace ShortFieldTagsNames; + const auto &layout = fid.get_layout(); + EKAT_REQUIRE_MSG(f.data_type() == DataType::RealType, + "Error! FieldAtHeight only supports Real data type field.\n" + " - field name: " + + fid.name() + + "\n" + " - field data type: " + + e2str(f.data_type()) + "\n"); + + // All good, create the diag output + FieldIdentifier d_fid(name(), layout.clone(), fid.get_units(), + fid.get_grid_name()); + m_diagnostic_output = Field(d_fid); + m_diagnostic_output.allocate_view(); + + // Let's also create the previous field + FieldIdentifier prev_fid(name() + "_prev", layout.clone(), fid.get_units(), + fid.get_grid_name()); + m_field_prev = Field(prev_fid); + m_field_prev.allocate_view(); +} +void AtmTendDiag::compute_diagnostic_impl() { + Real var_fill_value = constants::DefaultFillValue().value; + std::int64_t dt; + auto tts = m_diagnostic_output.get_header().get_tracking().get_time_stamp(); + + const auto &f = get_field_in(m_name); + + if(m_ts.is_valid()) { + dt = tts - m_ts; + auto ddt = static_cast(dt); + m_ts = tts; + m_field_prev.update(f, 1 / ddt, -1 / ddt); + m_diagnostic_output.deep_copy(m_field_prev); + } else { + m_diagnostic_output.deep_copy(var_fill_value); + m_ts = tts; + } + m_field_prev.deep_copy(f); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/atm_tend.hpp b/components/eamxx/src/diagnostics/atm_tend.hpp new file mode 100644 index 000000000000..f134fd3f6857 --- /dev/null +++ b/components/eamxx/src/diagnostics/atm_tend.hpp @@ -0,0 +1,50 @@ +#ifndef EAMXX_ATM_TEND_DIAG_HPP +#define EAMXX_ATM_TEND_DIAG_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/util/scream_time_stamp.hpp" + +namespace scream { + +/* + * This diagnostic will produce the atmospheric tendency. + */ + +class AtmTendDiag : public AtmosphereDiagnostic { + public: + // Constructors + AtmTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const; + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + protected: + void initialize_impl(const RunType /*run_type*/); + + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + // The tendency of what? + std::string m_name; + + // Store the previous field + Field m_field_prev; + + // Store a time stamp + util::TimeStamp m_ts; + +}; // class AtmTendDiag + +} // namespace scream + +#endif // EAMXX_ATM_TEND_DIAG_HPP diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index efb55980a2fb..07f90624db03 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -23,6 +23,7 @@ #include "diagnostics/aodvis.hpp" #include "diagnostics/number_path.hpp" #include "diagnostics/aerocom_cld.hpp" +#include "diagnostics/atm_tend.hpp" namespace scream { @@ -53,6 +54,7 @@ inline void register_diagnostics () { diag_factory.register_product("AerosolOpticalDepth550nm",&create_atmosphere_diagnostic); diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); diag_factory.register_product("AeroComCld",&create_atmosphere_diagnostic); + diag_factory.register_product("AtmTendDiag",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 2ea6bcffd88a..1dfb92fa7e29 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -71,4 +71,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) # Test AEROCOM_CLD CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") + # Test atm_tend + CreateDiagTest(atm_tend "atm_tend_test.cpp") + endif() diff --git a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp new file mode 100644 index 000000000000..5652215719ac --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp @@ -0,0 +1,111 @@ +#include "catch2/catch.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "share/field/field_utils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_universal_constants.hpp" + +namespace scream { + +std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, + const int nlevs) { + const int num_global_cols = ncols * comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names", vos_t{"Point Grid"}); + auto &pl = gm_params.sublist("Point Grid"); + pl.set("type", "point_grid"); + pl.set("aliases", vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm, gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("extraaci") { + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); + + const auto nondim = Units::nondimensional(); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 5; + const int ngcols = 1 * comm.size(); + + auto gm = create_gm(comm, ngcols, nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) qc, nc + FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; + FieldIdentifier qc_fid("qc", scalar2d_layout, kg / kg, grid->name()); + + Field qc(qc_fid); + qc.allocate_view(); + qc.get_header().get_tracking().update_time_stamp(t0); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(0, 0.05); + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto &diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + ekat::ParameterList params; + REQUIRE_THROWS( + diag_factory.create("AtmTendDiag", comm, params)); // No 'Tend Name' + + // TODO: The diag currently doesn't throw when given a phony name, need + // hardening! params.set("Tend Name", "NoWay"); REQUIRE_THROWS + // (diag_factory.create("AtmTendDiag",comm,params)); // Bad 'Tend Name' + + // Randomize + randomize(qc, engine, pdf); + // Create and set up the diagnostic + params.set("grid_name", grid->name()); + params.set("Tend Name", "qc"); + auto diag = diag_factory.create("AtmTendDiag", comm, params); + diag->set_grids(gm); + diag->set_required_field(qc); + diag->initialize(t0, RunType::Initial); + + auto qc_v = qc.get_view(); + qc_v(0, 0) = 5.0; + qc.sync_to_dev(); + + // Run diag + diag->compute_diagnostic(); + auto diag_f = diag->get_diagnostic(); + + Real var_fill_value = constants::DefaultFillValue().value; + + // Check result: diag should be filled with var_fill_value + auto some_field = qc.clone(); + some_field.deep_copy(var_fill_value); + REQUIRE(views_are_equal(diag_f, some_field)); + + util::TimeStamp t1({2024, 1, 2}, {0, 0, 0}); // a day later? + qc.get_header().get_tracking().update_time_stamp(t1); + qc_v(0, 0) = 29.0; + qc.sync_to_dev(); + // diag->initialize(t1, RunType::Initial); + // diag->update(t1, RunType::Initial); + diag->compute_diagnostic(); + diag_f = diag->get_diagnostic(); + auto diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 0) == 1.0 / 3600.0); +} + +} // namespace scream diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 3db201f424ba..8b7b861ff440 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1377,6 +1377,13 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { diag_name = "VaporFlux"; // split will return the list [X, ''], with X being whatever is before 'VapFlux' params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); + } else if (diag_field_name.find("_atm_tend")!=std::string::npos) { + diag_name = "AtmTendDiag"; + // TODO: not sure if this is needed? Can skip, but what to do inside diag? + // Set the grid_name + params.set("grid_name",get_field_manager("sim")->get_grid()->name()); + // split will return [X, ''], with X being whatever is before '_atm_tend' + params.set("Tend Name",ekat::split(diag_field_name,"_atm_tend").front()); } else if (diag_field_name=="PotentialTemperature" or diag_field_name=="LiqPotentialTemperature") { diag_name = "PotentialTemperature"; From 5025176dad03f4dddcff7c820237ebe80f2676d9 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 3 May 2024 23:32:44 -0400 Subject: [PATCH 122/904] address comments and harden tests --- components/eamxx/src/diagnostics/atm_tend.cpp | 47 +++++++++-------- components/eamxx/src/diagnostics/atm_tend.hpp | 5 +- .../src/diagnostics/tests/atm_tend_test.cpp | 51 +++++++++---------- 3 files changed, 52 insertions(+), 51 deletions(-) diff --git a/components/eamxx/src/diagnostics/atm_tend.cpp b/components/eamxx/src/diagnostics/atm_tend.cpp index e4903e6d5573..1420ea8f8478 100644 --- a/components/eamxx/src/diagnostics/atm_tend.cpp +++ b/components/eamxx/src/diagnostics/atm_tend.cpp @@ -9,11 +9,11 @@ namespace scream { AtmTendDiag::AtmTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereDiagnostic(comm, params) { - EKAT_REQUIRE_MSG(params.isParameter("Tend Name"), - "Error! AtmTendDiag requires 'Tend Name' in its " + EKAT_REQUIRE_MSG(params.isParameter("Field Name"), + "Error! AtmTendDiag requires 'Field Name' in its " "input parameters.\n"); - m_name = m_params.get("Tend Name"); + m_name = m_params.get("Field Name"); } std::string AtmTendDiag::name() const { return m_name + "_atm_tend"; } @@ -29,48 +29,55 @@ void AtmTendDiag::set_grids( void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { const auto &f = get_field_in(m_name); const auto &fid = f.get_header().get_identifier(); + const auto &gn = fid.get_grid_name(); // Sanity checks using namespace ShortFieldTagsNames; const auto &layout = fid.get_layout(); EKAT_REQUIRE_MSG(f.data_type() == DataType::RealType, - "Error! FieldAtHeight only supports Real data type field.\n" + "Error! AtmTendDiag only supports Real data type field.\n" " - field name: " + fid.name() + "\n" " - field data type: " + e2str(f.data_type()) + "\n"); + using namespace ekat::units; + // The units are the same except per second + auto diag_units = fid.get_units() / s; + // TODO: set the units string correctly by appending "/s" + // All good, create the diag output - FieldIdentifier d_fid(name(), layout.clone(), fid.get_units(), - fid.get_grid_name()); + FieldIdentifier d_fid(name(), layout.clone(), diag_units, gn); m_diagnostic_output = Field(d_fid); m_diagnostic_output.allocate_view(); // Let's also create the previous field - FieldIdentifier prev_fid(name() + "_prev", layout.clone(), fid.get_units(), - fid.get_grid_name()); - m_field_prev = Field(prev_fid); - m_field_prev.allocate_view(); + FieldIdentifier prev_fid(name() + "_prev", layout.clone(), diag_units, gn); + m_f_prev = Field(prev_fid); + m_f_prev.allocate_view(); } void AtmTendDiag::compute_diagnostic_impl() { Real var_fill_value = constants::DefaultFillValue().value; std::int64_t dt; - auto tts = m_diagnostic_output.get_header().get_tracking().get_time_stamp(); - const auto &f = get_field_in(m_name); + const auto &f = get_field_in(m_name); + const auto &curr_ts = f.get_header().get_tracking().get_time_stamp(); + const auto &prev_ts = m_f_prev.get_header().get_tracking().get_time_stamp(); - if(m_ts.is_valid()) { - dt = tts - m_ts; - auto ddt = static_cast(dt); - m_ts = tts; - m_field_prev.update(f, 1 / ddt, -1 / ddt); - m_diagnostic_output.deep_copy(m_field_prev); + if(prev_ts.is_valid()) { + // This diag was called before, so we have a valid value for m_f_prev, + // and can compute the tendency + dt = curr_ts - prev_ts; + m_f_prev.update(f, 1.0 / dt, -1.0 / dt); + m_diagnostic_output.deep_copy(m_f_prev); } else { + // This is the first time we evaluate this diag. We cannot compute a tend + // yet, so fill with an invalid value m_diagnostic_output.deep_copy(var_fill_value); - m_ts = tts; } - m_field_prev.deep_copy(f); + m_f_prev.deep_copy(f); + m_f_prev.get_header().get_tracking().update_time_stamp(curr_ts); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/atm_tend.hpp b/components/eamxx/src/diagnostics/atm_tend.hpp index f134fd3f6857..0cc707e7d4a5 100644 --- a/components/eamxx/src/diagnostics/atm_tend.hpp +++ b/components/eamxx/src/diagnostics/atm_tend.hpp @@ -38,10 +38,7 @@ class AtmTendDiag : public AtmosphereDiagnostic { std::string m_name; // Store the previous field - Field m_field_prev; - - // Store a time stamp - util::TimeStamp m_ts; + Field m_f_prev; }; // class AtmTendDiag diff --git a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp index 5652215719ac..79d1386bbaf8 100644 --- a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp +++ b/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp @@ -26,7 +26,7 @@ std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, return gm; } -TEST_CASE("extraaci") { +TEST_CASE("atm_tend") { using namespace ShortFieldTagsNames; using namespace ekat::units; @@ -36,11 +36,9 @@ TEST_CASE("extraaci") { // A time stamp util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); - const auto nondim = Units::nondimensional(); - // Create a grids manager - single column for these tests - constexpr int nlevs = 5; - const int ngcols = 1 * comm.size(); + constexpr int nlevs = 25; + const int ngcols = 25 * comm.size(); auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); @@ -51,11 +49,11 @@ TEST_CASE("extraaci") { Field qc(qc_fid); qc.allocate_view(); - qc.get_header().get_tracking().update_time_stamp(t0); // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(0, 0.05); + RPDF pdf(0.0, 200.0); + auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -65,47 +63,46 @@ TEST_CASE("extraaci") { ekat::ParameterList params; REQUIRE_THROWS( - diag_factory.create("AtmTendDiag", comm, params)); // No 'Tend Name' + diag_factory.create("AtmTendDiag", comm, params)); // No 'Field Name' - // TODO: The diag currently doesn't throw when given a phony name, need - // hardening! params.set("Tend Name", "NoWay"); REQUIRE_THROWS - // (diag_factory.create("AtmTendDiag",comm,params)); // Bad 'Tend Name' + Real var_fill_value = constants::DefaultFillValue().value; - // Randomize + // Set time for qc and randomize its values + qc.get_header().get_tracking().update_time_stamp(t0); randomize(qc, engine, pdf); + // Create and set up the diagnostic params.set("grid_name", grid->name()); - params.set("Tend Name", "qc"); + params.set("Field Name", "qc"); auto diag = diag_factory.create("AtmTendDiag", comm, params); diag->set_grids(gm); diag->set_required_field(qc); diag->initialize(t0, RunType::Initial); - auto qc_v = qc.get_view(); - qc_v(0, 0) = 5.0; - qc.sync_to_dev(); - // Run diag diag->compute_diagnostic(); auto diag_f = diag->get_diagnostic(); - Real var_fill_value = constants::DefaultFillValue().value; - // Check result: diag should be filled with var_fill_value auto some_field = qc.clone(); some_field.deep_copy(var_fill_value); REQUIRE(views_are_equal(diag_f, some_field)); - util::TimeStamp t1({2024, 1, 2}, {0, 0, 0}); // a day later? + some_field.deep_copy(qc); + + util::TimeStamp t1({2024, 1, 2}, {0, 0, 0}); // a day later + const Real a_day = 24.0 * 60.0 * 60.0; // seconds qc.get_header().get_tracking().update_time_stamp(t1); - qc_v(0, 0) = 29.0; - qc.sync_to_dev(); - // diag->initialize(t1, RunType::Initial); - // diag->update(t1, RunType::Initial); + randomize(qc, engine, pdf); + + // Run diag again diag->compute_diagnostic(); - diag_f = diag->get_diagnostic(); - auto diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 0) == 1.0 / 3600.0); + some_field.update(qc, 1.0 / a_day, -1.0 / a_day); + REQUIRE(views_are_equal(diag_f, some_field)); + + // This should fail (return false): + some_field.update(qc, 1.0 / a_day, -1.0 / a_day); + REQUIRE_FALSE(views_are_equal(diag_f, some_field)); } } // namespace scream From 5ecca368768512b95dc27868ba205a0d891eaed2 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 1 Jul 2024 16:19:17 -0700 Subject: [PATCH 123/904] rename the tend and use init_timestep --- .../eamxx/src/diagnostics/CMakeLists.txt | 2 +- .../{atm_tend.cpp => atm_backtend.cpp} | 42 +++++++++------ .../eamxx/src/diagnostics/atm_backtend.hpp | 53 +++++++++++++++++++ components/eamxx/src/diagnostics/atm_tend.hpp | 47 ---------------- .../src/diagnostics/register_diagnostics.hpp | 4 +- .../src/diagnostics/tests/CMakeLists.txt | 2 +- ...tm_tend_test.cpp => atm_backtend_test.cpp} | 14 ++--- .../eamxx/src/share/io/scorpio_output.cpp | 6 +-- 8 files changed, 92 insertions(+), 78 deletions(-) rename components/eamxx/src/diagnostics/{atm_tend.cpp => atm_backtend.cpp} (66%) create mode 100644 components/eamxx/src/diagnostics/atm_backtend.hpp delete mode 100644 components/eamxx/src/diagnostics/atm_tend.hpp rename components/eamxx/src/diagnostics/tests/{atm_tend_test.cpp => atm_backtend_test.cpp} (89%) diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index 75c6f420fc61..be51f4346155 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -20,7 +20,7 @@ set(DIAGNOSTIC_SRCS aodvis.cpp number_path.cpp aerocom_cld.cpp - atm_tend.cpp + atm_backtend.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/atm_tend.cpp b/components/eamxx/src/diagnostics/atm_backtend.cpp similarity index 66% rename from components/eamxx/src/diagnostics/atm_tend.cpp rename to components/eamxx/src/diagnostics/atm_backtend.cpp index 1420ea8f8478..7904d3aa797d 100644 --- a/components/eamxx/src/diagnostics/atm_tend.cpp +++ b/components/eamxx/src/diagnostics/atm_backtend.cpp @@ -1,4 +1,4 @@ -#include "diagnostics/atm_tend.hpp" +#include "diagnostics/atm_backtend.hpp" #include @@ -6,19 +6,19 @@ namespace scream { -AtmTendDiag::AtmTendDiag(const ekat::Comm &comm, - const ekat::ParameterList ¶ms) +AtmBackTendDiag::AtmBackTendDiag(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) : AtmosphereDiagnostic(comm, params) { - EKAT_REQUIRE_MSG(params.isParameter("Field Name"), - "Error! AtmTendDiag requires 'Field Name' in its " + EKAT_REQUIRE_MSG(params.isParameter("Tendency Name"), + "Error! AtmBackTendDiag requires 'Tendency Name' in its " "input parameters.\n"); - m_name = m_params.get("Field Name"); + m_name = m_params.get("Tendency Name"); } -std::string AtmTendDiag::name() const { return m_name + "_atm_tend"; } +std::string AtmBackTendDiag::name() const { return m_name + "_atm_tend"; } -void AtmTendDiag::set_grids( +void AtmBackTendDiag::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -26,7 +26,7 @@ void AtmTendDiag::set_grids( add_field(m_name, gname); } -void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { +void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { const auto &f = get_field_in(m_name); const auto &fid = f.get_header().get_identifier(); const auto &gn = fid.get_grid_name(); @@ -34,13 +34,14 @@ void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { // Sanity checks using namespace ShortFieldTagsNames; const auto &layout = fid.get_layout(); - EKAT_REQUIRE_MSG(f.data_type() == DataType::RealType, - "Error! AtmTendDiag only supports Real data type field.\n" - " - field name: " + - fid.name() + - "\n" - " - field data type: " + - e2str(f.data_type()) + "\n"); + EKAT_REQUIRE_MSG( + f.data_type() == DataType::RealType, + "Error! AtmBackTendDiag only supports Real data type field.\n" + " - field name: " + + fid.name() + + "\n" + " - field data type: " + + e2str(f.data_type()) + "\n"); using namespace ekat::units; // The units are the same except per second @@ -57,7 +58,14 @@ void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { m_f_prev = Field(prev_fid); m_f_prev.allocate_view(); } -void AtmTendDiag::compute_diagnostic_impl() { + +void AtmBackTendDiag::init_timestep(const util::TimeStamp &start_of_step) { + m_start_t = start_of_step; + auto f_curr = get_field_in(m_name); + m_f_prev.deep_copy(f_curr); +} + +void AtmBackTendDiag::compute_diagnostic_impl() { Real var_fill_value = constants::DefaultFillValue().value; std::int64_t dt; diff --git a/components/eamxx/src/diagnostics/atm_backtend.hpp b/components/eamxx/src/diagnostics/atm_backtend.hpp new file mode 100644 index 000000000000..d4698d82a052 --- /dev/null +++ b/components/eamxx/src/diagnostics/atm_backtend.hpp @@ -0,0 +1,53 @@ +#ifndef EAMXX_ATM_BACKTEND_DIAG_HPP +#define EAMXX_ATM_BACKTEND_DIAG_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/util/scream_time_stamp.hpp" + +namespace scream { + +/* + * This diagnostic will back out the atmosphere tendency of a given field. + */ + +class AtmBackTendDiag : public AtmosphereDiagnostic { + public: + // Constructors + AtmBackTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const; + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + // Let's override the init time step method + void init_timestep(const util::TimeStamp &start_of_step) override; + + // Let's override the initialize method to set the fields below + void initialize_impl(const RunType /*run_type*/) override; + + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + // The tendency of what? + std::string m_name; + + // Store the previous field + Field m_f_prev; + + // Store the timestamp of the start of the timestep + util::TimeStamp m_start_t; + +}; // class AtmBackTendDiag + +} // namespace scream + +#endif // EAMXX_ATM_BACKTEND_DIAG_HPP diff --git a/components/eamxx/src/diagnostics/atm_tend.hpp b/components/eamxx/src/diagnostics/atm_tend.hpp deleted file mode 100644 index 0cc707e7d4a5..000000000000 --- a/components/eamxx/src/diagnostics/atm_tend.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef EAMXX_ATM_TEND_DIAG_HPP -#define EAMXX_ATM_TEND_DIAG_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_time_stamp.hpp" - -namespace scream { - -/* - * This diagnostic will produce the atmospheric tendency. - */ - -class AtmTendDiag : public AtmosphereDiagnostic { - public: - // Constructors - AtmTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); - - // The name of the diagnostic - std::string name() const; - - // Set the grid - void set_grids(const std::shared_ptr grids_manager); - - protected: -#ifdef KOKKOS_ENABLE_CUDA - public: -#endif - void compute_diagnostic_impl(); - - protected: - void initialize_impl(const RunType /*run_type*/); - - // Keep track of field dimensions - int m_num_cols; - int m_num_levs; - - // The tendency of what? - std::string m_name; - - // Store the previous field - Field m_f_prev; - -}; // class AtmTendDiag - -} // namespace scream - -#endif // EAMXX_ATM_TEND_DIAG_HPP diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 07f90624db03..f22ec478be54 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -23,7 +23,7 @@ #include "diagnostics/aodvis.hpp" #include "diagnostics/number_path.hpp" #include "diagnostics/aerocom_cld.hpp" -#include "diagnostics/atm_tend.hpp" +#include "diagnostics/atm_backtend.hpp" namespace scream { @@ -54,7 +54,7 @@ inline void register_diagnostics () { diag_factory.register_product("AerosolOpticalDepth550nm",&create_atmosphere_diagnostic); diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); diag_factory.register_product("AeroComCld",&create_atmosphere_diagnostic); - diag_factory.register_product("AtmTendDiag",&create_atmosphere_diagnostic); + diag_factory.register_product("AtmBackTendDiag",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 1dfb92fa7e29..a684aa248fc0 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -72,6 +72,6 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") # Test atm_tend - CreateDiagTest(atm_tend "atm_tend_test.cpp") + CreateDiagTest(atm_backtend "atm_backtend_test.cpp") endif() diff --git a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp similarity index 89% rename from components/eamxx/src/diagnostics/tests/atm_tend_test.cpp rename to components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp index 79d1386bbaf8..e8e2b0c29c90 100644 --- a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp +++ b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp @@ -26,7 +26,7 @@ std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, return gm; } -TEST_CASE("atm_tend") { +TEST_CASE("atm_backtend") { using namespace ShortFieldTagsNames; using namespace ekat::units; @@ -37,8 +37,8 @@ TEST_CASE("atm_tend") { util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); // Create a grids manager - single column for these tests - constexpr int nlevs = 25; - const int ngcols = 25 * comm.size(); + constexpr int nlevs = 7; + const int ngcols = 2 * comm.size(); auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); @@ -62,8 +62,8 @@ TEST_CASE("atm_tend") { register_diagnostics(); ekat::ParameterList params; - REQUIRE_THROWS( - diag_factory.create("AtmTendDiag", comm, params)); // No 'Field Name' + REQUIRE_THROWS(diag_factory.create("AtmBackTendDiag", comm, + params)); // No 'Tendency Name' Real var_fill_value = constants::DefaultFillValue().value; @@ -73,8 +73,8 @@ TEST_CASE("atm_tend") { // Create and set up the diagnostic params.set("grid_name", grid->name()); - params.set("Field Name", "qc"); - auto diag = diag_factory.create("AtmTendDiag", comm, params); + params.set("Tendency Name", "qc"); + auto diag = diag_factory.create("AtmBackTendDiag", comm, params); diag->set_grids(gm); diag->set_required_field(qc); diag->initialize(t0, RunType::Initial); diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 8b7b861ff440..9c04a0d69d3b 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1377,13 +1377,13 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { diag_name = "VaporFlux"; // split will return the list [X, ''], with X being whatever is before 'VapFlux' params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); - } else if (diag_field_name.find("_atm_tend")!=std::string::npos) { - diag_name = "AtmTendDiag"; + } else if (diag_field_name.find("_atm_backtend")!=std::string::npos) { + diag_name = "AtmBackTendDiag"; // TODO: not sure if this is needed? Can skip, but what to do inside diag? // Set the grid_name params.set("grid_name",get_field_manager("sim")->get_grid()->name()); // split will return [X, ''], with X being whatever is before '_atm_tend' - params.set("Tend Name",ekat::split(diag_field_name,"_atm_tend").front()); + params.set("Tendency Name",ekat::split(diag_field_name,"_atm_backtend").front()); } else if (diag_field_name=="PotentialTemperature" or diag_field_name=="LiqPotentialTemperature") { diag_name = "PotentialTemperature"; From 86586bf98119316f9a45f6e5092a391a6d87e98e Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 1 Jul 2024 16:51:40 -0700 Subject: [PATCH 124/904] drop unneeded comments from code --- components/eamxx/src/diagnostics/atm_backtend.cpp | 5 ++--- components/eamxx/src/share/io/scorpio_output.cpp | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/diagnostics/atm_backtend.cpp b/components/eamxx/src/diagnostics/atm_backtend.cpp index 7904d3aa797d..c29bb665537a 100644 --- a/components/eamxx/src/diagnostics/atm_backtend.cpp +++ b/components/eamxx/src/diagnostics/atm_backtend.cpp @@ -46,7 +46,6 @@ void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { using namespace ekat::units; // The units are the same except per second auto diag_units = fid.get_units() / s; - // TODO: set the units string correctly by appending "/s" // All good, create the diag output FieldIdentifier d_fid(name(), layout.clone(), diag_units, gn); @@ -60,8 +59,8 @@ void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { } void AtmBackTendDiag::init_timestep(const util::TimeStamp &start_of_step) { - m_start_t = start_of_step; - auto f_curr = get_field_in(m_name); + m_start_t = start_of_step; + const auto &f_curr = get_field_in(m_name); m_f_prev.deep_copy(f_curr); } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 9c04a0d69d3b..811a5fb42c62 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1379,7 +1379,6 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); } else if (diag_field_name.find("_atm_backtend")!=std::string::npos) { diag_name = "AtmBackTendDiag"; - // TODO: not sure if this is needed? Can skip, but what to do inside diag? // Set the grid_name params.set("grid_name",get_field_manager("sim")->get_grid()->name()); // split will return [X, ''], with X being whatever is before '_atm_tend' From a1621df4fab561d262b0b0c592a1c69d43a0f9be Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 2 Jul 2024 08:07:22 -0700 Subject: [PATCH 125/904] better reorganization of logic --- .../eamxx/src/diagnostics/atm_backtend.cpp | 4 +-- .../eamxx/src/diagnostics/atm_backtend.hpp | 3 -- .../diagnostics/tests/atm_backtend_test.cpp | 32 +++++++++++-------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/diagnostics/atm_backtend.cpp b/components/eamxx/src/diagnostics/atm_backtend.cpp index c29bb665537a..4b147de2d6c8 100644 --- a/components/eamxx/src/diagnostics/atm_backtend.cpp +++ b/components/eamxx/src/diagnostics/atm_backtend.cpp @@ -59,9 +59,9 @@ void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { } void AtmBackTendDiag::init_timestep(const util::TimeStamp &start_of_step) { - m_start_t = start_of_step; const auto &f_curr = get_field_in(m_name); m_f_prev.deep_copy(f_curr); + m_f_prev.get_header().get_tracking().update_time_stamp(start_of_step); } void AtmBackTendDiag::compute_diagnostic_impl() { @@ -83,8 +83,6 @@ void AtmBackTendDiag::compute_diagnostic_impl() { // yet, so fill with an invalid value m_diagnostic_output.deep_copy(var_fill_value); } - m_f_prev.deep_copy(f); - m_f_prev.get_header().get_tracking().update_time_stamp(curr_ts); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/atm_backtend.hpp b/components/eamxx/src/diagnostics/atm_backtend.hpp index d4698d82a052..a8bd72883dbe 100644 --- a/components/eamxx/src/diagnostics/atm_backtend.hpp +++ b/components/eamxx/src/diagnostics/atm_backtend.hpp @@ -43,9 +43,6 @@ class AtmBackTendDiag : public AtmosphereDiagnostic { // Store the previous field Field m_f_prev; - // Store the timestamp of the start of the timestep - util::TimeStamp m_start_t; - }; // class AtmBackTendDiag } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp index e8e2b0c29c90..47086b5ddedc 100644 --- a/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp +++ b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp @@ -43,7 +43,7 @@ TEST_CASE("atm_backtend") { auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); - // Input (randomized) qc, nc + // Input (randomized) qc FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; FieldIdentifier qc_fid("qc", scalar2d_layout, kg / kg, grid->name()); @@ -88,21 +88,27 @@ TEST_CASE("atm_backtend") { some_field.deep_copy(var_fill_value); REQUIRE(views_are_equal(diag_f, some_field)); - some_field.deep_copy(qc); + const Real a_day = 24.0 * 60.0 * 60.0; // seconds - util::TimeStamp t1({2024, 1, 2}, {0, 0, 0}); // a day later - const Real a_day = 24.0 * 60.0 * 60.0; // seconds - qc.get_header().get_tracking().update_time_stamp(t1); - randomize(qc, engine, pdf); + constexpr int ntests = 10; - // Run diag again - diag->compute_diagnostic(); - some_field.update(qc, 1.0 / a_day, -1.0 / a_day); - REQUIRE(views_are_equal(diag_f, some_field)); + for(int itest = 2; itest < ntests; itest++) { + // Run diag again + some_field.deep_copy(qc); + + diag->init_timestep(t0); + + util::TimeStamp t1({2024, 1, itest}, {0, 0, 0}); // a day later + qc.get_header().get_tracking().update_time_stamp(t1); + randomize(qc, engine, pdf); + + diag->compute_diagnostic(); + some_field.update(qc, 1.0 / a_day, -1.0 / a_day); + REQUIRE(views_are_equal(diag_f, some_field)); - // This should fail (return false): - some_field.update(qc, 1.0 / a_day, -1.0 / a_day); - REQUIRE_FALSE(views_are_equal(diag_f, some_field)); + // reset t0 to t1 to keep iterating... + t0 = t1; + } } } // namespace scream From a598acd9e9fdd7df5f0a4a4ff5f2e3ee7dc64956 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 09:55:14 -0700 Subject: [PATCH 126/904] Adds description for top_lev and adds its value for 128 levs --- components/eamxx/cime_config/namelist_defaults_scream.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50019833d8ae..50188236e4e0 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -237,7 +237,8 @@ be lost if SCREAM_HACK_XML is not enabled. 0.001 - 6 + 6 + 0 From c889de02c584c90cee76fae26be5572e58b94599 Mon Sep 17 00:00:00 2001 From: dqwu Date: Tue, 2 Jul 2024 16:21:06 -0500 Subject: [PATCH 127/904] Fixing build error in anlgce.cmake (unknown EKAT_MPIRUN_EXE) Move two include statements to the end of anlgce.cmake to fix the build error "EKAT_MPIRUN_EXE is not known". --- components/eamxx/cmake/machine-files/anlgce.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index 079000059d19..e79f70015471 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) - # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) From c533709f4ff112434dc1cef5a4336ad664dbcc73 Mon Sep 17 00:00:00 2001 From: dqwu Date: Wed, 29 May 2024 12:11:12 -0500 Subject: [PATCH 128/904] Adding cmake file for ANL UB22 workstation Adding cmake file for building SCREAM on ANL UB22 workstation --- .../eamxx/cmake/machine-files/anlgce-ub22.cmake | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 components/eamxx/cmake/machine-files/anlgce-ub22.cmake diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake new file mode 100644 index 000000000000..e79f70015471 --- /dev/null +++ b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake @@ -0,0 +1,12 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Remove this if you are using a resource manager (slurm etc) +set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") + +# EKAT MPI settings +set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) From a9f16f1da78ac591faf91848de499b297a9c3a1b Mon Sep 17 00:00:00 2001 From: dqwu Date: Wed, 29 May 2024 13:33:50 -0500 Subject: [PATCH 129/904] Support for EAMXX/SCREAM testing in ANL GCE (UB22) Adding support for EAMXX standalone testing on ANL GCE (UB22) machine (Run EAMXX standalone tests on ANL GCE (UB22) using "./scripts/test-all-scream -m anlgce-ub22 ...") --- components/eamxx/scripts/machines_specs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 9536d415c0de..43e5ebfa083c 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,6 +82,10 @@ ["mpicxx","mpifort","mpicc"], "", ""), + "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), From 82ee85e25b93a06aae23765e6d178e608b9b7303 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 2 Jul 2024 13:27:34 -0600 Subject: [PATCH 130/904] Add P3 vars for MAM4xx --- .../src/physics/p3/disp/p3_main_impl_disp.cpp | 6 ++++-- .../src/physics/p3/eamxx_p3_process_interface.cpp | 14 +++++++++----- .../eamxx/src/physics/p3/impl/p3_main_impl.hpp | 14 ++++++++------ components/eamxx/src/physics/p3/p3_functions.hpp | 4 ++++ .../eamxx/src/physics/p3/p3_functions_f90.cpp | 2 +- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index c3692f3f814f..d20350b0788b 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -162,8 +162,8 @@ ::p3_main_internal_disp( qtend_ignore("qtend_ignore", nj, nk_pack), ntend_ignore("ntend_ignore", nj, nk_pack), // Variables still used in F90 but removed from C++ interface - mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), precip_total_tend("precip_total_tend", nj, nk_pack), - nevapr("nevapr", nj, nk_pack), qr_evap_tend("qr_evap_tend", nj, nk_pack), + mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), + qr_evap_tend("qr_evap_tend", nj, nk_pack), // cloud sedimentation v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), @@ -205,6 +205,8 @@ ::p3_main_internal_disp( auto rho_qi = diagnostic_outputs.rho_qi; auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; + auto precip_total_tend = diagnostic_outputs.precip_total_tend; + auto nevapr = diagnostic_outputs.nevapr; auto qv_prev = diagnostic_inputs.qv_prev; auto t_prev = diagnostic_inputs.t_prev; auto liq_ice_exchange = history_only.liq_ice_exchange; diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index d886478ac2af..acaecd7eef70 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -95,11 +95,13 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m add_field ("T_prev_micro_step", scalar3d_layout_mid, K, grid_name, ps); // Diagnostic Outputs: (all fields are just outputs w.r.t. P3) - add_field("precip_liq_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); - add_field("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); - add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name, ps); - add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name, ps); - add_field("eff_radius_qr", scalar3d_layout_mid, micron, grid_name, ps); + add_field("precip_liq_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); + add_field("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); + add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name, ps); + add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name, ps); + add_field("eff_radius_qr", scalar3d_layout_mid, micron, grid_name, ps); + add_field("precip_total_tend", scalar3d_layout_mid, kg/(kg*s), grid_name, ps); + add_field("nevapr", scalar3d_layout_mid, kg/(kg*s), grid_name, ps); // History Only: (all fields are just outputs and are really only meant for I/O purposes) // TODO: These should be averaged over subcycle as well. But there is no simple mechanism @@ -307,6 +309,8 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) diag_outputs.diag_eff_radius_qc = get_field_out("eff_radius_qc").get_view(); diag_outputs.diag_eff_radius_qi = get_field_out("eff_radius_qi").get_view(); diag_outputs.diag_eff_radius_qr = get_field_out("eff_radius_qr").get_view(); + diag_outputs.precip_total_tend = get_field_out("precip_total_tend").get_view(); + diag_outputs.nevapr = get_field_out("nevapr").get_view(); diag_outputs.precip_liq_surf = m_buffer.precip_liq_surf_flux; diag_outputs.precip_ice_surf = m_buffer.precip_ice_surf_flux; diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index b05b713aec29..ce5116cab2fa 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -142,9 +142,9 @@ ::p3_main_internal( qtend_ignore, ntend_ignore, // Variables still used in F90 but removed from C++ interface - mu_c, lamc, precip_total_tend, nevapr, qr_evap_tend; + mu_c, lamc, qr_evap_tend; - workspace.template take_many_and_reset<46>( + workspace.template take_many_and_reset<44>( { "mu_r", "T_atm", "lamr", "logn0r", "nu", "cdist", "cdist1", "cdistr", "inv_cld_frac_i", "inv_cld_frac_l", "inv_cld_frac_r", "qc_incld", "qr_incld", "qi_incld", "qm_incld", @@ -153,7 +153,7 @@ ::p3_main_internal( "rhofacr", "rhofaci", "acn", "qv_sat_l", "qv_sat_i", "sup", "qv_supersat_i", "tmparr1", "exner", "diag_equiv_reflectivity", "diag_vm_qi", "diag_diam_qi", "pratot", "prctot", "qtend_ignore", "ntend_ignore", - "mu_c", "lamc", "precip_total_tend", "nevapr", "qr_evap_tend" + "mu_c", "lamc", "qr_evap_tend" }, { &mu_r, &T_atm, &lamr, &logn0r, &nu, &cdist, &cdist1, &cdistr, @@ -163,7 +163,7 @@ ::p3_main_internal( &rhofacr, &rhofaci, &acn, &qv_sat_l, &qv_sat_i, &sup, &qv_supersat_i, &tmparr1, &exner, &diag_equiv_reflectivity, &diag_vm_qi, &diag_diam_qi, &pratot, &prctot, &qtend_ignore, &ntend_ignore, - &mu_c, &lamc, &precip_total_tend, &nevapr, &qr_evap_tend + &mu_c, &lamc, &qr_evap_tend }); // Get single-column subviews of all inputs, shouldn't need any i-indexing @@ -197,6 +197,8 @@ ::p3_main_internal( const auto orho_qi = ekat::subview(diagnostic_outputs.rho_qi, i); const auto oprecip_liq_flux = ekat::subview(diagnostic_outputs.precip_liq_flux, i); const auto oprecip_ice_flux = ekat::subview(diagnostic_outputs.precip_ice_flux, i); + const auto oprecip_total_tend = ekat::subview(diagnostic_outputs.precip_total_tend, i); + const auto onevapr = ekat::subview(diagnostic_outputs.nevapr, i); const auto oliq_ice_exchange = ekat::subview(history_only.liq_ice_exchange, i); const auto ovap_liq_exchange = ekat::subview(history_only.vap_liq_exchange, i); const auto ovap_ice_exchange = ekat::subview(history_only.vap_ice_exchange, i); @@ -216,7 +218,7 @@ ::p3_main_internal( &nc_incld, &nr_incld, &ni_incld, &bm_incld, &inv_rho, &prec, &rho, &rhofacr, &rhofaci, &acn, &qv_sat_l, &qv_sat_i, &sup, &qv_supersat_i, &tmparr1, &qtend_ignore, &ntend_ignore, - &mu_c, &lamc, &orho_qi, &oqv2qi_depos_tend, &precip_total_tend, &nevapr, &oprecip_liq_flux, &oprecip_ice_flux + &mu_c, &lamc, &orho_qi, &oqv2qi_depos_tend, &oprecip_total_tend, &onevapr, &oprecip_liq_flux, &oprecip_ice_flux }; // initialize @@ -252,7 +254,7 @@ ::p3_main_internal( oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, olatent_heat_vapor, olatent_heat_sublim, olatent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, - mu_r, lamr, logn0r, oqv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, + mu_r, lamr, logn0r, oqv2qi_depos_tend, oprecip_total_tend, onevapr, qr_evap_tend, ovap_liq_exchange, ovap_ice_exchange, oliq_ice_exchange, pratot, prctot, hydrometeorsPresent, nk, p3constants); diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 1749a4ae7346..47bb4560c2e1 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -191,6 +191,10 @@ struct Functions view_2d precip_liq_flux; // Grid-box average ice/snow flux [kg m^-2 s^-1] pverp view_2d precip_ice_flux; + // Total precipitation (rain + snow) [kg/kg/s] + view_2d precip_total_tend; + // Evaporation of total precipitation (rain + snow) [kg/kg/s] + view_2d nevapr; }; // This struct stores time stepping and grid-index-related information. diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 83aeb5b16d3d..eff3102f0e44 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -2048,7 +2048,7 @@ Int p3_main_f( inv_exner_d, qv_prev_d, t_prev_d}; P3F::P3DiagnosticOutputs diag_outputs{qv2qi_depos_tend_d, precip_liq_surf_d, precip_ice_surf_d, diag_eff_radius_qc_d, diag_eff_radius_qi_d, diag_eff_radius_qr_d, - rho_qi_d,precip_liq_flux_d, precip_ice_flux_d}; + rho_qi_d,precip_liq_flux_d, precip_ice_flux_d, precip_total_tend_d, nevapr_d}; P3F::P3Infrastructure infrastructure{dt, it, its, ite, kts, kte, do_predict_nc, do_prescribed_CCN, col_location_d}; P3F::P3HistoryOnly history_only{liq_ice_exchange_d, vap_liq_exchange_d, From 266a8e87a797f241dedd134a611341bc46a119cd Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 3 Jul 2024 02:35:01 -0500 Subject: [PATCH 131/904] Fix indexing of layers in high freq. output Previously, the layer above the one containing the desired depth was being selected, rather than the layer containing the depth. The default depth if no layer is found is now the deepest layer, rather than the first layer. This is because, if no layer is found, it means that even the deepest layer is shallower than the desired depth, and the only sensible default is the deepest layer. --- .../mpas_ocn_high_frequency_output.F | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F index 60a5dececcb3..5b1dd6aabc07 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F @@ -175,7 +175,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ type (mpas_pool_type), pointer :: highFrequencyOutputAMPool type (mpas_pool_type), pointer :: tracersPool - integer :: iLevel, iLevelTarget, iCell, iEdge, i, cell1, cell2, k, eoe + integer :: iLevel, iCell, iEdge, i, cell1, cell2, k, eoe integer :: iLevel0100, iLevel0250, iLevel0700, iLevel2000 real (kind=RKIND) :: sumLayerThickness integer, pointer :: nVertLevels, nCells, nEdges @@ -377,37 +377,41 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(highFrequencyOutputAMPool, 'columnIntegratedSpeed', columnIntegratedSpeed) ! find vertical level that is just above the 100 m reference level - iLevel0100 = 1 - do iLevel=2,nVertLevels + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0100 = nVertLevels + do iLevel=1,nVertLevels if(refBottomDepth(iLevel) > 100.0_RKIND) then - iLevel0100 = iLevel-1 + iLevel0100 = iLevel exit endif enddo ! find vertical level that is just above the 250 m reference level - iLevel0250 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0250 = nVertLevels do iLevel=iLevel0100,nVertLevels if(refBottomDepth(iLevel) > 250.0_RKIND) then - iLevel0250 = iLevel-1 + iLevel0250 = iLevel exit endif enddo ! find vertical level that is just above the 700 m reference level - iLevel0700 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0700 = nVertLevels do iLevel=iLevel0250,nVertLevels if(refBottomDepth(iLevel) > 700.0_RKIND) then - iLevel0700 = iLevel-1 + iLevel0700 = iLevel exit endif enddo ! find vertical level that is just above the 2000 m reference level - iLevel2000 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel2000 = nVertLevels do iLevel=iLevel0700,nVertLevels if(refBottomDepth(iLevel) > 2000.0_RKIND) then - iLevel2000 = iLevel-1 + iLevel2000 = iLevel exit endif enddo From 53b636a5cbd3d5796021f4983b62d3d73d27e770 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 3 Jul 2024 14:33:44 -0700 Subject: [PATCH 132/904] update docn_comp_mod.F90 --- .../data_comps/docn/src/docn_comp_mod.F90 | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index 002ed655b404..ac1278673656 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -1178,46 +1178,46 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) !------------------------------------------------------------------------------- ! RCEMIP phase 2 - Mock-Walker - ! MW_295dT1p25 - mean SST = 295 / dSST = 1.25 K + ! MW_295dT1p25 - mean SST = 295 K / dSST = 1.25 K if (sst_option == 11) then - mean_SST = 295 + mean_SST = 295 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT0p625 - mean SST = 300 / dSST = 0.625 K + ! MW_300dT0p625 - mean SST = 300 K / dSST = 0.625 K if (sst_option == 12) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 0.625 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT1p25 - mean SST = 300 / dSST = 1.25 K + ! MW_300dT1p25 - mean SST = 300 K / dSST = 1.25 K if (sst_option == 13) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT2p5 - mean SST = 300 / dSST = 2.5 K + ! MW_300dT2p5 - mean SST = 300 K / dSST = 2.5 K if (sst_option == 14) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 2.5 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_305dT1p25 - mean SST = 305 / dSST = 1.25 K + ! MW_305dT1p25 - mean SST = 305 K / dSST = 1.25 K if (sst_option == 15) then - mean_SST = 305 + mean_SST = 305 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) From 57fee69019b3ccc1ed5682966c486cf59bd430ad Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 3 Jul 2024 14:34:37 -0700 Subject: [PATCH 133/904] add TOA rad history variables --- .../eam/src/physics/crm/rrtmgp/radiation.F90 | 19 +++++++++++++++++++ .../eam/src/physics/rrtmgp/radiation.F90 | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/components/eam/src/physics/crm/rrtmgp/radiation.F90 b/components/eam/src/physics/crm/rrtmgp/radiation.F90 index b7b253d1b6c4..b80ea0c2fe10 100644 --- a/components/eam/src/physics/crm/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/crm/rrtmgp/radiation.F90 @@ -768,6 +768,18 @@ subroutine radiation_init(state) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2494,6 +2506,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2531,6 +2544,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) diff --git a/components/eam/src/physics/rrtmgp/radiation.F90 b/components/eam/src/physics/rrtmgp/radiation.F90 index 0c715000951d..5c87c3376d9d 100644 --- a/components/eam/src/physics/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/rrtmgp/radiation.F90 @@ -767,6 +767,18 @@ subroutine radiation_init(state,pbuf) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2375,6 +2387,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2412,6 +2425,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) From 778b6ac1b1221f83ecf1fbaf67827a523460442f Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Fri, 5 Jul 2024 13:59:21 -0600 Subject: [PATCH 134/904] Remove add_field() calls that are not needed by the Dry Deposition process. --- ...x_mam_dry_deposition_process_interface.cpp | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 5ac7489f1400..2968b9927ef5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -96,7 +96,6 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, @@ -115,22 +114,23 @@ void MAMDryDep::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - // add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); - add_field("obklen", scalar2d_layout, m, grid_name); - add_field("surfric", scalar2d_layout, m / s, grid_name); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - add_field("icefrac", scalar2d_layout, nondim, grid_name); - add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - add_field("fv", scalar2d_layout, m / s, grid_name); - add_field("ram1", scalar2d_layout, s / m, grid_name); + // TODO: The following are not used by drydep but to create a dry atmosphere object. + add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction + // + // TODO: Currently the following are scalar parameters that are fixed + // for the whole grid. Do these need to be column specific? + // add_field("obklen", scalar2d_layout, m, grid_name); + // add_field("surfric", scalar2d_layout, m / s, grid_name); + // add_field("landfrac", scalar2d_layout, nondim, grid_name); + // add_field("icefrac", scalar2d_layout, nondim, grid_name); + // add_field("ocnfrac", scalar2d_layout, nondim, grid_name); + // add_field("fv", scalar2d_layout, m / s, grid_name); + // add_field("ram1", scalar2d_layout, s / m, grid_name); // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -407,10 +407,7 @@ void fill_tracer_views( // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - using DryDep = mam4::DryDeposition; - - using MemberType = KT::MemberType; const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); From 636c7654d2fab21b3a7f19eede1a1f9106ff81b7 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Mar 2024 08:52:57 -0600 Subject: [PATCH 135/904] wet_scav - get code from balwinder branch. --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- .../eamxx_mam_wetscav_process_interface.cpp | 874 ++++++++++++++++++ .../eamxx_mam_wetscav_process_interface.hpp | 210 +++++ .../uncoupled/mam4_wet_scav/CMakeLists.txt | 30 + .../tests/uncoupled/mam4_wet_scav/input.yaml | 68 ++ .../mam4_wet_scav/mam4_wetscav_standalone.cpp | 69 ++ .../tests/uncoupled/mam4_wet_scav/output.yaml | 14 + 7 files changed, 1267 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp create mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt create mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml create mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp create mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/output.yaml diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 40f43cbf3a7e..dd8cf6e6b304 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -43,7 +43,8 @@ add_subdirectory(${EXTERNALS_SOURCE_DIR}/mam4xx ${CMAKE_BINARY_DIR}/externals/ma add_library(mam eamxx_mam_microphysics_process_interface.cpp eamxx_mam_optics_process_interface.cpp - eamxx_mam_aci_process_interface.cpp) + eamxx_mam_aci_process_interface.cpp + eamxx_mam_microphysics_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp new file mode 100644 index 000000000000..7229419b378a --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -0,0 +1,874 @@ +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +// NOTE: see the impl/ directory for the contents of the impl namespace +#include "impl/compute_particle_size.cpp" + +// Remove the following<<<< +#include +#include +//>>>>>>> + +/* +Future work: +Wirte comments +write in/outs for all variables clearly + + +*/ + +namespace scream { + +// ========================================================================================= +MAMWetscav::MAMWetscav(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ========================================================================================= +void MAMWetscav::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + + m_grid = grids_manager->get_grid("Physics"); + const auto &grid_name = m_grid->name(); + + ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank + nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + + // Layout for 2D (2d horiz) variable defined at mid-level and + // interfaces + const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + + // -- Input variables that exists in PBUF in EAM + static constexpr auto nondim = + Units::nondimensional(); // for variables that are fractions etc. + + // MUST FIXME: cldt and cldn are the same variables. They must be their + // previous step values. + add_field("cldn_prev_step", scalar3d_layout_mid, nondim, + grid_name); // layer cloud fraction [fraction] + add_field("cldst", scalar3d_layout_mid, nondim, + grid_name); //?? + add_field("rprdsh", scalar3d_layout_mid, kg / kg / s, + grid_name); // rain production, shallow convection + // [kg/kg/s] //NOT updated + add_field( + "rprddp", scalar3d_layout_mid, kg / kg / s, + grid_name); // rain production, deep convection [kg/kg/s] //NOT updated + add_field("evapcsh", scalar3d_layout_mid, kg / kg / s, + grid_name); // Evaporation rate of shallow convective + // //NOT updated precipitation >=0. [kg/kg/s] + add_field("evapcdp", scalar3d_layout_mid, kg / kg / s, + grid_name); // Evaporation rate of deep convective //NOT + // updated precipitation >=0. [kg/kg/s] + + // -- Input variables that exists in PBUF in EAM (in wetdep.F90) in the + // "inputs" data structure + // MUST FIXME: cldt and cldn are the same variables. They must be their + // previous step values. + add_field("cldt_prev_step", scalar3d_layout_mid, nondim, + grid_name); // total cloud fraction [fraction] + add_field( + "qme", scalar3d_layout_mid, kg / kg / s, + grid_name); // net condensation/evaporation of cloud water [kg/kg/s] + add_field("prain", scalar3d_layout_mid, kg / kg / s, + grid_name); // stratiform rain production rate [kg/kg/s] + add_field( + "evapr", scalar3d_layout_mid, kg / kg / s, + grid_name); // evaporation from stratiform rain [kg/kg/s] //NOT updated + + // -- Input variables that exists in PBUF in EAM (in wetdep.F90) + add_field("icwmrdp", scalar3d_layout_mid, kg / kg, + grid_name); // In cloud water mixing ratio, deep + // convection [kg/kg] //NOT updated + add_field("icwmrsh", scalar3d_layout_mid, kg / kg, + grid_name); // In cloud water mixing ratio, shallow + // convection [kg/kg] //NOT updated + add_field("rprddp", scalar3d_layout_mid, kg / kg / s, + grid_name); // Rain production, deep convection [kg/kg/s] + add_field( + "sh_frac", scalar3d_layout_mid, nondim, + grid_name); // Shallow convective cloud fraction [fraction] //NOT updated + add_field( + "dp_frac", scalar3d_layout_mid, nondim, + grid_name); // Deep convective cloud fraction [fraction] //NOT updated + add_field( + "icwmrsh", scalar3d_layout_mid, nondim, + grid_name); + add_field( + "icwmrdp", scalar3d_layout_mid, nondim, + grid_name); + add_field("evapcsh", scalar3d_layout_mid, kg / kg / s, + grid_name); // Evaporation rate of shallow convective + // precipitation >=0. [kg/kg/s] + add_field("evapcdp", scalar3d_layout_mid, kg / kg / s, + grid_name); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + + // -- surface fluxes (input/outpts) for the coupler's cam_out data struture + // for the land model + static constexpr auto m2 = m * m; + add_field( + "bcphiwet", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of hydrophilic black carbon [kg/m2/s] + add_field( + "bcphidry", scalar3d_layout_mid, kg / m2 / s, + grid_name); // dry deposition of hydrophilic black carbon [kg/m2/s] + add_field( + "ocphiwet", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of hydrophilic organic carbon [kg/m2/s] + add_field( + "ocphidry", scalar3d_layout_mid, kg / m2 / s, + grid_name); // dry deposition of hydrophilic organic carbon [kg/m2/s] + + add_field("dstwet1", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of dust (bin1) [kg/m2/s] + add_field("dstwet2", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of dust (bin2) [kg/m2/s] + add_field("dstwet3", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of dust (bin3) [kg/m2/s] + add_field("dstwet4", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of dust (bin4) [kg/m2/s] + + // -- input/ouputs from PBUF for updating particle size and water uptake by + // particles + static constexpr auto m3 = m2 * m; + add_field("dgncur_a", scalar3d_layout_mid, m, + grid_name); // aerosol particle diameter [m] + add_field("wetdens", scalar3d_layout_mid, kg / m3, + grid_name); // wet aerosol density [kg/m3] + add_field("qaerwat", scalar3d_layout_mid, kg / kg, + grid_name); // aerosol water [kg/kg] + add_field("dgnumwet", scalar3d_layout_mid, m, + grid_name); // wet aerosol diameter [m] + add_field("fracis", scalar3d_layout_mid, nondim, + grid_name); // fraction of transported species that are + // insoluble [fraction] + + // -- interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + // -- NOTE: Interstitial aerosols are updated in the interface using the + // "tendencies" from the wetscavenging process + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(imode); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(imode); + + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(imode, ispec); + if(strlen(cld_mmr_field_name) > 0) { + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these + // are NOT advected + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // Tracers group -- do we need this in addition to the tracers above? In any + // case, this call should be idempotent, so it can't hurt. + add_group("tracers", grid_name, 1, Bundling::Required); + + // The following fields are not needed by this process but we define them so + // that we can create MAM4xx class objects like atmosphere, prognostics etc. + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + grid_name); // Cloud fraction + add_field("pbl_height", scalar2d_layout_mid, m, + grid_name); // PBL height + + static constexpr auto s2 = s * s; + add_field("phis", scalar2d_layout_mid, m2 / s2, + grid_name); // surface geopotential + add_field("qv", scalar3d_layout_mid, q_unit, + grid_name); // specific humidity + add_field("nc", scalar3d_layout_mid, n_unit, + grid_name); // cloud water number conc + add_field("ni", scalar3d_layout_mid, n_unit, + grid_name); // cloud ice number conc + add_field("omega", scalar3d_layout_mid, Pa / s, + grid_name); // vertical pressure velocity +} + +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMWetscav::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMWetscav::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMWetscav."); +} + +// ========================================================================================= +void MAMWetscav::initialize_impl(const RunType run_type) { + // Gather runtime options + //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + // populate the wet atmosphere state with views from fields and + // the buffer (NOTE: wet atmosphere only has qv, qc, qi, nc, ni and omega) + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + + // -- Following wet atm variables are NOT used by the process but we still + // need them to + // -- create atmosphere object + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + // populate the dry atmosphere state with views from fields + // (NOTE: dry atmosphere has everything that wet + // atmosphere has along with z_surf, T_mid, p_mid, z_mid, z_iface, + // dz, p_del, cldfrac, w_updraft, pblh, phis) + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + + // How "buffer_" works: We use buffer to allocate memory for the members of + // dry_atm_ object. Here we are providing those memory locations to the + // dry_atm_ members. These members are computed from the above wet_atm_ or + // dry_atm_ members that are explicitly getting their values either from the + // input file or from other processes. These members are null at this point, + // they are assigned in "Kokkos::parallel_for("preprocess", scan_policy, + // preprocess_);" call in the run_impl + + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.dz = buffer_.dz; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.w_updraft = buffer_.w_updraft; + + // The following dry_atm_ members *may* not be used by the process but they + // are needed for creating MAM4xx class objects like Atmosphere + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.z_surf = 0.0; // MUST FIXME: for now + // ---- set wet/dry aerosol-related gas state data + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // Other required variables + cldn_prev_step_ = get_field_out("cldn_prev_step").get_view< Real **>(); + cldt_prev_step_ = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + cldst_ = get_field_out("cldst").get_view< Real **>();//?? + evapr_ = get_field_out("evapr").get_view< Real **>(); + rprdsh_ = + get_field_out("rprdsh").get_view(); // rain production, shallow + // convection [kg/kg/s] + evapcsh_ = + get_field_out("evapcsh") + .get_view(); // Evaporation rate of shallow convective + // precipitation >=0. [kg/kg/s] + + sh_frac_ = get_field_out("sh_frac") + .get_view(); // Shallow convective cloud fraction [fraction] + + rprddp_ = + get_field_out("rprddp") + .get_view(); // rain production, deep convection [kg/kg/s] + + evapcdp_ = get_field_out("evapcdp") + .get_view(); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + dp_frac_ = get_field_out("dp_frac") + .get_view(); // Deep convective cloud fraction [fraction] + + icwmrsh_ = get_field_out("icwmrsh") + .get_view(); // ?? + + icwmrdp_ = get_field_out("icwmrdp") + .get_view(); // ?? + + // set wet/dry aerosol state data (interstitial aerosols only) + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(imode); + wet_aero_.int_aero_nmr[imode] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[imode] = buffer_.dry_int_aero_nmr[imode]; + + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(imode); + wet_aero_.cld_aero_nmr[imode] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[imode] = wet_aero_.cld_aero_nmr[imode]; + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[imode][ispec] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[imode][ispec] = + buffer_.dry_int_aero_mmr[imode][ispec]; + } + + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(imode, ispec); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[imode][ispec] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[imode][ispec] = + buffer_.dry_cld_aero_mmr[imode][ispec]; + } + } + } + + // set up our preprocess/postprocess functors + // Here we initialize (not compute) objects in preprocess struct using the + // objects in the argument list + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); +} + +// ========================================================================================= +void MAMWetscav::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of all variables + // needed by this process or setting up MAM4xx classes and their objects + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + /*Fortran code: + call modal_aero_calcsize_sub(state%ncol, state%lchnk, state%q, state%pdel, + dt, & !in qqcw, ptend, dgnumdry_m=dgncur_a) !inout + + ----subroutine modal_aero_calcsize_sub(ncol, lchnk, state_q, pdel, deltat, + qqcw, ptend, do_adjust_in, & do_aitacc_transfer_in, list_idx_in, + update_mmr_in, dgnumdry_m) */ + // mam4::CalcSizeProcess process_(aero_config_); //initiate MAM4xx calcsize + // process + + /* ---------------------------------------------------------------------------------------- + * Compute particle size using the calcsize process + * ---------------------------------------------------------------------------------------- + */ + + // -- configure the process + /* + * -- NOTES: 1. Flags for the inter-mode particle transfer + * (do_aitacc_transfer) and size adjustment (do_adjust) are TRUE by default + * a. Size adjustment is only done by changing aerosol + * numbers in the modes. + * 2. Interstitial and cld borne aerosols (i.e. "tends") mmr will + * be updated (update_mmr is TRUE by default) + */ + // MUST: FIXME: Make sure state is not updated...only tendencies should be + // updated!! + static constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; + int nspec_amode[ntot_amode_]; + int lspectype_amode[maxd_aspectype][ntot_amode_]; + mam4::Real specdens_amode[maxd_aspectype]; + int lmassptr_amode[maxd_aspectype][ntot_amode_]; + Real spechygro[maxd_aspectype]; + + int numptr_amode[ntot_amode_]; + int mam_idx[ntot_amode_][mam4::ndrop::nspec_max]; + int mam_cnst_idx[ntot_amode_][mam4::ndrop::nspec_max]; + + mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, + numptr_amode, specdens_amode, spechygro, + mam_idx, mam_cnst_idx); + + Real inv_density[ntot_amode_][mam4::AeroConfig::num_aerosol_ids()] = {}; + Real num2vol_ratio_min[ntot_amode_] = {}; + Real num2vol_ratio_max[ntot_amode_] = {}; + Real num2vol_ratio_max_nmodes[ntot_amode_] = {}; + Real num2vol_ratio_min_nmodes[ntot_amode_] = {}; + Real num2vol_ratio_nom_nmodes[ntot_amode_] = {}; + Real dgnmin_nmodes[ntot_amode_] = {}; + Real dgnmax_nmodes[ntot_amode_] = {}; + Real dgnnom_nmodes[ntot_amode_] = {}; + Real mean_std_dev_nmodes[ntot_amode_] = {}; + + bool noxf_acc2ait[mam4::AeroConfig::num_aerosol_ids()] = {}; + int n_common_species_ait_accum = {}; + int ait_spec_in_acc[mam4::AeroConfig::num_aerosol_ids()] = {}; + int acc_spec_in_ait[mam4::AeroConfig::num_aerosol_ids()] = {}; + mam4::modal_aero_calcsize::init_calcsize( + inv_density, num2vol_ratio_min, num2vol_ratio_max, + num2vol_ratio_max_nmodes, num2vol_ratio_min_nmodes, + num2vol_ratio_nom_nmodes, dgnmin_nmodes, dgnmax_nmodes, dgnnom_nmodes, + mean_std_dev_nmodes, noxf_acc2ait, n_common_species_ait_accum, + ait_spec_in_acc, acc_spec_in_ait); + + // diagnostics for visible band summed over modes + + // Note: Need to compute inv density using indexing from e3sm + for(int imode = 0; imode < ntot_amode_; ++imode) { + const int nspec = nspec_amode[imode]; + for(int isp = 0; isp < nspec; ++isp) { + const int idx = lspectype_amode[isp][imode] - 1; + inv_density[imode][isp] = 1.0 / specdens_amode[idx]; + } // isp + } // imode + + view_2d state_q( + "state_q", nlev_, + nvars_); // MUST FIXME: make it is 3d view to avoid race condition + view_2d qqcw("qqcw", nlev_, nvars_); + + + //create a const column view of zeros + view_1d zeros_nlev("zeros_nlev", nlev_); + Kokkos::deep_copy(zeros_nlev,0); + const_view_1d const_zeros_nlev(zeros_nlev); + + mam4::AeroConfig aero_config; + mam4::WetDeposition::Config wetdep_config;// = wetdep_.Config(); + wetdep_.init(aero_config,wetdep_config);//FIXME: Should we call this in the initialize???? + + + + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + + // loop over atmosphere columns and compute aerosol particle size + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const ThreadTeam &team) { + const int icol = team.league_rank(); // column index*/ + + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0, nlev_), [&](int klev) { + view_1d state_q_k = ekat::subview( + state_q, klev); // 1d view of size (nvars_) for storing all + // gasses and aerosols + mam_coupling::state_q_for_column_at_one_lev(dry_aero_, icol, klev, + state_q_k); + + view_1d qqcw_k = + ekat::subview(qqcw, klev); // 1d view of size (nvars_) for + // storing all gasses and aerosols + mam_coupling::qqcw_for_column_at_one_lev(dry_aero_, icol, klev, + qqcw_k); + + static constexpr bool do_adjust = true, do_aitacc_transfer = true, + update_mmr = true; + + Real dgncur_c_kk[ntot_amode_] = {}; + Real dgnumdry_m_kk[ntot_amode_] = {}; + // Calculate aerosol size distribution parameters and aerosol + // water + // uptake + mam4::modal_aero_calcsize::modal_aero_calcsize_sub( + state_q_k.data(), // in + qqcw_k.data(), // in/out + dt, do_adjust, do_aitacc_transfer, update_mmr, lmassptr_amode, + numptr_amode, + inv_density, // in + num2vol_ratio_min, num2vol_ratio_max, + num2vol_ratio_max_nmodes, num2vol_ratio_min_nmodes, + num2vol_ratio_nom_nmodes, dgnmin_nmodes, dgnmax_nmodes, + dgnnom_nmodes, mean_std_dev_nmodes, + // outputs + noxf_acc2ait, n_common_species_ait_accum, ait_spec_in_acc, + acc_spec_in_ait, dgnumdry_m_kk, dgncur_c_kk); + + Real dgnumwet_m_kk[ntot_amode_] = {}; + Real qaerwat_m_kk[ntot_amode_] = {}; + // std::cout< qqcw_get_field(pbuf,mm,lchnk) + qqcw_sav(1:ncol,:,lspec) = fldcw(1:ncol,:) !RCE 2012/01/12 + + ! FIXME: Not sure if this is a bug or not as qqcw_tmp seem + different ! from the previous call and qqcw_tmp is always zero. May need ! + further check. - Shuaiqi Tang in refactoring for MAM4xx call wetdepa_v2( & + ncol, dt, state%pdel, & ! in dep_inputs%cmfdqr, + dep_inputs%evapc, dlf, dep_inputs%conicw, & ! in dep_inputs%prain, + dep_inputs%evapr, dep_inputs%totcond, & ! in dep_inputs%cldt, + dep_inputs%cldcu, & ! in dep_inputs%cldvcu, + dep_inputs%cldvst, & ! in sol_factb, sol_facti, + sol_factic, & ! in mam_prevap_resusp_optcc, + .true., scavcoefnv(:,:,jnv), f_act_conv, & ! in fldcw, qqcw_tmp, & ! in + fracis_cw, dqdt_tmp, iscavt, & ! out icscavt, isscavt, + bcscavt, bsscavt, rcscavt, rsscavt ) ! out + + ! resuspension goes to coarse mode + call calc_resusp_to_coarse( ncol, mm, & ! in + mmtoo_prevap_resusp, .false., & ! in + rcscavt, rsscavt, & ! in + dqdt_tmp, rtscavt_sv ) ! inout + + fldcw(1:ncol,:) = fldcw(1:ncol,:) + dqdt_tmp(1:ncol,:) * dt + + call calc_sfc_flux(dqdt_tmp(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFWET', + sflx, pcols, lchnk) aerdepwetcw(:ncol,mm) = sflx(:ncol) + + call calc_sfc_flux(icscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSIC', + sflx, pcols, lchnk) + + call calc_sfc_flux(isscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSIS', + sflx, pcols, lchnk) + + call calc_sfc_flux(bcscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSBC', + sflx, pcols, lchnk) + + call calc_sfc_flux(bsscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSBS', + sflx, pcols, lchnk) + + call calc_sfc_flux(rcscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSEC', + sflx, pcols, lchnk) + + call calc_sfc_flux(rsscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSES', + sflx, pcols, lchnk) + + endif lphase_jnmw_conditional + + enddo lspec_loop_aa ! lspec = 1, nspec_amode(m)+2 + enddo lphase_loop_aa ! lphase = 1, 2 + enddo mmode_loop_aa ! m = 1, ntot_amode + + ! if the user has specified prescribed aerosol dep fluxes then + ! do not set cam_out dep fluxes according to the prognostic aerosols + if (.not.aerodep_flx_prescribed()) then + call set_srf_wetdep(aerdepwetis, aerdepwetcw, cam_out) + endif + + call wetdep_inputs_unset(dep_inputs) + */ + std::cout << "End of wetscav run" << std::endl; +} + +// ========================================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp new file mode 100644 index 000000000000..bec616e260a1 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -0,0 +1,210 @@ +#ifndef EAMXX_MAM_WETSCAV_HPP +#define EAMXX_MAM_WETSCAV_HPP + +// For MAM4 aerosol configuration +#include + +// For declaring wetscav class derived from atm process class +#include "share/atm_process/atmosphere_process.hpp" + +// For MAM4 processes +#include + +// For MAM4 calcsize process (FIXME:should we include it in mam4 or mam_coupling??) +#include + +// For wetdep processes +#include + +// For component name +#include + +#include + +namespace scream { + +/* + * The class responsible to handle the aerosol wetscavenging + * + * The AD should store exactly ONE instance of this class stored + * in its list of subcomponents (the AD should make sure of this). + * + */ + +class MAMWetscav : public scream::AtmosphereProcess { + + using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; + + using const_view_2d = typename KT::template view_2d; + using const_view_1d = typename KT::template view_1d; // remove it if possible + + + // a thread team dispatched to a single vertical column + using ThreadTeam = mam4::ThreadTeam; + + public: + // Constructors + MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_wetscavenging"; } + + // Set the grid and input output variables + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + + + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol_in, const int nlev_in, + const mam_coupling::WetAtmosphere& wet_atm_in, + const mam_coupling::AerosolState& wet_aero_in, + const mam_coupling::DryAtmosphere& dry_atm_in, + const mam_coupling::AerosolState& dry_aero_in) { + ncol_pre_ = ncol_in; + nlev_pre_ = nlev_in; + wet_atm_pre_ = wet_atm_in; + wet_aero_pre_ = wet_aero_in; + dry_atm_pre_ = dry_atm_in; + dry_aero_pre_ = dry_aero_in; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const Kokkos::TeamPolicy::member_type& team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_vertical_layer_heights(team, dry_atm_pre_, i); + team.team_barrier(); // allows kernels below to use layer heights + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + team.team_barrier(); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + + }; // MAMWetscav::Preprocess + + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere& wet_atm, + const mam_coupling::AerosolState& wet_aero, + const mam_coupling::DryAtmosphere& dry_atm, + const mam_coupling::AerosolState& dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const Kokkos::TeamPolicy::member_type& team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, wet_aero_post_, i); + team.team_barrier(); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_post_, nlev_post_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // MAMWetscav::Postprocess + + + /* ----------------------------------------------- + * Local variables + * ------------------------------------------------ + */ + + // pre- and postprocessing scratch pads (for wet <-> dry conversions) + Preprocess preprocess_; + Postprocess postprocess_; + + + // Number of horizontal columns and vertical levels + int ncol_, nlev_; + + //Number of aerosol modes + static constexpr int ntot_amode_ = mam4::AeroConfig::num_modes(); + + //Extent for the e3sm's state vector for tracers + //--NOTE: The aerosol species are from index 16 to 40 ( or 15 to 39 in C++) + // but we define this variable from 0 to nvars_, where nvars_ is 39. + // Index 0 to 14 has no value + static constexpr int nvars_ = mam4::ndrop::nvars; + + + // atmospheric variables + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + view_2d cldn_prev_step_, cldt_prev_step_; // cloud fraction from the previous step, FIXME: they carry same info, we might remove one later + view_2d evapr_; + view_2d cldst_; + + view_2d rprdsh_; // rain production, shallow convection [kg/kg/s] + view_2d evapcsh_; + view_2d sh_frac_; + view_2d icwmrsh_; + + view_2d rprddp_; + view_2d evapcdp_; + view_2d dp_frac_; + view_2d icwmrdp_; + + + + // aerosol states + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // workspace manager for internal local variables + //ekat::WorkspaceManager workspace_mgr_; + mam_coupling::Buffer buffer_; + + mam4::WetDeposition wetdep_; + + + + std::shared_ptr m_grid; +}; // class MAMWetscav + +} // namespace scream + +#endif // EAMXX_MAM_WETSCAV_HPP diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt new file mode 100644 index 000000000000..1db8448fd845 --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt @@ -0,0 +1,30 @@ +include (ScreamUtils) + +# This test needs to be reworked for microphysics -- currently it's still using +# input for nucleation. + +# Create the test +CreateADUnitTest(mam4_wetscav_standalone + LABELS mam4_wetscav physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1) +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +set (RUN_T0 2021-10-12-45000) + +# ## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml new file mode 100644 index 000000000000..ce1c8d4eeefe --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml @@ -0,0 +1,68 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_wetscav] + mam_wetscav: + number_of_subcycles: ${NUM_SUBCYCLES} + compute_tendencies: [all] + lambda_low: 0.001 + +grids_manager: + Type: Mesh Free + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + cldn_prev_step: 0.5 + rprdsh: 1e-5 + rprddp: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + cldt_prev_step: 1e-5 + cldst: 1e-5 + qme: 1e-5 + prain: 1e-5 + evapr: 1e-5 + icwmrdp: 1e-5 + icwmrsh: 1e-5 + rprddp: 1e-5 + rprdsh: 1e-5 + sh_frac: 1e-5 + dp_frac: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + pbuf: 1e-5 # cloud-borne aerosols + dgncur_a: 1e-5 # aerosol particle diameter [m] + wetdens: 1e-5 # wet aerosol density [kg/m3] + qaerwat: 1e-5 # aerosol water [kg/kg] + dgnumwet: 1e-5 # wet aerosol diameter [m] + fracis: 1e-5 # fraction of transported species that are insoluble [fraction] + cldfrac_tot: 0.5 + pbl_height: 25.0 + phis : 0.1 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp b/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp new file mode 100644 index 000000000000..203b51681d05 --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp @@ -0,0 +1,69 @@ +#include + +#include "control/atmosphere_driver.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "physics/register_physics.hpp" +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/atm_process/atmosphere_process.hpp" + +#include "ekat/ekat_parse_yaml_file.hpp" +#include "ekat/logging/ekat_logger.hpp" + +#include + +namespace scream { + +TEST_CASE("mam4_wetscav-stand-alone", "") { + using namespace scream; + using namespace scream::control; + + // Create a comm + ekat::Comm atm_comm (MPI_COMM_WORLD); + + // Load ad parameter list + std::string fname = "input.yaml"; + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + logger.debug("yaml parsed."); + + // Time stepping parameters + const auto& ts = ad_params.sublist("time_stepping"); + const auto dt = ts.get("time_step"); + const auto nsteps = ts.get("number_of_steps"); + const auto t0_str = ts.get("run_t0"); + const auto t0 = util::str_to_time_stamp(t0_str); + + logger.info("running MAMWetscav standalone test with dt = {} for {} steps.", dt, nsteps); + EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); + + // Need to register products in the factory *before* we create any atm process or grids manager. + register_physics(); + register_mesh_free_grids_manager(); + register_diagnostics(); + logger.debug("products registered."); + + // Create the driver + AtmosphereDriver ad; + logger.debug("driver created."); + + // Init and run + ad.initialize(atm_comm,ad_params,t0); + logger.debug("driver initialized."); + + logger.info("Start time stepping loop ... [0%]"); + for (int i=0; i Date: Mon, 11 Mar 2024 09:31:06 -0600 Subject: [PATCH 136/904] wet_scav - update interface --- .../eamxx_mam_wetscav_process_interface.cpp | 7 +++++++ .../eamxx_mam_wetscav_process_interface.hpp | 21 +++++++++++++------ .../mam/impl/compute_particle_size.cpp | 9 ++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 components/eamxx/src/physics/mam/impl/compute_particle_size.cpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 7229419b378a..6aebcd75f89c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -1,4 +1,6 @@ +#include #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" +#include "scream_config.h" // for SCREAM_CIME_BUILD // NOTE: see the impl/ directory for the contents of the impl namespace #include "impl/compute_particle_size.cpp" @@ -27,6 +29,11 @@ MAMWetscav::MAMWetscav(const ekat::Comm &comm, */ } +AtmosphereProcessType MAMWetscav::type() const { + return AtmosphereProcessType::Physics; +} + +std::string MAMOptics::name() const { return "mam4_wet_scav"; } // ========================================================================================= void MAMWetscav::set_grids( const std::shared_ptr grids_manager) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index bec616e260a1..8a03529b5e1f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -20,6 +20,13 @@ #include #include +#ifndef KOKKOS_ENABLE_CUDA +#define protected_except_cuda public +#define private_except_cuda public +#else +#define protected_except_cuda protected +#define private_except_cuda private +#endif namespace scream { @@ -49,10 +56,10 @@ class MAMWetscav : public scream::AtmosphereProcess { MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The type of subcomponent - AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } - + AtmosphereProcessType + type() const override; // The name of the subcomponent - std::string name() const { return "mam_wetscavenging"; } + std::string name() const override; // Set the grid and input output variables void set_grids( @@ -95,12 +102,14 @@ class MAMWetscav : public scream::AtmosphereProcess { KOKKOS_INLINE_FUNCTION void operator()(const Kokkos::TeamPolicy::member_type& team) const { const int i = team.league_rank(); // column index - + // first, compute dry fields compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); + team.team_barrier(); + // second, we can use dry fields to compute dz, zmin, zint compute_vertical_layer_heights(team, dry_atm_pre_, i); - team.team_barrier(); // allows kernels below to use layer heights compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - team.team_barrier(); + team.team_barrier(); // allows kernels below to use layer heights } // operator() // number of horizontal columns and vertical levels diff --git a/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp b/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp new file mode 100644 index 000000000000..06a4c4cb3bea --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp @@ -0,0 +1,9 @@ +namespace scream::impl { + +/*KOKKOS_INLINE_FUNCTION +void compute_particle_size(int icol, int nlev_, //in + view_1d& state_q, view_1d& qqcw) { + +}*/ + +} // namespace scream::impl From 912e0bc1db38448d7d2b751de7c55ce082d3c247 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Mar 2024 11:54:44 -0600 Subject: [PATCH 137/904] wet_scav - using compute_calcsize_and_water_uptake_dr --- .../eamxx/src/physics/mam/CMakeLists.txt | 2 +- .../eamxx_mam_wetscav_process_interface.cpp | 91 +++++++++++++------ .../eamxx_mam_wetscav_process_interface.hpp | 17 ++-- .../eamxx/src/physics/register_physics.hpp | 2 + 4 files changed, 74 insertions(+), 38 deletions(-) diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index dd8cf6e6b304..3182e3837943 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -44,7 +44,7 @@ add_library(mam eamxx_mam_microphysics_process_interface.cpp eamxx_mam_optics_process_interface.cpp eamxx_mam_aci_process_interface.cpp - eamxx_mam_microphysics_process_interface.cpp) + eamxx_mam_wetscav_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 6aebcd75f89c..c778070f4487 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -33,7 +33,7 @@ AtmosphereProcessType MAMWetscav::type() const { return AtmosphereProcessType::Physics; } -std::string MAMOptics::name() const { return "mam4_wet_scav"; } +std::string MAMWetscav::name() const { return "mam4_wetscav"; } // ========================================================================================= void MAMWetscav::set_grids( const std::shared_ptr grids_manager) { @@ -371,7 +371,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { icwmrdp_ = get_field_out("icwmrdp") .get_view(); // ?? - + // set wet/dry aerosol state data (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { const char *int_nmr_field_name = @@ -418,7 +418,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - + printf("Working on wet_sav \n"); // preprocess input -- needs a scan for the calculation of all variables // needed by this process or setting up MAM4xx classes and their objects Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -450,6 +450,7 @@ void MAMWetscav::run_impl(const double dt) { */ // MUST: FIXME: Make sure state is not updated...only tendencies should be // updated!! +#if 0 static constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; int nspec_amode[ntot_amode_]; int lspectype_amode[maxd_aspectype][ntot_amode_]; @@ -490,8 +491,8 @@ void MAMWetscav::run_impl(const double dt) { // diagnostics for visible band summed over modes // Note: Need to compute inv density using indexing from e3sm - for(int imode = 0; imode < ntot_amode_; ++imode) { - const int nspec = nspec_amode[imode]; + // for(int imode = 0; imode < ntot_amode_; ++imode) { + // const int nspec = nspec_amode[imode]; for(int isp = 0; isp < nspec; ++isp) { const int idx = lspectype_amode[isp][imode] - 1; inv_density[imode][isp] = 1.0 / specdens_amode[idx]; @@ -503,18 +504,19 @@ void MAMWetscav::run_impl(const double dt) { nvars_); // MUST FIXME: make it is 3d view to avoid race condition view_2d qqcw("qqcw", nlev_, nvars_); - - //create a const column view of zeros + //create a const column view of zeros view_1d zeros_nlev("zeros_nlev", nlev_); Kokkos::deep_copy(zeros_nlev,0); const_view_1d const_zeros_nlev(zeros_nlev); +#endif + constexpr int pcnst = mam4::aero_model::pcnst; + constexpr int ntot_amode = mam4::AeroConfig::num_modes(); + constexpr int maxd_aspectype= mam4::ndrop::maxd_aspectype; mam4::AeroConfig aero_config; mam4::WetDeposition::Config wetdep_config;// = wetdep_.Config(); wetdep_.init(aero_config,wetdep_config);//FIXME: Should we call this in the initialize???? - - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -523,6 +525,51 @@ void MAMWetscav::run_impl(const double dt) { policy, KOKKOS_LAMBDA(const ThreadTeam &team) { const int icol = team.league_rank(); // column index*/ + // call wetdep for computing....add mod=re descriptive comment + // here? + auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); + // set surface state data + const haero::Surface + sfc{}; // sfc object is NEVER used in wetdep process + + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = + mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); + + // compute calcsize and + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0, nlev_), [&](int kk) { + + Real state_q[pcnst] = {}; + Real qqcw[pcnst] = {}; + + int nspec_amode[ntot_amode]; + int lspectype_amode[maxd_aspectype][ntot_amode]; + int lmassptr_amode[maxd_aspectype][ntot_amode]; + Real specdens_amode[maxd_aspectype]; + Real spechygro[maxd_aspectype]; + + mam4::utils::extract_stateq_from_prognostics(progs, atm, state_q, kk); + mam4::utils::extract_qqcw_from_prognostics(progs, qqcw, kk); + team.team_barrier(); + + Real mean_std_dev_nmodes[ntot_amode] = {}; + Real dgnumwet_m_kk[ntot_amode] = {}; + Real qaerwat_m_kk[ntot_amode] = {}; + + mam4::modal_aer_opt::compute_calcsize_and_water_uptake_dr( + dry_atm_.p_mid(icol, kk), dry_atm_.T_mid(icol, kk), + cldn_prev_step_(icol, kk), state_q, qqcw, dt, // in + nspec_amode, lspectype_amode, specdens_amode, lmassptr_amode, spechygro, + mean_std_dev_nmodes, dgnumwet_m_kk, qaerwat_m_kk); + + team.team_barrier(); + mam4::utils::inject_qqcw_to_prognostics(qqcw, progs, kk); + mam4::utils::inject_stateq_to_prognostics(state_q, progs, kk); + + }); // klev parallel_for loop + +#if 0 Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0, nlev_), [&](int klev) { view_1d state_q_k = ekat::subview( @@ -585,19 +632,7 @@ void MAMWetscav::run_impl(const double dt) { cldn_prev_step_(icol, klev), dgnumdry_m_kk, dgnumwet_m_kk, qaerwat_m_kk); }); // klev parallel_for loop - - // call wetdep for computing....add mod=re descriptive comment - // here? - - auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); - - // set surface state data - const haero::Surface - sfc{}; // sfc object is NEVER used in wetdep process - - // fetch column-specific subviews into aerosol prognostics - mam4::Prognostics progs = - mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); +#endif // set up diagnostics mam4::Diagnostics diags(nlev_); @@ -606,18 +641,16 @@ void MAMWetscav::run_impl(const double dt) { diags.shallow_convective_precipitation_evaporation = ekat::subview(evapcsh_, icol); - diags.deep_convective_cloud_fraction = ekat::subview(dp_frac_, - icol); + icol); //std::cout<<"BALLI:"< grids_manager) override; - + // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; void init_buffers(const ATMBufferManager &buffer_manager) override; @@ -111,7 +111,7 @@ class MAMWetscav : public scream::AtmosphereProcess { compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); team.team_barrier(); // allows kernels below to use layer heights } // operator() - + // number of horizontal columns and vertical levels int ncol_pre_, nlev_pre_; @@ -166,7 +166,7 @@ class MAMWetscav : public scream::AtmosphereProcess { Preprocess preprocess_; Postprocess postprocess_; - + // Number of horizontal columns and vertical levels int ncol_, nlev_; @@ -177,7 +177,8 @@ class MAMWetscav : public scream::AtmosphereProcess { //--NOTE: The aerosol species are from index 16 to 40 ( or 15 to 39 in C++) // but we define this variable from 0 to nvars_, where nvars_ is 39. // Index 0 to 14 has no value - static constexpr int nvars_ = mam4::ndrop::nvars; + // FIXME: do we need nvars_? + static constexpr int nvars_ = mam4::aero_model::pcnst; // atmospheric variables @@ -197,9 +198,9 @@ class MAMWetscav : public scream::AtmosphereProcess { view_2d evapcdp_; view_2d dp_frac_; view_2d icwmrdp_; - - + + // aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_; @@ -209,8 +210,8 @@ class MAMWetscav : public scream::AtmosphereProcess { mam4::WetDeposition wetdep_; - - + + std::shared_ptr m_grid; }; // class MAMWetscav diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 36afabf8de97..44becc2f3e6a 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -27,6 +27,7 @@ #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" #include "physics/mam/eamxx_mam_aci_process_interface.hpp" +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -64,6 +65,7 @@ inline void register_physics () { proc_factory.register_product("mam4_micro",&create_atmosphere_process); proc_factory.register_product("mam4_optics",&create_atmosphere_process); proc_factory.register_product("mam4_aci",&create_atmosphere_process); + proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 68c9a9dc45ce2a3174bc94dc1b925021a1bab7f4 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Mar 2024 11:57:33 -0600 Subject: [PATCH 138/904] wet_scav - delete old code. --- .../eamxx_mam_wetscav_process_interface.cpp | 128 +----------------- 1 file changed, 3 insertions(+), 125 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c778070f4487..d198a170646c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -418,6 +418,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + printf("Working on wet_sav \n"); // preprocess input -- needs a scan for the calculation of all variables // needed by this process or setting up MAM4xx classes and their objects @@ -450,65 +451,6 @@ void MAMWetscav::run_impl(const double dt) { */ // MUST: FIXME: Make sure state is not updated...only tendencies should be // updated!! -#if 0 - static constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; - int nspec_amode[ntot_amode_]; - int lspectype_amode[maxd_aspectype][ntot_amode_]; - mam4::Real specdens_amode[maxd_aspectype]; - int lmassptr_amode[maxd_aspectype][ntot_amode_]; - Real spechygro[maxd_aspectype]; - - int numptr_amode[ntot_amode_]; - int mam_idx[ntot_amode_][mam4::ndrop::nspec_max]; - int mam_cnst_idx[ntot_amode_][mam4::ndrop::nspec_max]; - - mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, - numptr_amode, specdens_amode, spechygro, - mam_idx, mam_cnst_idx); - - Real inv_density[ntot_amode_][mam4::AeroConfig::num_aerosol_ids()] = {}; - Real num2vol_ratio_min[ntot_amode_] = {}; - Real num2vol_ratio_max[ntot_amode_] = {}; - Real num2vol_ratio_max_nmodes[ntot_amode_] = {}; - Real num2vol_ratio_min_nmodes[ntot_amode_] = {}; - Real num2vol_ratio_nom_nmodes[ntot_amode_] = {}; - Real dgnmin_nmodes[ntot_amode_] = {}; - Real dgnmax_nmodes[ntot_amode_] = {}; - Real dgnnom_nmodes[ntot_amode_] = {}; - Real mean_std_dev_nmodes[ntot_amode_] = {}; - - bool noxf_acc2ait[mam4::AeroConfig::num_aerosol_ids()] = {}; - int n_common_species_ait_accum = {}; - int ait_spec_in_acc[mam4::AeroConfig::num_aerosol_ids()] = {}; - int acc_spec_in_ait[mam4::AeroConfig::num_aerosol_ids()] = {}; - mam4::modal_aero_calcsize::init_calcsize( - inv_density, num2vol_ratio_min, num2vol_ratio_max, - num2vol_ratio_max_nmodes, num2vol_ratio_min_nmodes, - num2vol_ratio_nom_nmodes, dgnmin_nmodes, dgnmax_nmodes, dgnnom_nmodes, - mean_std_dev_nmodes, noxf_acc2ait, n_common_species_ait_accum, - ait_spec_in_acc, acc_spec_in_ait); - - // diagnostics for visible band summed over modes - - // Note: Need to compute inv density using indexing from e3sm - // for(int imode = 0; imode < ntot_amode_; ++imode) { - // const int nspec = nspec_amode[imode]; - for(int isp = 0; isp < nspec; ++isp) { - const int idx = lspectype_amode[isp][imode] - 1; - inv_density[imode][isp] = 1.0 / specdens_amode[idx]; - } // isp - } // imode - - view_2d state_q( - "state_q", nlev_, - nvars_); // MUST FIXME: make it is 3d view to avoid race condition - view_2d qqcw("qqcw", nlev_, nvars_); - - //create a const column view of zeros - view_1d zeros_nlev("zeros_nlev", nlev_); - Kokkos::deep_copy(zeros_nlev,0); - const_view_1d const_zeros_nlev(zeros_nlev); -#endif constexpr int pcnst = mam4::aero_model::pcnst; constexpr int ntot_amode = mam4::AeroConfig::num_modes(); constexpr int maxd_aspectype= mam4::ndrop::maxd_aspectype; @@ -563,77 +505,13 @@ void MAMWetscav::run_impl(const double dt) { nspec_amode, lspectype_amode, specdens_amode, lmassptr_amode, spechygro, mean_std_dev_nmodes, dgnumwet_m_kk, qaerwat_m_kk); - team.team_barrier(); + mam4::utils::inject_qqcw_to_prognostics(qqcw, progs, kk); mam4::utils::inject_stateq_to_prognostics(state_q, progs, kk); }); // klev parallel_for loop -#if 0 - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0, nlev_), [&](int klev) { - view_1d state_q_k = ekat::subview( - state_q, klev); // 1d view of size (nvars_) for storing all - // gasses and aerosols - mam_coupling::state_q_for_column_at_one_lev(dry_aero_, icol, klev, - state_q_k); - - view_1d qqcw_k = - ekat::subview(qqcw, klev); // 1d view of size (nvars_) for - // storing all gasses and aerosols - mam_coupling::qqcw_for_column_at_one_lev(dry_aero_, icol, klev, - qqcw_k); - - static constexpr bool do_adjust = true, do_aitacc_transfer = true, - update_mmr = true; - - Real dgncur_c_kk[ntot_amode_] = {}; - Real dgnumdry_m_kk[ntot_amode_] = {}; - // Calculate aerosol size distribution parameters and aerosol - // water - // uptake - mam4::modal_aero_calcsize::modal_aero_calcsize_sub( - state_q_k.data(), // in - qqcw_k.data(), // in/out - dt, do_adjust, do_aitacc_transfer, update_mmr, lmassptr_amode, - numptr_amode, - inv_density, // in - num2vol_ratio_min, num2vol_ratio_max, - num2vol_ratio_max_nmodes, num2vol_ratio_min_nmodes, - num2vol_ratio_nom_nmodes, dgnmin_nmodes, dgnmax_nmodes, - dgnnom_nmodes, mean_std_dev_nmodes, - // outputs - noxf_acc2ait, n_common_species_ait_accum, ait_spec_in_acc, - acc_spec_in_ait, dgnumdry_m_kk, dgncur_c_kk); - - Real dgnumwet_m_kk[ntot_amode_] = {}; - Real qaerwat_m_kk[ntot_amode_] = {}; - // std::cout< Date: Mon, 11 Mar 2024 14:34:53 -0600 Subject: [PATCH 139/904] wet_scav - remove private member of parallel_for --- .../eamxx_mam_wetscav_process_interface.cpp | 56 +++++++++++++------ .../uncoupled/mam4_wet_scav/CMakeLists.txt | 2 +- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index d198a170646c..cb4ef5130932 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -412,6 +412,9 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // objects in the argument list preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // ========================================================================================= @@ -459,6 +462,23 @@ void MAMWetscav::run_impl(const double dt) { mam4::WetDeposition::Config wetdep_config;// = wetdep_.Config(); wetdep_.init(aero_config,wetdep_config);//FIXME: Should we call this in the initialize???? + // NOTE! we need a const mam_coupling::DryAtmosphere dry_atm for gpu access. + // We cannot use member of this class inside of the parallel_for + const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + const auto &dry_aero = dry_aero_; + const auto &cldn_prev_step =cldn_prev_step_; + const auto &rprdsh = rprdsh_; + const auto &evapcsh = evapcsh_; + const auto &dp_frac=dp_frac_; + const auto &sh_frac=sh_frac_; + const auto &icwmrsh=icwmrsh_; + const auto &cldt_prev_step=cldt_prev_step_; + const auto &cldst=cldst_; + const auto &evapr=evapr_; + const auto &rprddp=rprddp_; + const auto &evapcdp=evapcdp_; + const auto &icwmrdp=icwmrdp_; + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -469,14 +489,14 @@ void MAMWetscav::run_impl(const double dt) { // call wetdep for computing....add mod=re descriptive comment // here? - auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); + auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); // set surface state data const haero::Surface sfc{}; // sfc object is NEVER used in wetdep process // fetch column-specific subviews into aerosol prognostics mam4::Prognostics progs = - mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); + mam_coupling::aerosols_for_column(dry_aero, icol); // compute calcsize and Kokkos::parallel_for( @@ -492,6 +512,7 @@ void MAMWetscav::run_impl(const double dt) { Real spechygro[maxd_aspectype]; mam4::utils::extract_stateq_from_prognostics(progs, atm, state_q, kk); + mam4::utils::extract_qqcw_from_prognostics(progs, qqcw, kk); team.team_barrier(); @@ -500,47 +521,48 @@ void MAMWetscav::run_impl(const double dt) { Real qaerwat_m_kk[ntot_amode] = {}; mam4::modal_aer_opt::compute_calcsize_and_water_uptake_dr( - dry_atm_.p_mid(icol, kk), dry_atm_.T_mid(icol, kk), - cldn_prev_step_(icol, kk), state_q, qqcw, dt, // in + dry_atm.p_mid(icol, kk), dry_atm.T_mid(icol, kk), + cldn_prev_step(icol, kk), state_q, qqcw, dt, // in nspec_amode, lspectype_amode, specdens_amode, lmassptr_amode, spechygro, mean_std_dev_nmodes, dgnumwet_m_kk, qaerwat_m_kk); - mam4::utils::inject_qqcw_to_prognostics(qqcw, progs, kk); mam4::utils::inject_stateq_to_prognostics(state_q, progs, kk); - }); // klev parallel_for loop team.team_barrier(); // set up diagnostics mam4::Diagnostics diags(nlev_); diags.shallow_convective_precipitation_production = - ekat::subview(rprdsh_, icol); + ekat::subview(rprdsh, icol); diags.shallow_convective_precipitation_evaporation = - ekat::subview(evapcsh_, icol); + ekat::subview(evapcsh, icol); + - diags.deep_convective_cloud_fraction = ekat::subview(dp_frac_, + diags.deep_convective_cloud_fraction = ekat::subview(dp_frac, icol); //std::cout<<"BALLI:"< Date: Mon, 11 Mar 2024 17:56:18 -0600 Subject: [PATCH 140/904] wet_scav - add dig variables. Unit tests is working. --- .../eamxx_mam_wetscav_process_interface.cpp | 111 ++++++++++++------ .../eamxx_mam_wetscav_process_interface.hpp | 10 +- 2 files changed, 84 insertions(+), 37 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index cb4ef5130932..09a808b79b13 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -340,38 +340,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } - // Other required variables - cldn_prev_step_ = get_field_out("cldn_prev_step").get_view< Real **>(); - cldt_prev_step_ = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - cldst_ = get_field_out("cldst").get_view< Real **>();//?? - evapr_ = get_field_out("evapr").get_view< Real **>(); - rprdsh_ = - get_field_out("rprdsh").get_view(); // rain production, shallow - // convection [kg/kg/s] - evapcsh_ = - get_field_out("evapcsh") - .get_view(); // Evaporation rate of shallow convective - // precipitation >=0. [kg/kg/s] - - sh_frac_ = get_field_out("sh_frac") - .get_view(); // Shallow convective cloud fraction [fraction] - - rprddp_ = - get_field_out("rprddp") - .get_view(); // rain production, deep convection [kg/kg/s] - - evapcdp_ = get_field_out("evapcdp") - .get_view(); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] - dp_frac_ = get_field_out("dp_frac") - .get_view(); // Deep convective cloud fraction [fraction] - - icwmrsh_ = get_field_out("icwmrsh") - .get_view(); // ?? - - icwmrdp_ = get_field_out("icwmrdp") - .get_view(); // ?? - // set wet/dry aerosol state data (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { const char *int_nmr_field_name = @@ -407,6 +375,39 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } + // Other required variables + cldn_prev_step_ = get_field_out("cldn_prev_step").get_view< Real **>(); + cldt_prev_step_ = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + cldst_ = get_field_out("cldst").get_view< Real **>();//?? + evapr_ = get_field_out("evapr").get_view< Real **>(); + rprdsh_ = + get_field_out("rprdsh").get_view(); // rain production, shallow + // convection [kg/kg/s] + evapcsh_ = + get_field_out("evapcsh") + .get_view(); // Evaporation rate of shallow convective + // precipitation >=0. [kg/kg/s] + + sh_frac_ = get_field_out("sh_frac") + .get_view(); // Shallow convective cloud fraction [fraction] + + rprddp_ = + get_field_out("rprddp") + .get_view(); // rain production, deep convection [kg/kg/s] + + evapcdp_ = get_field_out("evapcdp") + .get_view(); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + dp_frac_ = get_field_out("dp_frac") + .get_view(); // Deep convective cloud fraction [fraction] + + icwmrsh_ = get_field_out("icwmrsh") + .get_view(); // ?? + + icwmrdp_ = get_field_out("icwmrdp") + .get_view(); // ?? + + // set up our preprocess/postprocess functors // Here we initialize (not compute) objects in preprocess struct using the // objects in the argument list @@ -415,6 +416,24 @@ void MAMWetscav::initialize_impl(const RunType run_type) { postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + // wetdep + constexpr int pcnst = mam4::aero_model::pcnst; + + tracer_mixing_ratio_work_ = view_3d("tracer_mixing_ratio_work_", ncol_, nlev_, pcnst); + d_tracer_mixing_ratio_dt_work_ = view_3d("d_tracer_mixing_ratio_dt_work_", ncol_, nlev_, pcnst); + + aerosol_wet_deposition_interstitial_work_ = view_2d("aerosol_wet_deposition_interstitial",ncol_, nlev_); + aerosol_wet_deposition_cloud_water_work_ = view_2d("aerosol_wet_deposition_cloud_water",ncol_, nlev_); + + for (int m = 0; m < mam_coupling::num_aero_modes(); m++) + { + wet_geometric_mean_diameter_i_work_[m] + = view_2d("wet_geometric_mean_diameter_i_work_1",ncol_, nlev_); + } + + total_convective_detrainment_work_= view_2d("total_convective_detrainment_work_",ncol_, nlev_); + } // ========================================================================================= @@ -455,7 +474,7 @@ void MAMWetscav::run_impl(const double dt) { // MUST: FIXME: Make sure state is not updated...only tendencies should be // updated!! constexpr int pcnst = mam4::aero_model::pcnst; - constexpr int ntot_amode = mam4::AeroConfig::num_modes(); + constexpr int ntot_amode = mam_coupling::num_aero_modes(); constexpr int maxd_aspectype= mam4::ndrop::maxd_aspectype; mam4::AeroConfig aero_config; @@ -478,7 +497,16 @@ void MAMWetscav::run_impl(const double dt) { const auto &rprddp=rprddp_; const auto &evapcdp=evapcdp_; const auto &icwmrdp=icwmrdp_; - + const auto &tracer_mixing_ratio_work=tracer_mixing_ratio_work_; + const auto &d_tracer_mixing_ratio_dt_work=d_tracer_mixing_ratio_dt_work_; + const auto &aerosol_wet_deposition_interstitial_work=aerosol_wet_deposition_interstitial_work_; + const auto &aerosol_wet_deposition_cloud_water_work=aerosol_wet_deposition_cloud_water_work_; + const auto &total_convective_detrainment_work=total_convective_detrainment_work_; + view_2d wet_geometric_mean_diameter_i_work[ntot_amode]; + for (int m = 0; m < ntot_amode; m++) + { + wet_geometric_mean_diameter_i_work[m] = wet_geometric_mean_diameter_i_work_[m]; + } const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -546,6 +574,7 @@ void MAMWetscav::run_impl(const double dt) { diags.shallow_convective_cloud_condensate = ekat::subview(icwmrsh_, icol); // FIXME: why are we setting deep_convective_cloud_fraction to zeros_nlev? // diags.deep_convective_cloud_fraction = zeros_nlev; + // FXIME: shallow_convective_cloud_fraction was previously set to dp_frac_icol. diags.shallow_convective_cloud_fraction = ekat::subview(cldt_prev_step, icol); diags.stratiform_cloud_fraction = ekat::subview(cldst, icol); diags.evaporation_of_falling_precipitation = ekat::subview(evapr, icol); @@ -556,7 +585,19 @@ void MAMWetscav::run_impl(const double dt) { ekat::subview(evapcdp, icol); diags.deep_convective_cloud_condensate = ekat::subview(icwmrdp, icol); - // setup tendencies + diags.tracer_mixing_ratio = ekat::subview(tracer_mixing_ratio_work, icol); + diags.d_tracer_mixing_ratio_dt = ekat::subview(d_tracer_mixing_ratio_dt_work, icol); + + diags.aerosol_wet_deposition_interstitial = ekat::subview(aerosol_wet_deposition_interstitial_work, icol); + diags.aerosol_wet_deposition_cloud_water = ekat::subview(aerosol_wet_deposition_cloud_water_work, icol); + + // FIXME where is dp computed? calcsize? + for (int m = 0; m < ntot_amode; m++) + { + diags.wet_geometric_mean_diameter_i[m] = ekat::subview(wet_geometric_mean_diameter_i_work[m], icol); + } + // // setup tendencies + diags.total_convective_detrainment = ekat::subview(total_convective_detrainment_work, icol); mam4::Tendencies tends{}; #if 1 wetdep_.compute_tendencies(aero_config, team, 0, dt, atm, sfc, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index b6a1dc67a1ca..e85a6c3f3c61 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -43,6 +43,7 @@ class MAMWetscav : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; using view_1d = typename KT::template view_1d; using view_2d = typename KT::template view_2d; + using view_3d = typename KT::template view_3d; using const_view_2d = typename KT::template view_2d; using const_view_1d = typename KT::template view_1d; // remove it if possible @@ -119,7 +120,6 @@ class MAMWetscav : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMWetscav::Preprocess // Postprocessing functor @@ -199,7 +199,13 @@ class MAMWetscav : public scream::AtmosphereProcess { view_2d dp_frac_; view_2d icwmrdp_; - + // wet dep + view_3d tracer_mixing_ratio_work_; + view_3d d_tracer_mixing_ratio_dt_work_; + view_2d aerosol_wet_deposition_interstitial_work_; + view_2d aerosol_wet_deposition_cloud_water_work_; + view_2d wet_geometric_mean_diameter_i_work_[mam4::AeroConfig::num_modes()]; + view_2d total_convective_detrainment_work_; // aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_; From 7d3cbff26cbe52e8c1a3f96c507f3443d31961cd Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Fri, 22 Mar 2024 14:10:37 -0600 Subject: [PATCH 141/904] wet_scav - update interface --- .../eamxx_mam_wetscav_process_interface.cpp | 260 +++++++----------- .../eamxx_mam_wetscav_process_interface.hpp | 39 +-- 2 files changed, 102 insertions(+), 197 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 09a808b79b13..48643fe084a3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -259,6 +259,19 @@ void MAMWetscav::set_grids( grid_name); // cloud ice number conc add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); // vertical pressure velocity + + // FIXME: units + add_field("dlf", scalar3d_layout_mid, n_unit, + grid_name); // + add_field("dp_ccf", scalar3d_layout_mid, n_unit, + grid_name); // + add_field("sh_ccf", scalar3d_layout_mid, n_unit, + grid_name); // + // + add_field("aerdepwetis", scalar3d_layout_mid, n_unit, + grid_name); // + add_field("aerdepwetcw", scalar3d_layout_mid, n_unit, + grid_name); // } // ========================================================================================= @@ -374,40 +387,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } } - - // Other required variables - cldn_prev_step_ = get_field_out("cldn_prev_step").get_view< Real **>(); - cldt_prev_step_ = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - cldst_ = get_field_out("cldst").get_view< Real **>();//?? - evapr_ = get_field_out("evapr").get_view< Real **>(); - rprdsh_ = - get_field_out("rprdsh").get_view(); // rain production, shallow - // convection [kg/kg/s] - evapcsh_ = - get_field_out("evapcsh") - .get_view(); // Evaporation rate of shallow convective - // precipitation >=0. [kg/kg/s] - - sh_frac_ = get_field_out("sh_frac") - .get_view(); // Shallow convective cloud fraction [fraction] - - rprddp_ = - get_field_out("rprddp") - .get_view(); // rain production, deep convection [kg/kg/s] - - evapcdp_ = get_field_out("evapcdp") - .get_view(); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] - dp_frac_ = get_field_out("dp_frac") - .get_view(); // Deep convective cloud fraction [fraction] - - icwmrsh_ = get_field_out("icwmrsh") - .get_view(); // ?? - - icwmrdp_ = get_field_out("icwmrdp") - .get_view(); // ?? - - // set up our preprocess/postprocess functors // Here we initialize (not compute) objects in preprocess struct using the // objects in the argument list @@ -419,21 +398,11 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // wetdep constexpr int pcnst = mam4::aero_model::pcnst; - - tracer_mixing_ratio_work_ = view_3d("tracer_mixing_ratio_work_", ncol_, nlev_, pcnst); - d_tracer_mixing_ratio_dt_work_ = view_3d("d_tracer_mixing_ratio_dt_work_", ncol_, nlev_, pcnst); - - aerosol_wet_deposition_interstitial_work_ = view_2d("aerosol_wet_deposition_interstitial",ncol_, nlev_); - aerosol_wet_deposition_cloud_water_work_ = view_2d("aerosol_wet_deposition_cloud_water",ncol_, nlev_); - - for (int m = 0; m < mam_coupling::num_aero_modes(); m++) - { - wet_geometric_mean_diameter_i_work_[m] - = view_2d("wet_geometric_mean_diameter_i_work_1",ncol_, nlev_); - } - - total_convective_detrainment_work_= view_2d("total_convective_detrainment_work_",ncol_, nlev_); - + // FIXME: qqcw_sav_ should be part of work_ + Kokkos::View + qqcw_sav_("qqcw_sav", mam4::nlev); + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); + work_ = view_2d("work", ncol_, work_len); } // ========================================================================================= @@ -473,40 +442,43 @@ void MAMWetscav::run_impl(const double dt) { */ // MUST: FIXME: Make sure state is not updated...only tendencies should be // updated!! - constexpr int pcnst = mam4::aero_model::pcnst; - constexpr int ntot_amode = mam_coupling::num_aero_modes(); - constexpr int maxd_aspectype= mam4::ndrop::maxd_aspectype; - - mam4::AeroConfig aero_config; - mam4::WetDeposition::Config wetdep_config;// = wetdep_.Config(); - wetdep_.init(aero_config,wetdep_config);//FIXME: Should we call this in the initialize???? - // NOTE! we need a const mam_coupling::DryAtmosphere dry_atm for gpu access. // We cannot use member of this class inside of the parallel_for const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; - const auto &cldn_prev_step =cldn_prev_step_; - const auto &rprdsh = rprdsh_; - const auto &evapcsh = evapcsh_; - const auto &dp_frac=dp_frac_; - const auto &sh_frac=sh_frac_; - const auto &icwmrsh=icwmrsh_; - const auto &cldt_prev_step=cldt_prev_step_; - const auto &cldst=cldst_; - const auto &evapr=evapr_; - const auto &rprddp=rprddp_; - const auto &evapcdp=evapcdp_; - const auto &icwmrdp=icwmrdp_; - const auto &tracer_mixing_ratio_work=tracer_mixing_ratio_work_; - const auto &d_tracer_mixing_ratio_dt_work=d_tracer_mixing_ratio_dt_work_; - const auto &aerosol_wet_deposition_interstitial_work=aerosol_wet_deposition_interstitial_work_; - const auto &aerosol_wet_deposition_cloud_water_work=aerosol_wet_deposition_cloud_water_work_; - const auto &total_convective_detrainment_work=total_convective_detrainment_work_; - view_2d wet_geometric_mean_diameter_i_work[ntot_amode]; - for (int m = 0; m < ntot_amode; m++) - { - wet_geometric_mean_diameter_i_work[m] = wet_geometric_mean_diameter_i_work_[m]; - } + const auto &qqcw_sav = qqcw_sav_; + const auto &work = work_; + + // inputs + auto dlf = get_field_in("dlf").get_view(); + auto dp_ccf = get_field_in("dp_ccf").get_view(); + auto sh_ccf = get_field_in("sh_ccf").get_view(); + auto cldn_prev_step = get_field_out("cldn_prev_step").get_view< Real **>(); + // where is cldt_prev_step used? + auto cldt_prev_step = get_field_in("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + auto cldst = get_field_in("cldst").get_view< Real **>();//?? + auto evapr = get_field_in("evapr").get_view< Real **>(); + auto rprdsh = get_field_in("rprdsh").get_view(); // rain production, shallow + // convection [kg/kg/s] + auto evapcsh = get_field_in("evapcsh").get_view(); // Evaporation rate of shallow convective + // precipitation >=0. [kg/kg/s] + auto sh_frac = get_field_in("sh_frac").get_view(); // Shallow convective cloud fraction [fraction] + auto rprddp = get_field_in("rprddp").get_view(); // rain production, deep convection [kg/kg/s] + auto evapcdp = get_field_in("evapcdp").get_view(); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + auto dp_frac = get_field_in("dp_frac") + .get_view(); // Deep convective cloud fraction [fraction] + + auto icwmrsh = get_field_in("icwmrsh") + .get_view(); // ?? + + auto icwmrdp = get_field_in("icwmrdp") + .get_view(); // ?? + + // outputs + const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); + const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -515,95 +487,61 @@ void MAMWetscav::run_impl(const double dt) { policy, KOKKOS_LAMBDA(const ThreadTeam &team) { const int icol = team.league_rank(); // column index*/ - // call wetdep for computing....add mod=re descriptive comment - // here? auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); // set surface state data - const haero::Surface - sfc{}; // sfc object is NEVER used in wetdep process - // fetch column-specific subviews into aerosol prognostics mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aero, icol); - - // compute calcsize and - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0, nlev_), [&](int kk) { - - Real state_q[pcnst] = {}; - Real qqcw[pcnst] = {}; - - int nspec_amode[ntot_amode]; - int lspectype_amode[maxd_aspectype][ntot_amode]; - int lmassptr_amode[maxd_aspectype][ntot_amode]; - Real specdens_amode[maxd_aspectype]; - Real spechygro[maxd_aspectype]; - - mam4::utils::extract_stateq_from_prognostics(progs, atm, state_q, kk); - - mam4::utils::extract_qqcw_from_prognostics(progs, qqcw, kk); - team.team_barrier(); - - Real mean_std_dev_nmodes[ntot_amode] = {}; - Real dgnumwet_m_kk[ntot_amode] = {}; - Real qaerwat_m_kk[ntot_amode] = {}; - - mam4::modal_aer_opt::compute_calcsize_and_water_uptake_dr( - dry_atm.p_mid(icol, kk), dry_atm.T_mid(icol, kk), - cldn_prev_step(icol, kk), state_q, qqcw, dt, // in - nspec_amode, lspectype_amode, specdens_amode, lmassptr_amode, spechygro, - mean_std_dev_nmodes, dgnumwet_m_kk, qaerwat_m_kk); - - mam4::utils::inject_qqcw_to_prognostics(qqcw, progs, kk); - mam4::utils::inject_stateq_to_prognostics(state_q, progs, kk); - }); // klev parallel_for loop - - team.team_barrier(); - // set up diagnostics - mam4::Diagnostics diags(nlev_); - diags.shallow_convective_precipitation_production = - ekat::subview(rprdsh, icol); - diags.shallow_convective_precipitation_evaporation = - ekat::subview(evapcsh, icol); - - - diags.deep_convective_cloud_fraction = ekat::subview(dp_frac, - icol); - //std::cout<<"BALLI:"< + qqcw_sav_; // aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_; - // workspace manager for internal local variables - //ekat::WorkspaceManager workspace_mgr_; mam_coupling::Buffer buffer_; - mam4::WetDeposition wetdep_; - - - std::shared_ptr m_grid; }; // class MAMWetscav From 8363b750b59881d56c981b8d6ac9297648c94050 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 26 Mar 2024 16:32:30 -0600 Subject: [PATCH 142/904] wet_scav - test is working in cpu/gpus. --- .../eamxx_mam_wetscav_process_interface.cpp | 51 +++++++++++-------- .../tests/uncoupled/mam4_wet_scav/input.yaml | 4 +- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 48643fe084a3..60f352f06e34 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -261,11 +261,11 @@ void MAMWetscav::set_grids( grid_name); // vertical pressure velocity // FIXME: units - add_field("dlf", scalar3d_layout_mid, n_unit, + add_field("dlf", scalar3d_layout_mid, n_unit, grid_name); // - add_field("dp_ccf", scalar3d_layout_mid, n_unit, + add_field("dp_ccf", scalar3d_layout_mid, n_unit, grid_name); // - add_field("sh_ccf", scalar3d_layout_mid, n_unit, + add_field("sh_ccf", scalar3d_layout_mid, n_unit, grid_name); // // add_field("aerdepwetis", scalar3d_layout_mid, n_unit, @@ -399,8 +399,10 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // wetdep constexpr int pcnst = mam4::aero_model::pcnst; // FIXME: qqcw_sav_ should be part of work_ - Kokkos::View - qqcw_sav_("qqcw_sav", mam4::nlev); + // we need to add columns + qqcw_sav_ =Kokkos::View("qqcw_sav", mam4::nlev); + + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); } @@ -449,30 +451,33 @@ void MAMWetscav::run_impl(const double dt) { const auto &qqcw_sav = qqcw_sav_; const auto &work = work_; - // inputs - auto dlf = get_field_in("dlf").get_view(); - auto dp_ccf = get_field_in("dp_ccf").get_view(); - auto sh_ccf = get_field_in("sh_ccf").get_view(); + // inputs/outputs + auto dlf = get_field_out("dlf").get_view(); + auto dp_ccf = get_field_out("dp_ccf").get_view(); + auto sh_ccf = get_field_out("sh_ccf").get_view(); + auto cldn_prev_step = get_field_out("cldn_prev_step").get_view< Real **>(); + // where is cldt_prev_step used? - auto cldt_prev_step = get_field_in("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - auto cldst = get_field_in("cldst").get_view< Real **>();//?? - auto evapr = get_field_in("evapr").get_view< Real **>(); - auto rprdsh = get_field_in("rprdsh").get_view(); // rain production, shallow - // convection [kg/kg/s] - auto evapcsh = get_field_in("evapcsh").get_view(); // Evaporation rate of shallow convective + auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + auto cldst = get_field_out("cldst").get_view< Real **>();//?? + auto evapr = get_field_out("evapr").get_view< Real **>(); + auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow + + // convection [kg/kg/s] + auto evapcsh = get_field_out("evapcsh").get_view(); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] - auto sh_frac = get_field_in("sh_frac").get_view(); // Shallow convective cloud fraction [fraction] - auto rprddp = get_field_in("rprddp").get_view(); // rain production, deep convection [kg/kg/s] - auto evapcdp = get_field_in("evapcdp").get_view(); // Evaporation rate of deep convective + auto sh_frac = get_field_out("sh_frac").get_view(); // Shallow convective cloud fraction [fraction] + auto rprddp = get_field_out("rprddp").get_view(); // rain production, deep convection [kg/kg/s] + auto evapcdp = get_field_out("evapcdp").get_view(); // Evaporation rate of deep convective // precipitation >=0. [kg/kg/s] - auto dp_frac = get_field_in("dp_frac") + auto dp_frac = get_field_out("dp_frac") .get_view(); // Deep convective cloud fraction [fraction] - auto icwmrsh = get_field_in("icwmrsh") + auto icwmrsh = get_field_out("icwmrsh") .get_view(); // ?? - auto icwmrdp = get_field_in("icwmrdp") + auto icwmrdp = get_field_out("icwmrdp") .get_view(); // ?? // outputs @@ -531,7 +536,8 @@ void MAMWetscav::run_impl(const double dt) { // diags.aerosol_wet_deposition_cloud_water; auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol);\ auto work_icol = ekat::subview(work, icol); - + // auto qqcw_sav_icol = ekat::subview(qqcw_sav,icol); +#if 1 mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, // inputs cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, @@ -542,6 +548,7 @@ void MAMWetscav::run_impl(const double dt) { aerdepwetis_icol, aerdepwetcw_icol, // FIXME remove qqcw_sav qqcw_sav, work_icol); +#endif }); // icol parallel_for loop /* diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml index ce1c8d4eeefe..c73e2dbaa9a6 100644 --- a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml @@ -61,7 +61,9 @@ initial_conditions: cldfrac_tot: 0.5 pbl_height: 25.0 phis : 0.1 - + dlf : 0.0 + dp_ccf : 0.0 + sh_ccf : 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From 604c2fa3c228ff7fd84b71daec008bddbca58f1d Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 27 Mar 2024 08:59:18 -0600 Subject: [PATCH 143/904] wet_scav - adding single-process. --- .../eamxx_mam_wetscav_process_interface.cpp | 6 +- .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../mam/wet_scav/CMakeLists.txt | 46 ++++++++++++ .../single-process/mam/wet_scav/input.yaml | 70 +++++++++++++++++++ .../mam/wet_scav/mam4_wetscav_standalone.cpp | 69 ++++++++++++++++++ .../single-process/mam/wet_scav/output.yaml | 14 ++++ 6 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/wet_scav/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp create mode 100644 components/eamxx/tests/single-process/mam/wet_scav/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 60f352f06e34..226dcb14d6e2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -64,6 +64,8 @@ void MAMWetscav::set_grids( // interfaces const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -77,6 +79,8 @@ void MAMWetscav::set_grids( "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice cloud water [kg/kg] wet + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure // -- Input variables that exists in PBUF in EAM static constexpr auto nondim = @@ -321,7 +325,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - + dry_atm_.p_int = get_field_in("p_int").get_view(); // How "buffer_" works: We use buffer to allocate memory for the members of // dry_atm_ object. Here we are providing those memory locations to the // dry_atm_ members. These members are computed from the above wet_atm_ or diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index c6b4e0748af3..3e1629ae0f36 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -21,6 +21,7 @@ if (SCREAM_ENABLE_MAM) # initial conditions. add_subdirectory(mam/optics) add_subdirectory(mam/aci) + add_subdirectory(mam/wet_scav) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt new file mode 100644 index 000000000000..dbddf4d26e29 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt @@ -0,0 +1,46 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_wetscav_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_wetscav physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} +) + +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 2 4 6) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_wetscav physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml new file mode 100644 index 000000000000..c73e2dbaa9a6 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -0,0 +1,70 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_wetscav] + mam_wetscav: + number_of_subcycles: ${NUM_SUBCYCLES} + compute_tendencies: [all] + lambda_low: 0.001 + +grids_manager: + Type: Mesh Free + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + cldn_prev_step: 0.5 + rprdsh: 1e-5 + rprddp: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + cldt_prev_step: 1e-5 + cldst: 1e-5 + qme: 1e-5 + prain: 1e-5 + evapr: 1e-5 + icwmrdp: 1e-5 + icwmrsh: 1e-5 + rprddp: 1e-5 + rprdsh: 1e-5 + sh_frac: 1e-5 + dp_frac: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + pbuf: 1e-5 # cloud-borne aerosols + dgncur_a: 1e-5 # aerosol particle diameter [m] + wetdens: 1e-5 # wet aerosol density [kg/m3] + qaerwat: 1e-5 # aerosol water [kg/kg] + dgnumwet: 1e-5 # wet aerosol diameter [m] + fracis: 1e-5 # fraction of transported species that are insoluble [fraction] + cldfrac_tot: 0.5 + pbl_height: 25.0 + phis : 0.1 + dlf : 0.0 + dp_ccf : 0.0 + sh_ccf : 0.0 +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp b/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp new file mode 100644 index 000000000000..203b51681d05 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp @@ -0,0 +1,69 @@ +#include + +#include "control/atmosphere_driver.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "physics/register_physics.hpp" +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/atm_process/atmosphere_process.hpp" + +#include "ekat/ekat_parse_yaml_file.hpp" +#include "ekat/logging/ekat_logger.hpp" + +#include + +namespace scream { + +TEST_CASE("mam4_wetscav-stand-alone", "") { + using namespace scream; + using namespace scream::control; + + // Create a comm + ekat::Comm atm_comm (MPI_COMM_WORLD); + + // Load ad parameter list + std::string fname = "input.yaml"; + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + logger.debug("yaml parsed."); + + // Time stepping parameters + const auto& ts = ad_params.sublist("time_stepping"); + const auto dt = ts.get("time_step"); + const auto nsteps = ts.get("number_of_steps"); + const auto t0_str = ts.get("run_t0"); + const auto t0 = util::str_to_time_stamp(t0_str); + + logger.info("running MAMWetscav standalone test with dt = {} for {} steps.", dt, nsteps); + EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); + + // Need to register products in the factory *before* we create any atm process or grids manager. + register_physics(); + register_mesh_free_grids_manager(); + register_diagnostics(); + logger.debug("products registered."); + + // Create the driver + AtmosphereDriver ad; + logger.debug("driver created."); + + // Init and run + ad.initialize(atm_comm,ad_params,t0); + logger.debug("driver initialized."); + + logger.info("Start time stepping loop ... [0%]"); + for (int i=0; i Date: Sat, 13 Apr 2024 20:16:45 -0600 Subject: [PATCH 144/904] wet_dep - update to match mam4xx interface. --- .../eamxx_mam_wetscav_process_interface.cpp | 122 ++++++++++++++---- .../eamxx_mam_wetscav_process_interface.hpp | 3 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 48 ++++++- 3 files changed, 147 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 226dcb14d6e2..be6e4d348cba 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -43,6 +43,9 @@ void MAMWetscav::set_grids( // Nevertheless, for output reasons, we like to see 'kg/kg'. auto q_unit = kg / kg; q_unit.set_string("kg/kg"); + // FIXME: units of tendencies; check units + auto dqdt_unit = kg / kg / s; + dqdt_unit.set_string("kg/kg/t"); auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); @@ -52,7 +55,7 @@ void MAMWetscav::set_grids( ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column - + const int nmodes = mam4::AeroConfig::num_modes();; // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -66,6 +69,9 @@ void MAMWetscav::set_grids( FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // Layout for 3D for nmodes + const FieldLayout scalar3d_layout_nmodes_mid{{COL, NMODES, LEV}, {ncol_, nmodes, nlev_}}; + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -180,13 +186,14 @@ void MAMWetscav::set_grids( // -- input/ouputs from PBUF for updating particle size and water uptake by // particles static constexpr auto m3 = m2 * m; - add_field("dgncur_a", scalar3d_layout_mid, m, - grid_name); // aerosol particle diameter [m] - add_field("wetdens", scalar3d_layout_mid, kg / m3, + add_field("dgncur_a", scalar3d_layout_nmodes_mid, m, + grid_name); // aerosol dry particle diameter [m] + add_field("wetdens", scalar3d_layout_nmodes_mid, kg / m3, grid_name); // wet aerosol density [kg/m3] - add_field("qaerwat", scalar3d_layout_mid, kg / kg, + add_field("qaerwat", scalar3d_layout_nmodes_mid, kg / kg, grid_name); // aerosol water [kg/kg] - add_field("dgnumwet", scalar3d_layout_mid, m, + // + add_field("dgnumwet", scalar3d_layout_nmodes_mid, m, grid_name); // wet aerosol diameter [m] add_field("fracis", scalar3d_layout_mid, nondim, grid_name); // fraction of transported species that are @@ -247,6 +254,33 @@ void MAMWetscav::set_grids( grid_name, "tracers"); } + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + std::string ptend_gas_name(mam_coupling::gas_mmr_field_name(g)); + ptend_gas_name += "ptend_"; + add_field(ptend_gas_name, scalar3d_layout_mid, dqdt_unit, + grid_name); + } + + // tendencies for interstitial aerosols + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + std::string ptend_num(mam_coupling::int_aero_nmr_field_name(imode)); + ptend_num += "ptend_"; + add_field(ptend_num, scalar3d_layout_mid, n_unit, + grid_name); + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + std::string ptend_int_mmr_field_name (int_mmr_field_name); + ptend_int_mmr_field_name += "ptend_"; + add_field(ptend_int_mmr_field_name, scalar3d_layout_mid, dqdt_unit, + grid_name); + } + } + } + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // Cloud fraction add_field("pbl_height", scalar2d_layout_mid, m, @@ -271,11 +305,11 @@ void MAMWetscav::set_grids( grid_name); // add_field("sh_ccf", scalar3d_layout_mid, n_unit, grid_name); // - // - add_field("aerdepwetis", scalar3d_layout_mid, n_unit, - grid_name); // - add_field("aerdepwetcw", scalar3d_layout_mid, n_unit, - grid_name); // + // FIXME: do we need to write aerdepwetis and aerdepwetcw? + // add_field("aerdepwetis", scalar3d_layout_mid, n_unit, + // grid_name); // + // add_field("aerdepwetcw", scalar3d_layout_mid, n_unit, + // grid_name); // } // ========================================================================================= @@ -349,7 +383,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.phis = get_field_in("phis").get_view(); - dry_atm_.z_surf = 0.0; // MUST FIXME: for now + dry_atm_.z_surf = 0.0; // ---- set wet/dry aerosol-related gas state data for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); @@ -391,6 +425,33 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } } + + // ---- set aerosol-related gas tendencies data + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + std::string ptend_mmr_field_name(mam_coupling::gas_mmr_field_name(g)); + ptend_mmr_field_name += "ptend_"; + dry_aero_tends_.gas_mmr[g] = get_field_out(ptend_mmr_field_name).get_view(); + } + + // set aerosol state tendencies data (interstitial aerosols only) + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + std::string ptend_int_nmr_field_name(mam_coupling::int_aero_nmr_field_name(imode)); + ptend_int_nmr_field_name +="ptend_"; + dry_aero_tends_.int_aero_nmr[imode] = + get_field_out(ptend_int_nmr_field_name).get_view(); + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + std::string ptend_int_nmr_field_name(int_mmr_field_name); + ptend_int_nmr_field_name +="ptend_"; + dry_aero_tends_.int_aero_mmr[imode][ispec] = + get_field_out(ptend_int_nmr_field_name).get_view(); + } + } + } + // set up our preprocess/postprocess functors // Here we initialize (not compute) objects in preprocess struct using the // objects in the argument list @@ -409,6 +470,8 @@ void MAMWetscav::initialize_impl(const RunType run_type) { const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); + aerdepwetis_ = view_2d("aerdepwetis", ncol_, mam4::aero_model::pcnst); + aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, mam4::aero_model::pcnst); } // ========================================================================================= @@ -454,6 +517,7 @@ void MAMWetscav::run_impl(const double dt) { const auto &dry_aero = dry_aero_; const auto &qqcw_sav = qqcw_sav_; const auto &work = work_; + const auto & dry_aero_tends= dry_aero_tends_; // inputs/outputs auto dlf = get_field_out("dlf").get_view(); @@ -485,8 +549,13 @@ void MAMWetscav::run_impl(const double dt) { .get_view(); // ?? // outputs - const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); - const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); + const auto aerdepwetis = aerdepwetis_;//get_field_out("aerdepwetis").get_view(); + const auto aerdepwetcw = aerdepwetcw_;// get_field_out("aerdepwetcw").get_view(); + + const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); + const auto dry_geometric_mean_diameter_i = get_field_out("dgncur_a").get_view(); + const auto qaerwat = get_field_out("qaerwat").get_view(); + const auto wetdens = get_field_out("wetdens").get_view(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -501,8 +570,8 @@ void MAMWetscav::run_impl(const double dt) { // fetch column-specific subviews into aerosol prognostics mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aero, icol); - // FIXME: we need to create tendecies... - mam4::Tendencies tends{}; + // fetch column-specific subviews into aerosol tendencies + mam4::Tendencies tends = mam_coupling::aerosols_tendencies_for_column(dry_aero_tends, icol); // shallow_convective_cloud_fraction auto cldn_prev_step_icol = ekat::subview(cldn_prev_step, icol); @@ -530,9 +599,9 @@ void MAMWetscav::run_impl(const double dt) { auto cldst_icol = ekat::subview(cldst, icol); // deep_convective_cloud_fraction; - auto dp_ccf_icol = ekat::subview(dp_ccf, icol); - // shallow_convective_cloud_fraction; - auto sh_ccf_icol = ekat::subview(sh_ccf, icol); + // auto dp_ccf_icol = ekat::subview(dp_ccf, icol); + // // shallow_convective_cloud_fraction; + // auto sh_ccf_icol = ekat::subview(sh_ccf, icol); // total_convective_detrainment; auto dlf_icol = ekat::subview(dlf, icol); //aerosol_wet_deposition_interstitial; @@ -541,18 +610,23 @@ void MAMWetscav::run_impl(const double dt) { auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol);\ auto work_icol = ekat::subview(work, icol); // auto qqcw_sav_icol = ekat::subview(qqcw_sav,icol); -#if 1 + auto wet_diameter_icol = ekat::subview(wet_geometric_mean_diameter_i,icol); + auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i,icol); + auto qaerwat_icol = ekat::subview(qaerwat,icol); + auto wetdens_icol = ekat::subview(wetdens,icol); + mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, // inputs - cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, - evapcsh_icol, dp_frac_icol, sh_frac_icol, dp_ccf_icol, sh_ccf_icol, - icwmrdp_col, icwmrsh_icol, evapr_icol, cldst_icol, + cldst_icol, cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, + evapcsh_icol, dp_frac_icol, sh_frac_icol, + icwmrdp_col, icwmrsh_icol, evapr_icol, dlf_icol, + wet_diameter_icol,dry_diameter_icol, + qaerwat_icol, wetdens_icol, // output aerdepwetis_icol, aerdepwetcw_icol, // FIXME remove qqcw_sav qqcw_sav, work_icol); -#endif }); // icol parallel_for loop /* diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index beb3d0c6789f..fc6886357858 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -181,9 +181,10 @@ class MAMWetscav : public scream::AtmosphereProcess { Kokkos::View qqcw_sav_; // aerosol states - mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; mam_coupling::Buffer buffer_; + mam_coupling::view_2d aerdepwetis_, aerdepwetcw_; std::shared_ptr m_grid; }; // class MAMWetscav diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 3a0011468826..99a116d4872e 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -562,6 +562,34 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, return progs; } + +// Given an AerosolState with views for dry aerosol quantities, creates a +// mam4::Tendencies object for the column with the given index with +// ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to +// mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Tendencies interstitial_aerosols_tendencies_for_column(const AerosolState& dry_aero, + const int column_index) { + constexpr int nlev = mam4::nlev; + mam4::Tendencies tends(nlev); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), + "int_aero_nmr not defined for dry aerosol state!"); + tends.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + tends.q_aero_i[m][a] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), + "gas_mmr not defined for dry aerosol state!"); + tends.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); + } + return tends; +} + // Given a dry aerosol state, creates a mam4::Prognostics object for the column // with the given index with interstitial and cloudborne aerosol views defined. // This object can be provided to mam4xx for the column. @@ -581,7 +609,25 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, } return progs; } - +// Given a dry aerosol state tendencies, creates a mam4::Tendencies object for the column +// with the given index with interstitial and cloudborne aerosol views defined. +// This object can be provided to mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Tendencies aerosols_tendencies_for_column(const AerosolState& dry_aero, + const int column_index) { + auto tends = interstitial_aerosols_tendencies_for_column(dry_aero, column_index); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), + "dry_cld_aero_nmr not defined for aerosol state!"); + tends.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.cld_aero_mmr[m][a].data()) { + tends.q_aero_c[m][a] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + } + } + } + return tends; +} // Given a thread team and a dry atmosphere state, dispatches threads from the // team to compute vertical layer heights and interfaces for the column with // the given index. From 1ac385d8c448dcefae71c4529799f1c9fd08695e Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sat, 13 Apr 2024 20:51:59 -0600 Subject: [PATCH 145/904] wet_dep - update mam4xx interface. --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 15 ++++----------- .../mam/eamxx_mam_wetscav_process_interface.hpp | 2 -- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index be6e4d348cba..7fa333f97bdb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -463,15 +463,10 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // wetdep constexpr int pcnst = mam4::aero_model::pcnst; - // FIXME: qqcw_sav_ should be part of work_ - // we need to add columns - qqcw_sav_ =Kokkos::View("qqcw_sav", mam4::nlev); - - const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); - aerdepwetis_ = view_2d("aerdepwetis", ncol_, mam4::aero_model::pcnst); - aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, mam4::aero_model::pcnst); + aerdepwetis_ = view_2d("aerdepwetis", ncol_, pcnst); + aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, pcnst); } // ========================================================================================= @@ -515,7 +510,6 @@ void MAMWetscav::run_impl(const double dt) { // We cannot use member of this class inside of the parallel_for const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; - const auto &qqcw_sav = qqcw_sav_; const auto &work = work_; const auto & dry_aero_tends= dry_aero_tends_; @@ -624,9 +618,8 @@ void MAMWetscav::run_impl(const double dt) { wet_diameter_icol,dry_diameter_icol, qaerwat_icol, wetdens_icol, // output - aerdepwetis_icol, aerdepwetcw_icol, - // FIXME remove qqcw_sav - qqcw_sav, work_icol); + aerdepwetis_icol, aerdepwetcw_icol, + work_icol); }); // icol parallel_for loop /* diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index fc6886357858..c09a789b6c7e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -178,8 +178,6 @@ class MAMWetscav : public scream::AtmosphereProcess { mam_coupling::DryAtmosphere dry_atm_; // wet dep view_2d work_; - Kokkos::View - qqcw_sav_; // aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; From c5c0cfab20ebe02a22eeab1aa0416b2ddba4adae Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sun, 14 Apr 2024 12:14:55 -0600 Subject: [PATCH 146/904] wet_dep - update name of tendencies. --- .../eamxx_mam_wetscav_process_interface.cpp | 34 ++++++++----------- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 7fa333f97bdb..1b8ee2d8e28f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -256,26 +256,23 @@ void MAMWetscav::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_gas_name(mam_coupling::gas_mmr_field_name(g)); - ptend_gas_name += "ptend_"; - add_field(ptend_gas_name, scalar3d_layout_mid, dqdt_unit, + std::string ptend_gas_name = "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); + add_field(ptend_gas_name, scalar3d_layout_mid, dqdt_unit, grid_name); } // tendencies for interstitial aerosols for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_num(mam_coupling::int_aero_nmr_field_name(imode)); - ptend_num += "ptend_"; - add_field(ptend_num, scalar3d_layout_mid, n_unit, + std::string ptend_num = "ptend_" +std::string(mam_coupling::int_aero_nmr_field_name(imode)); + add_field(ptend_num, scalar3d_layout_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_mmr_field_name (int_mmr_field_name); - ptend_int_mmr_field_name += "ptend_"; - add_field(ptend_int_mmr_field_name, scalar3d_layout_mid, dqdt_unit, + std::string ptend_int_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); + add_field(ptend_int_mmr_field_name, scalar3d_layout_mid, dqdt_unit, grid_name); } } @@ -428,15 +425,15 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // ---- set aerosol-related gas tendencies data for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_mmr_field_name(mam_coupling::gas_mmr_field_name(g)); - ptend_mmr_field_name += "ptend_"; + std::string ptend_mmr_field_name = "ptend_" + + std::string(mam_coupling::gas_mmr_field_name(g)); dry_aero_tends_.gas_mmr[g] = get_field_out(ptend_mmr_field_name).get_view(); } // set aerosol state tendencies data (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_int_nmr_field_name(mam_coupling::int_aero_nmr_field_name(imode)); - ptend_int_nmr_field_name +="ptend_"; + std::string ptend_int_nmr_field_name ="ptend_"+ + std::string(mam_coupling::int_aero_nmr_field_name(imode)); dry_aero_tends_.int_aero_nmr[imode] = get_field_out(ptend_int_nmr_field_name).get_view(); @@ -444,10 +441,9 @@ void MAMWetscav::initialize_impl(const RunType run_type) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_nmr_field_name(int_mmr_field_name); - ptend_int_nmr_field_name +="ptend_"; + std::string ptend_int_aero_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); dry_aero_tends_.int_aero_mmr[imode][ispec] = - get_field_out(ptend_int_nmr_field_name).get_view(); + get_field_out(ptend_int_aero_mmr_field_name).get_view(); } } } @@ -511,7 +507,7 @@ void MAMWetscav::run_impl(const double dt) { const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; const auto &work = work_; - const auto & dry_aero_tends= dry_aero_tends_; + const auto &dry_aero_tends= dry_aero_tends_; // inputs/outputs auto dlf = get_field_out("dlf").get_view(); @@ -565,8 +561,8 @@ void MAMWetscav::run_impl(const double dt) { mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aero, icol); // fetch column-specific subviews into aerosol tendencies - mam4::Tendencies tends = mam_coupling::aerosols_tendencies_for_column(dry_aero_tends, icol); - + // Note: we are only updating interstitial aerosols. + mam4::Tendencies tends = mam_coupling::interstitial_aerosols_tendencies_for_column(dry_aero_tends, icol); // shallow_convective_cloud_fraction auto cldn_prev_step_icol = ekat::subview(cldn_prev_step, icol); ///shallow_convective_precipitation_production diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 99a116d4872e..e48b5c9f2e1a 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -618,7 +618,7 @@ mam4::Tendencies aerosols_tendencies_for_column(const AerosolState& dry_aero, auto tends = interstitial_aerosols_tendencies_for_column(dry_aero, column_index); for (int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), - "dry_cld_aero_nmr not defined for aerosol state!"); + "Tendencies : dry_cld_aero_nmr not defined for aerosol state!"); tends.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.cld_aero_mmr[m][a].data()) { From 593c8b169a63f608d7403aafc3ca769a1bcd90f3 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 20 May 2024 14:16:18 -0600 Subject: [PATCH 147/904] wet_scav - rebase code --- .../eamxx_mam_wetscav_process_interface.cpp | 137 +++++++++--------- 1 file changed, 66 insertions(+), 71 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 1b8ee2d8e28f..123de4d95f65 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -55,37 +55,32 @@ void MAMWetscav::set_grids( ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column - const int nmodes = mam4::AeroConfig::num_modes();; - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; + const int nmodes = mam4::AeroConfig::num_modes(); - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and - // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + // layout for 3D (2d horiz X 1d vertical) variables at level midpoints/interfaces + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); - // Layout for 2D (2d horiz) variable defined at mid-level and - // interfaces - const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + // layout for 2D (1d horiz X 1d vertical) variables + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); - FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - - // Layout for 3D for nmodes - const FieldLayout scalar3d_layout_nmodes_mid{{COL, NMODES, LEV}, {ncol_, nmodes, nlev_}}; + // layout for 3D (ncol, nmodes, nlevs) + FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true,nmodes,"nmodes"); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, + add_field("T_mid", scalar3d_mid, K, grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, + add_field("p_mid", scalar3d_mid, Pa, grid_name); // pressure at mid points in [Pa - add_field("pseudo_density", scalar3d_layout_mid, Pa, + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); // pseudo density in [Pa] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); // ice cloud water [kg/kg] wet - add_field("p_int", scalar3d_layout_int, Pa, + add_field("p_int", scalar3d_int, Pa, grid_name); // total pressure // -- Input variables that exists in PBUF in EAM @@ -94,20 +89,20 @@ void MAMWetscav::set_grids( // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. - add_field("cldn_prev_step", scalar3d_layout_mid, nondim, + add_field("cldn_prev_step", scalar3d_mid, nondim, grid_name); // layer cloud fraction [fraction] - add_field("cldst", scalar3d_layout_mid, nondim, + add_field("cldst", scalar3d_mid, nondim, grid_name); //?? - add_field("rprdsh", scalar3d_layout_mid, kg / kg / s, + add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); // rain production, shallow convection // [kg/kg/s] //NOT updated add_field( - "rprddp", scalar3d_layout_mid, kg / kg / s, + "rprddp", scalar3d_mid, kg / kg / s, grid_name); // rain production, deep convection [kg/kg/s] //NOT updated - add_field("evapcsh", scalar3d_layout_mid, kg / kg / s, + add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of shallow convective // //NOT updated precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_layout_mid, kg / kg / s, + add_field("evapcdp", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of deep convective //NOT // updated precipitation >=0. [kg/kg/s] @@ -115,42 +110,42 @@ void MAMWetscav::set_grids( // "inputs" data structure // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. - add_field("cldt_prev_step", scalar3d_layout_mid, nondim, + add_field("cldt_prev_step", scalar3d_mid, nondim, grid_name); // total cloud fraction [fraction] add_field( - "qme", scalar3d_layout_mid, kg / kg / s, + "qme", scalar3d_mid, kg / kg / s, grid_name); // net condensation/evaporation of cloud water [kg/kg/s] - add_field("prain", scalar3d_layout_mid, kg / kg / s, + add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // stratiform rain production rate [kg/kg/s] add_field( - "evapr", scalar3d_layout_mid, kg / kg / s, + "evapr", scalar3d_mid, kg / kg / s, grid_name); // evaporation from stratiform rain [kg/kg/s] //NOT updated // -- Input variables that exists in PBUF in EAM (in wetdep.F90) - add_field("icwmrdp", scalar3d_layout_mid, kg / kg, + add_field("icwmrdp", scalar3d_mid, kg / kg, grid_name); // In cloud water mixing ratio, deep // convection [kg/kg] //NOT updated - add_field("icwmrsh", scalar3d_layout_mid, kg / kg, + add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); // In cloud water mixing ratio, shallow // convection [kg/kg] //NOT updated - add_field("rprddp", scalar3d_layout_mid, kg / kg / s, + add_field("rprddp", scalar3d_mid, kg / kg / s, grid_name); // Rain production, deep convection [kg/kg/s] add_field( - "sh_frac", scalar3d_layout_mid, nondim, + "sh_frac", scalar3d_mid, nondim, grid_name); // Shallow convective cloud fraction [fraction] //NOT updated add_field( - "dp_frac", scalar3d_layout_mid, nondim, + "dp_frac", scalar3d_mid, nondim, grid_name); // Deep convective cloud fraction [fraction] //NOT updated add_field( - "icwmrsh", scalar3d_layout_mid, nondim, + "icwmrsh", scalar3d_mid, nondim, grid_name); add_field( - "icwmrdp", scalar3d_layout_mid, nondim, + "icwmrdp", scalar3d_mid, nondim, grid_name); - add_field("evapcsh", scalar3d_layout_mid, kg / kg / s, + add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_layout_mid, kg / kg / s, + add_field("evapcdp", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of deep convective // precipitation >=0. [kg/kg/s] @@ -162,40 +157,40 @@ void MAMWetscav::set_grids( // for the land model static constexpr auto m2 = m * m; add_field( - "bcphiwet", scalar3d_layout_mid, kg / m2 / s, + "bcphiwet", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of hydrophilic black carbon [kg/m2/s] add_field( - "bcphidry", scalar3d_layout_mid, kg / m2 / s, + "bcphidry", scalar3d_mid, kg / m2 / s, grid_name); // dry deposition of hydrophilic black carbon [kg/m2/s] add_field( - "ocphiwet", scalar3d_layout_mid, kg / m2 / s, + "ocphiwet", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of hydrophilic organic carbon [kg/m2/s] add_field( - "ocphidry", scalar3d_layout_mid, kg / m2 / s, + "ocphidry", scalar3d_mid, kg / m2 / s, grid_name); // dry deposition of hydrophilic organic carbon [kg/m2/s] - add_field("dstwet1", scalar3d_layout_mid, kg / m2 / s, + add_field("dstwet1", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of dust (bin1) [kg/m2/s] - add_field("dstwet2", scalar3d_layout_mid, kg / m2 / s, + add_field("dstwet2", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of dust (bin2) [kg/m2/s] - add_field("dstwet3", scalar3d_layout_mid, kg / m2 / s, + add_field("dstwet3", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of dust (bin3) [kg/m2/s] - add_field("dstwet4", scalar3d_layout_mid, kg / m2 / s, + add_field("dstwet4", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of dust (bin4) [kg/m2/s] // -- input/ouputs from PBUF for updating particle size and water uptake by // particles static constexpr auto m3 = m2 * m; - add_field("dgncur_a", scalar3d_layout_nmodes_mid, m, + add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); // aerosol dry particle diameter [m] - add_field("wetdens", scalar3d_layout_nmodes_mid, kg / m3, + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); // wet aerosol density [kg/m3] - add_field("qaerwat", scalar3d_layout_nmodes_mid, kg / kg, + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); // aerosol water [kg/kg] // - add_field("dgnumwet", scalar3d_layout_nmodes_mid, m, + add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); // wet aerosol diameter [m] - add_field("fracis", scalar3d_layout_mid, nondim, + add_field("fracis", scalar3d_mid, nondim, grid_name); // fraction of transported species that are // insoluble [fraction] @@ -207,7 +202,7 @@ void MAMWetscav::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(imode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios @@ -216,7 +211,7 @@ void MAMWetscav::set_grids( // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { @@ -224,7 +219,7 @@ void MAMWetscav::set_grids( const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } @@ -234,7 +229,7 @@ void MAMWetscav::set_grids( if(strlen(cld_mmr_field_name) > 0) { // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these // are NOT advected - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } @@ -250,21 +245,21 @@ void MAMWetscav::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { std::string ptend_gas_name = "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); - add_field(ptend_gas_name, scalar3d_layout_mid, dqdt_unit, + add_field(ptend_gas_name, scalar3d_mid, dqdt_unit, grid_name); } // tendencies for interstitial aerosols for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { std::string ptend_num = "ptend_" +std::string(mam_coupling::int_aero_nmr_field_name(imode)); - add_field(ptend_num, scalar3d_layout_mid, n_unit, + add_field(ptend_num, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios @@ -272,40 +267,40 @@ void MAMWetscav::set_grids( mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { std::string ptend_int_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); - add_field(ptend_int_mmr_field_name, scalar3d_layout_mid, dqdt_unit, + add_field(ptend_int_mmr_field_name, scalar3d_mid, dqdt_unit, grid_name); } } } - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); // Cloud fraction - add_field("pbl_height", scalar2d_layout_mid, m, + add_field("pbl_height", scalar2d, m, grid_name); // PBL height static constexpr auto s2 = s * s; - add_field("phis", scalar2d_layout_mid, m2 / s2, + add_field("phis", scalar2d, m2 / s2, grid_name); // surface geopotential - add_field("qv", scalar3d_layout_mid, q_unit, + add_field("qv", scalar3d_mid, q_unit, grid_name); // specific humidity - add_field("nc", scalar3d_layout_mid, n_unit, + add_field("nc", scalar3d_mid, n_unit, grid_name); // cloud water number conc - add_field("ni", scalar3d_layout_mid, n_unit, + add_field("ni", scalar3d_mid, n_unit, grid_name); // cloud ice number conc - add_field("omega", scalar3d_layout_mid, Pa / s, + add_field("omega", scalar3d_mid, Pa / s, grid_name); // vertical pressure velocity // FIXME: units - add_field("dlf", scalar3d_layout_mid, n_unit, + add_field("dlf", scalar3d_mid, n_unit, grid_name); // - add_field("dp_ccf", scalar3d_layout_mid, n_unit, + add_field("dp_ccf", scalar3d_mid, n_unit, grid_name); // - add_field("sh_ccf", scalar3d_layout_mid, n_unit, + add_field("sh_ccf", scalar3d_mid, n_unit, grid_name); // // FIXME: do we need to write aerdepwetis and aerdepwetcw? - // add_field("aerdepwetis", scalar3d_layout_mid, n_unit, + // add_field("aerdepwetis", scalar3d_mid, n_unit, // grid_name); // - // add_field("aerdepwetcw", scalar3d_layout_mid, n_unit, + // add_field("aerdepwetcw", scalar3d_mid, n_unit, // grid_name); // } From a76689c251480912e9333078c993c147dd3fa8de Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 20 May 2024 17:06:21 -0600 Subject: [PATCH 148/904] wet_scav - adding missing input variable. --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 123de4d95f65..2f444f366782 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -142,6 +142,10 @@ void MAMWetscav::set_grids( add_field( "icwmrdp", scalar3d_mid, nondim, grid_name); + + add_field( + "prain", scalar3d_mid, nondim, + grid_name); add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] @@ -533,6 +537,9 @@ void MAMWetscav::run_impl(const double dt) { auto icwmrdp = get_field_out("icwmrdp") .get_view(); // ?? + auto prain = get_field_out("prain") + .get_view(); // ?? + // outputs const auto aerdepwetis = aerdepwetis_;//get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = aerdepwetcw_;// get_field_out("aerdepwetcw").get_view(); @@ -599,13 +606,15 @@ void MAMWetscav::run_impl(const double dt) { auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i,icol); auto qaerwat_icol = ekat::subview(qaerwat,icol); auto wetdens_icol = ekat::subview(wetdens,icol); + auto prain_icol = ekat::subview(prain,icol); + mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, // inputs cldst_icol, cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, - dlf_icol, + dlf_icol, prain_icol, wet_diameter_icol,dry_diameter_icol, qaerwat_icol, wetdens_icol, // output From 5cd64aef926b0a7633797e60068445cf78978e36 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 20 May 2024 20:52:16 -0600 Subject: [PATCH 149/904] wet_scav - removing old comments and variables that not needed. --- .../eamxx_mam_wetscav_process_interface.cpp | 371 +++--------------- 1 file changed, 46 insertions(+), 325 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 2f444f366782..9ac0720d8ab3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -2,9 +2,6 @@ #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" #include "scream_config.h" // for SCREAM_CIME_BUILD -// NOTE: see the impl/ directory for the contents of the impl namespace -#include "impl/compute_particle_size.cpp" - // Remove the following<<<< #include #include @@ -65,8 +62,9 @@ void MAMWetscav::set_grids( FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); // layout for 3D (ncol, nmodes, nlevs) - FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true,nmodes,"nmodes"); + FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); + // FieldLayout scalar2d = m_grid->get_2d_scalar_layout // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -89,9 +87,9 @@ void MAMWetscav::set_grids( // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. - add_field("cldn_prev_step", scalar3d_mid, nondim, + add_field("cldn", scalar3d_mid, nondim, grid_name); // layer cloud fraction [fraction] - add_field("cldst", scalar3d_mid, nondim, + add_field("cldt", scalar3d_mid, nondim, grid_name); //?? add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); // rain production, shallow convection @@ -110,11 +108,13 @@ void MAMWetscav::set_grids( // "inputs" data structure // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. - add_field("cldt_prev_step", scalar3d_mid, nondim, - grid_name); // total cloud fraction [fraction] - add_field( - "qme", scalar3d_mid, kg / kg / s, - grid_name); // net condensation/evaporation of cloud water [kg/kg/s] + // add_field("cldt_prev_step", scalar3d_mid, nondim, + // grid_name); // total cloud fraction [fraction] + + // FIXME: we do not need qme + // add_field( + // "qme", scalar3d_mid, kg / kg / s, + // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // stratiform rain production rate [kg/kg/s] add_field( @@ -128,30 +128,27 @@ void MAMWetscav::set_grids( add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); // In cloud water mixing ratio, shallow // convection [kg/kg] //NOT updated - add_field("rprddp", scalar3d_mid, kg / kg / s, - grid_name); // Rain production, deep convection [kg/kg/s] + // add_field("rprddp", scalar3d_mid, kg / kg / s, + // grid_name); // Rain production, deep convection [kg/kg/s] add_field( "sh_frac", scalar3d_mid, nondim, grid_name); // Shallow convective cloud fraction [fraction] //NOT updated add_field( "dp_frac", scalar3d_mid, nondim, grid_name); // Deep convective cloud fraction [fraction] //NOT updated + // in cloud water mixing ratio, deep shallow [kg/kg] add_field( "icwmrsh", scalar3d_mid, nondim, grid_name); - add_field( - "icwmrdp", scalar3d_mid, nondim, - grid_name); - - add_field( - "prain", scalar3d_mid, nondim, - grid_name); - add_field("evapcsh", scalar3d_mid, kg / kg / s, - grid_name); // Evaporation rate of shallow convective - // precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_mid, kg / kg / s, - grid_name); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] + // add_field( + // "icwmrdp", scalar3d_mid, nondim, + // grid_name); + // add_field("evapcsh", scalar3d_mid, kg / kg / s, + // grid_name); // Evaporation rate of shallow convective + // // precipitation >=0. [kg/kg/s] + // add_field("evapcdp", scalar3d_mid, kg / kg / s, + // grid_name); // Evaporation rate of deep convective + // // precipitation >=0. [kg/kg/s] // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process @@ -295,15 +292,17 @@ void MAMWetscav::set_grids( grid_name); // vertical pressure velocity // FIXME: units - add_field("dlf", scalar3d_mid, n_unit, - grid_name); // - add_field("dp_ccf", scalar3d_mid, n_unit, - grid_name); // - add_field("sh_ccf", scalar3d_mid, n_unit, + add_field("dlf", scalar3d_mid, kg / kg / s, grid_name); // + // add_field("dp_ccf", scalar3d_mid, n_unit, + // grid_name); // + // add_field("sh_ccf", scalar3d_mid, n_unit, + // grid_name); // + // FIXME: do we need to write aerdepwetis and aerdepwetcw? // add_field("aerdepwetis", scalar3d_mid, n_unit, // grid_name); // + // add_field("aerdepwetcw", scalar3d_mid, n_unit, // grid_name); // } @@ -460,7 +459,10 @@ void MAMWetscav::initialize_impl(const RunType run_type) { constexpr int pcnst = mam4::aero_model::pcnst; const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); + // FIXME: do I need to move these variables to FM? + // aerosol wet deposition (interstitial) [kg/m2/s] aerdepwetis_ = view_2d("aerdepwetis", ncol_, pcnst); + // aerosol wet deposition (cloud water) [kg/m2/s] aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, pcnst); } @@ -474,35 +476,6 @@ void MAMWetscav::run_impl(const double dt) { // needed by this process or setting up MAM4xx classes and their objects Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - - /*Fortran code: - call modal_aero_calcsize_sub(state%ncol, state%lchnk, state%q, state%pdel, - dt, & !in qqcw, ptend, dgnumdry_m=dgncur_a) !inout - - ----subroutine modal_aero_calcsize_sub(ncol, lchnk, state_q, pdel, deltat, - qqcw, ptend, do_adjust_in, & do_aitacc_transfer_in, list_idx_in, - update_mmr_in, dgnumdry_m) */ - // mam4::CalcSizeProcess process_(aero_config_); //initiate MAM4xx calcsize - // process - - /* ---------------------------------------------------------------------------------------- - * Compute particle size using the calcsize process - * ---------------------------------------------------------------------------------------- - */ - - // -- configure the process - /* - * -- NOTES: 1. Flags for the inter-mode particle transfer - * (do_aitacc_transfer) and size adjustment (do_adjust) are TRUE by default - * a. Size adjustment is only done by changing aerosol - * numbers in the modes. - * 2. Interstitial and cld borne aerosols (i.e. "tends") mmr will - * be updated (update_mmr is TRUE by default) - */ - // MUST: FIXME: Make sure state is not updated...only tendencies should be - // updated!! - // NOTE! we need a const mam_coupling::DryAtmosphere dry_atm for gpu access. - // We cannot use member of this class inside of the parallel_for const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; const auto &work = work_; @@ -510,14 +483,14 @@ void MAMWetscav::run_impl(const double dt) { // inputs/outputs auto dlf = get_field_out("dlf").get_view(); - auto dp_ccf = get_field_out("dp_ccf").get_view(); - auto sh_ccf = get_field_out("sh_ccf").get_view(); + // auto dp_ccf = get_field_out("dp_ccf").get_view(); + // auto sh_ccf = get_field_out("sh_ccf").get_view(); - auto cldn_prev_step = get_field_out("cldn_prev_step").get_view< Real **>(); + auto cldn = get_field_out("cldn").get_view< Real **>(); // where is cldt_prev_step used? auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - auto cldst = get_field_out("cldst").get_view< Real **>();//?? + auto cldt = get_field_out("cldt").get_view< Real **>();//?? auto evapr = get_field_out("evapr").get_view< Real **>(); auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow @@ -540,9 +513,11 @@ void MAMWetscav::run_impl(const double dt) { auto prain = get_field_out("prain") .get_view(); // ?? + const auto aerdepwetis = aerdepwetis_;// + const auto aerdepwetcw = aerdepwetcw_;// // outputs - const auto aerdepwetis = aerdepwetis_;//get_field_out("aerdepwetis").get_view(); - const auto aerdepwetcw = aerdepwetcw_;// get_field_out("aerdepwetcw").get_view(); + // const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); + // const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); const auto dry_geometric_mean_diameter_i = get_field_out("dgncur_a").get_view(); @@ -566,7 +541,7 @@ void MAMWetscav::run_impl(const double dt) { // Note: we are only updating interstitial aerosols. mam4::Tendencies tends = mam_coupling::interstitial_aerosols_tendencies_for_column(dry_aero_tends, icol); // shallow_convective_cloud_fraction - auto cldn_prev_step_icol = ekat::subview(cldn_prev_step, icol); + auto cldn_icol = ekat::subview(cldn, icol); ///shallow_convective_precipitation_production auto rprdsh_icol = ekat::subview(rprdsh, icol); // deep_convective_precipitation_production @@ -581,40 +556,28 @@ void MAMWetscav::run_impl(const double dt) { auto sh_frac_icol = ekat::subview(sh_frac, icol); // FIXME: what is this? - // diags.deep_convective_cloud_condensate auto icwmrdp_col = ekat::subview(icwmrdp, icol); - // diags.shallow_convective_cloud_condensate auto icwmrsh_icol = ekat::subview(icwmrsh, icol); - // diags.evaporation_of_falling_precipitation = auto evapr_icol = ekat::subview(evapr, icol); - // diags.stratiform_cloud_fraction = - auto cldst_icol = ekat::subview(cldst, icol); - - // deep_convective_cloud_fraction; - // auto dp_ccf_icol = ekat::subview(dp_ccf, icol); - // // shallow_convective_cloud_fraction; - // auto sh_ccf_icol = ekat::subview(sh_ccf, icol); - // total_convective_detrainment; + auto cldt_icol = ekat::subview(cldt, icol); + auto dlf_icol = ekat::subview(dlf, icol); - //aerosol_wet_deposition_interstitial; auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); - // diags.aerosol_wet_deposition_cloud_water; auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol);\ auto work_icol = ekat::subview(work, icol); - // auto qqcw_sav_icol = ekat::subview(qqcw_sav,icol); auto wet_diameter_icol = ekat::subview(wet_geometric_mean_diameter_i,icol); auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i,icol); auto qaerwat_icol = ekat::subview(qaerwat,icol); auto wetdens_icol = ekat::subview(wetdens,icol); auto prain_icol = ekat::subview(prain,icol); - mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, // inputs - cldst_icol, cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, + cldt_icol, cldn_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, dlf_icol, prain_icol, + // in/out wet_diameter_icol,dry_diameter_icol, qaerwat_icol, wetdens_icol, // output @@ -622,248 +585,6 @@ void MAMWetscav::run_impl(const double dt) { work_icol); }); // icol parallel_for loop - /* - call calc_sfc_flux(rprdsh(:ncol,:), state%pdel(:ncol,:), - rprdshsum(:ncol)) ! output the last argument call - calc_sfc_flux(rprddp(:ncol,:), state%pdel(:ncol,:), rprddpsum(:ncol)) ! - output the last argument call calc_sfc_flux(evapcsh(:ncol,:), - state%pdel(:ncol,:), evapcshsum(:ncol)) ! output the last argument call - calc_sfc_flux(evapcdp(:ncol,:), state%pdel(:ncol,:), evapcdpsum(:ncol)) ! - output the last argument - - ! initiate variables - qsrflx_mzaer2cnvpr(:,:,:) = 0.0_r8 - aerdepwetis(:,:) = 0.0_r8 - aerdepwetcw(:,:) = 0.0_r8 - qqcw_tmp(:,:) = 0.0_r8 - ! below-cloud scavcoef = 0.0 for cloud-borne species - scavcoefnv(:,:,0) = 0.0_r8 - ! resuspension goes to a different phase or mode - rtscavt_sv(:,:,:) = 0.0_r8 - - ! examine if there is precipitation falling from above in each grid - call examine_prec_exist ( ncol, state%pdel, & ! in - dep_inputs%prain, dep_inputs%cmfdqr,& ! in - dep_inputs%evapr, & ! in - isprx ) ! out - - ! calculate the mass-weighted sol_factic for coarse mode species - call set_f_act_coarse( ncol, & ! in - state%q, ptend%q, dt, & ! in - f_act_conv_coarse, f_act_conv_coarse_dust, & ! out - f_act_conv_coarse_nacl ) ! out - - mmode_loop_aa: do mtmp = 1, ntot_amode ! main loop over aerosol modes - - ! for mam4, do accum, aitken, pcarbon, then coarse - ! so change the order of 3 and 4 here - imode = mode_order_change(mtmp) - - ! loop over interstitial (1) and cloud-borne (2) forms - !BSINGH (09/12/2014):Do cloudborne first for unified convection scheme - so !that the resuspension of cloudborne can be saved then applied to - interstitial (RCE) lphase_loop_aa: do lphase = 2,1,-1 ! do cloudborne (2) - first then interstitial (1) - - if (lphase == 1) then ! interstial aerosol - call modal_aero_bcscavcoef_get( imode, ncol, isprx, dgnumwet, & - scavcoefnv(:,:,1), scavcoefnv(:,:,2) ) - endif - call define_act_frac ( lphase, imode, & ! in - sol_facti, sol_factic, sol_factb, f_act_conv) ! out - - ! REASTER 08/12/2015 - changed ordering (mass then number) for - prevap resuspend to coarse lspec_loop_aa: do lspec = 1, - nspec_amode(imode)+2 ! loop over number + chem constituents + water - - call index_ordering ( & - lspec, imode, lphase, & ! in - jaeronumb, jaeromass, jaerowater, & ! in - mm, jnv, jnummaswtr ) ! out - - if (mm <= 0 .or. jnummaswtr == jaerowater ) cycle ! by pass wet - aerosols - - ! mam_prevap_resusp_optcc values control the prevap_resusp - calculations in wetdepa_v2: ! 0 = no resuspension ! 130 = non-linear - resuspension of aerosol mass based on scavenged aerosol mass ! 230 = - non-linear resuspension of aerosol number based on raindrop number ! the - 130 thru 230 all use the new prevap_resusp code block in subr wetdepa_v2 - ! - mam_prevap_resusp_optcc = mam_prevap_resusp_no - - if ( jnummaswtr == jaeromass ) then ! dry mass - mam_prevap_resusp_optcc = mam_prevap_resusp_mass - elseif ( jnummaswtr == jaeronumb .and. lphase == 1 .and. imode == - modeptr_coarse ) then ! number mam_prevap_resusp_optcc = - mam_prevap_resusp_num endif - - ! set f_act_conv for interstitial (lphase=1) coarse mode species - ! for the convective in-cloud, we conceptually treat the coarse - dust and seasalt ! as being externally mixed, and apply ! f_act_conv = - f_act_conv_coarse_dust/nacl to dust/seasalt ! number and sulfate are - conceptually partitioned to the dust and seasalt ! on a mass basis, so the - f_act_conv for number and sulfate are ! mass-weighted averages of the - values used for dust/seasalt if ((lphase == 1) .and. (imode == - modeptr_coarse)) then f_act_conv = f_act_conv_coarse if (jnummaswtr == - jaeromass) then if (lmassptr_amode(lspec,imode) == - lptr_dust_a_amode(imode)) then f_act_conv = f_act_conv_coarse_dust elseif - (lmassptr_amode(lspec,imode) == lptr_nacl_a_amode(imode)) then f_act_conv = - f_act_conv_coarse_nacl endif endif endif - - lphase_jnmw_conditional: if (lphase == 1) then - ptend%lq(mm) = .true. - ! q_tmp reflects changes from modal_aero_calcsize and is the - "most current" q q_tmp(1:ncol,:) = state%q(1:ncol,:,mm) + - ptend%q(1:ncol,:,mm)*dt !Feed in the saved cloudborne mixing ratios from - phase 2 qqcw_in(:,:) = qqcw_sav(:,:,lspec) - - call wetdepa_v2( & - ncol, dt, state%pdel, & ! in dep_inputs%cmfdqr, - dep_inputs%evapc, dlf, dep_inputs%conicw, & ! in dep_inputs%prain, - dep_inputs%evapr, dep_inputs%totcond, & ! in dep_inputs%cldt, - dep_inputs%cldcu, & ! in dep_inputs%cldvcu, - dep_inputs%cldvst, & ! in sol_factb, sol_facti, - sol_factic, & ! in mam_prevap_resusp_optcc, - .false., scavcoefnv(:,:,jnv), f_act_conv, & ! in q_tmp, qqcw_in(:,:), & ! - in fracis(:,:,mm), dqdt_tmp, iscavt, & ! out - icscavt, isscavt, bcscavt, bsscavt, rcscavt, rsscavt ) ! - out - - ! resuspension goes to coarse mode - call calc_resusp_to_coarse( ncol, mm, & ! in - mmtoo_prevap_resusp, .true., & ! in - rcscavt, rsscavt, & ! in - dqdt_tmp, rtscavt_sv ) ! inout - - ptend%q(1:ncol,:,mm) = ptend%q(1:ncol,:,mm) + - dqdt_tmp(1:ncol,:) - - call outfld( trim(cnst_name(mm))//'WET', dqdt_tmp(:,:), pcols, - lchnk) call outfld( trim(cnst_name(mm))//'SIC', icscavt, pcols, lchnk) call - outfld( trim(cnst_name(mm))//'SIS', isscavt, pcols, lchnk) call outfld( - trim(cnst_name(mm))//'SBC', bcscavt, pcols, lchnk) call outfld( - trim(cnst_name(mm))//'SBS', bsscavt, pcols, lchnk) - - call calc_sfc_flux(dqdt_tmp(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx aerdepwetis(:ncol,mm) = sflx(:ncol) - - call calc_sfc_flux(icscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx sflxic = sflx - - call calc_sfc_flux(isscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name(mm))//'SFSIS', sflx, - pcols, lchnk) - - call calc_sfc_flux(bcscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name(mm))//'SFSBC', sflx, - pcols, lchnk) sflxbc = sflx - - call calc_sfc_flux(bsscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name(mm))//'SFSBS', sflx, - pcols, lchnk) - - ! here the prevap resuspension is in rcscavt & rsscavt and - column integral is written to history !BSINGH(09/15/2014):Following two - nested do-loops are new additions for unified convection - !BSINGH(09/15/2014):After these do-loops, code was added by - RCE, the comments by RCE are kept as it is call - calc_sfc_flux(rcscavt(:ncol,:), state%pdel(:ncol,:), sflx(:ncol)) ! output - sflx sflxec = sflx - - call calc_sfc_flux(rsscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name(mm))//'SFSES', sflx, - pcols, lchnk) - - ! apportion convective surface fluxes to deep and shallow conv - ! this could be done more accurately in subr wetdepa - ! since deep and shallow rarely occur simultaneously, and - these ! fields are just diagnostics, this approximate method is adequate - ! only do this for interstitial aerosol, because conv clouds - to not ! affect the stratiform-cloudborne aerosol call - apportion_sfc_flux_deep ( ncol, & ! in - rprddpsum,rprdshsum,evapcdpsum,evapcshsum,& ! in - sflxbc, sflxec, & ! in - sflxbcdp, sflxecdp ) ! out - - call outfld( trim(cnst_name(mm))//'SFSBD', sflxbcdp, pcols, - lchnk) ! when ma_convproc_intr is used, convective in-cloud wet removal is - done there ! the convective (total and deep) precip-evap-resuspension - includes in- and below-cloud ! contributions, so pass the below-cloud - contribution to ma_convproc_intr qsrflx_mzaer2cnvpr(1:ncol,mm,1) = sflxec( - 1:ncol) qsrflx_mzaer2cnvpr(1:ncol,mm,2) = sflxecdp(1:ncol) - - elseif (lphase == 2) then lphase_jnmw_conditional - ! There is no cloud-borne aerosol water in the model, so this - code block ! should NEVER execute for lspec = nspec_amode(m)+1 (i.e., - jnummaswtr = 2). ! The code only worked because the "do lspec" loop cycles - when lspec = nspec_amode(m)+1, ! but that does not make the code correct. - fldcw => qqcw_get_field(pbuf,mm,lchnk) - qqcw_sav(1:ncol,:,lspec) = fldcw(1:ncol,:) !RCE 2012/01/12 - - ! FIXME: Not sure if this is a bug or not as qqcw_tmp seem - different ! from the previous call and qqcw_tmp is always zero. May need ! - further check. - Shuaiqi Tang in refactoring for MAM4xx call wetdepa_v2( & - ncol, dt, state%pdel, & ! in dep_inputs%cmfdqr, - dep_inputs%evapc, dlf, dep_inputs%conicw, & ! in dep_inputs%prain, - dep_inputs%evapr, dep_inputs%totcond, & ! in dep_inputs%cldt, - dep_inputs%cldcu, & ! in dep_inputs%cldvcu, - dep_inputs%cldvst, & ! in sol_factb, sol_facti, - sol_factic, & ! in mam_prevap_resusp_optcc, - .true., scavcoefnv(:,:,jnv), f_act_conv, & ! in fldcw, qqcw_tmp, & ! in - fracis_cw, dqdt_tmp, iscavt, & ! out icscavt, isscavt, - bcscavt, bsscavt, rcscavt, rsscavt ) ! out - - ! resuspension goes to coarse mode - call calc_resusp_to_coarse( ncol, mm, & ! in - mmtoo_prevap_resusp, .false., & ! in - rcscavt, rsscavt, & ! in - dqdt_tmp, rtscavt_sv ) ! inout - - fldcw(1:ncol,:) = fldcw(1:ncol,:) + dqdt_tmp(1:ncol,:) * dt - - call calc_sfc_flux(dqdt_tmp(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFWET', - sflx, pcols, lchnk) aerdepwetcw(:ncol,mm) = sflx(:ncol) - - call calc_sfc_flux(icscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSIC', - sflx, pcols, lchnk) - - call calc_sfc_flux(isscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSIS', - sflx, pcols, lchnk) - - call calc_sfc_flux(bcscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSBC', - sflx, pcols, lchnk) - - call calc_sfc_flux(bsscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSBS', - sflx, pcols, lchnk) - - call calc_sfc_flux(rcscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSEC', - sflx, pcols, lchnk) - - call calc_sfc_flux(rsscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSES', - sflx, pcols, lchnk) - - endif lphase_jnmw_conditional - - enddo lspec_loop_aa ! lspec = 1, nspec_amode(m)+2 - enddo lphase_loop_aa ! lphase = 1, 2 - enddo mmode_loop_aa ! m = 1, ntot_amode - - ! if the user has specified prescribed aerosol dep fluxes then - ! do not set cam_out dep fluxes according to the prognostic aerosols - if (.not.aerodep_flx_prescribed()) then - call set_srf_wetdep(aerdepwetis, aerdepwetcw, cam_out) - endif - - call wetdep_inputs_unset(dep_inputs) - */ std::cout << "End of wetscav run" << std::endl; } From 711199bac4b1870f284fcc6c0029fda6d9d09ea9 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 23 May 2024 14:26:34 -0600 Subject: [PATCH 150/904] wet_scav - updating code after rebase. --- .../physics/mam/eamxx_mam_wetscav_process_interface.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 9ac0720d8ab3..a3cc31338d8b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -39,13 +39,8 @@ void MAMWetscav::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. auto q_unit = kg / kg; - q_unit.set_string("kg/kg"); - // FIXME: units of tendencies; check units auto dqdt_unit = kg / kg / s; - dqdt_unit.set_string("kg/kg/t"); - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); m_grid = grids_manager->get_grid("Physics"); const auto &grid_name = m_grid->name(); @@ -115,7 +110,7 @@ void MAMWetscav::set_grids( // add_field( // "qme", scalar3d_mid, kg / kg / s, // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] - add_field("prain", scalar3d_mid, kg / kg / s, + add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // stratiform rain production rate [kg/kg/s] add_field( "evapr", scalar3d_mid, kg / kg / s, @@ -489,7 +484,7 @@ void MAMWetscav::run_impl(const double dt) { auto cldn = get_field_out("cldn").get_view< Real **>(); // where is cldt_prev_step used? - auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + // auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? auto cldt = get_field_out("cldt").get_view< Real **>();//?? auto evapr = get_field_out("evapr").get_view< Real **>(); auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow From 1b8052812bc2fa34c0ee02b898a0bd80f13fbc97 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 23 May 2024 16:06:35 -0600 Subject: [PATCH 151/904] wet_scav - Adding missing variables in input file. --- components/eamxx/tests/single-process/mam/wet_scav/input.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index c73e2dbaa9a6..02290fed670b 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -27,11 +27,13 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} cldn_prev_step: 0.5 + cldn: 0.5 rprdsh: 1e-5 rprddp: 1e-5 evapcsh: 1e-5 evapcdp: 1e-5 cldt_prev_step: 1e-5 + cldt: 1e-5 cldst: 1e-5 qme: 1e-5 prain: 1e-5 From 8ec139cc95a0440ab6bd43ac896d36dba3b50b4c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 09:04:48 -0600 Subject: [PATCH 152/904] wet_scav - Moving 'aerdepwetis' and 'aerdepwetcw' to the field manager and adding them to the output.yaml file. --- .../eamxx_mam_wetscav_process_interface.cpp | 33 ++++++++----------- .../single-process/mam/wet_scav/output.yaml | 6 ++-- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index a3cc31338d8b..5542ff23d709 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -47,7 +47,8 @@ void MAMWetscav::set_grids( ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column - const int nmodes = mam4::AeroConfig::num_modes(); + const int nmodes = mam4::AeroConfig::num_modes(); // Number of modes + constexpr int pcnst = mam4::aero_model::pcnst; // layout for 3D (2d horiz X 1d vertical) variables at level midpoints/interfaces FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); @@ -59,7 +60,8 @@ void MAMWetscav::set_grids( // layout for 3D (ncol, nmodes, nlevs) FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); - // FieldLayout scalar2d = m_grid->get_2d_scalar_layout + // layout for 2D (ncol, pcnst) + FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "pcnst"); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -293,13 +295,12 @@ void MAMWetscav::set_grids( // grid_name); // // add_field("sh_ccf", scalar3d_mid, n_unit, // grid_name); // - - // FIXME: do we need to write aerdepwetis and aerdepwetcw? - // add_field("aerdepwetis", scalar3d_mid, n_unit, - // grid_name); // - - // add_field("aerdepwetcw", scalar3d_mid, n_unit, - // grid_name); // + // aerosol wet deposition (interstitial) + add_field("aerdepwetis", scalar2d_pconst, kg/m2/s, + grid_name); // + // aerosol wet deposition (cloud water) + add_field("aerdepwetcw", scalar2d_pconst, kg/m2/s, + grid_name); // } // ========================================================================================= @@ -451,14 +452,9 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_aero_); // wetdep - constexpr int pcnst = mam4::aero_model::pcnst; + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); - // FIXME: do I need to move these variables to FM? - // aerosol wet deposition (interstitial) [kg/m2/s] - aerdepwetis_ = view_2d("aerdepwetis", ncol_, pcnst); - // aerosol wet deposition (cloud water) [kg/m2/s] - aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, pcnst); } // ========================================================================================= @@ -507,12 +503,9 @@ void MAMWetscav::run_impl(const double dt) { auto prain = get_field_out("prain") .get_view(); // ?? - - const auto aerdepwetis = aerdepwetis_;// - const auto aerdepwetcw = aerdepwetcw_;// // outputs - // const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); - // const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); + const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); + const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); const auto dry_geometric_mean_diameter_i = get_field_out("dgncur_a").get_view(); diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 3dc34fdac75d..c63885d069d3 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -5,10 +5,10 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid - + - aerdepwetis + - aerdepwetcw output_control: - Frequency: 2 + Frequency: 2 frequency_units: nsteps MPI Ranks in Filename: true ... From 7ced79ab1557e90210d36ec1997be3037e0019d7 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 09:11:05 -0600 Subject: [PATCH 153/904] wet_scav - Removing headers that are part of mam_coupling and unused views. --- .../mam/eamxx_mam_wetscav_process_interface.hpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index c09a789b6c7e..6f15752953cd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -7,15 +7,6 @@ // For declaring wetscav class derived from atm process class #include "share/atm_process/atmosphere_process.hpp" -// For MAM4 processes -#include - -// For MAM4 calcsize process (FIXME:should we include it in mam4 or mam_coupling??) -#include - -// For wetdep processes -#include - // For component name #include @@ -182,7 +173,6 @@ class MAMWetscav : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; mam_coupling::Buffer buffer_; - mam_coupling::view_2d aerdepwetis_, aerdepwetcw_; std::shared_ptr m_grid; }; // class MAMWetscav From 6b6a79d7be4526d6d7a1376d4faddc4d4ef96c36 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 09:17:16 -0600 Subject: [PATCH 154/904] wet_scav - Adding output fields. --- .../eamxx/tests/single-process/mam/wet_scav/output.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index c63885d069d3..e9ad22d7119f 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -7,6 +7,10 @@ Fields: Field Names: - aerdepwetis - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat output_control: Frequency: 2 frequency_units: nsteps From d39659a7d960f22fcc70b4f227806b16c83e92d0 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 09:54:48 -0600 Subject: [PATCH 155/904] wet_scav - Adding outputs and running test for 3 steps. --- .../single-process/mam/wet_scav/input.yaml | 8 +-- .../single-process/mam/wet_scav/output.yaml | 52 ++++++++++++++++++- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 02290fed670b..698966f0559b 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -6,7 +6,7 @@ driver_options: time_stepping: time_step: ${ATM_TIME_STEP} run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} + number_of_steps: 3 atmosphere_processes: atm_procs_list: [mam4_wetscav] @@ -26,8 +26,8 @@ grids_manager: initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - cldn_prev_step: 0.5 - cldn: 0.5 + cldn_prev_step: 0.5 + cldn: 0.5 rprdsh: 1e-5 rprddp: 1e-5 evapcsh: 1e-5 @@ -35,7 +35,7 @@ initial_conditions: cldt_prev_step: 1e-5 cldt: 1e-5 cldst: 1e-5 - qme: 1e-5 + qme: 1e-5 prain: 1e-5 evapr: 1e-5 icwmrdp: 1e-5 diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index e9ad22d7119f..4912603943d2 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -11,8 +11,58 @@ Fields: - dgncur_a - wetdens - qaerwat + - ptend_bc_a1 + - ptend_bc_a3 + - ptend_bc_a4 + - ptend_dst_a1 + - ptend_dst_a3 + - ptend_mom_a1 + - ptend_mom_a2 + - ptend_mom_a3 + - ptend_mom_a4 + - ptend_nacl_a1 + - ptend_nacl_a2 + - ptend_nacl_a3 + - ptend_num_a1 + - ptend_num_a2 + - ptend_num_a3 + - ptend_num_a4 + - ptend_pom_a1 + - ptend_pom_a3 + - ptend_pom_a4 + - ptend_so4_a1 + - ptend_so4_a2 + - ptend_so4_a3 + - ptend_soa_a1 + - ptend_soa_a2 + - ptend_soa_a3 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - pom_c1 + - pom_c3 + - pom_c4 + - so4_c1 + - so4_c2 + - so4_c3 + - soa_c1 + - soa_c2 + - soa_c3 output_control: - Frequency: 2 + Frequency: 1 frequency_units: nsteps MPI Ranks in Filename: true ... From 6e81baa54c22b4f6d4d8ebed15ada3fb6882f7ed Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 10:03:54 -0600 Subject: [PATCH 156/904] wet_sav- removin unused input. --- components/eamxx/tests/single-process/mam/wet_scav/input.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 698966f0559b..bf7fe8e5380e 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -54,7 +54,6 @@ initial_conditions: dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] - pbuf: 1e-5 # cloud-borne aerosols dgncur_a: 1e-5 # aerosol particle diameter [m] wetdens: 1e-5 # wet aerosol density [kg/m3] qaerwat: 1e-5 # aerosol water [kg/kg] From 5b0b24399625a6f9f5c25e4537db031e8e512d00 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 10:05:59 -0600 Subject: [PATCH 157/904] wet_sav - Adding FIXME. --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 5542ff23d709..037c6d17acb5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -150,7 +150,7 @@ void MAMWetscav::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- - + // FIXME: we have not added code to update the surface fluxes. // -- surface fluxes (input/outpts) for the coupler's cam_out data struture // for the land model static constexpr auto m2 = m * m; From 6e553176938fff232725166274a5b2745bbe416c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 10:11:42 -0600 Subject: [PATCH 158/904] wet_sav - Updating comments. --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 037c6d17acb5..418e8b3bf267 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -11,8 +11,6 @@ Future work: Wirte comments write in/outs for all variables clearly - - */ namespace scream { @@ -474,9 +472,6 @@ void MAMWetscav::run_impl(const double dt) { // inputs/outputs auto dlf = get_field_out("dlf").get_view(); - // auto dp_ccf = get_field_out("dp_ccf").get_view(); - // auto sh_ccf = get_field_out("sh_ccf").get_view(); - auto cldn = get_field_out("cldn").get_view< Real **>(); // where is cldt_prev_step used? @@ -496,13 +491,12 @@ void MAMWetscav::run_impl(const double dt) { .get_view(); // Deep convective cloud fraction [fraction] auto icwmrsh = get_field_out("icwmrsh") - .get_view(); // ?? - + .get_view(); //in cloud water mixing ratio, shallow convection auto icwmrdp = get_field_out("icwmrdp") - .get_view(); // ?? + .get_view(); // in cloud water mixing ratio, deep convection auto prain = get_field_out("prain") - .get_view(); // ?? + .get_view(); // stratiform rain production rate // outputs const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); From 8cd04d2c33b50f0a74d1886881a38faef970c391 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 10:25:10 -0600 Subject: [PATCH 159/904] wet_scav - Setting Frequency=2 --- components/eamxx/tests/single-process/mam/wet_scav/input.yaml | 2 +- components/eamxx/tests/single-process/mam/wet_scav/output.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index bf7fe8e5380e..c8def99dd627 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -6,7 +6,7 @@ driver_options: time_stepping: time_step: ${ATM_TIME_STEP} run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: 3 + number_of_steps: ${NUM_STEPS} atmosphere_processes: atm_procs_list: [mam4_wetscav] diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 4912603943d2..13493a002483 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -62,7 +62,7 @@ Fields: - soa_c2 - soa_c3 output_control: - Frequency: 1 + Frequency: 2 frequency_units: nsteps MPI Ranks in Filename: true ... From e91b2e25fe4ccb4769700025038efbf791707916 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 11:40:17 -0600 Subject: [PATCH 160/904] wet_scav - Adding multi-process test for mam4_wetsav. --- .../dynamics_physics/CMakeLists.txt | 2 +- .../CMakeLists.txt | 95 ++++++++++ .../input.yaml | 107 ++++++++++++ .../output.yaml | 164 ++++++++++++++++++ 4 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml diff --git a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 64ac2187c280..b9b58f2fe86f 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -13,7 +13,7 @@ if (SCREAM_DOUBLE_PRECISION) # initial conditions. #add_subdirectory(homme_mam4xx_pg2) add_subdirectory(mam/homme_shoc_cld_p3_mam_optics_rrtmgp) + add_subdirectory(mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav) endif() endif() endif() - diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..41a915b92656 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt @@ -0,0 +1,95 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 41) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction shoc spa p3 scream_rrtmgp mam ${dynLibName} + LABELS dynamics shoc cld p3 rrtmgp physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 2) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 1) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/init/spa_init_ne2np4.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc + scream/init/${EAMxx_tests_IC_FILE_72lev} + cam/topo/${EAMxx_tests_TOPO_FILE} +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp spa mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml new file mode 100644 index 000000000000..bfbec956d852 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -0,0 +1,107 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_liq_surf_mass: 0.0 + precip_ice_surf_mass: 0.0 + cldn: 0.5 + rprdsh: 1e-5 + rprddp: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + cldt: 1e-5 + prain: 1e-5 + evapr: 1e-5 + icwmrdp: 1e-5 + icwmrsh: 1e-5 + rprddp: 1e-5 + rprdsh: 1e-5 + sh_frac: 1e-5 + dp_frac: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + dgncur_a: 1e-5 # aerosol particle diameter [m] + wetdens: 1e-5 # wet aerosol density [kg/m3] + qaerwat: 1e-5 # aerosol water [kg/kg] + dgnumwet: 1e-5 # wet aerosol diameter [m] + fracis: 1e-5 # fraction of transported species that are insoluble [fraction] + cldfrac_tot: 0.5 + pbl_height: 25.0 + phis : 0.1 + dlf : 0.0 + dp_ccf : 0.0 + sh_ccf : 0.0 + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mac_aero_mic,rrtmgp] + schedule_type: Sequential + Type: Group + mac_aero_mic: + atm_procs_list: [shoc,CldFraction,spa,p3,mam4_wetscav] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + spa: + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc + p3: + max_total_ni: 740.0e3 + shoc: + check_flux_state_consistency: true + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + mam_wetscav: + number_of_subcycles: ${NUM_SUBCYCLES} + compute_tendencies: [all] + lambda_low: 0.001 + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml new file mode 100644 index 000000000000..f85225d1274f --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml @@ -0,0 +1,164 @@ +%YAML 1.1 +--- +filename_prefix: homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav_output +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + # SHOC + - cldfrac_liq + - eddy_diff_mom + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # SPA + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + - nccn + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + HOMME + - horiz_winds + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + HOMME + - T_mid + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + # Diagnostics + - T_mid_at_lev_2 + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_500mb + - T_mid_at_500hPa + - T_mid_at_50000Pa + - PotentialTemperature + - AtmosphereDensity + - Exner + - VirtualTemperature + - z_int + - geopotential_int_at_lev_2 + - z_mid_at_500mb + - geopotential_mid + - dz + - DryStaticEnergy + - SeaLevelPressure + - LiqWaterPath + - IceWaterPath + - VapWaterPath + - RainWaterPath + - RimeWaterPath + - ShortwaveCloudForcing + - LongwaveCloudForcing + - RelativeHumidity + - ZonalVapFlux + - MeridionalVapFlux + - PotentialTemperature_at_model_top + - PotentialTemperature_at_500mb + # MAM4_WETSCAV + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat + - ptend_bc_a1 + - ptend_bc_a3 + - ptend_bc_a4 + - ptend_dst_a1 + - ptend_dst_a3 + - ptend_mom_a1 + - ptend_mom_a2 + - ptend_mom_a3 + - ptend_mom_a4 + - ptend_nacl_a1 + - ptend_nacl_a2 + - ptend_nacl_a3 + - ptend_num_a1 + - ptend_num_a2 + - ptend_num_a3 + - ptend_num_a4 + - ptend_pom_a1 + - ptend_pom_a3 + - ptend_pom_a4 + - ptend_so4_a1 + - ptend_so4_a2 + - ptend_so4_a3 + - ptend_soa_a1 + - ptend_soa_a2 + - ptend_soa_a3 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - pom_c1 + - pom_c3 + - pom_c4 + - so4_c1 + - so4_c2 + - so4_c3 + - soa_c1 + - soa_c2 + - soa_c3 + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 2570f0613d383e2bfad5f375dfae29093651dfe0 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 13:11:10 -0600 Subject: [PATCH 161/904] wet_sav - Adding wetscav to namelist. --- cime_config/tests.py | 1 + .../cime_config/namelist_defaults_scream.xml | 62 +++++++++++++++++++ .../scream/mam4xx/wetscav/shell_commands | 7 +++ 3 files changed, 70 insertions(+) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands diff --git a/cime_config/tests.py b/cime_config/tests.py index 06c16ca176c9..b4fb91cc5ee1 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -724,6 +724,7 @@ "tests" : ( "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", ) }, diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50188236e4e0..66036956f920 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -261,6 +261,10 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/poly_rrtmg_c20240206.nc + + + + @@ -530,6 +534,64 @@ be lost if SCREAM_HACK_XML is not enabled. 0 0.001 900.0 + + + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 0.5 + 25.0 + 0.1 + 0.0 + 0.0 + 0.0 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands new file mode 100644 index 000000000000..821feb10b7d7 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -0,0 +1,7 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,spa,p3,mam4_wetscav" -b From 7b1471db26a6c0c29d4a861b82f9d29bbbc13a0c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sat, 1 Jun 2024 09:36:13 -0600 Subject: [PATCH 162/904] wet_scav - Removing initial values of pbl_height and phis from namelist_defaults_scream.xml. Upgrading MAM4xx. --- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 66036956f920..79fcc64f3848 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -587,8 +587,6 @@ be lost if SCREAM_HACK_XML is not enabled. 1e-5 1e-5 0.5 - 25.0 - 0.1 0.0 0.0 0.0 From 25d4a916f9778707662c7d17286cacbf09eec65d Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sat, 1 Jun 2024 12:07:03 -0600 Subject: [PATCH 163/904] wet_scav - Adding line in CMakeLists. --- .../eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt index dbddf4d26e29..0624c8ed2020 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt @@ -7,7 +7,8 @@ CreateADUnitTest(${TEST_BASE_NAME} LABELS mam4_wetscav physics LIBS mam MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} -) + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} + ) # Set AD configurable options SetVarDependingOnTestSize(NUM_STEPS 2 4 6) From 3e6bf35a2bb579a68c4e9a69fd4038e81ed38094 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sun, 2 Jun 2024 10:08:25 -0600 Subject: [PATCH 164/904] wet_scav - Update interstitial aerosol state. --- .../eamxx_mam_wetscav_process_interface.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 418e8b3bf267..0fa9d1d4f252 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -459,8 +459,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - - printf("Working on wet_sav \n"); // preprocess input -- needs a scan for the calculation of all variables // needed by this process or setting up MAM4xx classes and their objects Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -509,6 +507,9 @@ void MAMWetscav::run_impl(const double dt) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + // Making a local copy of 'nlev_' because we cannot use a member of a class inside a parallel_for. + const int nlev= nlev_; + // loop over atmosphere columns and compute aerosol particle size Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { @@ -565,9 +566,20 @@ void MAMWetscav::run_impl(const double dt) { // output aerdepwetis_icol, aerdepwetcw_icol, work_icol); + team.team_barrier(); + // update interstitial aerosol state + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + const auto q_aero_i = progs.q_aero_i[m][a]; + const auto tends_q_aero_i =tends.q_aero_i[m][a]; + q_aero_i(kk) += tends_q_aero_i(kk) * dt; + } + } + }); + }); // icol parallel_for loop - std::cout << "End of wetscav run" << std::endl; } // ========================================================================================= From a55b7a78144ca774328f526f7119975aa61a5986 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 3 Jun 2024 19:54:35 -0700 Subject: [PATCH 165/904] A working CIME case with wetscav in the process list at the right place --- .../testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands index 821feb10b7d7..f0e96c413985 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -4,4 +4,4 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,spa,p3,mam4_wetscav" -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_wetscav,rrtmgp" -b From 7299556aab7b1bab086632ed9f7ee741b526e383 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 4 Jun 2024 14:09:48 -0700 Subject: [PATCH 166/904] Fixes single process test --- .../mam/wet_scav/CMakeLists.txt | 20 +++--- .../mam/wet_scav/mam4_wetscav_standalone.cpp | 69 ------------------- .../single-process/mam/wet_scav/output.yaml | 3 +- 3 files changed, 9 insertions(+), 83 deletions(-) delete mode 100644 components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp diff --git a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt index 0624c8ed2020..3881df3e059d 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt @@ -2,38 +2,34 @@ include (ScreamUtils) set (TEST_BASE_NAME mam4_wetscav_standalone) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + # Create the test CreateADUnitTest(${TEST_BASE_NAME} LABELS mam4_wetscav physics LIBS mam MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} - ) +) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 2 4 6) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) set (ATM_TIME_STEP 1800) set (RUN_T0 2021-10-12-45000) +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) -# Ensure test input files are present in the data dir -set (TEST_INPUT_FILES - scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc -) -foreach (file IN ITEMS ${TEST_INPUT_FILES}) - GetInputFile(${file}) -endforeach() - # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} - FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} LABELS mam4_wetscav physics META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 @@ -42,6 +38,6 @@ CompareNCFilesFamilyMpi ( if (SCREAM_ENABLE_BASELINE_TESTS) # Compare one of the output files with the baselines. # Note: one is enough, since we already check that np1 is BFB with npX - set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) endif() diff --git a/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp b/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp deleted file mode 100644 index 203b51681d05..000000000000 --- a/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/register_physics.hpp" -#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/logging/ekat_logger.hpp" - -#include - -namespace scream { - -TEST_CASE("mam4_wetscav-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - logger.debug("yaml parsed."); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - logger.info("running MAMWetscav standalone test with dt = {} for {} steps.", dt, nsteps); - EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); - - // Need to register products in the factory *before* we create any atm process or grids manager. - register_physics(); - register_mesh_free_grids_manager(); - register_diagnostics(); - logger.debug("products registered."); - - // Create the driver - AtmosphereDriver ad; - logger.debug("driver created."); - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - logger.debug("driver initialized."); - - logger.info("Start time stepping loop ... [0%]"); - for (int i=0; i Date: Tue, 4 Jun 2024 17:22:27 -0700 Subject: [PATCH 167/904] Fixes file for the mam4xx standalone tests --- .../tests/single-process/mam/wet_scav/input.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index c8def99dd627..f1e0194106a2 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -10,21 +10,20 @@ time_stepping: atmosphere_processes: atm_procs_list: [mam4_wetscav] - mam_wetscav: - number_of_subcycles: ${NUM_SUBCYCLES} - compute_tendencies: [all] - lambda_low: 0.001 grids_manager: Type: Mesh Free - grids_names: [Physics] - Physics: + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: type: point_grid + aliases: [Physics] number_of_global_columns: 218 number_of_vertical_levels: 72 initial_conditions: - Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} cldn_prev_step: 0.5 cldn: 0.5 From f65792df7fb381c141acc85b8b78759caa4f28c7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 10 Jun 2024 05:02:12 -0700 Subject: [PATCH 168/904] Removing uncoupled test files since they are renamed as single process tests --- .../uncoupled/mam4_wet_scav/CMakeLists.txt | 30 -------- .../tests/uncoupled/mam4_wet_scav/input.yaml | 70 ------------------- .../mam4_wet_scav/mam4_wetscav_standalone.cpp | 69 ------------------ .../tests/uncoupled/mam4_wet_scav/output.yaml | 14 ---- 4 files changed, 183 deletions(-) delete mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt delete mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml delete mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp delete mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/output.yaml diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt deleted file mode 100644 index 8558f6fbb056..000000000000 --- a/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -include (ScreamUtils) - -# This test needs to be reworked for microphysics -- currently it's still using -# input for nucleation. - -# Create the test -CreateADUnitTest(mam4_wetscav_standalone - LABELS mam4_wetscav physics - LIBS mam - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} -) - -# Set AD configurable options -set (ATM_TIME_STEP 1) -SetVarDependingOnTestSize(NUM_STEPS 2 2 2) -set (RUN_T0 2021-10-12-45000) - -# ## Copy (and configure) yaml files needed by tests -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) - -# Ensure test input files are present in the data dir -set (TEST_INPUT_FILES - scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc -) -foreach (file IN ITEMS ${TEST_INPUT_FILES}) - GetInputFile(${file}) -endforeach() diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml deleted file mode 100644 index c73e2dbaa9a6..000000000000 --- a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml +++ /dev/null @@ -1,70 +0,0 @@ -%YAML 1.1 ---- -driver_options: - atmosphere_dag_verbosity_level: 5 - -time_stepping: - time_step: ${ATM_TIME_STEP} - run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} - -atmosphere_processes: - atm_procs_list: [mam4_wetscav] - mam_wetscav: - number_of_subcycles: ${NUM_SUBCYCLES} - compute_tendencies: [all] - lambda_low: 0.001 - -grids_manager: - Type: Mesh Free - grids_names: [Physics] - Physics: - type: point_grid - number_of_global_columns: 218 - number_of_vertical_levels: 72 - -initial_conditions: - Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc - topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - cldn_prev_step: 0.5 - rprdsh: 1e-5 - rprddp: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 - cldt_prev_step: 1e-5 - cldst: 1e-5 - qme: 1e-5 - prain: 1e-5 - evapr: 1e-5 - icwmrdp: 1e-5 - icwmrsh: 1e-5 - rprddp: 1e-5 - rprdsh: 1e-5 - sh_frac: 1e-5 - dp_frac: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] - pbuf: 1e-5 # cloud-borne aerosols - dgncur_a: 1e-5 # aerosol particle diameter [m] - wetdens: 1e-5 # wet aerosol density [kg/m3] - qaerwat: 1e-5 # aerosol water [kg/kg] - dgnumwet: 1e-5 # wet aerosol diameter [m] - fracis: 1e-5 # fraction of transported species that are insoluble [fraction] - cldfrac_tot: 0.5 - pbl_height: 25.0 - phis : 0.1 - dlf : 0.0 - dp_ccf : 0.0 - sh_ccf : 0.0 -# The parameters for I/O control -Scorpio: - output_yaml_files: ["output.yaml"] -... diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp b/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp deleted file mode 100644 index 203b51681d05..000000000000 --- a/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/register_physics.hpp" -#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/logging/ekat_logger.hpp" - -#include - -namespace scream { - -TEST_CASE("mam4_wetscav-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - logger.debug("yaml parsed."); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - logger.info("running MAMWetscav standalone test with dt = {} for {} steps.", dt, nsteps); - EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); - - // Need to register products in the factory *before* we create any atm process or grids manager. - register_physics(); - register_mesh_free_grids_manager(); - register_diagnostics(); - logger.debug("products registered."); - - // Create the driver - AtmosphereDriver ad; - logger.debug("driver created."); - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - logger.debug("driver initialized."); - - logger.info("Start time stepping loop ... [0%]"); - for (int i=0; i Date: Thu, 20 Jun 2024 14:42:13 -0700 Subject: [PATCH 169/904] Clang format and makes dp_frac and input, updates mam4xx submodule to point to a branch --- .../eamxx_mam_wetscav_process_interface.cpp | 242 ++++++++++-------- 1 file changed, 131 insertions(+), 111 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 0fa9d1d4f252..9656ac4e2fb6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -1,5 +1,7 @@ -#include #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +#include + #include "scream_config.h" // for SCREAM_CIME_BUILD // Remove the following<<<< @@ -36,19 +38,20 @@ void MAMWetscav::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; + auto q_unit = kg / kg; auto dqdt_unit = kg / kg / s; - auto n_unit = 1 / kg; // units of number mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers m_grid = grids_manager->get_grid("Physics"); const auto &grid_name = m_grid->name(); ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column - const int nmodes = mam4::AeroConfig::num_modes(); // Number of modes + const int nmodes = mam4::AeroConfig::num_modes(); // Number of modes constexpr int pcnst = mam4::aero_model::pcnst; - // layout for 3D (2d horiz X 1d vertical) variables at level midpoints/interfaces + // layout for 3D (2d horiz X 1d vertical) variables at level + // midpoints/interfaces FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); @@ -56,7 +59,8 @@ void MAMWetscav::set_grids( FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); // layout for 3D (ncol, nmodes, nlevs) - FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); + FieldLayout scalar3d_mid_nmodes = + m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); // layout for 2D (ncol, pcnst) FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "pcnst"); @@ -80,12 +84,15 @@ void MAMWetscav::set_grids( static constexpr auto nondim = Units::nondimensional(); // for variables that are fractions etc. + // Deep convective cloud fraction [fraction] //NOT updated + add_field("dp_frac", scalar3d_mid, nondim, grid_name); + // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. add_field("cldn", scalar3d_mid, nondim, - grid_name); // layer cloud fraction [fraction] + grid_name); // layer cloud fraction [fraction] add_field("cldt", scalar3d_mid, nondim, - grid_name); //?? + grid_name); //?? add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); // rain production, shallow convection // [kg/kg/s] //NOT updated @@ -111,7 +118,7 @@ void MAMWetscav::set_grids( // "qme", scalar3d_mid, kg / kg / s, // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] add_field("prain", scalar3d_mid, kg / kg / s, - grid_name); // stratiform rain production rate [kg/kg/s] + grid_name); // stratiform rain production rate [kg/kg/s] add_field( "evapr", scalar3d_mid, kg / kg / s, grid_name); // evaporation from stratiform rain [kg/kg/s] //NOT updated @@ -124,17 +131,14 @@ void MAMWetscav::set_grids( grid_name); // In cloud water mixing ratio, shallow // convection [kg/kg] //NOT updated // add_field("rprddp", scalar3d_mid, kg / kg / s, - // grid_name); // Rain production, deep convection [kg/kg/s] + // grid_name); // Rain production, deep convection + // [kg/kg/s] add_field( "sh_frac", scalar3d_mid, nondim, grid_name); // Shallow convective cloud fraction [fraction] //NOT updated - add_field( - "dp_frac", scalar3d_mid, nondim, - grid_name); // Deep convective cloud fraction [fraction] //NOT updated + // in cloud water mixing ratio, deep shallow [kg/kg] - add_field( - "icwmrsh", scalar3d_mid, nondim, - grid_name); + add_field("icwmrsh", scalar3d_mid, nondim, grid_name); // add_field( // "icwmrdp", scalar3d_mid, nondim, // grid_name); @@ -198,8 +202,8 @@ void MAMWetscav::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(imode); - add_field(int_nmr_field_name, scalar3d_mid, n_unit, - grid_name, "tracers"); + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios const char *cld_nmr_field_name = @@ -207,16 +211,15 @@ void MAMWetscav::set_grids( // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - add_field(cld_nmr_field_name, scalar3d_mid, n_unit, - grid_name); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit, - grid_name, "tracers"); + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -225,8 +228,7 @@ void MAMWetscav::set_grids( if(strlen(cld_mmr_field_name) > 0) { // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these // are NOT advected - add_field(cld_mmr_field_name, scalar3d_mid, q_unit, - grid_name); + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } } @@ -241,30 +243,31 @@ void MAMWetscav::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, - grid_name, "tracers"); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); } // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_gas_name = "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); - add_field(ptend_gas_name, scalar3d_mid, dqdt_unit, - grid_name); + std::string ptend_gas_name = + "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); + add_field(ptend_gas_name, scalar3d_mid, dqdt_unit, grid_name); } // tendencies for interstitial aerosols for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_num = "ptend_" +std::string(mam_coupling::int_aero_nmr_field_name(imode)); - add_field(ptend_num, scalar3d_mid, n_unit, - grid_name); + std::string ptend_num = + "ptend_" + std::string(mam_coupling::int_aero_nmr_field_name(imode)); + add_field(ptend_num, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); + std::string ptend_int_mmr_field_name = + "ptend_" + std::string(int_mmr_field_name); add_field(ptend_int_mmr_field_name, scalar3d_mid, dqdt_unit, - grid_name); + grid_name); } } } @@ -288,16 +291,16 @@ void MAMWetscav::set_grids( // FIXME: units add_field("dlf", scalar3d_mid, kg / kg / s, - grid_name); // + grid_name); // // add_field("dp_ccf", scalar3d_mid, n_unit, // grid_name); // // add_field("sh_ccf", scalar3d_mid, n_unit, // grid_name); // // aerosol wet deposition (interstitial) - add_field("aerdepwetis", scalar2d_pconst, kg/m2/s, + add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); // // aerosol wet deposition (cloud water) - add_field("aerdepwetcw", scalar2d_pconst, kg/m2/s, + add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); // } @@ -417,15 +420,16 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // ---- set aerosol-related gas tendencies data for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_mmr_field_name = "ptend_" + - std::string(mam_coupling::gas_mmr_field_name(g)); - dry_aero_tends_.gas_mmr[g] = get_field_out(ptend_mmr_field_name).get_view(); + std::string ptend_mmr_field_name = + "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); + dry_aero_tends_.gas_mmr[g] = + get_field_out(ptend_mmr_field_name).get_view(); } - // set aerosol state tendencies data (interstitial aerosols only) + // set aerosol state tendencies data (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_int_nmr_field_name ="ptend_"+ - std::string(mam_coupling::int_aero_nmr_field_name(imode)); + std::string ptend_int_nmr_field_name = + "ptend_" + std::string(mam_coupling::int_aero_nmr_field_name(imode)); dry_aero_tends_.int_aero_nmr[imode] = get_field_out(ptend_int_nmr_field_name).get_view(); @@ -433,7 +437,8 @@ void MAMWetscav::initialize_impl(const RunType run_type) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_aero_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); + std::string ptend_int_aero_mmr_field_name = + "ptend_" + std::string(int_mmr_field_name); dry_aero_tends_.int_aero_mmr[imode][ispec] = get_field_out(ptend_int_aero_mmr_field_name).get_view(); } @@ -452,7 +457,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // wetdep const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); - work_ = view_2d("work", ncol_, work_len); + work_ = view_2d("work", ncol_, work_len); } // ========================================================================================= @@ -465,50 +470,65 @@ void MAMWetscav::run_impl(const double dt) { Kokkos::fence(); const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; - const auto &work = work_; - const auto &dry_aero_tends= dry_aero_tends_; + const auto &work = work_; + const auto &dry_aero_tends = dry_aero_tends_; // inputs/outputs - auto dlf = get_field_out("dlf").get_view(); - auto cldn = get_field_out("cldn").get_view< Real **>(); + auto dlf = get_field_out("dlf").get_view(); + auto cldn = get_field_out("cldn").get_view(); // where is cldt_prev_step used? - // auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - auto cldt = get_field_out("cldt").get_view< Real **>();//?? - auto evapr = get_field_out("evapr").get_view< Real **>(); - auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow - - // convection [kg/kg/s] - auto evapcsh = get_field_out("evapcsh").get_view(); // Evaporation rate of shallow convective + // auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); + // //FIXME: Is it same as cldn_prev_step?? + auto cldt = get_field_out("cldt").get_view(); //?? + auto evapr = get_field_out("evapr").get_view(); + auto rprdsh = + get_field_out("rprdsh").get_view(); // rain production, shallow + + // convection [kg/kg/s] + auto evapcsh = + get_field_out("evapcsh") + .get_view(); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] - auto sh_frac = get_field_out("sh_frac").get_view(); // Shallow convective cloud fraction [fraction] - auto rprddp = get_field_out("rprddp").get_view(); // rain production, deep convection [kg/kg/s] - auto evapcdp = get_field_out("evapcdp").get_view(); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] - auto dp_frac = get_field_out("dp_frac") - .get_view(); // Deep convective cloud fraction [fraction] + auto sh_frac = + get_field_out("sh_frac") + .get_view(); // Shallow convective cloud fraction [fraction] + auto rprddp = + get_field_out("rprddp") + .get_view(); // rain production, deep convection [kg/kg/s] + auto evapcdp = + get_field_out("evapcdp") + .get_view(); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + // Deep convective cloud fraction [fraction] + auto dp_frac = get_field_in("dp_frac").get_view(); auto icwmrsh = get_field_out("icwmrsh") - .get_view(); //in cloud water mixing ratio, shallow convection - auto icwmrdp = get_field_out("icwmrdp") - .get_view(); // in cloud water mixing ratio, deep convection + .get_view(); // in cloud water mixing ratio, + // shallow convection + auto icwmrdp = + get_field_out("icwmrdp") + .get_view(); // in cloud water mixing ratio, deep convection auto prain = get_field_out("prain") - .get_view(); // stratiform rain production rate + .get_view(); // stratiform rain production rate // outputs const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); - const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); - const auto dry_geometric_mean_diameter_i = get_field_out("dgncur_a").get_view(); + const auto wet_geometric_mean_diameter_i = + get_field_out("dgnumwet").get_view(); + const auto dry_geometric_mean_diameter_i = + get_field_out("dgncur_a").get_view(); const auto qaerwat = get_field_out("qaerwat").get_view(); const auto wetdens = get_field_out("wetdens").get_view(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - // Making a local copy of 'nlev_' because we cannot use a member of a class inside a parallel_for. - const int nlev= nlev_; + // Making a local copy of 'nlev_' because we cannot use a member of a class + // inside a parallel_for. + const int nlev = nlev_; // loop over atmosphere columns and compute aerosol particle size Kokkos::parallel_for( @@ -522,15 +542,17 @@ void MAMWetscav::run_impl(const double dt) { mam_coupling::aerosols_for_column(dry_aero, icol); // fetch column-specific subviews into aerosol tendencies // Note: we are only updating interstitial aerosols. - mam4::Tendencies tends = mam_coupling::interstitial_aerosols_tendencies_for_column(dry_aero_tends, icol); + mam4::Tendencies tends = + mam_coupling::interstitial_aerosols_tendencies_for_column( + dry_aero_tends, icol); // shallow_convective_cloud_fraction auto cldn_icol = ekat::subview(cldn, icol); - ///shallow_convective_precipitation_production + /// shallow_convective_precipitation_production auto rprdsh_icol = ekat::subview(rprdsh, icol); // deep_convective_precipitation_production auto rprddp_icol = ekat::subview(rprddp, icol); // deep_convective_precipitation_evaporation - auto evapcdp_icol = ekat::subview(evapcdp, icol); + auto evapcdp_icol = ekat::subview(evapcdp, icol); // shallow_convective_precipitation_evaporation = auto evapcsh_icol = ekat::subview(evapcsh, icol); // deep_convective_cloud_fraction @@ -539,47 +561,45 @@ void MAMWetscav::run_impl(const double dt) { auto sh_frac_icol = ekat::subview(sh_frac, icol); // FIXME: what is this? - auto icwmrdp_col = ekat::subview(icwmrdp, icol); + auto icwmrdp_col = ekat::subview(icwmrdp, icol); auto icwmrsh_icol = ekat::subview(icwmrsh, icol); - auto evapr_icol = ekat::subview(evapr, icol); - auto cldt_icol = ekat::subview(cldt, icol); - - auto dlf_icol = ekat::subview(dlf, icol); - auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); - auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol);\ - auto work_icol = ekat::subview(work, icol); - auto wet_diameter_icol = ekat::subview(wet_geometric_mean_diameter_i,icol); - auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i,icol); - auto qaerwat_icol = ekat::subview(qaerwat,icol); - auto wetdens_icol = ekat::subview(wetdens,icol); - auto prain_icol = ekat::subview(prain,icol); - - mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, - // inputs - cldt_icol, cldn_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, - evapcsh_icol, dp_frac_icol, sh_frac_icol, - icwmrdp_col, icwmrsh_icol, evapr_icol, - dlf_icol, prain_icol, - // in/out - wet_diameter_icol,dry_diameter_icol, - qaerwat_icol, wetdens_icol, - // output - aerdepwetis_icol, aerdepwetcw_icol, - work_icol); - team.team_barrier(); - // update interstitial aerosol state - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + auto evapr_icol = ekat::subview(evapr, icol); + auto cldt_icol = ekat::subview(cldt, icol); + + auto dlf_icol = ekat::subview(dlf, icol); + auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); + auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol); + auto work_icol = ekat::subview(work, icol); + auto wet_diameter_icol = + ekat::subview(wet_geometric_mean_diameter_i, icol); + auto dry_diameter_icol = + ekat::subview(dry_geometric_mean_diameter_i, icol); + auto qaerwat_icol = ekat::subview(qaerwat, icol); + auto wetdens_icol = ekat::subview(wetdens, icol); + auto prain_icol = ekat::subview(prain, icol); + + mam4::wetdep::aero_model_wetdep( + team, atm, progs, tends, dt, + // inputs + cldt_icol, cldn_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, + evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, + evapr_icol, dlf_icol, prain_icol, + // in/out + wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, + // output + aerdepwetis_icol, aerdepwetcw_icol, work_icol); + team.team_barrier(); + // update interstitial aerosol state + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - for(int a = 0; a < mam4::num_species_mode(m); ++a) { - const auto q_aero_i = progs.q_aero_i[m][a]; - const auto tends_q_aero_i =tends.q_aero_i[m][a]; - q_aero_i(kk) += tends_q_aero_i(kk) * dt; - } - } - }); - + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + const auto q_aero_i = progs.q_aero_i[m][a]; + const auto tends_q_aero_i = tends.q_aero_i[m][a]; + q_aero_i(kk) += tends_q_aero_i(kk) * dt; + } + } + }); }); // icol parallel_for loop - } // ========================================================================================= From 14efd0d666759460bc440474964e275d478f713f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 20 Jun 2024 19:55:15 -0700 Subject: [PATCH 170/904] Makes cldt intent-in (include changes to the mam4xx submodule) --- .../eamxx_mam_wetscav_process_interface.cpp | 56 ++++++++----------- .../single-process/mam/wet_scav/input.yaml | 8 +-- 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 9656ac4e2fb6..e71bda90ca94 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -81,18 +81,18 @@ void MAMWetscav::set_grids( grid_name); // total pressure // -- Input variables that exists in PBUF in EAM - static constexpr auto nondim = - Units::nondimensional(); // for variables that are fractions etc. + // for variables that are fractions etc. + static constexpr auto nondim = Units::nondimensional(); - // Deep convective cloud fraction [fraction] //NOT updated + // Deep convective cloud fraction [fraction] add_field("dp_frac", scalar3d_mid, nondim, grid_name); - // MUST FIXME: cldt and cldn are the same variables. They must be their - // previous step values. - add_field("cldn", scalar3d_mid, nondim, - grid_name); // layer cloud fraction [fraction] - add_field("cldt", scalar3d_mid, nondim, - grid_name); //?? + // Shallow convective cloud fraction [fraction] //NOT updated + add_field("sh_frac", scalar3d_mid, nondim, grid_name); + + // Total cloud fraction [fraction] + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); + add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); // rain production, shallow convection // [kg/kg/s] //NOT updated @@ -108,10 +108,6 @@ void MAMWetscav::set_grids( // -- Input variables that exists in PBUF in EAM (in wetdep.F90) in the // "inputs" data structure - // MUST FIXME: cldt and cldn are the same variables. They must be their - // previous step values. - // add_field("cldt_prev_step", scalar3d_mid, nondim, - // grid_name); // total cloud fraction [fraction] // FIXME: we do not need qme // add_field( @@ -133,9 +129,6 @@ void MAMWetscav::set_grids( // add_field("rprddp", scalar3d_mid, kg / kg / s, // grid_name); // Rain production, deep convection // [kg/kg/s] - add_field( - "sh_frac", scalar3d_mid, nondim, - grid_name); // Shallow convective cloud fraction [fraction] //NOT updated // in cloud water mixing ratio, deep shallow [kg/kg] add_field("icwmrsh", scalar3d_mid, nondim, grid_name); @@ -473,14 +466,18 @@ void MAMWetscav::run_impl(const double dt) { const auto &work = work_; const auto &dry_aero_tends = dry_aero_tends_; - // inputs/outputs - auto dlf = get_field_out("dlf").get_view(); - auto cldn = get_field_out("cldn").get_view(); + // inputs + // Shallow convective cloud fraction [fraction] + auto sh_frac = get_field_in("sh_frac").get_view(); + + // Deep convective cloud fraction [fraction] + auto dp_frac = get_field_in("dp_frac").get_view(); + + // Total cloud fraction + auto cldt = get_field_in("cldfrac_liq").get_view(); //?? - // where is cldt_prev_step used? - // auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); - // //FIXME: Is it same as cldn_prev_step?? - auto cldt = get_field_out("cldt").get_view(); //?? + // inputs/outputs + auto dlf = get_field_out("dlf").get_view(); auto evapr = get_field_out("evapr").get_view(); auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow @@ -490,9 +487,6 @@ void MAMWetscav::run_impl(const double dt) { get_field_out("evapcsh") .get_view(); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] - auto sh_frac = - get_field_out("sh_frac") - .get_view(); // Shallow convective cloud fraction [fraction] auto rprddp = get_field_out("rprddp") .get_view(); // rain production, deep convection [kg/kg/s] @@ -500,8 +494,6 @@ void MAMWetscav::run_impl(const double dt) { get_field_out("evapcdp") .get_view(); // Evaporation rate of deep convective // precipitation >=0. [kg/kg/s] - // Deep convective cloud fraction [fraction] - auto dp_frac = get_field_in("dp_frac").get_view(); auto icwmrsh = get_field_out("icwmrsh") .get_view(); // in cloud water mixing ratio, @@ -545,8 +537,6 @@ void MAMWetscav::run_impl(const double dt) { mam4::Tendencies tends = mam_coupling::interstitial_aerosols_tendencies_for_column( dry_aero_tends, icol); - // shallow_convective_cloud_fraction - auto cldn_icol = ekat::subview(cldn, icol); /// shallow_convective_precipitation_production auto rprdsh_icol = ekat::subview(rprdsh, icol); // deep_convective_precipitation_production @@ -581,9 +571,9 @@ void MAMWetscav::run_impl(const double dt) { mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, // inputs - cldt_icol, cldn_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, - evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, - evapr_icol, dlf_icol, prain_icol, + cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, + dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, + dlf_icol, prain_icol, // in/out wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, // output diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index f1e0194106a2..0096ea4cc635 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -25,14 +25,10 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - cldn_prev_step: 0.5 - cldn: 0.5 rprdsh: 1e-5 rprddp: 1e-5 evapcsh: 1e-5 evapcdp: 1e-5 - cldt_prev_step: 1e-5 - cldt: 1e-5 cldst: 1e-5 qme: 1e-5 prain: 1e-5 @@ -41,8 +37,8 @@ initial_conditions: icwmrsh: 1e-5 rprddp: 1e-5 rprdsh: 1e-5 - sh_frac: 1e-5 - dp_frac: 1e-5 + sh_frac: 0. + dp_frac: 0. evapcsh: 1e-5 evapcdp: 1e-5 bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] From 6ada930cc3355152519342f26f48372c8f0a8aff Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 20 Jun 2024 21:23:10 -0700 Subject: [PATCH 171/904] Makes some more convective fields intent-in (updates submodule mam4xx) --- .../eamxx_mam_wetscav_process_interface.cpp | 77 +++++++++---------- .../single-process/mam/wet_scav/input.yaml | 12 +-- 2 files changed, 39 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index e71bda90ca94..817b8eeaed7e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -80,8 +80,7 @@ void MAMWetscav::set_grids( add_field("p_int", scalar3d_int, Pa, grid_name); // total pressure - // -- Input variables that exists in PBUF in EAM - // for variables that are fractions etc. + // For variables that are non dimensional (e.g., fractions etc.) static constexpr auto nondim = Units::nondimensional(); // Deep convective cloud fraction [fraction] @@ -93,18 +92,21 @@ void MAMWetscav::set_grids( // Total cloud fraction [fraction] add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); - add_field("rprdsh", scalar3d_mid, kg / kg / s, - grid_name); // rain production, shallow convection - // [kg/kg/s] //NOT updated - add_field( - "rprddp", scalar3d_mid, kg / kg / s, - grid_name); // rain production, deep convection [kg/kg/s] //NOT updated - add_field("evapcsh", scalar3d_mid, kg / kg / s, - grid_name); // Evaporation rate of shallow convective - // //NOT updated precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_mid, kg / kg / s, - grid_name); // Evaporation rate of deep convective //NOT - // updated precipitation >=0. [kg/kg/s] + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); + + // Evaporation rate of deep precipitation >=0. [kg/kg/s] + add_field("evapcdp", scalar3d_mid, kg / kg / s, grid_name); + + // Rain production, shallow convection [kg/kg/s] + add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); + + // Rain production, deep convection [kg/kg/s] + add_field("rprddp", scalar3d_mid, kg / kg / s, grid_name); + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // ------------------------------------------------------------------------------------------------------------------------- // -- Input variables that exists in PBUF in EAM (in wetdep.F90) in the // "inputs" data structure @@ -126,22 +128,12 @@ void MAMWetscav::set_grids( add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); // In cloud water mixing ratio, shallow // convection [kg/kg] //NOT updated - // add_field("rprddp", scalar3d_mid, kg / kg / s, - // grid_name); // Rain production, deep convection - // [kg/kg/s] // in cloud water mixing ratio, deep shallow [kg/kg] add_field("icwmrsh", scalar3d_mid, nondim, grid_name); // add_field( // "icwmrdp", scalar3d_mid, nondim, // grid_name); - // add_field("evapcsh", scalar3d_mid, kg / kg / s, - // grid_name); // Evaporation rate of shallow convective - // // precipitation >=0. [kg/kg/s] - // add_field("evapcdp", scalar3d_mid, kg / kg / s, - // grid_name); // Evaporation rate of deep convective - // // precipitation >=0. [kg/kg/s] - // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -466,7 +458,9 @@ void MAMWetscav::run_impl(const double dt) { const auto &work = work_; const auto &dry_aero_tends = dry_aero_tends_; - // inputs + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- // Shallow convective cloud fraction [fraction] auto sh_frac = get_field_in("sh_frac").get_view(); @@ -474,26 +468,25 @@ void MAMWetscav::run_impl(const double dt) { auto dp_frac = get_field_in("dp_frac").get_view(); // Total cloud fraction - auto cldt = get_field_in("cldfrac_liq").get_view(); //?? + auto cldt = get_field_in("cldfrac_liq").get_view(); + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + auto evapcsh = get_field_in("evapcsh").get_view(); + + // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] + auto evapcdp = get_field_in("evapcdp").get_view(); - // inputs/outputs + // Rain production, shallow convection [kg/kg/s] + auto rprdsh = get_field_in("rprdsh").get_view(); + + // Rain production, deep convection [kg/kg/s] + auto rprddp = get_field_in("rprddp").get_view(); + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "Updated" or pure inputs/outputs for the process + // ------------------------------------------------------------------------------------------------------------------------- auto dlf = get_field_out("dlf").get_view(); auto evapr = get_field_out("evapr").get_view(); - auto rprdsh = - get_field_out("rprdsh").get_view(); // rain production, shallow - - // convection [kg/kg/s] - auto evapcsh = - get_field_out("evapcsh") - .get_view(); // Evaporation rate of shallow convective - // precipitation >=0. [kg/kg/s] - auto rprddp = - get_field_out("rprddp") - .get_view(); // rain production, deep convection [kg/kg/s] - auto evapcdp = - get_field_out("evapcdp") - .get_view(); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] auto icwmrsh = get_field_out("icwmrsh") .get_view(); // in cloud water mixing ratio, diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 0096ea4cc635..46bf2db5740b 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -25,22 +25,18 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - rprdsh: 1e-5 - rprddp: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 + rprdsh: 0. + rprddp: 0. + evapcsh: 0. + evapcdp: 0. cldst: 1e-5 qme: 1e-5 prain: 1e-5 evapr: 1e-5 icwmrdp: 1e-5 icwmrsh: 1e-5 - rprddp: 1e-5 - rprdsh: 1e-5 sh_frac: 0. dp_frac: 0. - evapcsh: 1e-5 - evapcdp: 1e-5 bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] From 04ae76410354e6b4d7a5c86ebacbdac6f99ace01 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 21 Jun 2024 07:01:01 -0700 Subject: [PATCH 172/904] Fixes some more vars to be intent-in (mam4xx submodule updated) --- .../eamxx_mam_wetscav_process_interface.cpp | 46 +++++++++---------- .../single-process/mam/wet_scav/input.yaml | 4 +- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 817b8eeaed7e..ac8a1fae4419 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -104,6 +104,16 @@ void MAMWetscav::set_grids( // Rain production, deep convection [kg/kg/s] add_field("rprddp", scalar3d_mid, kg / kg / s, grid_name); + // In cloud water mixing ratio, deep convection [kg/kg] + add_field("icwmrdp", scalar3d_mid, kg / kg, grid_name); + + // In cloud water mixing ratio, shallow convection [kg/kg] + add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); + + // evaporation from stratiform rain [kg/kg/s] + // FIXME: Get it from P3 + add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -117,23 +127,9 @@ void MAMWetscav::set_grids( // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // stratiform rain production rate [kg/kg/s] - add_field( - "evapr", scalar3d_mid, kg / kg / s, - grid_name); // evaporation from stratiform rain [kg/kg/s] //NOT updated // -- Input variables that exists in PBUF in EAM (in wetdep.F90) - add_field("icwmrdp", scalar3d_mid, kg / kg, - grid_name); // In cloud water mixing ratio, deep - // convection [kg/kg] //NOT updated - add_field("icwmrsh", scalar3d_mid, kg / kg, - grid_name); // In cloud water mixing ratio, shallow - // convection [kg/kg] //NOT updated - - // in cloud water mixing ratio, deep shallow [kg/kg] - add_field("icwmrsh", scalar3d_mid, nondim, grid_name); - // add_field( - // "icwmrdp", scalar3d_mid, nondim, - // grid_name); + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -482,18 +478,20 @@ void MAMWetscav::run_impl(const double dt) { // Rain production, deep convection [kg/kg/s] auto rprddp = get_field_in("rprddp").get_view(); + // In cloud water mixing ratio, deep convection + auto icwmrdp = get_field_in("icwmrdp").get_view(); + + // In cloud water mixing ratio, shallow convection + auto icwmrsh = get_field_in("icwmrsh").get_view(); + + // evaporation from stratiform rain [kg/kg/s] + // FIXME: Get it from P3 + auto evapr = get_field_in("evapr").get_view(); + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "Updated" or pure inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- - auto dlf = get_field_out("dlf").get_view(); - auto evapr = get_field_out("evapr").get_view(); - - auto icwmrsh = get_field_out("icwmrsh") - .get_view(); // in cloud water mixing ratio, - // shallow convection - auto icwmrdp = - get_field_out("icwmrdp") - .get_view(); // in cloud water mixing ratio, deep convection + auto dlf = get_field_out("dlf").get_view(); auto prain = get_field_out("prain") .get_view(); // stratiform rain production rate diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 46bf2db5740b..13447577cf8f 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -33,8 +33,8 @@ initial_conditions: qme: 1e-5 prain: 1e-5 evapr: 1e-5 - icwmrdp: 1e-5 - icwmrsh: 1e-5 + icwmrdp: 0. + icwmrsh: 0. sh_frac: 0. dp_frac: 0. bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] From b4ffbe9b0ce30a7d1779c472506a26939f37643a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 12:53:07 -0700 Subject: [PATCH 173/904] Moved around variables to match I/O-1 --- .../eamxx_mam_wetscav_process_interface.cpp | 181 ++++++++++-------- 1 file changed, 97 insertions(+), 84 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index ac8a1fae4419..b003cb22e855 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -64,34 +64,72 @@ void MAMWetscav::set_grids( // layout for 2D (ncol, pcnst) FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "pcnst"); - // ------------------------------------------------------------------------------------------------------------------------- + + // -------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_mid, Pa, - grid_name); // pressure at mid points in [Pa - add_field("pseudo_density", scalar3d_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qc", scalar3d_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("p_int", scalar3d_int, Pa, - grid_name); // total pressure + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg] + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + // planetary boundary layer height + add_field("pbl_height", scalar2d, m, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] (FIXME: Get it from P3) + add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); + + // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) + add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // For variables that are non dimensional (e.g., fractions etc.) static constexpr auto nondim = Units::nondimensional(); + //----------- Variables from macrophysics scheme ------------- + + // Total cloud fraction [fraction] + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); + + //----------- Variables from convective scheme ------------- + + // Following variables are from convective parameterization (not implemented + // yet in EAMxx), so should be zero for now + // Deep convective cloud fraction [fraction] add_field("dp_frac", scalar3d_mid, nondim, grid_name); // Shallow convective cloud fraction [fraction] //NOT updated add_field("sh_frac", scalar3d_mid, nondim, grid_name); - // Total cloud fraction [fraction] - add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); - // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); @@ -110,26 +148,6 @@ void MAMWetscav::set_grids( // In cloud water mixing ratio, shallow convection [kg/kg] add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); - // evaporation from stratiform rain [kg/kg/s] - // FIXME: Get it from P3 - add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); - - // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "updated" or inputs/outputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - - // -- Input variables that exists in PBUF in EAM (in wetdep.F90) in the - // "inputs" data structure - - // FIXME: we do not need qme - // add_field( - // "qme", scalar3d_mid, kg / kg / s, - // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] - add_field("prain", scalar3d_mid, kg / kg / s, - grid_name); // stratiform rain production rate [kg/kg/s] - - // -- Input variables that exists in PBUF in EAM (in wetdep.F90) - // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -137,48 +155,37 @@ void MAMWetscav::set_grids( // -- surface fluxes (input/outpts) for the coupler's cam_out data struture // for the land model static constexpr auto m2 = m * m; - add_field( - "bcphiwet", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of hydrophilic black carbon [kg/m2/s] - add_field( - "bcphidry", scalar3d_mid, kg / m2 / s, - grid_name); // dry deposition of hydrophilic black carbon [kg/m2/s] - add_field( - "ocphiwet", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of hydrophilic organic carbon [kg/m2/s] - add_field( - "ocphidry", scalar3d_mid, kg / m2 / s, - grid_name); // dry deposition of hydrophilic organic carbon [kg/m2/s] - - add_field("dstwet1", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of dust (bin1) [kg/m2/s] - add_field("dstwet2", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of dust (bin2) [kg/m2/s] - add_field("dstwet3", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of dust (bin3) [kg/m2/s] - add_field("dstwet4", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of dust (bin4) [kg/m2/s] - - // -- input/ouputs from PBUF for updating particle size and water uptake by - // particles - static constexpr auto m3 = m2 * m; - add_field("dgncur_a", scalar3d_mid_nmodes, m, - grid_name); // aerosol dry particle diameter [m] - add_field("wetdens", scalar3d_mid_nmodes, kg / m3, - grid_name); // wet aerosol density [kg/m3] - add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, - grid_name); // aerosol water [kg/kg] - // - add_field("dgnumwet", scalar3d_mid_nmodes, m, - grid_name); // wet aerosol diameter [m] - add_field("fracis", scalar3d_mid, nondim, - grid_name); // fraction of transported species that are - // insoluble [fraction] - // -- interstitial and cloudborne aerosol tracers of interest: mass (q) and + // Wet deposition of hydrophilic black carbon [kg/m2/s] + add_field("bcphiwet", scalar3d_mid, kg / m2 / s, grid_name); + + // Dry deposition of hydrophilic black carbon [kg/m2/s] + add_field("bcphidry", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of hydrophilic organic carbon [kg/m2/s] + add_field("ocphiwet", scalar3d_mid, kg / m2 / s, grid_name); + + // Dry deposition of hydrophilic organic carbon [kg/m2/s] + add_field("ocphidry", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin1) [kg/m2/s] + add_field("dstwet1", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin2) [kg/m2/s] + add_field("dstwet2", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin3) [kg/m2/s] + add_field("dstwet3", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin4) [kg/m2/s] + add_field("dstwet4", scalar3d_mid, kg / m2 / s, grid_name); + + // Interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios - // -- NOTE: Interstitial aerosols are updated in the interface using the + + // NOTE: Interstitial aerosols are updated in the interface using the // "tendencies" from the wetscavenging process + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = @@ -228,6 +235,20 @@ void MAMWetscav::set_grids( "tracers"); } + static constexpr auto m3 = m2 * m; + add_field("dgncur_a", scalar3d_mid_nmodes, m, + grid_name); // aerosol dry particle diameter [m] + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, + grid_name); // wet aerosol density [kg/m3] + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, + grid_name); // aerosol water [kg/kg] + // + add_field("dgnumwet", scalar3d_mid_nmodes, m, + grid_name); // wet aerosol diameter [m] + add_field("fracis", scalar3d_mid, nondim, + grid_name); // fraction of transported species that are + // insoluble [fraction] + // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { std::string ptend_gas_name = @@ -261,14 +282,6 @@ void MAMWetscav::set_grids( static constexpr auto s2 = s * s; add_field("phis", scalar2d, m2 / s2, grid_name); // surface geopotential - add_field("qv", scalar3d_mid, q_unit, - grid_name); // specific humidity - add_field("nc", scalar3d_mid, n_unit, - grid_name); // cloud water number conc - add_field("ni", scalar3d_mid, n_unit, - grid_name); // cloud ice number conc - add_field("omega", scalar3d_mid, Pa / s, - grid_name); // vertical pressure velocity // FIXME: units add_field("dlf", scalar3d_mid, kg / kg / s, From abd75db1ca56d4a1ebc71ebf0365be03521e4a83 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 19:27:55 -0700 Subject: [PATCH 174/904] Removes ptend from output FM and some cleanup for I/O-2 --- .../eamxx_mam_wetscav_process_interface.cpp | 94 ++++++------------- .../single-process/mam/wet_scav/output.yaml | 25 ----- 2 files changed, 28 insertions(+), 91 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index b003cb22e855..148353e793ea 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -148,9 +148,9 @@ void MAMWetscav::set_grids( // In cloud water mixing ratio, shallow convection [kg/kg] add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); - // ------------------------------------------------------------------------------------------------------------------------- + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process - // ------------------------------------------------------------------------------------------------------------------------- + // --------------------------------------------------------------------- // FIXME: we have not added code to update the surface fluxes. // -- surface fluxes (input/outpts) for the coupler's cam_out data struture // for the land model @@ -235,47 +235,26 @@ void MAMWetscav::set_grids( "tracers"); } + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- static constexpr auto m3 = m2 * m; - add_field("dgncur_a", scalar3d_mid_nmodes, m, - grid_name); // aerosol dry particle diameter [m] - add_field("wetdens", scalar3d_mid_nmodes, kg / m3, - grid_name); // wet aerosol density [kg/m3] - add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, - grid_name); // aerosol water [kg/kg] - // - add_field("dgnumwet", scalar3d_mid_nmodes, m, - grid_name); // wet aerosol diameter [m] - add_field("fracis", scalar3d_mid, nondim, - grid_name); // fraction of transported species that are - // insoluble [fraction] - // aerosol-related gases: mass mixing ratios - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_gas_name = - "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); - add_field(ptend_gas_name, scalar3d_mid, dqdt_unit, grid_name); - } + // Aerosol dry particle diameter [m] + add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); - // tendencies for interstitial aerosols - for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_num = - "ptend_" + std::string(mam_coupling::int_aero_nmr_field_name(imode)); - add_field(ptend_num, scalar3d_mid, n_unit, grid_name); - for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(imode, ispec); - if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_mmr_field_name = - "ptend_" + std::string(int_mmr_field_name); - add_field(ptend_int_mmr_field_name, scalar3d_mid, dqdt_unit, - grid_name); - } - } - } + // wet aerosol density [kg/m3] + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); + + // aerosol water [kg/kg] + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); + + // wet aerosol diameter [m] + add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); + + // fraction of transported species that are insoluble [fraction] + add_field("fracis", scalar3d_mid, nondim, grid_name); - add_field("cldfrac_tot", scalar3d_mid, nondim, - grid_name); // Cloud fraction add_field("pbl_height", scalar2d, m, grid_name); // PBL height @@ -283,13 +262,9 @@ void MAMWetscav::set_grids( add_field("phis", scalar2d, m2 / s2, grid_name); // surface geopotential - // FIXME: units - add_field("dlf", scalar3d_mid, kg / kg / s, - grid_name); // - // add_field("dp_ccf", scalar3d_mid, n_unit, - // grid_name); // - // add_field("sh_ccf", scalar3d_mid, n_unit, - // grid_name); // + add_field("dlf", scalar3d_mid, kg / kg / s, + grid_name); // + // aerosol wet deposition (interstitial) add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); // @@ -366,7 +341,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // The following dry_atm_ members *may* not be used by the process but they // are needed for creating MAM4xx class objects like Atmosphere - dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_liq").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.phis = get_field_in("phis").get_view(); dry_atm_.z_surf = 0.0; @@ -412,30 +387,17 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } - // ---- set aerosol-related gas tendencies data + // Alllocate aerosol-related gas tendencies for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_mmr_field_name = - "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); - dry_aero_tends_.gas_mmr[g] = - get_field_out(ptend_mmr_field_name).get_view(); + Kokkos::resize(dry_aero_tends_.gas_mmr[g], ncol_, nlev_); } - // set aerosol state tendencies data (interstitial aerosols only) + // Allocate aerosol state tendencies (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_int_nmr_field_name = - "ptend_" + std::string(mam_coupling::int_aero_nmr_field_name(imode)); - dry_aero_tends_.int_aero_nmr[imode] = - get_field_out(ptend_int_nmr_field_name).get_view(); + Kokkos::resize(dry_aero_tends_.int_aero_nmr[imode], ncol_, nlev_); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(imode, ispec); - if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_aero_mmr_field_name = - "ptend_" + std::string(int_mmr_field_name); - dry_aero_tends_.int_aero_mmr[imode][ispec] = - get_field_out(ptend_int_aero_mmr_field_name).get_view(); - } + Kokkos::resize(dry_aero_tends_.int_aero_mmr[imode][ispec], ncol_, nlev_); } } @@ -504,7 +466,7 @@ void MAMWetscav::run_impl(const double dt) { // ------------------------------------------------------------------------------------------------------------------------- // These variables are "Updated" or pure inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- - auto dlf = get_field_out("dlf").get_view(); + auto dlf = get_field_in("dlf").get_view(); auto prain = get_field_out("prain") .get_view(); // stratiform rain production rate diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index b9f8bc107fc1..9cd0a16bc414 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -11,31 +11,6 @@ Fields: - dgncur_a - wetdens - qaerwat - - ptend_bc_a1 - - ptend_bc_a3 - - ptend_bc_a4 - - ptend_dst_a1 - - ptend_dst_a3 - - ptend_mom_a1 - - ptend_mom_a2 - - ptend_mom_a3 - - ptend_mom_a4 - - ptend_nacl_a1 - - ptend_nacl_a2 - - ptend_nacl_a3 - - ptend_num_a1 - - ptend_num_a2 - - ptend_num_a3 - - ptend_num_a4 - - ptend_pom_a1 - - ptend_pom_a3 - - ptend_pom_a4 - - ptend_so4_a1 - - ptend_so4_a2 - - ptend_so4_a3 - - ptend_soa_a1 - - ptend_soa_a2 - - ptend_soa_a3 - bc_c1 - bc_c3 - bc_c4 From 6c819c8aba0687b8f9ec9a848448af360bc70b80 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 19:43:06 -0700 Subject: [PATCH 175/904] Done with the initialize_impl method cleanup --- .../eamxx_mam_wetscav_process_interface.cpp | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 148353e793ea..daa203ab7b88 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -100,9 +100,15 @@ void MAMWetscav::set_grids( // Layer thickness(pdel) [Pa] at midpoints add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - // planetary boundary layer height + // planetary boundary layer height [m] add_field("pbl_height", scalar2d, m, grid_name); + static constexpr auto m2 = m * m; + static constexpr auto s2 = s * s; + + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); + //----------- Variables from microphysics scheme ------------- // Evaporation from stratiform rain [kg/kg/s] (FIXME: Get it from P3) @@ -148,13 +154,15 @@ void MAMWetscav::set_grids( // In cloud water mixing ratio, shallow convection [kg/kg] add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); + // Detraining cld H20 from deep convection [kg/ks/s] + add_field("dlf", scalar3d_mid, kg / kg / s, grid_name); + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- // FIXME: we have not added code to update the surface fluxes. // -- surface fluxes (input/outpts) for the coupler's cam_out data struture // for the land model - static constexpr auto m2 = m * m; // Wet deposition of hydrophilic black carbon [kg/m2/s] add_field("bcphiwet", scalar3d_mid, kg / m2 / s, grid_name); @@ -255,22 +263,10 @@ void MAMWetscav::set_grids( // fraction of transported species that are insoluble [fraction] add_field("fracis", scalar3d_mid, nondim, grid_name); - add_field("pbl_height", scalar2d, m, - grid_name); // PBL height - - static constexpr auto s2 = s * s; - add_field("phis", scalar2d, m2 / s2, - grid_name); // surface geopotential - - add_field("dlf", scalar3d_mid, kg / kg / s, - grid_name); // - - // aerosol wet deposition (interstitial) - add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, - grid_name); // - // aerosol wet deposition (cloud water) - add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, - grid_name); // + // aerosol wet deposition (interstitial) [kg/m2/s] + add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); + // aerosol wet deposition (cloud water) [kg/m2/s] + add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); } // ========================================================================================= From 3e73b8ffc0fa82c734421f4a42dbbc9ed5e1a001 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 22:38:54 -0700 Subject: [PATCH 176/904] Rebased on current master --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 5 ++--- externals/mam4xx | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index daa203ab7b88..ef7661c2ba85 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -297,7 +297,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); // populate the wet atmosphere state with views from fields and - // the buffer (NOTE: wet atmosphere only has qv, qc, qi, nc, ni and omega) + // the buffer (NOTE: wet atmosphere only has qv, qc, qi, nc, ni) wet_atm_.qc = get_field_in("qc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); @@ -307,8 +307,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { wet_atm_.qv = get_field_in("qv").get_view(); wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); - // populate the dry atmosphere state with views from fields // (NOTE: dry atmosphere has everything that wet // atmosphere has along with z_surf, T_mid, p_mid, z_mid, z_iface, @@ -317,6 +315,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); // How "buffer_" works: We use buffer to allocate memory for the members of // dry_atm_ object. Here we are providing those memory locations to the // dry_atm_ members. These members are computed from the above wet_atm_ or diff --git a/externals/mam4xx b/externals/mam4xx index a8aa56ba915c..1b5f3b6f4cd9 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e +Subproject commit 1b5f3b6f4cd90b18869e20acc4e68663b3c4e0e8 From f52f6284305ae10a87ad9c2d46d686a9a9811dae Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Jul 2024 06:47:26 -0700 Subject: [PATCH 177/904] Completed a round of cleanup- variables rearranged to match i/o-3 --- .../eamxx_mam_wetscav_process_interface.cpp | 114 ++++++++++-------- .../single-process/mam/wet_scav/input.yaml | 23 ++-- externals/mam4xx | 2 +- 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index ef7661c2ba85..713078d76623 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -115,7 +115,7 @@ void MAMWetscav::set_grids( add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) - add_field("prain", scalar3d_mid, kg / kg / s, grid_name); + add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // For variables that are non dimensional (e.g., fractions etc.) static constexpr auto nondim = Units::nondimensional(); @@ -154,7 +154,7 @@ void MAMWetscav::set_grids( // In cloud water mixing ratio, shallow convection [kg/kg] add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); - // Detraining cld H20 from deep convection [kg/ks/s] + // Detraining cld H20 from deep convection [kg/kg/s] add_field("dlf", scalar3d_mid, kg / kg / s, grid_name); // --------------------------------------------------------------------- @@ -293,37 +293,38 @@ void MAMWetscav::init_buffers(const ATMBufferManager &buffer_manager) { // ========================================================================================= void MAMWetscav::initialize_impl(const RunType run_type) { - // Gather runtime options - //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- - // populate the wet atmosphere state with views from fields and - // the buffer (NOTE: wet atmosphere only has qv, qc, qi, nc, ni) + // store fields only to be converted to dry mmrs in wet_atm_ wet_atm_.qc = get_field_in("qc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); - // -- Following wet atm variables are NOT used by the process but we still - // need them to - // -- create atmosphere object - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - // populate the dry atmosphere state with views from fields + // Following wet atm variables are NOT used by the process but we still + // need them to create atmosphere object + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields // (NOTE: dry atmosphere has everything that wet // atmosphere has along with z_surf, T_mid, p_mid, z_mid, z_iface, - // dz, p_del, cldfrac, w_updraft, pblh, phis) - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); + // dz, p_del, cldfrac, w_updraft, pblh, phis and omega) + dry_atm_.z_surf = 0; + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_liq").get_view(); + + // The following dry_atm_ members *may* not be used by the process but they + // are needed for creating MAM4xx class objects like Atmosphere dry_atm_.omega = get_field_in("omega").get_view(); - // How "buffer_" works: We use buffer to allocate memory for the members of - // dry_atm_ object. Here we are providing those memory locations to the - // dry_atm_ members. These members are computed from the above wet_atm_ or - // dry_atm_ members that are explicitly getting their values either from the - // input file or from other processes. These members are null at this point, - // they are assigned in "Kokkos::parallel_for("preprocess", scan_policy, - // preprocess_);" call in the run_impl + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.qv = buffer_.qv_dry; dry_atm_.qc = buffer_.qc_dry; dry_atm_.nc = buffer_.nc_dry; @@ -334,12 +335,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_atm_.z_iface = buffer_.z_iface; dry_atm_.w_updraft = buffer_.w_updraft; - // The following dry_atm_ members *may* not be used by the process but they - // are needed for creating MAM4xx class objects like Atmosphere - dry_atm_.cldfrac = get_field_in("cldfrac_liq").get_view(); - dry_atm_.pblh = get_field_in("pbl_height").get_view(); - dry_atm_.phis = get_field_in("phis").get_view(); - dry_atm_.z_surf = 0.0; // ---- set wet/dry aerosol-related gas state data for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); @@ -382,6 +377,10 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } + //--------------------------------------------------------------------------------- + // Allocate memory + // (Kokkos::resize only works on host to allocates memory) + //--------------------------------------------------------------------------------- // Alllocate aerosol-related gas tendencies for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { Kokkos::resize(dry_aero_tends_.gas_mmr[g], ncol_, nlev_); @@ -396,29 +395,31 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } - // set up our preprocess/postprocess functors - // Here we initialize (not compute) objects in preprocess struct using the - // objects in the argument list + // Allocate work array + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); + work_ = view_2d("work", ncol_, work_len); + + //--------------------------------------------------------------------------------- + // Setup preprocessing and post processing + //--------------------------------------------------------------------------------- + // set up our preprocess and postprocess functors preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - - // wetdep - - const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); - work_ = view_2d("work", ncol_, work_len); } // ========================================================================================= void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // preprocess input -- needs a scan for the calculation of all variables // needed by this process or setting up MAM4xx classes and their objects Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; const auto &work = work_; @@ -427,15 +428,17 @@ void MAMWetscav::run_impl(const double dt) { // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - // Shallow convective cloud fraction [fraction] + + //----------- Variables from convective scheme ------------- + + // Following variables are from convective parameterization (not implemented + // yet in EAMxx), so should be zero for now + auto sh_frac = get_field_in("sh_frac").get_view(); // Deep convective cloud fraction [fraction] auto dp_frac = get_field_in("dp_frac").get_view(); - // Total cloud fraction - auto cldt = get_field_in("cldfrac_liq").get_view(); - // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] auto evapcsh = get_field_in("evapcsh").get_view(); @@ -454,18 +457,24 @@ void MAMWetscav::run_impl(const double dt) { // In cloud water mixing ratio, shallow convection auto icwmrsh = get_field_in("icwmrsh").get_view(); - // evaporation from stratiform rain [kg/kg/s] - // FIXME: Get it from P3 + // Detraining cld H20 from deep convection [kg/kg/s] + auto dlf = get_field_in("dlf").get_view(); + + //----------- Variables from macrophysics scheme ------------- + // Total cloud fraction + auto cldt = get_field_in("cldfrac_liq").get_view(); + + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] auto evapr = get_field_in("evapr").get_view(); + // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) + auto prain = get_field_in("prain").get_view(); // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "Updated" or pure inputs/outputs for the process + // These variables are "Computed" or pure outputs for the process // ------------------------------------------------------------------------------------------------------------------------- - auto dlf = get_field_in("dlf").get_view(); - auto prain = get_field_out("prain") - .get_view(); // stratiform rain production rate - // outputs const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); @@ -483,7 +492,7 @@ void MAMWetscav::run_impl(const double dt) { // inside a parallel_for. const int nlev = nlev_; - // loop over atmosphere columns and compute aerosol particle size + // Loop over atmosphere columns Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { const int icol = team.league_rank(); // column index*/ @@ -535,9 +544,8 @@ void MAMWetscav::run_impl(const double dt) { cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, dlf_icol, prain_icol, - // in/out + // outputs wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, - // output aerdepwetis_icol, aerdepwetcw_icol, work_icol); team.team_barrier(); // update interstitial aerosol state diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 13447577cf8f..c481165d29ad 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -25,18 +25,22 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables from P3 + prain: 1e-5 + evapr: 1e-5 + #variables from convection scheme (they should all be zero) rprdsh: 0. rprddp: 0. evapcsh: 0. evapcdp: 0. - cldst: 1e-5 - qme: 1e-5 - prain: 1e-5 - evapr: 1e-5 icwmrdp: 0. icwmrsh: 0. sh_frac: 0. dp_frac: 0. + dlf : 0. + #surface fluxes bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] @@ -45,17 +49,6 @@ initial_conditions: dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] - dgncur_a: 1e-5 # aerosol particle diameter [m] - wetdens: 1e-5 # wet aerosol density [kg/m3] - qaerwat: 1e-5 # aerosol water [kg/kg] - dgnumwet: 1e-5 # wet aerosol diameter [m] - fracis: 1e-5 # fraction of transported species that are insoluble [fraction] - cldfrac_tot: 0.5 - pbl_height: 25.0 - phis : 0.1 - dlf : 0.0 - dp_ccf : 0.0 - sh_ccf : 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] diff --git a/externals/mam4xx b/externals/mam4xx index 1b5f3b6f4cd9..f5700d5f2ac6 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 1b5f3b6f4cd90b18869e20acc4e68663b3c4e0e8 +Subproject commit f5700d5f2ac6a6a7f845ee73a50a2bd1d268e22c From 870e9785261f6b26d4c0b4c253213e0ca1981628 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Jul 2024 09:41:56 -0700 Subject: [PATCH 178/904] Adds a multi-process test: p3-wetscav --- .../eamxx_mam_wetscav_process_interface.cpp | 24 ++++++++++--------- .../src/share/io/scream_scorpio_interface.cpp | 4 ++-- .../multi-process/physics_only/CMakeLists.txt | 1 + .../single-process/mam/wet_scav/input.yaml | 4 ++-- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 713078d76623..c581a5757064 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -10,9 +10,10 @@ //>>>>>>> /* -Future work: -Wirte comments -write in/outs for all variables clearly +----------------------------------------------------------------- +NOTES: +1. We should connect surface fluxes and add code to update the fluxes +2. Identify diagnostic variables and remove them from FM */ namespace scream { @@ -112,10 +113,11 @@ void MAMWetscav::set_grids( //----------- Variables from microphysics scheme ------------- // Evaporation from stratiform rain [kg/kg/s] (FIXME: Get it from P3) - add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); + add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) - add_field("prain", scalar3d_mid, kg / kg / s, grid_name); + add_field("precip_total_tend", scalar3d_mid, kg / kg / s, + grid_name); // For variables that are non dimensional (e.g., fractions etc.) static constexpr auto nondim = Units::nondimensional(); @@ -425,9 +427,9 @@ void MAMWetscav::run_impl(const double dt) { const auto &work = work_; const auto &dry_aero_tends = dry_aero_tends_; - // ------------------------------------------------------------------------------------------------------------------------- + // --------------------------------------------------------------- // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- + // --------------------------------------------------------------- //----------- Variables from convective scheme ------------- @@ -467,10 +469,10 @@ void MAMWetscav::run_impl(const double dt) { //----------- Variables from microphysics scheme ------------- // Evaporation from stratiform rain [kg/kg/s] - auto evapr = get_field_in("evapr").get_view(); + auto nevapr = get_field_in("nevapr").get_view(); // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) - auto prain = get_field_in("prain").get_view(); + auto prain = get_field_in("precip_total_tend").get_view(); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "Computed" or pure outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -523,7 +525,7 @@ void MAMWetscav::run_impl(const double dt) { auto icwmrdp_col = ekat::subview(icwmrdp, icol); auto icwmrsh_icol = ekat::subview(icwmrsh, icol); - auto evapr_icol = ekat::subview(evapr, icol); + auto nevapr_icol = ekat::subview(nevapr, icol); auto cldt_icol = ekat::subview(cldt, icol); auto dlf_icol = ekat::subview(dlf, icol); @@ -542,7 +544,7 @@ void MAMWetscav::run_impl(const double dt) { team, atm, progs, tends, dt, // inputs cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, - dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, + dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, nevapr_icol, dlf_icol, prain_icol, // outputs wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index cb2b501f2b5c..68a432819ec3 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - for (auto& it : s.files) { + /*for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - } + }*/ s.files.clear(); for (auto& it : s.decomps) { diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index b9c13234796b..36b43c47c173 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -8,6 +8,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) + add_subdirectory(mam/p3_mam4_wetscav) endif() endif() diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index c481165d29ad..f2a62a15823e 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -28,8 +28,8 @@ initial_conditions: pbl_height: 25.0 phis : 0.1 #variables from P3 - prain: 1e-5 - evapr: 1e-5 + precip_total_tend: 1e-5 + nevapr: 1e-5 #variables from convection scheme (they should all be zero) rprdsh: 0. rprddp: 0. From 8a81e1c18bf670523cc4908a6d0e5a4daaeae359 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Jul 2024 09:43:17 -0700 Subject: [PATCH 179/904] Adds missing files for the new multi-process test --- .../mam/p3_mam4_wetscav/CMakeLists.txt | 41 ++++++++++++ .../mam/p3_mam4_wetscav/input.yaml | 67 +++++++++++++++++++ .../mam/p3_mam4_wetscav/output.yaml | 66 ++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..7e13932e0632 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME p3_mam4_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS p3 mam + LABELS p3 physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep p3 dt<=300s +set (P3_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${P3_MAX_DT} - 1) / ${P3_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS p3 physics mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml new file mode 100644 index 000000000000..c2c422ca2e94 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml @@ -0,0 +1,67 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic, mam4_wetscav] + schedule_type: Sequential + mac_mic: + atm_procs_list: [p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables needed by mam4_wetscav + #variables from convection scheme (they should all be zero) + rprdsh: 0. + rprddp: 0. + evapcsh: 0. + evapcdp: 0. + icwmrdp: 0. + icwmrsh: 0. + sh_frac: 0. + dp_frac: 0. + dlf : 0. + #surface fluxes + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + + #variable required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml new file mode 100644 index 000000000000..d8d8e7694709 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -0,0 +1,66 @@ +%YAML 1.1 +--- +filename_prefix: shoc_mam4_aci_output +Averaging Type: Instant +Field Names: + - cldfrac_liq + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 64fc5a3c82945c1209833f3d9c3d9c990bacbab0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Jul 2024 10:59:30 -0700 Subject: [PATCH 180/904] Adds another multi-process test for wetscav --- .../multi-process/physics_only/CMakeLists.txt | 1 + .../mam/p3_mam4_wetscav/input.yaml | 4 +- .../mam/p3_mam4_wetscav/output.yaml | 2 +- .../shoc_cldfrac_p3_wetscav/CMakeLists.txt | 41 +++++++ .../mam/shoc_cldfrac_p3_wetscav/input.yaml | 83 +++++++++++++++ .../mam/shoc_cldfrac_p3_wetscav/output.yaml | 100 ++++++++++++++++++ 6 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 36b43c47c173..d3b584bd09e4 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -9,6 +9,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) add_subdirectory(mam/p3_mam4_wetscav) + add_subdirectory(mam/shoc_cldfrac_p3_wetscav) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml index c2c422ca2e94..f5d2dc036c0b 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml @@ -37,7 +37,7 @@ initial_conditions: pbl_height: 25.0 phis : 0.1 #variables needed by mam4_wetscav - #variables from convection scheme (they should all be zero) + #--variables from convection scheme (they should all be zero) rprdsh: 0. rprddp: 0. evapcsh: 0. @@ -47,7 +47,7 @@ initial_conditions: sh_frac: 0. dp_frac: 0. dlf : 0. - #surface fluxes + #--surface fluxes bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml index d8d8e7694709..c26db8eb6814 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_mam4_aci_output +filename_prefix: p3_mam4_wetscav_output Averaging Type: Instant Field Names: - cldfrac_liq diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..b5a6be819ce2 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_cldfrac_p3_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 mam + LABELS shoc cld p3 physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 physics PEM mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml new file mode 100644 index 000000000000..f5f491a33807 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml @@ -0,0 +1,83 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic, mam4_wetscav] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables needed by mam4_wetscav + #--variables from convection scheme (they should all be zero) + rprdsh: 0. + rprddp: 0. + evapcsh: 0. + evapcdp: 0. + icwmrdp: 0. + icwmrsh: 0. + sh_frac: 0. + dp_frac: 0. + dlf : 0. + #--surface fluxes + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + + #variable required for shoc + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variable required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml new file mode 100644 index 000000000000..8f7d04661aa4 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -0,0 +1,100 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cldfrac_p3_mam4_wetscav_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # wetscav + - T_mid + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 0ec3227dd7312ab040502093ce698e0a760e92c9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 4 Jul 2024 23:50:24 -0700 Subject: [PATCH 181/904] updated submodule to a newer version --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index f5700d5f2ac6..e949c1cb50b1 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit f5700d5f2ac6a6a7f845ee73a50a2bd1d268e22c +Subproject commit e949c1cb50b16eeafe0161caafc4c044171b7975 From d00cd5035485018a0d07cf69554b9be7945d85fc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 00:43:26 -0700 Subject: [PATCH 182/904] Cleans wetscav hpp file and minor changes to the cpp file --- .../eamxx_mam_wetscav_process_interface.cpp | 19 +--- .../eamxx_mam_wetscav_process_interface.hpp | 101 +++++++++--------- 2 files changed, 52 insertions(+), 68 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c581a5757064..1dc6265942c3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -1,19 +1,11 @@ #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" -#include - -#include "scream_config.h" // for SCREAM_CIME_BUILD - -// Remove the following<<<< -#include -#include -//>>>>>>> - /* ----------------------------------------------------------------- NOTES: 1. We should connect surface fluxes and add code to update the fluxes 2. Identify diagnostic variables and remove them from FM +3. Add assert statements to check output ranges */ namespace scream { @@ -27,12 +19,9 @@ MAMWetscav::MAMWetscav(const ekat::Comm &comm, */ } -AtmosphereProcessType MAMWetscav::type() const { - return AtmosphereProcessType::Physics; -} - -std::string MAMWetscav::name() const { return "mam4_wetscav"; } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMWetscav::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index 6f15752953cd..ea2de8681081 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -10,7 +10,6 @@ // For component name #include -#include #ifndef KOKKOS_ENABLE_CUDA #define protected_except_cuda public #define private_except_cuda public @@ -30,28 +29,21 @@ namespace scream { */ class MAMWetscav : public scream::AtmosphereProcess { - - using KT = ekat::KokkosTypes; - using view_1d = typename KT::template view_1d; + using KT = ekat::KokkosTypes; using view_2d = typename KT::template view_2d; - using view_3d = typename KT::template view_3d; - - using const_view_2d = typename KT::template view_2d; - using const_view_1d = typename KT::template view_1d; // remove it if possible - // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; public: - // Constructors + // Constructor MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The type of subcomponent - AtmosphereProcessType - type() const override; + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + // The name of the subcomponent - std::string name() const override; + std::string name() const { return "mam4_wetscav"; } // Set the grid and input output variables void set_grids( @@ -70,7 +62,6 @@ class MAMWetscav : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; - // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. @@ -79,10 +70,10 @@ class MAMWetscav : public scream::AtmosphereProcess { // on host: initializes preprocess functor with necessary state data void initialize(const int ncol_in, const int nlev_in, - const mam_coupling::WetAtmosphere& wet_atm_in, - const mam_coupling::AerosolState& wet_aero_in, - const mam_coupling::DryAtmosphere& dry_atm_in, - const mam_coupling::AerosolState& dry_aero_in) { + const mam_coupling::WetAtmosphere &wet_atm_in, + const mam_coupling::AerosolState &wet_aero_in, + const mam_coupling::DryAtmosphere &dry_atm_in, + const mam_coupling::AerosolState &dry_aero_in) { ncol_pre_ = ncol_in; nlev_pre_ = nlev_in; wet_atm_pre_ = wet_atm_in; @@ -92,26 +83,29 @@ class MAMWetscav : public scream::AtmosphereProcess { } KOKKOS_INLINE_FUNCTION - void operator()(const Kokkos::TeamPolicy::member_type& team) const { - const int i = team.league_rank(); // column index + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index // first, compute dry fields compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); team.team_barrier(); // second, we can use dry fields to compute dz, zmin, zint compute_vertical_layer_heights(team, dry_atm_pre_, i); compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - team.team_barrier(); // allows kernels below to use layer heights - } // operator() + // allows kernels below to use layer heights operator() + team.team_barrier(); + } - // number of horizontal columns and vertical levels + // Number of horizontal columns and vertical levels int ncol_pre_, nlev_pre_; - // local atmospheric and aerosol state data + // Local atmospheric and aerosol state data mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; - mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMWetscav::Preprocess + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMWetscav::Preprocess // Postprocessing functor struct Postprocess { @@ -119,24 +113,26 @@ class MAMWetscav : public scream::AtmosphereProcess { // on host: initializes postprocess functor with necessary state data void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere& wet_atm, - const mam_coupling::AerosolState& wet_aero, - const mam_coupling::DryAtmosphere& dry_atm, - const mam_coupling::AerosolState& dry_aero) { - ncol_post_ = ncol; - nlev_post_ = nlev; - wet_atm_post_ = wet_atm; + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; wet_aero_post_ = wet_aero; - dry_atm_post_ = dry_atm; + dry_atm_post_ = dry_atm; dry_aero_post_ = dry_aero; } KOKKOS_INLINE_FUNCTION - void operator()(const Kokkos::TeamPolicy::member_type& team) const { - const int i = team.league_rank(); // column index - compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, wet_aero_post_, i); + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); team.team_barrier(); - } // operator() + } // operator() // number of horizontal columns and vertical levels int ncol_post_, nlev_post_; @@ -144,33 +140,32 @@ class MAMWetscav : public scream::AtmosphereProcess { // local atmospheric and aerosol state data mam_coupling::WetAtmosphere wet_atm_post_; mam_coupling::DryAtmosphere dry_atm_post_; - mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; - }; // MAMWetscav::Postprocess - - - /* ----------------------------------------------- - * Local variables - * ------------------------------------------------ - */ + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // MAMWetscav::Postprocess + private: + // ----------------------------------------------- + // Local variables + // ------------------------------------------------ // pre- and postprocessing scratch pads (for wet <-> dry conversions) Preprocess preprocess_; Postprocess postprocess_; - // Number of horizontal columns and vertical levels int ncol_, nlev_; - //Number of aerosol modes + // Number of aerosol modes static constexpr int ntot_amode_ = mam4::AeroConfig::num_modes(); - // atmospheric variables + // Atmospheric variables mam_coupling::WetAtmosphere wet_atm_; mam_coupling::DryAtmosphere dry_atm_; - // wet dep + + // Work arrays view_2d work_; - // aerosol states - mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; + + // Aerosol states + mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; mam_coupling::Buffer buffer_; From dd7e21b66f666f06b679d974247a3578b1e72e82 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 01:32:23 -0700 Subject: [PATCH 183/904] Adds descriptive var names; removes conv vars from FM; fixes input and namelist yaml files with new var names --- .../cime_config/namelist_defaults_scream.xml | 37 ++---- .../eamxx_mam_wetscav_process_interface.cpp | 115 +++++++----------- .../eamxx_mam_wetscav_process_interface.hpp | 6 +- .../input.yaml | 47 ++----- .../mam/p3_mam4_wetscav/input.yaml | 26 ++-- .../mam/shoc_cldfrac_p3_wetscav/input.yaml | 26 ++-- .../single-process/mam/wet_scav/input.yaml | 26 ++-- 7 files changed, 94 insertions(+), 189 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 79fcc64f3848..0cd7f023bf14 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -561,35 +561,14 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0 - 0.5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 0.5 - 0.0 - 0.0 - 0.0 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 1dc6265942c3..768a1b827a44 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -53,7 +53,8 @@ void MAMWetscav::set_grids( m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); // layout for 2D (ncol, pcnst) - FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "pcnst"); + FieldLayout scalar2d_pconst = + m_grid->get_2d_vector_layout(pcnst, "num_phys_constants"); // -------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process @@ -101,10 +102,10 @@ void MAMWetscav::set_grids( //----------- Variables from microphysics scheme ------------- - // Evaporation from stratiform rain [kg/kg/s] (FIXME: Get it from P3) + // Evaporation from stratiform rain [kg/kg/s] add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); - // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) + // Stratiform rain production rate [kg/kg/s] add_field("precip_total_tend", scalar3d_mid, kg / kg / s, grid_name); @@ -116,38 +117,6 @@ void MAMWetscav::set_grids( // Total cloud fraction [fraction] add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); - //----------- Variables from convective scheme ------------- - - // Following variables are from convective parameterization (not implemented - // yet in EAMxx), so should be zero for now - - // Deep convective cloud fraction [fraction] - add_field("dp_frac", scalar3d_mid, nondim, grid_name); - - // Shallow convective cloud fraction [fraction] //NOT updated - add_field("sh_frac", scalar3d_mid, nondim, grid_name); - - // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] - add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); - - // Evaporation rate of deep precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_mid, kg / kg / s, grid_name); - - // Rain production, shallow convection [kg/kg/s] - add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); - - // Rain production, deep convection [kg/kg/s] - add_field("rprddp", scalar3d_mid, kg / kg / s, grid_name); - - // In cloud water mixing ratio, deep convection [kg/kg] - add_field("icwmrdp", scalar3d_mid, kg / kg, grid_name); - - // In cloud water mixing ratio, shallow convection [kg/kg] - add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); - - // Detraining cld H20 from deep convection [kg/kg/s] - add_field("dlf", scalar3d_mid, kg / kg / s, grid_name); - // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- @@ -156,28 +125,32 @@ void MAMWetscav::set_grids( // for the land model // Wet deposition of hydrophilic black carbon [kg/m2/s] - add_field("bcphiwet", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_hydrophilic_bc", scalar3d_mid, kg / m2 / s, + grid_name); // Dry deposition of hydrophilic black carbon [kg/m2/s] - add_field("bcphidry", scalar3d_mid, kg / m2 / s, grid_name); + add_field("drydep_hydrophilic_bc", scalar3d_mid, kg / m2 / s, + grid_name); // Wet deposition of hydrophilic organic carbon [kg/m2/s] - add_field("ocphiwet", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_hydrophilic_oc", scalar3d_mid, kg / m2 / s, + grid_name); // Dry deposition of hydrophilic organic carbon [kg/m2/s] - add_field("ocphidry", scalar3d_mid, kg / m2 / s, grid_name); + add_field("drydep_hydrophilic_oc", scalar3d_mid, kg / m2 / s, + grid_name); // Wet deposition of dust (bin1) [kg/m2/s] - add_field("dstwet1", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_dust_bin1", scalar3d_mid, kg / m2 / s, grid_name); // Wet deposition of dust (bin2) [kg/m2/s] - add_field("dstwet2", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_dust_bin2", scalar3d_mid, kg / m2 / s, grid_name); // Wet deposition of dust (bin3) [kg/m2/s] - add_field("dstwet3", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_dust_bin3", scalar3d_mid, kg / m2 / s, grid_name); // Wet deposition of dust (bin4) [kg/m2/s] - add_field("dstwet4", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_dust_bin4", scalar3d_mid, kg / m2 / s, grid_name); // Interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -193,11 +166,11 @@ void MAMWetscav::set_grids( "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios + // Note: Do *not* add cld borne aerosols to the "tracer" group as these are + // not advected const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(imode); - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { @@ -210,6 +183,8 @@ void MAMWetscav::set_grids( } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // Note: Do *not* add cld borne aerosols to the "tracer" group as these + // are not advected const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(imode, ispec); if(strlen(cld_mmr_field_name) > 0) { @@ -220,10 +195,6 @@ void MAMWetscav::set_grids( } } - // Tracers group -- do we need this in addition to the tracers above? In any - // case, this call should be idempotent, so it can't hurt. - add_group("tracers", grid_name, 1, Bundling::Required); - // The following fields are not needed by this process but we define them so // that we can create MAM4xx class objects like atmosphere, prognostics etc. @@ -260,14 +231,9 @@ void MAMWetscav::set_grids( add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); } -// ========================================================================================= -// ON HOST, returns the number of bytes of device memory needed by the above -// Buffer type given the number of columns and vertical levels -size_t MAMWetscav::requested_buffer_size_in_bytes() const { - return mam_coupling::buffer_size(ncol_, nlev_); -} - -// ========================================================================================= +// ================================================================ +// INIT_BUFFERS +// ================================================================ // ON HOST, initializeŃ• the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. @@ -282,7 +248,9 @@ void MAMWetscav::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMWetscav."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMWetscav::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- // Input fields read in from IC file, namelist or other processes @@ -401,7 +369,9 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_aero_); } -// ========================================================================================= +// ================================================================ +// RUN_IMPL +// ================================================================ void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -425,31 +395,40 @@ void MAMWetscav::run_impl(const double dt) { // Following variables are from convective parameterization (not implemented // yet in EAMxx), so should be zero for now - auto sh_frac = get_field_in("sh_frac").get_view(); + auto sh_frac = view_2d("sh_frac", ncol_, nlev_); + Kokkos::deep_copy(sh_frac, 0); // Deep convective cloud fraction [fraction] - auto dp_frac = get_field_in("dp_frac").get_view(); + auto dp_frac = view_2d("dp_frac", ncol_, nlev_); + Kokkos::deep_copy(dp_frac, 0); // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] - auto evapcsh = get_field_in("evapcsh").get_view(); + auto evapcsh = view_2d("evapcsh", ncol_, nlev_); + Kokkos::deep_copy(evapcsh, 0); // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] - auto evapcdp = get_field_in("evapcdp").get_view(); + auto evapcdp = view_2d("evapcdp", ncol_, nlev_); + Kokkos::deep_copy(evapcdp, 0); // Rain production, shallow convection [kg/kg/s] - auto rprdsh = get_field_in("rprdsh").get_view(); + auto rprdsh = view_2d("rprdsh", ncol_, nlev_); + Kokkos::deep_copy(rprdsh, 0); // Rain production, deep convection [kg/kg/s] - auto rprddp = get_field_in("rprddp").get_view(); + auto rprddp = view_2d("rprddp", ncol_, nlev_); + Kokkos::deep_copy(rprddp, 0); // In cloud water mixing ratio, deep convection - auto icwmrdp = get_field_in("icwmrdp").get_view(); + auto icwmrdp = view_2d("icwmrdp", ncol_, nlev_); + Kokkos::deep_copy(icwmrdp, 0); // In cloud water mixing ratio, shallow convection - auto icwmrsh = get_field_in("icwmrsh").get_view(); + auto icwmrsh = view_2d("icwmrsh", ncol_, nlev_); + Kokkos::deep_copy(icwmrsh, 0); // Detraining cld H20 from deep convection [kg/kg/s] - auto dlf = get_field_in("dlf").get_view(); + auto dlf = view_2d("dlf", ncol_, nlev_); + Kokkos::deep_copy(dlf, 0); //----------- Variables from macrophysics scheme ------------- // Total cloud fraction diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index ea2de8681081..8bd81115d8a3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -50,7 +50,11 @@ class MAMWetscav : public scream::AtmosphereProcess { const std::shared_ptr grids_manager) override; // management of common atm process memory - size_t requested_buffer_size_in_bytes() const override; + // ON HOST, returns the number of bytes of device memory needed by the above + // Buffer type given the number of columns and vertical levels + size_t requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); + } void init_buffers(const ATMBufferManager &buffer_manager) override; // Initialize variables diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml index bfbec956d852..fc15d9ce2528 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -11,45 +11,18 @@ time_stepping: initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - surf_evap: 0.0 - surf_sens_flux: 0.0 - precip_liq_surf_mass: 0.0 - precip_ice_surf_mass: 0.0 - cldn: 0.5 - rprdsh: 1e-5 - rprddp: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 - cldt: 1e-5 - prain: 1e-5 - evapr: 1e-5 - icwmrdp: 1e-5 - icwmrsh: 1e-5 - rprddp: 1e-5 - rprdsh: 1e-5 - sh_frac: 1e-5 - dp_frac: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] - dgncur_a: 1e-5 # aerosol particle diameter [m] - wetdens: 1e-5 # wet aerosol density [kg/m3] - qaerwat: 1e-5 # aerosol water [kg/kg] - dgnumwet: 1e-5 # wet aerosol diameter [m] - fracis: 1e-5 # fraction of transported species that are insoluble [fraction] - cldfrac_tot: 0.5 pbl_height: 25.0 phis : 0.1 - dlf : 0.0 - dp_ccf : 0.0 - sh_ccf : 0.0 + #variables needed by mam4_wetscav + #--surface fluxes + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] atmosphere_processes: atm_procs_list: [homme,physics] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml index f5d2dc036c0b..f0d9b7c89a3b 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml @@ -37,25 +37,15 @@ initial_conditions: pbl_height: 25.0 phis : 0.1 #variables needed by mam4_wetscav - #--variables from convection scheme (they should all be zero) - rprdsh: 0. - rprddp: 0. - evapcsh: 0. - evapcdp: 0. - icwmrdp: 0. - icwmrsh: 0. - sh_frac: 0. - dp_frac: 0. - dlf : 0. #--surface fluxes - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] #variable required for p3 precip_ice_surf_mass: 0.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml index f5f491a33807..804939a22058 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml @@ -49,25 +49,15 @@ initial_conditions: pbl_height: 25.0 phis : 0.1 #variables needed by mam4_wetscav - #--variables from convection scheme (they should all be zero) - rprdsh: 0. - rprddp: 0. - evapcsh: 0. - evapcdp: 0. - icwmrdp: 0. - icwmrsh: 0. - sh_frac: 0. - dp_frac: 0. - dlf : 0. #--surface fluxes - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] #variable required for shoc surf_sens_flux: 0.0 diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index f2a62a15823e..d1fd9f3659da 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -30,25 +30,15 @@ initial_conditions: #variables from P3 precip_total_tend: 1e-5 nevapr: 1e-5 - #variables from convection scheme (they should all be zero) - rprdsh: 0. - rprddp: 0. - evapcsh: 0. - evapcdp: 0. - icwmrdp: 0. - icwmrsh: 0. - sh_frac: 0. - dp_frac: 0. - dlf : 0. #surface fluxes - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From e6610dd5e5a4fdc657c7f39da0543f5e66b00778 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 01:42:56 -0700 Subject: [PATCH 184/904] Adds const for input vars, removes scorpio temp change --- .../eamxx_mam_wetscav_process_interface.cpp | 39 +++++++++---------- .../mam/impl/compute_particle_size.cpp | 9 ----- .../src/share/io/scream_scorpio_interface.cpp | 4 +- 3 files changed, 21 insertions(+), 31 deletions(-) delete mode 100644 components/eamxx/src/physics/mam/impl/compute_particle_size.cpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 768a1b827a44..8d4b6cda29c3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -439,11 +439,11 @@ void MAMWetscav::run_impl(const double dt) { // Evaporation from stratiform rain [kg/kg/s] auto nevapr = get_field_in("nevapr").get_view(); - // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) + // Stratiform rain production rate [kg/kg/s] auto prain = get_field_in("precip_total_tend").get_view(); - // ------------------------------------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------ // These variables are "Computed" or pure outputs for the process - // ------------------------------------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------ const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); @@ -465,9 +465,9 @@ void MAMWetscav::run_impl(const double dt) { // Loop over atmosphere columns Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { - const int icol = team.league_rank(); // column index*/ + const int icol = team.league_rank(); // column index - auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + const auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); // set surface state data // fetch column-specific subviews into aerosol prognostics mam4::Prognostics progs = @@ -478,25 +478,24 @@ void MAMWetscav::run_impl(const double dt) { mam_coupling::interstitial_aerosols_tendencies_for_column( dry_aero_tends, icol); /// shallow_convective_precipitation_production - auto rprdsh_icol = ekat::subview(rprdsh, icol); + const auto rprdsh_icol = ekat::subview(rprdsh, icol); // deep_convective_precipitation_production - auto rprddp_icol = ekat::subview(rprddp, icol); + const auto rprddp_icol = ekat::subview(rprddp, icol); // deep_convective_precipitation_evaporation - auto evapcdp_icol = ekat::subview(evapcdp, icol); + const auto evapcdp_icol = ekat::subview(evapcdp, icol); // shallow_convective_precipitation_evaporation = - auto evapcsh_icol = ekat::subview(evapcsh, icol); + const auto evapcsh_icol = ekat::subview(evapcsh, icol); // deep_convective_cloud_fraction - auto dp_frac_icol = ekat::subview(dp_frac, icol); + const auto dp_frac_icol = ekat::subview(dp_frac, icol); // shallow_convective_cloud_fraction = - auto sh_frac_icol = ekat::subview(sh_frac, icol); - // FIXME: what is this? + const auto sh_frac_icol = ekat::subview(sh_frac, icol); - auto icwmrdp_col = ekat::subview(icwmrdp, icol); - auto icwmrsh_icol = ekat::subview(icwmrsh, icol); - auto nevapr_icol = ekat::subview(nevapr, icol); - auto cldt_icol = ekat::subview(cldt, icol); + const auto icwmrdp_col = ekat::subview(icwmrdp, icol); + const auto icwmrsh_icol = ekat::subview(icwmrsh, icol); + const auto nevapr_icol = ekat::subview(nevapr, icol); + const auto cldt_icol = ekat::subview(cldt, icol); - auto dlf_icol = ekat::subview(dlf, icol); + const auto dlf_icol = ekat::subview(dlf, icol); auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol); auto work_icol = ekat::subview(work, icol); @@ -504,9 +503,9 @@ void MAMWetscav::run_impl(const double dt) { ekat::subview(wet_geometric_mean_diameter_i, icol); auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i, icol); - auto qaerwat_icol = ekat::subview(qaerwat, icol); - auto wetdens_icol = ekat::subview(wetdens, icol); - auto prain_icol = ekat::subview(prain, icol); + auto qaerwat_icol = ekat::subview(qaerwat, icol); + auto wetdens_icol = ekat::subview(wetdens, icol); + const auto prain_icol = ekat::subview(prain, icol); mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, diff --git a/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp b/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp deleted file mode 100644 index 06a4c4cb3bea..000000000000 --- a/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp +++ /dev/null @@ -1,9 +0,0 @@ -namespace scream::impl { - -/*KOKKOS_INLINE_FUNCTION -void compute_particle_size(int icol, int nlev_, //in - view_1d& state_q, view_1d& qqcw) { - -}*/ - -} // namespace scream::impl diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 68a432819ec3..cb2b501f2b5c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - /*for (auto& it : s.files) { + for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - }*/ + } s.files.clear(); for (auto& it : s.decomps) { From 8c7d2a7ca862d7ff08d0cc9fbeea5f4923a4fb21 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 01:48:38 -0700 Subject: [PATCH 185/904] Cleans up namelist xml file and removes double ICs --- .../cime_config/namelist_defaults_scream.xml | 46 ++++--------------- 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 0cd7f023bf14..fb4dc5913d04 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -494,7 +494,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0,0.0 2.6e-08 - 0.0 + 0.0 0.0 0.0 0.0 @@ -519,6 +519,15 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0 + + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 0.0 @@ -534,41 +543,6 @@ be lost if SCREAM_HACK_XML is not enabled. 0 0.001 900.0 - - - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 From 83a85eb310647d09c9de458ab5c0f2077e57ea7d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 02:00:46 -0700 Subject: [PATCH 186/904] Removes a left over comment --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 8d4b6cda29c3..c748432af134 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -188,8 +188,6 @@ void MAMWetscav::set_grids( const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(imode, ispec); if(strlen(cld_mmr_field_name) > 0) { - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these - // are NOT advected add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } From 140ee78b4495a2c08a22c7379454e08ec605802f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 12:55:33 -0700 Subject: [PATCH 187/904] Minor change in comments --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c748432af134..1a0d4b7412b9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -211,21 +211,22 @@ void MAMWetscav::set_grids( // Aerosol dry particle diameter [m] add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); - // wet aerosol density [kg/m3] + // Wet aerosol density [kg/m3] add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); - // aerosol water [kg/kg] + // Aerosol water [kg/kg] add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); - // wet aerosol diameter [m] + // Wet aerosol diameter [m] add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); - // fraction of transported species that are insoluble [fraction] + // Fraction of transported species that are insoluble [fraction] add_field("fracis", scalar3d_mid, nondim, grid_name); - // aerosol wet deposition (interstitial) [kg/m2/s] + // Aerosol wet deposition (interstitial) [kg/m2/s] add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); - // aerosol wet deposition (cloud water) [kg/m2/s] + + // Aerosol wet deposition (cloud water) [kg/m2/s] add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); } From aed9ffcc67688ad812873b0afaa72b5ded9965aa Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Fri, 5 Jul 2024 14:01:58 -0600 Subject: [PATCH 188/904] wet_scav - Removing Kokkos::resize. --- .../physics/mam/eamxx_mam_wetscav_process_interface.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 1a0d4b7412b9..bef4d6c7463d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -337,19 +337,18 @@ void MAMWetscav::initialize_impl(const RunType run_type) { //--------------------------------------------------------------------------------- // Allocate memory - // (Kokkos::resize only works on host to allocates memory) //--------------------------------------------------------------------------------- // Alllocate aerosol-related gas tendencies for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - Kokkos::resize(dry_aero_tends_.gas_mmr[g], ncol_, nlev_); + dry_aero_tends_.gas_mmr[g] = view_2d("gas_mmr",ncol_, nlev_); } // Allocate aerosol state tendencies (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - Kokkos::resize(dry_aero_tends_.int_aero_nmr[imode], ncol_, nlev_); + dry_aero_tends_.int_aero_nmr[imode] = view_2d("int_aero_nmr",ncol_, nlev_); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { - Kokkos::resize(dry_aero_tends_.int_aero_mmr[imode][ispec], ncol_, nlev_); + dry_aero_tends_.int_aero_mmr[imode][ispec] = view_2d("int_aero_mmr",ncol_, nlev_); } } From 019a54cc36a99d55e1657c6c7cb2ac8b0502b2db Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 16:31:13 -0700 Subject: [PATCH 189/904] Fixes some test fails: remove old output varnames, fixed output nc file name --- .../eamxx_mam_wetscav_process_interface.cpp | 7 +++--- .../input.yaml | 4 +++ .../output.yaml | 25 ------------------- .../mam/shoc_cldfrac_p3_wetscav/output.yaml | 2 +- externals/mam4xx | 2 +- 5 files changed, 10 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index bef4d6c7463d..20672e61f0f5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -340,15 +340,16 @@ void MAMWetscav::initialize_impl(const RunType run_type) { //--------------------------------------------------------------------------------- // Alllocate aerosol-related gas tendencies for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - dry_aero_tends_.gas_mmr[g] = view_2d("gas_mmr",ncol_, nlev_); + dry_aero_tends_.gas_mmr[g] = view_2d("gas_mmr", ncol_, nlev_); } // Allocate aerosol state tendencies (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - dry_aero_tends_.int_aero_nmr[imode] = view_2d("int_aero_nmr",ncol_, nlev_); + dry_aero_tends_.int_aero_nmr[imode] = view_2d("int_aero_nmr", ncol_, nlev_); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { - dry_aero_tends_.int_aero_mmr[imode][ispec] = view_2d("int_aero_mmr",ncol_, nlev_); + dry_aero_tends_.int_aero_mmr[imode][ispec] = + view_2d("int_aero_mmr", ncol_, nlev_); } } diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml index fc15d9ce2528..c9b493a2958c 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -13,6 +13,10 @@ initial_conditions: topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} pbl_height: 25.0 phis : 0.1 + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_liq_surf_mass: 0.0 + precip_ice_surf_mass: 0.0 #variables needed by mam4_wetscav #--surface fluxes wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml index f85225d1274f..afe8437a2cf4 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml @@ -107,31 +107,6 @@ Fields: - dgncur_a - wetdens - qaerwat - - ptend_bc_a1 - - ptend_bc_a3 - - ptend_bc_a4 - - ptend_dst_a1 - - ptend_dst_a3 - - ptend_mom_a1 - - ptend_mom_a2 - - ptend_mom_a3 - - ptend_mom_a4 - - ptend_nacl_a1 - - ptend_nacl_a2 - - ptend_nacl_a3 - - ptend_num_a1 - - ptend_num_a2 - - ptend_num_a3 - - ptend_num_a4 - - ptend_pom_a1 - - ptend_pom_a3 - - ptend_pom_a4 - - ptend_so4_a1 - - ptend_so4_a2 - - ptend_so4_a3 - - ptend_soa_a1 - - ptend_soa_a2 - - ptend_soa_a3 - bc_c1 - bc_c3 - bc_c4 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml index 8f7d04661aa4..1ae53e6c1c30 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cldfrac_p3_mam4_wetscav_output +filename_prefix: shoc_cldfrac_p3_wetscav_output Averaging Type: Instant Field Names: # SHOC diff --git a/externals/mam4xx b/externals/mam4xx index e949c1cb50b1..cafdbc1b8813 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit e949c1cb50b16eeafe0161caafc4c044171b7975 +Subproject commit cafdbc1b88134038aff12c1cb00110c6397a4544 From fe66519678d9b1e718e493948d624e9983a5e71c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 16:36:26 -0700 Subject: [PATCH 190/904] Adds TODO to a comment as some editors can highlight these comments --- components/eamxx/src/share/io/scream_scorpio_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index cb2b501f2b5c..68a432819ec3 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - for (auto& it : s.files) { + /*for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - } + }*/ s.files.clear(); for (auto& it : s.decomps) { From af58453c13ea3ea0426d57f8a2844fcd3ba1dc44 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 16:49:32 -0700 Subject: [PATCH 191/904] Fixes previous commit: Adds a TODO comment here --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 2 +- components/eamxx/src/share/io/scream_scorpio_interface.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 20672e61f0f5..c6153794257a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -391,7 +391,7 @@ void MAMWetscav::run_impl(const double dt) { //----------- Variables from convective scheme ------------- - // Following variables are from convective parameterization (not implemented + // TODO: Following variables are from convective parameterization (not implemented // yet in EAMxx), so should be zero for now auto sh_frac = view_2d("sh_frac", ncol_, nlev_); diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 68a432819ec3..cb2b501f2b5c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - /*for (auto& it : s.files) { + for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - }*/ + } s.files.clear(); for (auto& it : s.decomps) { From 4d3b7bc26ccf11cfda080c80a1464cc65cb3d184 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 18:01:22 -0700 Subject: [PATCH 192/904] Adds a new function to add additional tracers for MAM4xx --- .../testmods_dirs/scream/mam4xx/aci/shell_commands | 8 +++++--- .../scream/mam4xx/optics/shell_commands | 9 +++++---- .../scream/mam4xx/wetscav/shell_commands | 12 ++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index 68eaf51698b3..658c94a3cf67 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -1,7 +1,9 @@ -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh #modify initial condition file to get aerosol species ICs $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands index 1c22bd9ee454..224c16586104 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands @@ -1,7 +1,8 @@ - -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_optics,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands index f0e96c413985..97f442f88d85 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -1,7 +1,11 @@ +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" - +#------------------------------------------------------ +#Update IC file and add wetscav process +#------------------------------------------------------ $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_wetscav,rrtmgp" -b From 6de6c33682b86c3d4caae1a2ac6a18879e4b53fc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 18:02:08 -0700 Subject: [PATCH 193/904] Adds the new script for computing number of tracers --- .../scream/mam4xx/update_eamxx_num_tracers.sh | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh new file mode 100644 index 000000000000..930af526eb8d --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh @@ -0,0 +1,22 @@ +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +# Additional MAM4xx tracers (MAM4xx adds 31 tracers) +ADDITIONAL_MAM4xx_TRACERS=31 + +# Original CMAKE options in env_build.xml +orig_cmake_opt=`./xmlquery --value SCREAM_CMAKE_OPTIONS` + +# Extract the number of tracers +orig_tracer_num=$(echo $orig_cmake_opt | grep -oP 'SCREAM_NUM_TRACERS \K[0-9]+') + +# Update number of tracers +new_tracer_num=$((orig_tracer_num + ADDITIONAL_MAM4xx_TRACERS)) + +# Form the new CMake options string by replacing the original number with the new number +new_cmake_opt=$(echo $orig_cmake_opt | sed "s/SCREAM_NUM_TRACERS $orig_tracer_num/SCREAM_NUM_TRACERS $new_tracer_num/") + +# Update cmake options string +./xmlchange SCREAM_CMAKE_OPTIONS="$new_cmake_opt" \ No newline at end of file From 54f71108424698eb2d78d89a88c4119ce30c69a2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 6 Jul 2024 20:56:14 -0700 Subject: [PATCH 194/904] Fixes script to update tracer number in scream cmake options --- .../testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh | 6 ++++-- .../testmods_dirs/scream/mam4xx/wetscav/shell_commands | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) mode change 100644 => 100755 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh old mode 100644 new mode 100755 index 930af526eb8d..c533123847ef --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh @@ -1,5 +1,7 @@ +#!/bin/sh + #------------------------------------------------------ -# MAM4xx adds additionaltracers to the simulation +# MAM4xx adds additional tracers to the simulation # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ @@ -19,4 +21,4 @@ new_tracer_num=$((orig_tracer_num + ADDITIONAL_MAM4xx_TRACERS)) new_cmake_opt=$(echo $orig_cmake_opt | sed "s/SCREAM_NUM_TRACERS $orig_tracer_num/SCREAM_NUM_TRACERS $new_tracer_num/") # Update cmake options string -./xmlchange SCREAM_CMAKE_OPTIONS="$new_cmake_opt" \ No newline at end of file +`./xmlchange SCREAM_CMAKE_OPTIONS="$new_cmake_opt"` diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands index 97f442f88d85..7b5b9a876503 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -1,8 +1,10 @@ +#!/bin/sh #------------------------------------------------------ # MAM4xx adds additionaltracers to the simulation # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ #Update IC file and add wetscav process From ffffd51a7f0808d8c1471198f984d246fb74b7b6 Mon Sep 17 00:00:00 2001 From: noel Date: Wed, 3 Jul 2024 15:38:51 -0700 Subject: [PATCH 195/904] update module versions on pm-gpu --- cime_config/machines/config_machines.xml | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 5c6caed992c2..4a988f9f5402 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -368,6 +368,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -378,24 +379,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/22.7 + nvidia/24.5 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - gcc-mixed/11.2.0 + gcc-native-mixed/12.3 @@ -407,12 +408,12 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + cray-libsci/23.12.5 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 From 135190ec84df237b145f2e3375fac7872f04b671 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 8 Jul 2024 16:48:39 -0700 Subject: [PATCH 196/904] Updating muller-gpu module versions to be similar as that of pm-gpu --- cime_config/machines/config_machines.xml | 26 ++++++++---------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 4a988f9f5402..c04dff5976cd 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -678,6 +678,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -688,26 +689,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 - + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 - cudatoolkit/11.7 - + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 + gcc-native-mixed/12.3 @@ -719,20 +718,13 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 - evp-patch From dd2b0c9559b1c03116dcb78ae9c35506a66b8e93 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 9 Jul 2024 07:17:24 -0600 Subject: [PATCH 197/904] Fix some input/output arrays as suggested by Balwinder. --- ...x_mam_dry_deposition_process_interface.cpp | 108 +++++++++--------- ...x_mam_dry_deposition_process_interface.hpp | 21 ++-- .../single-process/mam/drydep/input.yaml | 14 +-- 3 files changed, 74 insertions(+), 69 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 2968b9927ef5..bb61390c6b7a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -16,27 +16,6 @@ MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. */ - std::string nname = "Obukhov_length"; - if(params.isParameter(nname)) - parameters_.Obukhov_length_ = params.get(nname); - nname = "surface_friction_velocty"; - if(params.isParameter(nname)) - parameters_.surface_friction_velocty_ = params.get(nname); - nname = "land_fraction"; - if(params.isParameter(nname)) - parameters_.land_fraction_ = params.get(nname); - nname = "ice_fraction"; - if(params.isParameter(nname)) - parameters_.ice_fraction_ = params.get(nname); - nname = "ocean_fraction"; - if(params.isParameter(nname)) - parameters_.ocean_fraction_ = params.get(nname); - nname = "friction_velocity"; - if(params.isParameter(nname)) - parameters_.friction_velocity_ = params.get(nname); - nname = "aerodynamical_resistance"; - if(params.isParameter(nname)) - parameters_.aerodynamical_resistance_ = params.get(nname); } // ========================================================================================= @@ -79,14 +58,16 @@ void MAMDryDep::set_grids( // Layout for tracers. const int pcnst = mam4::aero_model::pcnst; FieldLayout scalar4d_layout_q{ - {COL, LEV, NMODES}, {ncol_, nlev_, pcnst}}; + {COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; FieldLayout scalar4d_layout_qqcw_tends{ - {COL, NMODES, LEV}, {ncol_, pcnst, nlev_}}; + {COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; - // Memory to format teh wet_aero.int_aero_nmr and wet_aero.int_aero_mmr + // Tempary memory to format the wet_aero.int_aero_nmr and wet_aero.int_aero_mmr // views into the order expected by mam4xx. add_field("qtracers", scalar4d_layout_q, kg / kg, grid_name); add_field("d_qtracers_dt", scalar4d_layout_q, kg / kg / s, grid_name); + + // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] @@ -114,23 +95,21 @@ void MAMDryDep::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + // add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); // TODO: The following are not used by drydep but to create a dry atmosphere object. add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - // - // TODO: Currently the following are scalar parameters that are fixed - // for the whole grid. Do these need to be column specific? - // add_field("obklen", scalar2d_layout, m, grid_name); - // add_field("surfric", scalar2d_layout, m / s, grid_name); - // add_field("landfrac", scalar2d_layout, nondim, grid_name); - // add_field("icefrac", scalar2d_layout, nondim, grid_name); - // add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - // add_field("fv", scalar2d_layout, m / s, grid_name); - // add_field("ram1", scalar2d_layout, s / m, grid_name); + + add_field("Obukhov_length", scalar2d_layout, m, grid_name); + add_field("surface_friction_velocty", scalar2d_layout, m / s, grid_name); + add_field("land_fraction", scalar2d_layout, nondim, grid_name); + add_field("ice_fraction", scalar2d_layout, nondim, grid_name); + add_field("ocean_fraction", scalar2d_layout, nondim, grid_name); + add_field("friction_velocity", scalar2d_layout, m / s, grid_name); + add_field("aerodynamical_resistance", scalar2d_layout, s / m, grid_name); // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -212,6 +191,15 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.p_del = get_field_in("pseudo_density").get_view(); dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); + + obklen_ = get_field_in("Obukhov_length").get_view(); + surfric_ = get_field_in("surface_friction_velocty").get_view(); + landfrac_ = get_field_in("land_fraction").get_view(); + icefrac_ = get_field_in("ice_fraction").get_view(); + ocnfrac_ = get_field_in("ocean_fraction").get_view(); + friction_velocity_ = get_field_in("friction_velocity").get_view(); + aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.z_iface = buffer_.z_iface; @@ -227,8 +215,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dgncur_awet_ = get_field_out("dgncur_awet").get_view(); wet_dens_ = get_field_out("wetdens").get_view(); - qtracers_ = get_field_out("qtracers").get_view(); - d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + qtracers_ = get_field_out("qtracers").get_view(); + d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols").get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols").get_view(); tendencies_ = get_field_out("Tendencies").get_view(); @@ -282,11 +271,17 @@ namespace { KOKKOS_INLINE_FUNCTION void compute_tendencies( const MAMDryDep::KT::MemberType &team, - const MAMDryDep::Parameters ¶meters, const mam4::DryDeposition &dry_deposition, const double dt, - MAMDryDep::view_3d qtracers, - MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::const_view_1d obklen, + const MAMDryDep::const_view_1d surfric, + const MAMDryDep::const_view_1d landfrac, + const MAMDryDep::const_view_1d icefrac, + const MAMDryDep::const_view_1d ocnfrac, + const MAMDryDep::const_view_1d friction_velocity, + const MAMDryDep::const_view_1d aerodynamical_resistance, + const MAMDryDep::view_3d qtracers, + const MAMDryDep::view_3d d_qtracers_dt, const MAMDryDep::view_2d dgncur_awet[mam_coupling::num_aero_modes()], const MAMDryDep::view_2d wet_dens[mam_coupling::num_aero_modes()], const mam_coupling::DryAtmosphere &dry_atm, @@ -317,13 +312,14 @@ void compute_tendencies( for (int i=0; i; using view_3d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; using const_view_2d = Field::view_dev_t; using const_view_3d = Field::view_dev_t; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); @@ -42,11 +43,19 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_3d d_qtracers_dt_; view_3d dgncur_awet_; view_3d wet_dens_; - view_3d tendencies_; + view_3d tendencies_; view_2d aerdepdrycw_; view_2d aerdepdryis_; + const_view_1d obklen_; + const_view_1d surfric_; + const_view_1d landfrac_; + const_view_1d icefrac_; + const_view_1d ocnfrac_; + const_view_1d friction_velocity_; + const_view_1d aerodynamical_resistance_; + public: using KT = ekat::KokkosTypes; @@ -121,19 +130,9 @@ class MAMDryDep final : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess - struct Parameters { - Real Obukhov_length_ = 0.20723257141035126e+03; - Real surface_friction_velocty_ = 0.39900396673305327; - Real land_fraction_ = 0.1; - Real ice_fraction_ = 0.6; - Real ocean_fraction_ = 0.3; - Real friction_velocity_ = 0.46637129718055864; - Real aerodynamical_resistance_ = 0.91147859222259044e+02; - }; private: // pre- and postprocessing scratch pads Preprocess preprocess_; - Parameters parameters_; }; // MAMDryDep } // namespace scream diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 1163046ccce5..8b110dc1634d 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -27,13 +27,13 @@ initial_conditions: T_mid: 273.0 dgncur_awet: 1e-5 wetdens: 1e-5 - obklen: 1e-5 - surfric: 1e-5 - landfrac: 1.0 - icefrac: 0.0 - ocnfrac: 0.0 - fv: 1e-5 - ram1: 1e-5 + Obukhov_length: 1e-5 + surface_friction_velocty: 1e-5 + land_fraction: 1.0 + ice_fraction: 0.0 + ocean_fraction: 0.0 + friction_velocity: 1e-5 + aerodynamical_resistance: 1e-5 w_updraft: 1e-5 pbl_height: 0.0 From 4f116b40bfa9b2e22b83fdde3606a477729f6ebd Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 9 Jul 2024 16:21:57 -0600 Subject: [PATCH 198/904] clean up for PR --- docs/dev-guide/adding-grid-support/index.md | 1 - docs/dev-guide/some-tools/mkatmsrffile.md | 2 -- mkdocs.yaml | 2 -- 3 files changed, 5 deletions(-) delete mode 100644 docs/dev-guide/some-tools/mkatmsrffile.md diff --git a/docs/dev-guide/adding-grid-support/index.md b/docs/dev-guide/adding-grid-support/index.md index 4decc3976d33..f7c25043747a 100644 --- a/docs/dev-guide/adding-grid-support/index.md +++ b/docs/dev-guide/adding-grid-support/index.md @@ -6,4 +6,3 @@ - [Atmosphere Grid Overview ](../../EAM/tech-guide/atmosphere-grid-overview.md) - [Grid Description File Overview ](adding-grid-support-grid-types.md) - [Step-by-Step Guide to Support a New Grid ](adding-grid-support-step-by-step-guide.md) -- [Some test ](../some-tools/mkatmsrffile.md) diff --git a/docs/dev-guide/some-tools/mkatmsrffile.md b/docs/dev-guide/some-tools/mkatmsrffile.md deleted file mode 100644 index 1c0fec1cf2bd..000000000000 --- a/docs/dev-guide/some-tools/mkatmsrffile.md +++ /dev/null @@ -1,2 +0,0 @@ - ---8<-- 'components/eam/tools/mkatmsrffile/test.md' diff --git a/mkdocs.yaml b/mkdocs.yaml index 66a694115b9c..5c57a22993c3 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -9,7 +9,6 @@ nav: - Development: - Introduction: 'dev-guide/index.md' - Adding Support for New Grids: 'dev-guide/adding-grid-support/index.md' - - mkatmsrffile: 'dev-guide/some-tools/mkatmsrffile.md' - Components: '*include ./components/*/mkdocs.yml' - Tools: '*include ./tools/*/mkdocs.yml' - More Information: @@ -58,7 +57,6 @@ markdown_extensions: - pymdownx.details - pymdownx.highlight - pymdownx.superfences - - pymdownx.snippets - pymdownx.tabbed: alternate_style: true - pymdownx.arithmatex: From 7197f506d95ba203c3e9e27412b1e67870cd0c21 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 10 Jul 2024 09:44:44 -0600 Subject: [PATCH 199/904] More code cleanup. --- ...x_mam_dry_deposition_process_interface.cpp | 230 +++++++++--------- ...x_mam_dry_deposition_process_interface.hpp | 11 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- 3 files changed, 115 insertions(+), 128 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index bb61390c6b7a..eced2cf49643 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -3,9 +3,50 @@ #include "mam4xx/drydep.hpp" /* -Future work: -Wirte comments -write in/outs for all variables clearly +Inputs: + +Atmosphere: + temperature "T_mid" + pressure "p_mid" + interface_pressure "p_int" + hydrostatic_dp "pseudo_density" + +Diagnostics: + tracer_mixing_ratio "qtracers" + wet_geometric_mean_diameter_i "dgncur_awet" + wet_density "wetdens" + +Diagnostic Scalar Parameters, one per column: + "Obukhov_length" + "surface_friction_velocty" + "land_fraction" + "ice_fraction" + "ocean_fraction" + "friction_velocity" + "aerodynamical_resistance" + +Prognostics: + n_mode_c "mam_coupling::cld_aero_nmr_field_name(m)" + q_aero_c "mam_coupling::int_aero_nmr_field_name(m)" + + + +Outputs: + +Diagnostics: + d_tracer_mixing_ratio_dt "d_qtracers_dt" + deposition_flux_of_cloud_borne_aerosols "deposition_flux_of_cloud_borne_aerosols" + deposition_flux_of_interstitial_aerosols "deposition_flux_of_interstitial_aerosols" + +Computed internally, could be exposed if needed by another process + vlc_grv(nlev) : dep velocity of gravitational settling [m/s] + vlc_trb(nlev) : dep velocity of turbulent dry deposition [m/s] + vlc_dry(nlev) : dep velocity, sum of vlc_grv and vlc_trb [m/s] + +Tendencies: + n_mode_c "Tendencies" + q_aero_c "Tendencies" + */ namespace scream { @@ -31,6 +72,8 @@ void MAMDryDep::set_grids( auto m3 = m * m * m; // meter cubed + auto nondim = ekat::units::Units::nondimensional(); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); @@ -40,8 +83,6 @@ void MAMDryDep::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - auto nondim = ekat::units::Units::nondimensional(); - // Layout for 2D (2d horiz) variable const FieldLayout scalar2d_layout{{COL}, {ncol_}}; @@ -51,9 +92,10 @@ void MAMDryDep::set_grids( const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + // at mid points const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ - {NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; // mid points + {NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; // Layout for tracers. const int pcnst = mam4::aero_model::pcnst; @@ -95,7 +137,7 @@ void MAMDryDep::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - // + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); @@ -192,13 +234,13 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); - obklen_ = get_field_in("Obukhov_length").get_view(); - surfric_ = get_field_in("surface_friction_velocty").get_view(); - landfrac_ = get_field_in("land_fraction").get_view(); - icefrac_ = get_field_in("ice_fraction").get_view(); - ocnfrac_ = get_field_in("ocean_fraction").get_view(); - friction_velocity_ = get_field_in("friction_velocity").get_view(); - aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); + obukhov_length_ = get_field_in("Obukhov_length").get_view(); + surface_friction_velocty_ = get_field_in("surface_friction_velocty").get_view(); + land_fraction_ = get_field_in("land_fraction").get_view(); + ice_fraction_ = get_field_in("ice_fraction").get_view(); + ocean_fraction_ = get_field_in("ocean_fraction").get_view(); + friction_velocity_ = get_field_in("friction_velocity").get_view(); + aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.z_mid = buffer_.z_mid; @@ -268,74 +310,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_, dry_aero_); } namespace { -KOKKOS_INLINE_FUNCTION -void compute_tendencies( - const MAMDryDep::KT::MemberType &team, - const mam4::DryDeposition &dry_deposition, - const double dt, - const MAMDryDep::const_view_1d obklen, - const MAMDryDep::const_view_1d surfric, - const MAMDryDep::const_view_1d landfrac, - const MAMDryDep::const_view_1d icefrac, - const MAMDryDep::const_view_1d ocnfrac, - const MAMDryDep::const_view_1d friction_velocity, - const MAMDryDep::const_view_1d aerodynamical_resistance, - const MAMDryDep::view_3d qtracers, - const MAMDryDep::view_3d d_qtracers_dt, - const MAMDryDep::view_2d dgncur_awet[mam_coupling::num_aero_modes()], - const MAMDryDep::view_2d wet_dens[mam_coupling::num_aero_modes()], - const mam_coupling::DryAtmosphere &dry_atm, - const mam_coupling::AerosolState &dry_aero, - const mam_coupling::AerosolState &wet_aero, - MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, - MAMDryDep::view_3d tendencies) -{ - const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); - const Real t = 0; - const int icol = team.league_rank(); - - const mam4::AeroConfig aero_config; - mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); - mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::Surface surf; - mam4::Diagnostics diags; - mam4::Tendencies tends; - - diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); - diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - - for (int i=0; i::get_default_team_policy(ncol, nlev); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { const int num_aero_modes = mam_coupling::num_aero_modes(); - MAMDryDep::view_2d dgncur_awet[num_aero_modes]; - MAMDryDep::view_2d wet_dens[num_aero_modes]; - - for (int i=0; i::get_default_team_policy(ncol, nlev); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { - const int column = team.league_rank(); - compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, column); + const int num_aero_species = mam_coupling::num_aero_species(); + const int icol = team.league_rank(); + const Real t = 0; + compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); + team.team_barrier(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int lev) { for (int mode = 0; mode < num_aero_modes; ++mode) { int icnst = mam4::ConvProc::numptrcw_amode(mode); - qtracers(column, lev, icnst) = wet_aero.int_aero_nmr[mode](column,lev); + qtracers(icol, lev, icnst) = wet_aero.int_aero_nmr[mode](icol,lev); for (int species = 0; species < num_aero_species; ++species) { icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); if (-1 < icnst) { - qtracers(column, lev, icnst) = wet_aero.int_aero_mmr[mode][species](column,lev); + qtracers(icol, lev, icnst) = wet_aero.int_aero_mmr[mode][species](icol,lev); } } } }); + team.team_barrier(); + + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + mam4::Surface surf; + mam4::Diagnostics diags; + mam4::Tendencies tends; + + for (int i=0; i(gas_mmr_names(gas)); + return gas_mmr_names(gas); } // This type stores multi-column views related specifically to the wet From 56e9d2efd44924b09ed99eca0331d97999ab3326 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 10 Jul 2024 13:58:10 -0500 Subject: [PATCH 200/904] PAM fixes following SCREAM merge into E3SM --- components/eam/src/physics/crm/pam/CMakeLists.txt | 6 ++++-- components/eam/src/physics/crm/pam/pam_driver.cpp | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 95075debd4fd..4a8f88f669b0 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -11,6 +11,9 @@ set(PAM_DRIVER_SRC add_library(pam_driver ${PAM_DRIVER_SRC}) +set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) +add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) + if (${USE_CUDA}) # PAM will be CUDA-linked with device symbols resolved at library creation set_target_properties(pam_driver @@ -37,7 +40,6 @@ set(PAM_MICRO p3) set(PAM_SGS shoc) set(PAM_RAD forced) set(PAM_SCREAM_USE_CXX True) -set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) # removing this once the cime cmake_macros aren't using obsolete # Kokkos settings like KOKKOS_OPTIONS. @@ -124,7 +126,7 @@ target_compile_options(pam_driver PUBLIC ) if (PAM_SCREAM_USE_CXX) - target_link_libraries(pam_driver pam_core physics dynamics pam_scream_cxx_interfaces ekat p3 shoc physics_share scream_share) + target_link_libraries(pam_driver pam_core physics dynamics pam_scream_cxx_interfaces ekat p3 shoc physics_share scream_share eamxx_physics) else() target_link_libraries(pam_driver pam_core physics dynamics ) endif() diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index 5fd1c5071d16..c7038343f08a 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -70,13 +70,13 @@ inline int pam_driver_set_subcycle_timestep( pam::PamCoupler &coupler, real crm_ real crm_dz = input_zint(k_gcm,n) - input_zint(k_gcm+1,n); real cfl_u = uvel_max(k,n)*crm_dt_fixed/crm_dx; real cfl_w = wvel_max(k,n)*crm_dt_fixed/crm_dz; - cfl_max(k,n) = max(cfl_u,cfl_w); + cfl_max(k,n) = std::max(cfl_u,cfl_w); }); // calculate final CFL across ensemble real cfl_loc = yakl::intrinsics::maxval(cfl_max); - cfl = max(cfl,cfl_loc); + cfl = std::max(cfl,cfl_loc); // update number of subcycles and time step - num_subcycle = max(num_subcycle,max(1,static_cast(ceil(cfl/0.7)))); + num_subcycle = std::max(num_subcycle,std::max(1,static_cast(ceil(cfl/0.7)))); real crm_dt_subcycle = crm_dt_fixed / num_subcycle; coupler.set_option("crm_dt",crm_dt_subcycle); // check for excessive subcylcing - don't exit, just print From d56618a94cc405089b10323334c7a94f6ad854f7 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 12:22:14 -0600 Subject: [PATCH 201/904] resolved: registry topodrag --- components/mpas-ocean/src/Registry.xml | 21 ++++++++++++++- .../src/shared/mpas_ocn_diagnostics.F | 26 ++++++++++++++----- .../mpas-ocean/src/shared/mpas_ocn_forcing.F | 19 +++++++++++++- .../shared/mpas_ocn_surface_bulk_forcing.F | 18 +++++++++---- .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 20 ++++++++++---- .../src/shared/mpas_ocn_thick_surface_flux.F | 26 ++++++++++++++----- 6 files changed, 105 insertions(+), 25 deletions(-) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 825fe607a5b4..0ed2b0c42462 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -702,6 +702,10 @@ description="The length scale of exponential decay of river runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$." possible_values="Any positive real number." /> + + @@ -3347,6 +3352,9 @@ + + - + + \date 15 September 2011 !> \details !> This routine computes the horizontal advection tendency for -!> thicknes based on current state and user choices of forcings. +!> thickness based on current state and user choices of forcings. ! !----------------------------------------------------------------------- subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoefficientsRunoff, & - surfaceThicknessFlux, surfaceThicknessFluxRunoff, tend, err)!{{{ + transmissionCoefficientsSubglacialRunoff, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & + surfaceThicknessFluxSubglacialRunoff, tend, err)!{{{ !----------------------------------------------------------------- ! ! input variables @@ -83,11 +84,13 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe real (kind=RKIND), dimension(:,:), intent(in) :: & transmissionCoefficients, &!< Input: Coefficients for the transmission of surface fluxes - transmissionCoefficientsRunoff !< Input: Coefficients for the transmission of surface fluxes due to river runoff + transmissionCoefficientsRunoff, &!< Input: Coefficients for the transmission of surface fluxes due to river runoff + transmissionCoefficientsSubglacialRunoff !< Input: Coefficients for the transmission of surface fluxes due to river runoff real (kind=RKIND), dimension(:), intent(in) :: & surfaceThicknessFlux, &!< Input: surface flux of thickness - surfaceThicknessFluxRunoff !< Input: surface flux of thickness due to river runoff + surfaceThicknessFluxRunoff, &!< Input: surface flux of thickness due to river runoff + surfaceThicknessFluxSubglacialRunoff !< Input: surface flux of thickness due to subglacial runoff !----------------------------------------------------------------- @@ -124,10 +127,11 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe call mpas_timer_start("thick surface flux") #ifdef MPAS_OPENACC - !$acc enter data copyin(transmissionCoefficients, transmissionCoefficientsRunoff) + !$acc enter data copyin(transmissionCoefficients, transmissionCoefficientsRunoff, transmissionCoefficientsSubglacialRunoff) !$acc parallel loop & !$acc present(tend, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & + !$acc surfaceThicknessFluxSubglacialRunoff, transmissionCoefficientsSubglacialRunoff, & !$acc transmissionCoefficients, transmissionCoefficientsRunoff, minLevelCell, maxLevelCell) & !$acc private(k, remainingFlux, remainingFluxRunoff) #else @@ -137,12 +141,15 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe do iCell = 1, nCellsOwned remainingFlux = 1.0_RKIND remainingFluxRunoff = 1.0_RKIND + remainingFluxSubglacialRunoff = 1.0_RKIND do k = minLevelCell(iCell), maxLevelCell(iCell) remainingFlux = remainingFlux - transmissionCoefficients(k, iCell) remainingFluxRunoff = remainingFluxRunoff - transmissionCoefficientsRunoff(k, iCell) + remainingFluxSubglacialRunoff = remainingFluxSubglacialRunoff - transmissionCoefficientsSubglacialRunoff(k, iCell) tend(k, iCell) = tend(k, iCell) + surfaceThicknessFlux(iCell) * transmissionCoefficients(k, iCell) & - + surfaceThicknessFluxRunoff(iCell) * transmissionCoefficientsRunoff(k, iCell) + + surfaceThicknessFluxRunoff(iCell) * transmissionCoefficientsRunoff(k, iCell) & + + surfaceThicknessFluxSubglacialRunoff(iCell) * transmissionCoefficientsSubglacialRunoff(k, iCell) end do if(maxLevelCell(iCell) > 0 .and. remainingFlux > 0.0_RKIND) then @@ -153,6 +160,11 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe tend(maxLevelCell(iCell), iCell) = tend(maxLevelCell(iCell), iCell) & + remainingFluxRunoff * surfaceThicknessFluxRunoff(iCell) end if + + if(maxLevelCell(iCell) > 0 .and. remainingFluxSubglacialRunoff > 0.0_RKIND) then + tend(maxLevelCell(iCell), iCell) = tend(maxLevelCell(iCell), iCell) & + + remainingFluxSubglacialRunoff * surfaceThicknessFluxSubglacialRunoff(iCell) + end if end do #ifndef MPAS_OPENACC !$omp end do @@ -160,7 +172,7 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe #endif #ifdef MPAS_OPENACC - !$acc exit data delete(transmissionCoefficients, transmissionCoefficientsRunoff) + !$acc exit data delete(transmissionCoefficients, transmissionCoefficientsRunoff, transmissionCoefficientsSubglacialRunoff) #endif call mpas_timer_stop("thick surface flux") From f57b015c4c3532f94787fca00f3ed1a7a0ddb054 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 12:23:19 -0600 Subject: [PATCH 202/904] resolved: gitignore --- .gitignore | 13 +++++++++++++ .../mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 13 +++++++++++++ .../src/shared/mpas_ocn_diagnostics_variables.F | 13 ++++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d7a1fabfd6bc..d2fcc80bbb71 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,16 @@ components/eamxx/docs/common/eamxx_params.md components/eamxx/src/python/build components/eamxx/src/python/build_src components/eamxx/src/python/dist + +# OS generated files # +###################### +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# idea folder +.idea/ diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index a8807b995067..0ca331b57b68 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -4442,6 +4442,19 @@ subroutine ocn_validate_state(domain, timeLevel)!{{{ call ocn_write_field_statistics(debugUnit, fieldName, minValue, maxValue) end if + ! Test riverRunoffFlux + fieldName = 'subglacialRunoffFlux' + minValue = HUGE(minValue) + maxValue = -HUGE(maxValue) + call mpas_pool_get_array(forcingPool, fieldName, real1DArr) + if ( associated(real1DArr) ) then + do iCell = 1, nCellsSolve + minValue = min( minValue, real1DArr(iCell) ) + maxValue = max( maxValue, real1DArr(iCell) ) + end do + call ocn_write_field_statistics(debugUnit, fieldName, minValue, maxValue) + end if + ! Test seaIceSalinityFlux fieldName = 'seaIceSalinityFlux' minValue = HUGE(minValue) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F index 30a62f68c8b9..38f32f88e17c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F @@ -85,6 +85,7 @@ module ocn_diagnostics_variables real (kind=RKIND), dimension(:), pointer :: sfcFlxAttCoeff real (kind=RKIND), dimension(:), pointer :: surfaceFluxAttenuationCoefficientRunoff + real (kind=RKIND), dimension(:), pointer :: surfaceFluxAttenuationCoefficientSubglacialRunoff real (kind=RKIND), dimension(:), pointer :: landIceFrictionVelocity real (kind=RKIND), dimension(:), pointer :: velocityTidalRMS @@ -680,6 +681,9 @@ subroutine ocn_diagnostics_variables_init(domain, jenkinsOn, hollandJenkinsOn, e call mpas_pool_get_array(diagnosticsPool, & 'surfaceFluxAttenuationCoefficientRunoff', & surfaceFluxAttenuationCoefficientRunoff) + call mpas_pool_get_array(diagnosticsPool, & + 'surfaceFluxAttenuationCoefficientSubglacialRunoff', & + surfaceFluxAttenuationCoefficientSubglacialRunoff) call mpas_pool_get_array(diagnosticsPool, & 'boundaryLayerDepth', & boundaryLayerDepth) @@ -964,7 +968,8 @@ subroutine ocn_diagnostics_variables_init(domain, jenkinsOn, hollandJenkinsOn, e !$acc Time_bnds, & !$acc simulationStartTime, & !$acc boundaryLayerDepthSmooth, & - !$acc surfaceFluxAttenuationCoefficientRunoff & + !$acc surfaceFluxAttenuationCoefficientRunoff, & + !$acc surfaceFluxAttenuationCoefficientSubglacialRunoff & !$acc ) end subroutine ocn_diagnostics_variables_init!}}} @@ -1223,7 +1228,8 @@ subroutine ocn_diagnostics_variables_destroy(err) !{{{ !$acc Time_bnds, & !$acc simulationStartTime, & !$acc boundaryLayerDepthSmooth, & - !$acc surfaceFluxAttenuationCoefficientRunoff & + !$acc surfaceFluxAttenuationCoefficientRunoff, & + !$acc surfaceFluxAttenuationCoefficientSubglacialRunoff & !$acc ) ! Nullify pointers @@ -1425,7 +1431,8 @@ subroutine ocn_diagnostics_variables_destroy(err) !{{{ Time_bnds, & simulationStartTime, & boundaryLayerDepthSmooth, & - surfaceFluxAttenuationCoefficientRunoff) + surfaceFluxAttenuationCoefficientRunoff, & + surfaceFluxAttenuationCoefficientSubglacialRunoff) end subroutine ocn_diagnostics_variables_destroy!}}} From b4198e827e55af9f6c29cfe3510de83de20a1a46 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 18 Oct 2023 17:28:44 -0600 Subject: [PATCH 203/904] subglacial runoff for tracers, part 1 --- .../src/shared/mpas_ocn_diagnostics.F | 22 ++++++++-- .../shared/mpas_ocn_surface_bulk_forcing.F | 21 +++++++--- .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 31 +++++++++++--- .../src/shared/mpas_ocn_thick_surface_flux.F | 2 +- .../mpas_ocn_tracer_surface_flux_to_tend.F | 40 ++++++++++++++++++- 5 files changed, 98 insertions(+), 18 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 0ca331b57b68..4e75d743130a 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3298,6 +3298,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & real (kind=RKIND) :: & fracAbsorbed, &! fraction of sfc flux absorbed fracAbsorbedRunoff, &! same for runoff + fracAbsorbedSubglacialRunoff, &! same for subglacial runoff sumSurfaceStressSquared ! sum of sfc stress squared ! pointers for variable/pool retrievals @@ -3313,6 +3314,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & penetrativeTemperatureFlux, &! various sfc flux components surfaceThicknessFlux, & surfaceThicknessFluxRunoff, & + surfaceThicknessFluxSubglacialRunoff, & rainTemperatureFlux, & evapTemperatureFlux, & icebergTemperatureFlux, & @@ -3325,6 +3327,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & normalVelocity, &! normal velocity activeTracersSurfaceFlux, &! sfc flux of active tracers (T,S) activeTracersSurfaceFluxRunoff, &! flx of tracers in runoff + activeTracersSurfaceFluxSubglacialRunoff, &! flx of tracers in subglacial runoff nonLocalSurfaceTracerFlux ! non-local flux of tracers real (kind=RKIND), dimension(:,:,:), pointer :: & @@ -3388,11 +3391,16 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & call mpas_pool_get_array(tracersSurfaceFluxPool, & 'activeTracersSurfaceFluxRunoff', & activeTracersSurfaceFluxRunoff) + call mpas_pool_get_array(tracersSurfaceFluxPool, & + 'activeTracersSurfaceFluxSubglacialRunoff', & + activeTracersSurfaceFluxSubglacialRunoff) call mpas_pool_get_array(forcingPool, 'surfaceThicknessFlux', & surfaceThicknessFlux) call mpas_pool_get_array(forcingPool, 'surfaceThicknessFluxRunoff', & surfaceThicknessFluxRunoff) + call mpas_pool_get_array(forcingPool, 'surfaceThicknessFluxSubglacialRunoff', & + surfaceThicknessFluxSubglacialRunoff) call mpas_pool_get_array(forcingPool, 'penetrativeTemperatureFlux', & penetrativeTemperatureFlux) call mpas_pool_get_array(forcingPool, 'surfaceStress', & @@ -3465,7 +3473,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & !$omp parallel !$omp do schedule(runtime) & - !$omp private(kmin, fracAbsorbed, fracAbsorbedRunoff, & + !$omp private(kmin, fracAbsorbed, fracAbsorbedRunoff, fracAbsorbedSubglacialRunoff, & !$omp sumSurfaceStressSquared, i, iEdge) do iCell = 1, nCells @@ -3479,6 +3487,10 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & - exp( max(-100.0_RKIND, & -layerThickness(kmin, iCell)/ & config_flux_attenuation_coefficient_runoff)) + fracAbsorbedSubglacialRunoff = 1.0_RKIND & + - exp( max(-100.0_RKIND, & + -layerThickness(kmin, iCell)/ & + config_flux_attenuation_coefficient_subglacial_runoff)) ! Store the total tracer flux below in ! nonLocalSurfaceTemperatureFlux for use in the CVMix nonlocal @@ -3493,13 +3505,17 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & seaIceTemperatureFlux(iCell) + & icebergTemperatureFlux(iCell)) & - fracAbsorbedRunoff* & - activeTracersSurfaceFluxRunoff(indexTempFlux,iCell) + activeTracersSurfaceFluxRunoff(indexTempFlux,iCell) & + - fracAbsorbedSubglacialRunoff* & + activeTracersSurfaceFluxSubglacialRunoff(indexTempFlux,iCell) nonLocalSurfaceTracerFlux(indexSaltFlux,iCell) = & activeTracersSurfaceFlux(indexSaltFlux,iCell) & - fracAbsorbed*surfaceThicknessFlux(iCell)* & activeTracers(indexSaltFlux,kmin,iCell) & - fracAbsorbedRunoff*surfaceThicknessFluxRunoff(iCell)* & + activeTracers(indexSaltFlux,kmin,iCell) & + - fracAbsorbedSubglacialRunoff*surfaceThicknessFluxSubglacialRunoff(iCell)* & activeTracers(indexSaltFlux,kmin,iCell) surfaceBuoyancyForcing(iCell) = & @@ -4442,7 +4458,7 @@ subroutine ocn_validate_state(domain, timeLevel)!{{{ call ocn_write_field_statistics(debugUnit, fieldName, minValue, maxValue) end if - ! Test riverRunoffFlux + ! Test subglacialRunoffFlux fieldName = 'subglacialRunoffFlux' minValue = HUGE(minValue) maxValue = -HUGE(maxValue) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index ca0ca9fc4b89..f37305381792 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -78,7 +78,7 @@ module ocn_surface_bulk_forcing !----------------------------------------------------------------------- subroutine ocn_surface_bulk_forcing_tracers(meshPool, groupName, forcingPool, tracerGroup, & - tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxRemoved, dt, layerThickness, err)!{{{ + tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxSubglacialRunoff, tracersSurfaceFluxRemoved, dt, layerThickness, err)!{{{ !----------------------------------------------------------------- ! @@ -98,6 +98,8 @@ subroutine ocn_surface_bulk_forcing_tracers(meshPool, groupName, forcingPool, tr real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFlux !< Input/Output: Surface flux for tracer group real (kind=RKIND), dimension(:,:), intent(inout) :: & tracersSurfaceFluxRunoff !< Input/Output: Surface flux for tracer group due to river runoff + real (kind=RKIND), dimension(:,:), intent(inout) :: & + tracersSurfaceFluxSubglacialRunoff !< Input/Output: Surface flux for tracer group due to subglacial runoff real (kind=RKIND), dimension(:,:), intent(inout) :: & tracersSurfaceFluxRemoved !< Input/Output: Accumulator for ignored Surface flux for tracer group real (kind=RKIND), dimension(:,:), intent(in) :: layerThickness @@ -122,7 +124,7 @@ subroutine ocn_surface_bulk_forcing_tracers(meshPool, groupName, forcingPool, tr call mpas_timer_start("bulk_" // trim(groupName)) if ( trim(groupName) == 'activeTracers' ) then call ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracerGroup, & - tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err) + tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxSubglacialRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err) end if call mpas_timer_stop("bulk_" // trim(groupName)) @@ -426,7 +428,7 @@ end subroutine ocn_surface_bulk_forcing_init!}}} !----------------------------------------------------------------------- subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracerGroup, & - tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err)!{{{ + tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxSubglacialRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err)!{{{ !----------------------------------------------------------------- ! @@ -443,6 +445,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer type (mpas_pool_type), intent(inout) :: forcingPool !< Input: Forcing information real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFlux real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFluxRunoff + real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFluxSubglacialRunoff real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFluxRemoved real (kind=RKIND), dimension(:,:,:), intent(inout) :: tracerGroup real (kind=RKIND), dimension(:,:), intent(in) :: layerThickness @@ -470,7 +473,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer type(mpas_pool_type),pointer :: tracersSurfaceFluxPool real (kind=RKIND), dimension(:), pointer :: latentHeatFlux, sensibleHeatFlux, longWaveHeatFluxUp, longWaveHeatFluxDown, & - seaIceHeatFlux, icebergHeatFlux, evaporationFlux, riverRunoffFlux + seaIceHeatFlux, icebergHeatFlux, evaporationFlux, riverRunoffFlux, subglacialRunoffFlux real (kind=RKIND), dimension(:), pointer :: seaIceFreshWaterFlux, icebergFreshWaterFlux, seaIceSalinityFlux, iceRunoffFlux real (kind=RKIND), dimension(:), pointer :: shortWaveHeatFlux, penetrativeTemperatureFlux real (kind=RKIND), dimension(:), pointer :: snowFlux, rainFlux @@ -572,6 +575,11 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer tracersSurfaceFluxRunoff(index_temperature_flux,iCell) = riverRunoffFlux(iCell) & * max(tracerGroup(index_temperature_flux,minLevelCell(iCell),iCell), 0.0_RKIND) / rho_sw + ! Subglacial runoff can have temperature below 0.0C, since it can come out at great depth + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & + ocn_freezing_temperature(salinity=0.0_RKIND, pressure=interfacePressure(iCell), & + inLandIceCavity=.true.) / rho_sw + ! Accumulate fluxes that use the freezing point ! mrp performance note: should call ocn_freezing_temperature just once here seaIceTemperatureFlux(iCell) = seaIceFreshWaterFlux(iCell) * & @@ -587,9 +595,10 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer tracersSurfaceFlux(index_temperature_flux, iCell) = tracersSurfaceFlux(index_temperature_flux, iCell) & + surfaceTemperatureFluxWithoutRunoff - ! add runoff contribution for sending through coupler + ! add river and subglacial runoff contribution for sending through coupler totalFreshWaterTemperatureFlux(iCell) = surfaceTemperatureFluxWithoutRunoff & - + tracersSurfaceFluxRunoff(index_temperature_flux,iCell) + + tracersSurfaceFluxRunoff(index_temperature_flux,iCell) & + + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) ! Fields with zero temperature are not accumulated. These include: ! snowFlux diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index 10188a3be11c..2d3dd66cbdb9 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -134,7 +134,8 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ real (kind=RKIND), dimension(:,:), pointer, contiguous :: & tendThick, &! accumulated layer thickness tendency fractionAbsorbed, &! fraction of sfc flux absorbed - fractionAbsorbedRunoff ! fraction of runoff flux absorbed + fractionAbsorbedRunoff, &! fraction of sfc flux absorbed + fractionAbsorbedSubglacialRunoff ! fraction of subglacial runoff flux absorbed !----------------------------------------------------------------- ! local variables @@ -161,6 +162,10 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ fractionAbsorbed) call mpas_pool_get_array(forcingPool, 'fractionAbsorbedRunoff', & fractionAbsorbedRunoff) + call mpas_pool_get_array(forcingPool, 'fractionAbsorbedSubglacialRunoff', & + fractionAbsorbedSubglacialRunoff) + + ! ! layer thickness tendency: @@ -610,9 +615,11 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & real (kind=RKIND), dimension(:,:), pointer, contiguous :: & layerThickness, &! layer thickness tracerGroupSurfaceFlux, &! tracer flux at surface - fractionAbsorbed, &! frac sfc flux aborbed - fractionAbsorbedRunoff, &! frac runoff flux aborbed + fractionAbsorbed, &! frac sfc flux absorbed + fractionAbsorbedRunoff, &! frac runoff flux absorbed + fractionAbsorbedSubglacialRunoff, &! frac subglacial runoff flux absorbed tracerGroupSurfaceFluxRunoff, &! runoff flux + tracerGroupSurfaceFluxSubglacialRunoff, &! subglacial runoff flux tracerGroupSurfaceFluxRemoved,&! total sfc flux absorbed nonLocalSurfaceTracerFlux ! non-local fluxes (eg KPP) @@ -674,6 +681,8 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & fractionAbsorbed) call mpas_pool_get_array(forcingPool, 'fractionAbsorbedRunoff', & fractionAbsorbedRunoff) + call mpas_pool_get_array(forcingPool, 'fractionAbsorbedSubglacialRunoff', & + fractionAbsorbedSubglacialRunoff) ! allocate and transfer data not specific to tracer groups allocate(normalThicknessFlux(nVertLevels, nEdgesAll+1)) @@ -798,6 +807,14 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & modifiedGroupName, & tracerGroupSurfaceFluxRunoff) + ! Get surface flux due to subglacial runoff array + ! only active tracers have subglacial runoff flux for now, + ! but we still need to associate for ALL tracers + modifiedGroupName = groupName // "SurfaceFluxSubglacialRunoff" + call mpas_pool_get_array(tracersSurfaceFluxPool, & + modifiedGroupName, & + tracerGroupSurfaceFluxSubglacialRunoff) + ! Get surface flux removed array to keep track of how much ! flux is ignored modifiedGroupName = groupName // "SurfaceFluxRemoved" @@ -824,6 +841,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & do n=1,nTracersGroup tracerGroupSurfaceFlux (n,iCell) = 0.0_RKIND tracerGroupSurfaceFluxRunoff (n,iCell) = 0.0_RKIND + tracerGroupSurfaceFluxSubglacialRunoff (n,iCell) = 0.0_RKIND tracerGroupSurfaceFluxRemoved(n,iCell) = 0.0_RKIND end do end do @@ -839,6 +857,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & forcingPool, tracerGroup, & tracerGroupSurfaceFlux, & tracerGroupSurfaceFluxRunoff, & + tracerGroupSurfaceFluxSubglacialRunoff, & tracerGroupSurfaceFluxRemoved, & dt, layerThickness, err) @@ -1198,9 +1217,9 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & endif ! compute budgets call ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, & - fractionAbsorbedRunoff, layerThickness, & - tracerGroupSurfaceFlux, & - tracerGroupSurfaceFluxRunoff, & + fractionAbsorbedRunoff, fractionAbsorbedSubglacialRunoff, & + layerThickness, tracerGroupSurfaceFlux, & + tracerGroupSurfaceFluxRunoff, tracerGroupSurfaceFluxSubglacialRunoff, & tracerGroupTend, err) ! diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F index 043ed7b2862b..180b4f79a753 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F @@ -85,7 +85,7 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe real (kind=RKIND), dimension(:,:), intent(in) :: & transmissionCoefficients, &!< Input: Coefficients for the transmission of surface fluxes transmissionCoefficientsRunoff, &!< Input: Coefficients for the transmission of surface fluxes due to river runoff - transmissionCoefficientsSubglacialRunoff !< Input: Coefficients for the transmission of surface fluxes due to river runoff + transmissionCoefficientsSubglacialRunoff !< Input: Coefficients for the transmission of surface fluxes due to subglacial runoff real (kind=RKIND), dimension(:), intent(in) :: & surfaceThicknessFlux, &!< Input: surface flux of thickness diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F index 57e2949bac9b..4303d1b78ff2 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F @@ -71,8 +71,8 @@ module ocn_tracer_surface_flux_to_tend ! !----------------------------------------------------------------------- - subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbsorbedRunoff, layerThickness, & - surfaceTracerFlux, surfaceTracerFluxRunoff, tend, err)!{{{ + subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbsorbedRunoff, fractionAbsorbedSubglacialRunoff, & + layerThickness, surfaceTracerFlux, surfaceTracerFluxRunoff, surfaceTracerFluxSubglacialRunoff, tend, err)!{{{ !----------------------------------------------------------------- ! ! input variables @@ -97,6 +97,13 @@ subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbso real (kind=RKIND), dimension(:,:), intent(in) :: & fractionAbsorbedRunoff !< Input: Coefficients for the application of surface fluxes due to river runoff + real (kind=RKIND), dimension(:,:), intent(in), pointer :: & + surfaceTracerFluxSubglacialRunoff !< Input: surface tracer fluxes from subglacial runoff + + real (kind=RKIND), dimension(:,:), intent(in) :: & + fractionAbsorbedSubglacialRunoff !< Input: Coefficients for the application of surface fluxes due to subglacial runoff + + !----------------------------------------------------------------- ! ! input/output variables @@ -197,7 +204,36 @@ subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbso call mpas_timer_stop("surface_tracer_runoff_flux") end if + ! now do subglacial runoff component + + if (associated(surfaceTracerFluxSubglacialRunoff)) then + call mpas_timer_start("surface_tracer_subglacial_runoff_flux") + + !$omp parallel + !$omp do schedule(runtime) private(remainingFlux, k, iTracer) + do iCell = 1, nCells + remainingFlux = 1.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + remainingFlux = remainingFlux - fractionAbsorbedSubglacialRunoff(k, iCell) + do iTracer = 1, nTracers + tend(iTracer, k, iCell) = tend(iTracer, k, iCell) + & + surfaceTracerFluxSubglacialRunoff(iTracer, iCell) * fractionAbsorbedSubglacialRunoff(k, iCell) + end do + end do + + if(maxLevelCell(iCell) > 0 .and. remainingFlux > 0.0_RKIND) then + do iTracer = 1, nTracers + tend(iTracer, maxLevelCell(iCell), iCell) = tend(iTracer, maxLevelCell(iCell), iCell) & + + surfaceTracerFluxSubglacialRunoff(iTracer, iCell) * remainingFlux + end do + end if + end do + !$omp end do + !$omp end parallel + + call mpas_timer_stop("surface_tracer_subglacial_runoff_flux") + end if !-------------------------------------------------------------------- end subroutine ocn_tracer_surface_flux_tend!}}} From 5cb8a57d5b11a2220b317d28ba2bad76b332457f Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 19 Oct 2023 11:40:43 -0600 Subject: [PATCH 204/904] subglacial runoff for tracers, part 2 - registry --- .../src/tracer_groups/Registry_CFC.xml | 9 ++ .../src/tracer_groups/Registry_DMS.xml | 9 ++ .../tracer_groups/Registry_MacroMolecules.xml | 12 ++ .../tracer_groups/Registry_activeTracers.xml | 8 ++ .../tracer_groups/Registry_debugTracers.xml | 11 ++ .../src/tracer_groups/Registry_ecosys.xml | 128 +++++++++++++++++- .../src/tracer_groups/Registry_idealAge.xml | 5 + 7 files changed, 181 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/tracer_groups/Registry_CFC.xml b/components/mpas-ocean/src/tracer_groups/Registry_CFC.xml index d77859dcef86..b0adb7d595f5 100644 --- a/components/mpas-ocean/src/tracer_groups/Registry_CFC.xml +++ b/components/mpas-ocean/src/tracer_groups/Registry_CFC.xml @@ -95,6 +95,15 @@ description="CFC12 Surface Flux Due to Runoff" /> + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/components/mpas-ocean/src/tracer_groups/Registry_idealAge.xml b/components/mpas-ocean/src/tracer_groups/Registry_idealAge.xml index c6adca3ea636..3a45bc10b794 100644 --- a/components/mpas-ocean/src/tracer_groups/Registry_idealAge.xml +++ b/components/mpas-ocean/src/tracer_groups/Registry_idealAge.xml @@ -69,6 +69,11 @@ description="Flux of iAge through the ocean surface due to river runoff. Positive into ocean." /> + + + Date: Thu, 19 Oct 2023 15:43:37 -0600 Subject: [PATCH 205/904] subglacial runoff freezing temperature at landicepressure --- components/mpas-ocean/src/Registry.xml | 8 ++++++++ .../mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 0ed2b0c42462..3a11826169d4 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -3787,6 +3787,14 @@ description="Global sum of fresh water flux from river runoff from the coupler that was removed due to config_remove_AIS_coupler_runoff option. Positive into the ocean." packages="thicknessBulkPKG" /> + + Date: Tue, 9 Jul 2024 12:27:07 -0600 Subject: [PATCH 206/904] resolved: compsets tmix --- components/mpas-ocean/bld/build-namelist | 7 +++++ .../mpas-ocean/bld/build-namelist-section | 1 + .../namelist_defaults_mpaso.xml | 1 + .../namelist_definition_mpaso.xml | 8 ++++++ components/mpas-ocean/cime_config/buildnml | 27 +++++++++++++++++++ .../cime_config/config_component.xml | 13 +++++++++ .../cime_config/config_compsets.xml | 10 +++++++ components/mpas-ocean/src/Registry.xml | 2 +- 8 files changed, 68 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 1c55bcb7be1d..ad8e268fbb0a 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -54,6 +54,9 @@ OPTIONS -ocn_ismf variable for defining how the ocn model will handle ice shelf melt fluxes Options are: none, data, internal, coupled + -ocn_sgr variable for defining how the ocn model will handle subglacial + runoff + Options are: none, data, coupled -decomp_prefix decomp_prefix variable -date_stamp date_stamp variable -cfg_grid Directory containing MPASO configuration scripts. @@ -111,6 +114,7 @@ my %opts = ( help => 0, ocn_forcing => undef, ocn_iceberg => undef, ocn_ismf => undef, + ocn_sgr => undef, decomp_prefix => undef, date_stamp => undef, ocn_bgc => undef, @@ -137,6 +141,7 @@ GetOptions( "ocn_forcing=s" => \$opts{'ocn_forcing'}, "ocn_iceberg=s" => \$opts{'ocn_iceberg'}, "ocn_ismf=s" => \$opts{'ocn_ismf'}, + "ocn_sgr=s" => \$opts{'ocn_sgr'}, "decomp_prefix=s" => \$opts{'decomp_prefix'}, "date_stamp=s" => \$opts{'date_stamp'}, "ocn_bgc=s" => \$opts{'ocn_bgc'}, @@ -180,6 +185,7 @@ my $OCN_GRID = $opts{'ocn_grid'}; my $OCN_FORCING = $opts{'ocn_forcing'}; my $OCN_ICEBERG = $opts{'ocn_iceberg'}; my $OCN_ISMF = $opts{'ocn_ismf'}; +my $OCN_SGR = $opts{'ocn_sgr'}; my $decomp_prefix = $opts{'decomp_prefix'}; my $date_stamp = $opts{'date_stamp'}; my $ocn_bgc = $opts{'ocn_bgc'}; @@ -702,6 +708,7 @@ add_default($nl, 'config_use_bulk_wind_stress'); add_default($nl, 'config_use_bulk_thickness_flux'); add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); +add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index cc1695243faa..7a1d06b35a7c 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -224,6 +224,7 @@ add_default($nl, 'config_use_bulk_wind_stress'); add_default($nl, 'config_use_bulk_thickness_flux'); add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); +add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index c7441c6b2214..5686c0e0a012 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -342,6 +342,7 @@ .true. 0.001 10.0 +10.0 .false. diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index f6c2f9c5b765..ac2bfd411e4b 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1188,6 +1188,14 @@ Valid values: Any positive real number. Default: Defined in namelist_defaults.xml + +The length scale of exponential decay of subglacial runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$. + +Valid values: Any positive real number. +Default: Defined in namelist_defaults.xml + + diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 4c7ed2c0a311..7ee71aee3a33 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -35,6 +35,7 @@ def buildnml(case, caseroot, compname): ocn_forcing = case.get_value("MPASO_FORCING") ocn_iceberg = case.get_value("MPASO_ICEBERG") ocn_ismf = case.get_value("MPASO_ISMF") + ocn_sgr = case.get_value("MPASO_SGR") ocn_bgc = case.get_value("MPASO_BGC") ocn_wave = case.get_value("MPASO_WAVE") ocn_tidal_mixing = case.get_value("MPASO_TIDAL_MIXING") @@ -76,6 +77,7 @@ def buildnml(case, caseroot, compname): analysis_mask_file = '' eco_forcing_file = '' u_tidal_rms_file = '' + data_sgr_file = '' if ocn_grid == 'oEC60to30v3': decomp_date = '230424' @@ -140,6 +142,8 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_paolo2023.oQU240wLI.20240404.nc' if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.oQU240wLI.20240221.nc' + if ocn_sgr == 'data': + data_sgr_file = 'oQU240wLI_DSGR.nc' elif ocn_grid == 'oQU120': decomp_date = '230424' @@ -352,6 +356,8 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_adusumilli2020.ECwISC30to60E2r1.230429.nc' if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.ECwISC30to60E2r1.20240221.nc' + if ocn_sgr == 'data': + data_sgr_file = 'ECwISC30to60E2r1_DSGR.nc' elif ocn_grid == 'IcoswISC30E3r5': decomp_date = '20231120' @@ -436,6 +442,9 @@ def buildnml(case, caseroot, compname): if u_tidal_rms_file != '': input_list.write("u_tidal_rms = {}/ocn/mpas-o/{}/{}\n".format(din_loc_root, ocn_mask, u_tidal_rms_file)) + if data_sgr_file != '': + input_list.write("subglacial_runoff = {}/ocn/mpas-o/{}/{}\n".format(din_loc_root, ocn_mask, data_sgr_file)) + #-------------------------------------------------------------------- # Invoke mpas build-namelist - output will go in $CASEBUILD/mpasoconf #-------------------------------------------------------------------- @@ -495,6 +504,7 @@ def buildnml(case, caseroot, compname): sysmod += " -ocn_forcing '{}'".format(ocn_forcing) sysmod += " -ocn_iceberg '{}'".format(ocn_iceberg) sysmod += " -ocn_ismf '{}'".format(ocn_ismf) + sysmod += " -ocn_sgr '{}'".format(ocn_sgr) sysmod += " -ocn_bgc '{}'".format(ocn_bgc) sysmod += " -ocn_wave '{}'".format(ocn_wave) sysmod += " -ocn_tidal_mixing '{}'".format(ocn_tidal_mixing) @@ -702,6 +712,19 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') + if data_sgr_file != '': + lines.append('') + lines.append('') + lines.append(' ') + lines.append('') + lines.append('') + if analysis_mask_file != '': lines.append('') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -1533,6 +1557,7 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -1598,6 +1623,7 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -1812,6 +1838,7 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') diff --git a/components/mpas-ocean/cime_config/config_component.xml b/components/mpas-ocean/cime_config/config_component.xml index e95b63682d95..eb8e09519829 100644 --- a/components/mpas-ocean/cime_config/config_component.xml +++ b/components/mpas-ocean/cime_config/config_component.xml @@ -39,6 +39,19 @@ Option to describe the MPASO prescribed tidal mixing + + char + none,data,coupled + none + + none + data + + case_comp + env_case.xml + Option to describe how MPASO will handle subglacial runoff fluxes + + char false,true diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index 448aa7a82b57..27b0218f4a26 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -87,6 +87,11 @@ 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + + GMPAS-JRA1p5-DIB-PISMF-DSGR + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + + GMPAS-JRA1p5-DIB-DISMF 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV @@ -97,6 +102,11 @@ 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + + GMPAS-JRA1p5-DIB-DISMF-DSGR + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + + GMPAS-JRA1p4 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 3a11826169d4..5e36e828f32b 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -702,7 +702,7 @@ description="The length scale of exponential decay of river runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$." possible_values="Any positive real number." /> - From 0fccd7005f50caf729c14c2df81b79224e5c3b11 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Fri, 20 Oct 2023 14:36:43 -0600 Subject: [PATCH 207/904] typo --- components/mpas-ocean/src/Registry.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 5e36e828f32b..f56799eaaf07 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -703,7 +703,7 @@ possible_values="Any positive real number." /> From f820afb71ee2e22ae6326f0e7c5bda7bf67c3eef Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Fri, 20 Oct 2023 15:13:14 -0600 Subject: [PATCH 208/904] more compsets --- components/mpas-ocean/cime_config/config_compsets.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index 27b0218f4a26..766b1397bcf2 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -37,6 +37,11 @@ 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV + + GMPAS-NYF-PISMF-DSGR + 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYFAIS45_SGLC_SWAV + + GMPAS-NYF-DISMF 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV @@ -142,6 +147,11 @@ 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV + + GMPAS-IAF-PISMF-DSGR + 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%IAFAIS45_SGLC_SWAV + + GMPAS-IAF-DISMF 2000_DATM%IAF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV From 71dc2ee55e2b9beabfd713e421f6fed87301b6b8 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 23 Oct 2023 16:53:14 -0600 Subject: [PATCH 209/904] typo: remainingFluxSubglacialRunoff --- .../mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F index 180b4f79a753..1f7c67b88e09 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F @@ -118,7 +118,7 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe integer :: iCell, k - real (kind=RKIND) :: remainingFlux, remainingFluxRunoff + real (kind=RKIND) :: remainingFlux, remainingFluxRunoff, remainingFluxSubglacialRunoff err = 0 @@ -133,10 +133,10 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe !$acc present(tend, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & !$acc surfaceThicknessFluxSubglacialRunoff, transmissionCoefficientsSubglacialRunoff, & !$acc transmissionCoefficients, transmissionCoefficientsRunoff, minLevelCell, maxLevelCell) & - !$acc private(k, remainingFlux, remainingFluxRunoff) + !$acc private(k, remainingFlux, remainingFluxRunoff, remainingFluxSubglacialRunoff) #else !$omp parallel - !$omp do schedule(runtime) private(remainingFlux, remainingFluxRunoff, k) + !$omp do schedule(runtime) private(remainingFlux, remainingFluxRunoff, remainingFluxSubglacialRunoff, k) #endif do iCell = 1, nCellsOwned remainingFlux = 1.0_RKIND From 2637229a9be48f2d82474f961d0dc18132da86c3 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 26 Oct 2023 10:58:22 -0600 Subject: [PATCH 210/904] updated sg units --- components/mpas-ocean/src/Registry.xml | 2 +- .../mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index f56799eaaf07..ad8ce6752441 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -3775,7 +3775,7 @@ description="Fresh water flux from river runoff at cell centers from coupler. Positive into the ocean." packages="thicknessBulkPKG" /> - diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 4714400951d2..c3a8a6691ddd 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -351,7 +351,7 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur surfaceThicknessFlux(iCell) = surfaceThicknessFlux(iCell) + ( snowFlux(iCell) + rainFlux(iCell) + evaporationFlux(iCell) & + seaIceFreshWaterFlux(iCell) + icebergFreshWaterFlux(iCell) + iceRunoffFlux(iCell) ) / rho_sw surfaceThicknessFluxRunoff(iCell) = riverRunoffFlux(iCell) / rho_sw - surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / rho_sw + surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / areaCell(iCell) end do #ifndef MPAS_OPENACC !$omp end do @@ -581,7 +581,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer ! Subglacial runoff can have temperature below 0.0C, since it can come out at great depth tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & - inLandIceCavity=.true.) / rho_sw + inLandIceCavity=.true.) / areaCell(iCell) ! Accumulate fluxes that use the freezing point ! mrp performance note: should call ocn_freezing_temperature just once here From 98dc48b3a0c3f04948e9225f0c7ce808573a35f2 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 31 Oct 2023 14:08:09 -0600 Subject: [PATCH 211/904] change config_flux_attenuation_coefficient_subglacial_runoff default to 1 mm --- .../mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml | 2 +- components/mpas-ocean/src/Registry.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 5686c0e0a012..66a6482c54aa 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -342,7 +342,7 @@ .true. 0.001 10.0 -10.0 +0.001 .false. diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index ad8ce6752441..76e868860a41 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -702,7 +702,7 @@ description="The length scale of exponential decay of river runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$." possible_values="Any positive real number." /> - From 239697803ded5efafc22d345fe5fe6adc0d05582 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 31 Oct 2023 14:53:00 -0600 Subject: [PATCH 212/904] add namelist option to include some parts of runoff only: config_use_SGR_opt --- .../mpas-ocean/bld/build-namelist-section | 1 + .../namelist_defaults_mpaso.xml | 1 + .../namelist_definition_mpaso.xml | 8 ++++++++ components/mpas-ocean/src/Registry.xml | 4 ++++ .../shared/mpas_ocn_surface_bulk_forcing.F | 20 +++++++++++++++---- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index 7a1d06b35a7c..b8f544edabaf 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -225,6 +225,7 @@ add_default($nl, 'config_use_bulk_thickness_flux'); add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); +add_default($nl, 'config_use_SGR_opt'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 66a6482c54aa..92709a4ee45d 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -343,6 +343,7 @@ 0.001 10.0 0.001 +0 .false. diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index ac2bfd411e4b..fc6374cc2a7f 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1196,6 +1196,14 @@ Valid values: Any positive real number. Default: Defined in namelist_defaults.xml + +Option to use or not subglacial runoff (SGR). + +Valid values: 0 - don't use SGR, 1 - use both mass and TS from SGR, 2 - use mass only from SGR , 3 - use TS only from SGR +Default: Defined in namelist_defaults.xml + + diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 76e868860a41..083db34082e4 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -706,6 +706,10 @@ description="The length scale of exponential decay of subglacial runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$." possible_values="Any positive real number." /> + Date: Tue, 31 Oct 2023 14:56:51 -0600 Subject: [PATCH 213/904] more basic compsets --- components/mpas-ocean/cime_config/config_compsets.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index 766b1397bcf2..6950fd8c2179 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -42,6 +42,11 @@ 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYFAIS45_SGLC_SWAV + + GMPAS-NYF-DSGR + 2000_DATM%NYF_SLND_MPASSI_MPASO%DATMFORCEDDSGR_DROF%NYFAIS45_SGLC_SWAV + + GMPAS-NYF-DISMF 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV @@ -97,6 +102,11 @@ 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + + GMPAS-JRA1p5-DSGR + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%DATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + + GMPAS-JRA1p5-DIB-DISMF 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV From 9b786dd97402994a88eca8c1eaa9a6c5663d06aa Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 31 Oct 2023 15:35:05 -0600 Subject: [PATCH 214/904] forgotten bld --- components/mpas-ocean/bld/build-namelist | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index ad8e268fbb0a..f70d6204be5f 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -709,6 +709,7 @@ add_default($nl, 'config_use_bulk_thickness_flux'); add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); +add_default($nl, 'config_use_SGR_opt'); ############################ # Namelist group: coupling # From de04001fe002b961ad230936314e17fc7fbe693d Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 1 Nov 2023 14:31:42 -0600 Subject: [PATCH 215/904] no sg in kpp option --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 4e75d743130a..999d69337abb 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3487,10 +3487,14 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & - exp( max(-100.0_RKIND, & -layerThickness(kmin, iCell)/ & config_flux_attenuation_coefficient_runoff)) - fracAbsorbedSubglacialRunoff = 1.0_RKIND & + if (config_use_SGR_opt == -1) then + fracAbsorbedSubglacialRunoff = 1.0_RKIND & - exp( max(-100.0_RKIND, & -layerThickness(kmin, iCell)/ & config_flux_attenuation_coefficient_subglacial_runoff)) + else + fracAbsorbedSubglacialRunoff = 0.0_RKIND + end if ! Store the total tracer flux below in ! nonLocalSurfaceTemperatureFlux for use in the CVMix nonlocal From 0ab238461d06ad3d605033731346d955c6bf2799 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 1 Nov 2023 17:50:46 -0600 Subject: [PATCH 216/904] config_use_SGR_opt -1 correction --- .../bld/namelist_files/namelist_definition_mpaso.xml | 2 +- components/mpas-ocean/src/Registry.xml | 2 +- .../mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index fc6374cc2a7f..c38d6edd97ad 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1200,7 +1200,7 @@ Default: Defined in namelist_defaults.xml category="forcing" group="forcing"> Option to use or not subglacial runoff (SGR). -Valid values: 0 - don't use SGR, 1 - use both mass and TS from SGR, 2 - use mass only from SGR , 3 - use TS only from SGR +Valid values: 0 - don't use SGR, 1 - use both mass and TS from SGR, 2 - use mass only from SGR , 3 - use TS only from SGR, -1 use all + KPP Default: Defined in namelist_defaults.xml diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 083db34082e4..953fa1cb39b4 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -708,7 +708,7 @@ /> diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index a12e628b2cc6..de9898c60997 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -352,7 +352,7 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur + seaIceFreshWaterFlux(iCell) + icebergFreshWaterFlux(iCell) + iceRunoffFlux(iCell) ) / rho_sw surfaceThicknessFluxRunoff(iCell) = riverRunoffFlux(iCell) / rho_sw - if ((config_use_SGR_opt == 1) .or. (config_use_SGR_opt == 2)) then + if ((config_use_SGR_opt == 1) .or. (config_use_SGR_opt == 2) .or. (config_use_SGR_opt == -1)) then surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / areaCell(iCell) else surfaceThicknessFluxSubglacialRunoff(iCell) = 0.0_RKIND @@ -587,7 +587,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer * max(tracerGroup(index_temperature_flux,minLevelCell(iCell),iCell), 0.0_RKIND) / rho_sw ! Subglacial runoff can have temperature below 0.0C, since it can come out at great depth - if ((config_use_SGR_opt == 1) .or. (config_use_SGR_opt == 3)) then + if ((config_use_SGR_opt == 1) .or. (config_use_SGR_opt == 3) .or. (config_use_SGR_opt == -1)) then tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & inLandIceCavity=.true.) / areaCell(iCell) From 81d948887f769ad72ac92a37ad6d78474b467f2c Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 8 Nov 2023 10:38:13 -0700 Subject: [PATCH 217/904] config read in forcing --- components/mpas-ocean/src/shared/mpas_ocn_forcing.F | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F index f530f67dfa62..8617cc4f8435 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F @@ -27,6 +27,7 @@ module ocn_forcing use mpas_dmpar use ocn_constants use ocn_diagnostics_variables + use ocn_config !IV comment implicit none private @@ -197,6 +198,8 @@ subroutine ocn_forcing_build_fraction_absorbed_array(meshPool, statePool, forcin end do end do + call mpas_log_write('config_use_SGR_opt: $r', realArgs=(/config_use_SGR_opt/)) + end subroutine ocn_forcing_build_fraction_absorbed_array!}}} !*********************************************************************** From 4745779efa824c568f84e4054e5bbf75f9e45031 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 8 Nov 2023 12:28:44 -0700 Subject: [PATCH 218/904] sgr_flux_vertical_location and SGR to sgr --- components/mpas-ocean/bld/build-namelist | 3 +- .../mpas-ocean/bld/build-namelist-section | 3 +- .../namelist_defaults_mpaso.xml | 3 +- .../namelist_definition_mpaso.xml | 10 +++- components/mpas-ocean/src/Registry.xml | 10 ++-- .../src/shared/mpas_ocn_diagnostics.F | 2 +- .../mpas-ocean/src/shared/mpas_ocn_forcing.F | 53 +++++++++++++------ .../shared/mpas_ocn_surface_bulk_forcing.F | 4 +- 8 files changed, 63 insertions(+), 25 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index f70d6204be5f..b413599f1778 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -709,7 +709,8 @@ add_default($nl, 'config_use_bulk_thickness_flux'); add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); -add_default($nl, 'config_use_SGR_opt'); +add_default($nl, 'config_sgr_flux_vertical_location'); +add_default($nl, 'config_use_sgr_opt'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index b8f544edabaf..9c0ad217b91f 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -225,7 +225,8 @@ add_default($nl, 'config_use_bulk_thickness_flux'); add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); -add_default($nl, 'config_use_SGR_opt'); +add_default($nl, 'config_sgr_flux_vertical_location'); +add_default($nl, 'config_use_sgr_opt'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 92709a4ee45d..5b3bf93522a6 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -343,7 +343,8 @@ 0.001 10.0 0.001 -0 +'top' +0 .false. diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index c38d6edd97ad..7629e5663eff 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1196,7 +1196,15 @@ Valid values: Any positive real number. Default: Defined in namelist_defaults.xml - +Selects the vertical location where subglacial runoff is fluxed + +Valid values: 'top','uniform', 'bottom'. +Default: Defined in namelist_defaults.xml + + + Option to use or not subglacial runoff (SGR). diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 953fa1cb39b4..88928169b1ce 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -706,9 +706,13 @@ description="The length scale of exponential decay of subglacial runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$." possible_values="Any positive real number." /> - + diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 999d69337abb..a7d03f5dcc89 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3487,7 +3487,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & - exp( max(-100.0_RKIND, & -layerThickness(kmin, iCell)/ & config_flux_attenuation_coefficient_runoff)) - if (config_use_SGR_opt == -1) then + if (config_use_sgr_opt == -1) then fracAbsorbedSubglacialRunoff = 1.0_RKIND & - exp( max(-100.0_RKIND, & -layerThickness(kmin, iCell)/ & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F index 8617cc4f8435..e642e82e2e67 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F @@ -184,21 +184,44 @@ subroutine ocn_forcing_build_fraction_absorbed_array(meshPool, statePool, forcin ! now do subglacial runoff separately - do iCell = 1, nCells - zTop = 0.0_RKIND - transmissionCoeffTop = ocn_forcing_transmission(zTop, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) - do k = minLevelCell(iCell), maxLevelCell(iCell) - zBot = zTop - layerThickness(k,iCell) - transmissionCoeffBot = ocn_forcing_transmission(zBot, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) - - fractionAbsorbedSubglacialRunoff(k, iCell) = transmissionCoeffTop - transmissionCoeffBot - - zTop = zBot - transmissionCoeffTop = transmissionCoeffBot - end do - end do - - call mpas_log_write('config_use_SGR_opt: $r', realArgs=(/config_use_SGR_opt/)) + if ( trim(config_sgr_flux_vertical_location) == 'top' ) then + do iCell = 1, nCells + zTop = 0.0_RKIND + transmissionCoeffTop = ocn_forcing_transmission(zTop, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) + do k = minLevelCell(iCell), maxLevelCell(iCell) + zBot = zTop - layerThickness(k,iCell) + transmissionCoeffBot = ocn_forcing_transmission(zBot, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) + fractionAbsorbedSubglacialRunoff(k, iCell) = transmissionCoeffTop - transmissionCoeffBot + zTop = zBot + transmissionCoeffTop = transmissionCoeffBot + end do + end do + else if ( trim(config_sgr_flux_vertical_location) == 'uniform' ) then + do iCell = 1, nCells + ! calculate total thickness + zTop = 0.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + zTop = zTop + layerThickness(k,iCell) + end do + ! distribute flux evenly throughout water column + zBot = 0.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + fractionAbsorbedSubglacialRunoff(k, iCell) = layerThickness(k,iCell) / zTop + end do + end do + else if ( trim(config_sgr_flux_vertical_location) == 'bottom' ) then + do iCell = 1, nCells + zTop = 0.0_RKIND + transmissionCoeffTop = ocn_forcing_transmission(zTop, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) + do k = maxLevelCell(iCell), minLevelCell(iCell), -1 + zBot = zTop - layerThickness(k,iCell) + transmissionCoeffBot = ocn_forcing_transmission(zBot, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) + fractionAbsorbedSubglacialRunoff(k, iCell) = transmissionCoeffTop - transmissionCoeffBot + zTop = zBot + transmissionCoeffTop = transmissionCoeffBot + end do + end do + end if end subroutine ocn_forcing_build_fraction_absorbed_array!}}} diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index de9898c60997..f6156609df09 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -352,7 +352,7 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur + seaIceFreshWaterFlux(iCell) + icebergFreshWaterFlux(iCell) + iceRunoffFlux(iCell) ) / rho_sw surfaceThicknessFluxRunoff(iCell) = riverRunoffFlux(iCell) / rho_sw - if ((config_use_SGR_opt == 1) .or. (config_use_SGR_opt == 2) .or. (config_use_SGR_opt == -1)) then + if ((config_use_sgr_opt == 1) .or. (config_use_sgr_opt == 2) .or. (config_use_sgr_opt == -1)) then surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / areaCell(iCell) else surfaceThicknessFluxSubglacialRunoff(iCell) = 0.0_RKIND @@ -587,7 +587,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer * max(tracerGroup(index_temperature_flux,minLevelCell(iCell),iCell), 0.0_RKIND) / rho_sw ! Subglacial runoff can have temperature below 0.0C, since it can come out at great depth - if ((config_use_SGR_opt == 1) .or. (config_use_SGR_opt == 3) .or. (config_use_SGR_opt == -1)) then + if ((config_use_sgr_opt == 1) .or. (config_use_sgr_opt == 3) .or. (config_use_sgr_opt == -1)) then tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & inLandIceCavity=.true.) / areaCell(iCell) From e55b516afd33df71a1789b7a32b6f918a5e82b94 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 16 Nov 2023 18:39:57 -0700 Subject: [PATCH 219/904] more sgr testing options for TS separation --- components/mpas-ocean/bld/build-namelist | 6 ++- .../mpas-ocean/bld/build-namelist-section | 6 ++- .../namelist_defaults_mpaso.xml | 7 +++- .../namelist_definition_mpaso.xml | 37 +++++++++++++++++-- components/mpas-ocean/src/Registry.xml | 23 ++++++++++-- .../src/shared/mpas_ocn_diagnostics.F | 2 +- .../shared/mpas_ocn_surface_bulk_forcing.F | 29 ++++++++------- 7 files changed, 87 insertions(+), 23 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index b413599f1778..e1d2548dffeb 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -710,7 +710,11 @@ add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); add_default($nl, 'config_sgr_flux_vertical_location'); -add_default($nl, 'config_use_sgr_opt'); +add_default($nl, 'config_use_sgr_opt_kpp'); +add_default($nl, 'config_use_sgr_opt_temp'); +add_default($nl, 'config_use_sgr_opt_salt'); +add_default($nl, 'config_sgr_temperature'); +add_default($nl, 'config_sgr_salinity'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index 9c0ad217b91f..0d44d3f328b6 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -226,7 +226,11 @@ add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); add_default($nl, 'config_sgr_flux_vertical_location'); -add_default($nl, 'config_use_sgr_opt'); +add_default($nl, 'config_use_sgr_opt_kpp'); +add_default($nl, 'config_use_sgr_opt_temp'); +add_default($nl, 'config_use_sgr_opt_salt'); +add_default($nl, 'config_sgr_temperature'); +add_default($nl, 'config_sgr_salinity'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 5b3bf93522a6..a2c6465c6d1c 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -344,7 +344,12 @@ 10.0 0.001 'top' -0 +0 +'sgr' +'sgr' +0.0 +0.0 + .false. diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index 7629e5663eff..8152b1fec724 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1204,14 +1204,45 @@ Valid values: 'top','uniform', 'bottom'. Default: Defined in namelist_defaults.xml - -Option to use or not subglacial runoff (SGR). +Option to use or not subglacial runoff (sgr) in kpp calculation. -Valid values: 0 - don't use SGR, 1 - use both mass and TS from SGR, 2 - use mass only from SGR , 3 - use TS only from SGR, -1 use all + KPP +Valid values: 0 - don't use, 1 - use Default: Defined in namelist_defaults.xml + +Option to choose sgr temperature + +Valid values: 'sgr','current', 'prescribed'. +Default: Defined in namelist_defaults.xml + + + +Option to choose sgr salinity + +Valid values: 'sgr','current', 'prescribed'. +Default: Defined in namelist_defaults.xml + + + +Option to choose sgr temperature, applied when config_use_sgr_opt_temp = 'prescribed' + +Valid values: Any real number. +Default: Defined in namelist_defaults.xml + + + +Option to choose sgr salinity, applied when config_use_sgr_opt_temp = 'prescribed' + +Valid values: Any real number. +Default: Defined in namelist_defaults.xml + diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 88928169b1ce..b22a9e7f5b94 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -710,10 +710,27 @@ description="Selects the vertical location where subglacial runoff is fluxed." possible_values="'top','uniform', 'bottom'" /> - + + + + + Date: Tue, 9 Jul 2024 12:31:41 -0600 Subject: [PATCH 220/904] resolved: conservation check --- components/mpas-ocean/cime_config/buildnml | 3 ++ .../Registry_conservation_check.xml | 17 +++++++++++ .../mpas_ocn_conservation_check.F | 28 ++++++++++++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 7ee71aee3a33..9566cc56bbb8 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -1192,14 +1192,17 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') + lines.append('') lines.append('') lines.append('') lines.append('') lines.append('') lines.append('') lines.append('') + lines.append('') lines.append('') lines.append('') + lines.append('') lines.append('') lines.append('') lines.append('') diff --git a/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml b/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml index 0fd2d28d1c6e..646daf743eee 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml @@ -106,6 +106,9 @@ + @@ -148,6 +151,10 @@ + + @@ -287,14 +298,17 @@ + + + @@ -343,14 +357,17 @@ + + + diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index bdd07e799b5d..612bb2078bf0 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -828,8 +828,10 @@ subroutine mass_conservation(domain, err) accumulatedEvaporationFlux, & accumulatedSeaIceFlux, & accumulatedRiverRunoffFlux, & + accumulatedSubglacialRunoffFlux, & accumulatedIceRunoffFlux, & accumulatedRemovedRiverRunoffFlux, & + accumulatedRemovedSubglacialRunoffFlux, & accumulatedRemovedIceRunoffFlux, & accumulatedIcebergFlux, & accumulatedFrazilFlux, & @@ -852,8 +854,10 @@ subroutine mass_conservation(domain, err) evaporationFlux, & seaIceFreshwaterFlux, & riverRunoffFlux, & + subglacialRunoffFlux, & iceRunoffFlux, & removedRiverRunoffFlux, & + removedSubglacialRunoffFlux, & removedIceRunoffFlux, & icebergFreshwaterFlux, & accumulatedFrazilIceMassNew, & @@ -877,7 +881,7 @@ subroutine mass_conservation(domain, err) iCell, ierr, k integer, parameter :: & - nSums = 12 + nSums = 14 integer, dimension(:), pointer :: minLevelCell, maxLevelCell @@ -900,8 +904,10 @@ subroutine mass_conservation(domain, err) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedEvaporationFlux", accumulatedEvaporationFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSeaIceFlux", accumulatedSeaIceFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRiverRunoffFlux", accumulatedRiverRunoffFlux) + call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSubglacialRunoffFlux", accumulatedSubglacialRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedIceRunoffFlux", accumulatedIceRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedRiverRunoffFlux",accumulatedRemovedRiverRunoffFlux) + call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedSubglacialRunoffFlux",accumulatedRemovedSubglacialRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedIceRunoffFlux", accumulatedRemovedIceRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedIcebergFlux", accumulatedIcebergFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedFrazilFlux", accumulatedFrazilFlux) @@ -937,8 +943,10 @@ subroutine mass_conservation(domain, err) call mpas_pool_get_array(forcingPool, 'evaporationFlux', evaporationFlux) call mpas_pool_get_array(forcingPool, 'seaIceFreshWaterFlux', seaIceFreshwaterFlux) call mpas_pool_get_array(forcingPool, 'riverRunoffFlux', riverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'subglacialRunoffFlux', subglacialRunoffFlux) call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) call mpas_pool_get_array(forcingPool, 'removedRiverRunoffFlux', removedRiverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'removedSubglacialRunoffFlux', removedSubglacialRunoffFlux) call mpas_pool_get_array(forcingPool, 'removedIceRunoffFlux', removedIceRunoffFlux) call mpas_pool_get_array(forcingPool, 'icebergFreshWaterFlux', icebergFreshwaterFlux) call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) @@ -956,6 +964,8 @@ subroutine mass_conservation(domain, err) sumArray( 7) = sumArray( 7) + areaCell(iCell) * removedRiverRunoffFlux(iCell) sumArray( 8) = sumArray( 8) + areaCell(iCell) * removedIceRunoffFlux(iCell) sumArray( 9) = sumArray( 9) + areaCell(iCell) * icebergFreshwaterFlux(iCell) + sumArray(13) = sumArray(13) + areaCell(iCell) * subglacialRunoffFlux(iCell) + sumArray(14) = sumArray(14) + areaCell(iCell) * removedSubglacialRunoffFlux(iCell) enddo if (config_use_frazil_ice_formation) then @@ -1002,6 +1012,8 @@ subroutine mass_conservation(domain, err) accumulatedFrazilFlux = accumulatedFrazilFlux + sumArrayOut(10) accumulatedLandIceFlux = accumulatedLandIceFlux + sumArrayOut(11) accumulatedLandIceFrazilFlux = accumulatedLandIceFrazilFlux + sumArrayOut(12) + accumulatedSubglacialRunoffFlux = accumulatedSubglacialRunoffFlux + sumArrayOut(13) + accumulatedRemovedSubglacialRunoffFlux = accumulatedRemovedSubglacialRunoffFlux + sumArrayOut(14) ! cleanup deallocate(sumArray) @@ -1021,8 +1033,10 @@ subroutine mass_conservation(domain, err) accumulatedEvaporationFlux = accumulatedEvaporationFlux /accumulatedFluxCounter accumulatedSeaIceFlux = accumulatedSeaIceFlux /accumulatedFluxCounter accumulatedRiverRunoffFlux = accumulatedRiverRunoffFlux /accumulatedFluxCounter + accumulatedSubglacialRunoffFlux = accumulatedSubglacialRunoffFlux /accumulatedFluxCounter accumulatedIceRunoffFlux = accumulatedIceRunoffFlux /accumulatedFluxCounter accumulatedRemovedRiverRunoffFlux = accumulatedRemovedRiverRunoffFlux /accumulatedFluxCounter + accumulatedRemovedSubglacialRunoffFlux = accumulatedRemovedSubglacialRunoffFlux /accumulatedFluxCounter accumulatedRemovedIceRunoffFlux = accumulatedRemovedIceRunoffFlux /accumulatedFluxCounter accumulatedIcebergFlux = accumulatedIcebergFlux /accumulatedFluxCounter accumulatedFrazilFlux = accumulatedFrazilFlux /accumulatedFluxCounter @@ -1049,6 +1063,7 @@ subroutine mass_conservation(domain, err) + accumulatedEvaporationFlux & + accumulatedSeaIceFlux & + accumulatedRiverRunoffFlux & + + accumulatedSubglacialRunoffFlux & + accumulatedIceRunoffFlux & + accumulatedIcebergFlux & + accumulatedFrazilFlux & @@ -1083,10 +1098,15 @@ subroutine mass_conservation(domain, err) v=accumulatedIcebergFlux ; write(m,"('icebergFreshwaterFlux ',es16.8,' x2o_Fioi_bergw wberg ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v v=accumulatedEvaporationFlux ; write(m,"('evaporationFlux ',es16.8,' x2o_Foxx_evap wevap ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v v=accumulatedRiverRunoffFlux ; write(m,"('riverRunoffFlux ',es16.8,' x2o_Foxx_rofl wrunoff ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v +v=accumulatedSubglacialRunoffFlux ; write(m,"('subglacialRunoffFlux ',es16.8,' x2o_Foxx_rofl wsgr ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v + if (landIceFreshwaterFluxesOn) then v=accumulatedRemovedRiverRunoffFlux; write(m,"('removedRiverRunoffFlux ',es16.8,' wrunoff ',f16.8)") v,v*c; call mpas_log_write(m); v=accumulatedRiverRunoffFlux+accumulatedRemovedRiverRunoffFlux; write(m,"(' SUM: river runoff',es16.8,' x2o_Foxx_rofl wrunoff SUM ',f16.8)") v,v*c; call mpas_log_write(m) +v=accumulatedRemovedSubglacialRunoffFlux; write(m,"('removedSubglacialRunoffFlux ',es16.8,' wsgr ',f16.8)") v,v*c; call mpas_log_write(m); +v=accumulatedSubglacialRunoffFlux+accumulatedRemovedSubglacialRunoffFlux; + write(m,"(' SUM: subglacial runoff',es16.8,' x2o_Foxx_rofl wsgr SUM ',f16.8)") v,v*c; call mpas_log_write(m) endif v=accumulatedIceRunoffFlux ; write(m,"('iceRunoffFlux ',es16.8,' x2o_Foxx_rofi wfrzrof ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v if (landIceFreshwaterFluxesOn) then @@ -2240,8 +2260,10 @@ subroutine reset_accumulated_variables(domain) accumulatedEvaporationFlux, & accumulatedSeaIceFlux, & accumulatedRiverRunoffFlux, & + accumulatedSubglacialRunoffFlux, & accumulatedIceRunoffFlux, & accumulatedRemovedRiverRunoffFlux, & + accumulatedRemovedSubglacialRunoffFlux, & accumulatedRemovedIceRunoffFlux, & accumulatedIcebergFlux, & accumulatedFrazilFlux, & @@ -2310,8 +2332,10 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedEvaporationFlux", accumulatedEvaporationFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSeaIceFlux", accumulatedSeaIceFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRiverRunoffFlux", accumulatedRiverRunoffFlux) + call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSubglacialRunoffFlux", accumulatedSubglacialRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedIceRunoffFlux", accumulatedIceRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedRiverRunoffFlux",accumulatedRemovedRiverRunoffFlux) + call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedSubglacialRunoffFlux",accumulatedRemovedSubglacialRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedIceRunoffFlux", accumulatedRemovedIceRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedIcebergFlux", accumulatedIcebergFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedFrazilFlux", accumulatedFrazilFlux) @@ -2323,8 +2347,10 @@ subroutine reset_accumulated_variables(domain) accumulatedEvaporationFlux = 0.0_RKIND accumulatedSeaIceFlux = 0.0_RKIND accumulatedRiverRunoffFlux = 0.0_RKIND + accumulatedSubglacialRunoffFlux = 0.0_RKIND accumulatedIceRunoffFlux = 0.0_RKIND accumulatedRemovedRiverRunoffFlux = 0.0_RKIND + accumulatedRemovedSubglacialRunoffFlux = 0.0_RKIND accumulatedRemovedIceRunoffFlux = 0.0_RKIND accumulatedIcebergFlux = 0.0_RKIND accumulatedFrazilFlux = 0.0_RKIND From af89287b4221cc4a514ccad511b92b850561f323 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 22 Nov 2023 14:20:29 -0700 Subject: [PATCH 221/904] removed redundant removedSubglacialRunoffFlux --- components/mpas-ocean/cime_config/buildnml | 1 - components/mpas-ocean/src/Registry.xml | 8 -------- .../mpas_ocn_conservation_check.F | 15 +-------------- 3 files changed, 1 insertion(+), 23 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 9566cc56bbb8..fca676ae03d5 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -1202,7 +1202,6 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') - lines.append('') lines.append('') lines.append('') lines.append('') diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index b22a9e7f5b94..7e3b1d125dae 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -3812,14 +3812,6 @@ description="Global sum of fresh water flux from river runoff from the coupler that was removed due to config_remove_AIS_coupler_runoff option. Positive into the ocean." packages="thicknessBulkPKG" /> - - Date: Mon, 27 Nov 2023 16:57:56 -0700 Subject: [PATCH 222/904] correct sgr units in flux add up --- .../src/analysis_members/mpas_ocn_conservation_check.F | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index cc1f2f3e288e..18ba8d6989de 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -28,6 +28,7 @@ module ocn_conservation_check use ocn_constants use ocn_config + use ocn_mesh implicit none private @@ -960,7 +961,7 @@ subroutine mass_conservation(domain, err) sumArray( 7) = sumArray( 7) + areaCell(iCell) * removedRiverRunoffFlux(iCell) sumArray( 8) = sumArray( 8) + areaCell(iCell) * removedIceRunoffFlux(iCell) sumArray( 9) = sumArray( 9) + areaCell(iCell) * icebergFreshwaterFlux(iCell) - sumArray(13) = sumArray(13) + areaCell(iCell) * subglacialRunoffFlux(iCell) + sumArray(13) = sumArray(13) + areaCell(iCell) * subglacialRunoffFlux(iCell) / areaCell(iCell) * rho_sw enddo if (config_use_frazil_ice_formation) then From 957d8b218d3cf06b2c4d94325d48a8507c0bdcfd Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 27 Nov 2023 17:46:10 -0700 Subject: [PATCH 223/904] energy conservation check --- .../mpas_ocn_conservation_check.F | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index 18ba8d6989de..e793babb51a1 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -428,6 +428,7 @@ subroutine energy_conservation(domain, err) accumulatedEvapTemperatureFlux, & accumulatedSeaIceTemperatureFlux, & accumulatedRiverRunoffTemperatureFlux, & + accumulatedSubglacialRunoffTemperatureFlux, & accumulatedIcebergTemperatureFlux real(kind=RKIND), dimension(:), allocatable :: & @@ -464,7 +465,8 @@ subroutine energy_conservation(domain, err) real(kind=RKIND), dimension(:,:), pointer :: & - activeTracersSurfaceFluxRunoff + activeTracersSurfaceFluxRunoff, & + activeTracersSurfaceFluxSubglacialRunoff type (MPAS_timeInterval_type) :: & timeStepESMF @@ -481,7 +483,7 @@ subroutine energy_conservation(domain, err) ierr integer, parameter :: & - nSums = 19 + nSums = 20 character(len=160) :: & m @@ -514,6 +516,7 @@ subroutine energy_conservation(domain, err) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedEvapTemperatureFlux", accumulatedEvapTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSeaIceTemperatureFlux", accumulatedSeaIceTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedRiverRunoffTemperatureFlux", accumulatedRiverRunoffTemperatureFlux) + call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSubglacialRunoffTemperatureFlux", accumulatedSubglacialRunoffTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedIcebergTemperatureFlux", accumulatedIcebergTemperatureFlux) !------------------------------------------------------------- @@ -557,6 +560,7 @@ subroutine energy_conservation(domain, err) call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_temperatureSurfaceFlux', index_temperature_flux) call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxRunoff', activeTracersSurfaceFluxRunoff) + call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxSubglacialRunoff', activeTracersSurfaceFluxSubglacialRunoff) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMassNew, 2) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMassOld, 1) @@ -581,6 +585,7 @@ subroutine energy_conservation(domain, err) ! river runoff temperature flux sumArray(14) = sumArray(14) + areaCell(iCell) * activeTracersSurfaceFluxRunoff(index_temperature_flux,iCell) sumArray(15) = sumArray(15) + areaCell(iCell) * icebergTemperatureFlux(iCell) + sumArray(20) = sumArray(20) + areaCell(iCell) * activeTracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) enddo ! iCell @@ -635,6 +640,7 @@ subroutine energy_conservation(domain, err) accumulatedLandIceHeatFlux = accumulatedLandIceHeatFlux + sumArrayOut(17) accumulatedLandIceFrazilHeatFlux = accumulatedLandIceFrazilHeatFlux + sumArrayOut(18) accumulatedRemovedIceRunoffHeatFlux = accumulatedRemovedIceRunoffHeatFlux + sumArrayOut(19) + accumulatedSubglacialRunoffTemperatureFlux = accumulatedSubglacialRunoffTemperatureFlux + sumArrayOut(20) ! cleanup deallocate(sumArray) @@ -664,6 +670,7 @@ subroutine energy_conservation(domain, err) accumulatedEvapTemperatureFlux = accumulatedEvapTemperatureFlux /accumulatedFluxCounter accumulatedSeaIceTemperatureFlux = accumulatedSeaIceTemperatureFlux /accumulatedFluxCounter accumulatedRiverRunoffTemperatureFlux = accumulatedRiverRunoffTemperatureFlux /accumulatedFluxCounter + accumulatedSubglacialRunoffTemperatureFlux = accumulatedSubglacialRunoffTemperatureFlux /accumulatedFluxCounter accumulatedIcebergTemperatureFlux = accumulatedIcebergTemperatureFlux /accumulatedFluxCounter accumulatedLandIceFrazilHeatFlux = accumulatedLandIceFrazilHeatFlux /accumulatedFluxCounter accumulatedRemovedIceRunoffHeatFlux = accumulatedRemovedIceRunoffHeatFlux /accumulatedFluxCounter @@ -698,6 +705,7 @@ subroutine energy_conservation(domain, err) + accumulatedEvapTemperatureFlux *rho_sw*cp_sw & + accumulatedSeaIceTemperatureFlux *rho_sw*cp_sw & + accumulatedRiverRunoffTemperatureFlux *rho_sw*cp_sw & + + accumulatedSubglacialRunoffTemperatureFlux *rho_sw*cp_sw & + accumulatedIcebergTemperatureFlux*rho_sw*cp_sw ! note, accumulatedLandIceFrazilHeatFlux not added because already in accumulatedFrazilHeatFlux @@ -761,6 +769,7 @@ subroutine energy_conservation(domain, err) v=accumulatedEvapTemperatureFlux *rho_sw*cp_sw; write(m,"('EvapTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedSeaIceTemperatureFlux *rho_sw*cp_sw; write(m,"('SeaIceTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedRiverRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('RiverRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v +v=accumulatedSubglacialRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('RiverRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedIcebergTemperatureFlux*rho_sw*cp_sw; write(m,"('IcebergTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v write(m,"('SUM IMPLICIT HEAT FLUXES ',es16.8,' hh20temp ',f16.8,es16.8)") s, s/A; call mpas_log_write(m) s = s + explicitHeatFluxSum @@ -2243,6 +2252,7 @@ subroutine reset_accumulated_variables(domain) accumulatedEvapTemperatureFlux, & accumulatedSeaIceTemperatureFlux, & accumulatedRiverRunoffTemperatureFlux, & + accumulatedSubglacialRunoffTemperatureFlux, & accumulatedIcebergTemperatureFlux real(kind=RKIND), pointer :: & @@ -2292,6 +2302,7 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedEvapTemperatureFlux", accumulatedEvapTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSeaIceTemperatureFlux", accumulatedSeaIceTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedRiverRunoffTemperatureFlux", accumulatedRiverRunoffTemperatureFlux) + call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSubglacialRunoffTemperatureFlux", accumulatedSubglacialRunoffTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedIcebergTemperatureFlux", accumulatedIcebergTemperatureFlux) accumulatedFluxCounter = 0 @@ -2310,6 +2321,7 @@ subroutine reset_accumulated_variables(domain) accumulatedEvapTemperatureFlux = 0.0_RKIND accumulatedSeaIceTemperatureFlux = 0.0_RKIND accumulatedRiverRunoffTemperatureFlux = 0.0_RKIND + accumulatedSubglacialRunoffTemperatureFlux = 0.0_RKIND accumulatedIcebergTemperatureFlux = 0.0_RKIND accumulatedLandIceFrazilHeatFlux = 0.0_RKIND accumulatedRemovedIceRunoffHeatFlux = 0.0_RKIND From ecb06580badc28a55316eea6caae273eb1fd415b Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 12:34:50 -0600 Subject: [PATCH 224/904] resolved: conservation check #2 --- components/mpas-ocean/cime_config/buildnml | 1 + .../Registry_conservation_check.xml | 5 +++ .../mpas_ocn_conservation_check.F | 32 ++++++++++++++++--- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index fca676ae03d5..725c26d28181 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -1208,6 +1208,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') + lines.append('') lines.append('') lines.append('') lines.append('') diff --git a/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml b/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml index 646daf743eee..8a79ac12629e 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml @@ -205,6 +205,9 @@ + + @@ -375,6 +379,7 @@ + diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index e793babb51a1..cdba7b357f2d 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -1180,10 +1180,14 @@ subroutine salt_conservation(domain, err) absoluteSaltError, & relativeSaltError + real(kind=RKIND), dimension(:,:), pointer :: & + activeTracersSurfaceFluxSubglacialRunoff + real(kind=RKIND), pointer :: & accumulatedSeaIceSalinityFlux, & - accumulatedFrazilSalinityFlux ! accumulatedLandIceFrazilSalinityFlux is not present because it is always 0 + accumulatedFrazilSalinityFlux, & + accumulatedSubglacialRunoffSalinityFlux real(kind=RKIND), dimension(:), allocatable :: & sumArray, & @@ -1204,14 +1208,15 @@ subroutine salt_conservation(domain, err) dt, dtAvg, v, A, s, c integer, pointer :: & - nCellsSolve + nCellsSolve, & + index_salinity_flux integer :: & iCell, & ierr integer, parameter :: & - nSums = 3 + nSums = 4 logical, pointer :: & activeTracersBulkRestoringPKG @@ -1229,6 +1234,7 @@ subroutine salt_conservation(domain, err) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSeaIceSalinityFlux", accumulatedSeaIceSalinityFlux) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedFrazilSalinityFlux", accumulatedFrazilSalinityFlux) + call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSubglacialRunoffSalinityFlux", accumulatedSubglacialRunoffSalinityFlux) !------------------------------------------------------------- ! Net salt flux to ice @@ -1254,11 +1260,17 @@ subroutine salt_conservation(domain, err) call mpas_pool_get_array(forcingPool, 'seaIceSalinityFlux', seaIceSalinityFlux) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceSalinity', accumulatedFrazilIceSalinityOld, 1) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceSalinity', accumulatedFrazilIceSalinityNew, 2) + call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxSubglacialRunoff', activeTracersSurfaceFluxSubglacialRunoff) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_salinitySurfaceFlux', index_salinity_flux) + + do iCell = 1, nCellsSolve ! salt flux to ocean sumArray(1) = sumArray(1) + areaCell(iCell) * seaIceSalinityFlux(iCell) + ! subglacial runoff + sumArray(4) = sumArray(4) + areaCell(iCell) * activeTracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) enddo ! iCell if (config_use_frazil_ice_formation) then @@ -1285,6 +1297,7 @@ subroutine salt_conservation(domain, err) ! accumulate fluxes accumulatedSeaIceSalinityFlux = accumulatedSeaIceSalinityFlux + sumArrayOut(1) accumulatedFrazilSalinityFlux = accumulatedFrazilSalinityFlux + sumArrayOut(2) + accumulatedSubglacialRunoffSalinityFlux = accumulatedSubglacialRunoffSalinityFlux + sumArrayOut(4) ! cleanup deallocate(sumArray) @@ -1301,6 +1314,7 @@ subroutine salt_conservation(domain, err) ! Average the fluxes accumulatedSeaIceSalinityFlux = accumulatedSeaIceSalinityFlux /accumulatedFluxCounter accumulatedFrazilSalinityFlux = accumulatedFrazilSalinityFlux /accumulatedFluxCounter + accumulatedSubglacialRunoffSalinityFlux = accumulatedSubglacialRunoffSalinityFlux /accumulatedFluxCounter ! get initial salt content call MPAS_pool_get_array(conservationCheckSaltAMPool, "initialSalt", initialSalt) @@ -1317,7 +1331,8 @@ subroutine salt_conservation(domain, err) call MPAS_pool_get_array(conservationCheckSaltAMPool, "netSaltFlux", netSaltFlux) netSaltFlux = accumulatedSeaIceSalinityFlux & - + accumulatedFrazilSalinityFlux + + accumulatedFrazilSalinityFlux & + + accumulatedSubglacialRunoffSalinityFlux ! compute the final salt error call MPAS_pool_get_array(conservationCheckSaltAMPool, "absoluteSaltError", absoluteSaltError) @@ -1347,6 +1362,8 @@ subroutine salt_conservation(domain, err) .and.config_frazil_under_land_ice) then v=0; write(m,"('LandIceFrazilSalinityFlux',es16.8,' (already in wmelt, do not sum) ',f16.8)") v,v*c; call mpas_log_write(m); !no sum: s=s+v end if +v=accumulatedSubglacialRunoffSalinityFlux ; write(m,"('SubglacialRunoffSalinityFlux ',es16.8,' x2o_Fioi_salt salt ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v + write(m,"('SUM VOLUME FLUXES ',es16.8,' ',f16.8,es16.8)") s, s*c; call mpas_log_write(m) call mpas_log_write(' ') @@ -2272,7 +2289,8 @@ subroutine reset_accumulated_variables(domain) real(kind=RKIND), pointer :: & accumulatedSeaIceSalinityFlux, & - accumulatedFrazilSalinityFlux + accumulatedFrazilSalinityFlux, & + accumulatedSubglacialRunoffSalinityFlux real(kind=RKIND), pointer :: & accumulatedCarbonSourceSink, & @@ -2362,9 +2380,13 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSeaIceSalinityFlux", accumulatedSeaIceSalinityFlux) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedFrazilSalinityFlux", accumulatedFrazilSalinityFlux) + call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedLandIceFrazilSalinityFlux", accumulatedLandIceFrazilSalinityFlux) + call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSubglacialRunoffSalinityFlux", accumulatedSubglacialRunoffSalinityFlux) + accumulatedSeaIceSalinityFlux = 0.0_RKIND accumulatedFrazilSalinityFlux = 0.0_RKIND + accumulatedSubglacialRunoffSalinityFlux = 0.0_RKIND call MPAS_pool_get_subpool(domain % blocklist % structs, "conservationCheckCarbonAM", conservationCheckCarbonAMPool) From a430272a1d1c41468387cc47e1a3038f2aa7187e Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 28 Nov 2023 09:54:29 -0700 Subject: [PATCH 225/904] salt: added tracersSurfaceFluxPool definition --- .../src/analysis_members/mpas_ocn_conservation_check.F | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index cdba7b357f2d..1278342ceb2d 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -1170,7 +1170,8 @@ subroutine salt_conservation(domain, err) conservationCheckSaltAMPool, & meshPool, & statePool, & - forcingPool + forcingPool, & + tracersSurfaceFluxPool real(kind=RKIND), pointer :: & initialSalt, & @@ -1260,6 +1261,7 @@ subroutine salt_conservation(domain, err) call mpas_pool_get_array(forcingPool, 'seaIceSalinityFlux', seaIceSalinityFlux) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceSalinity', accumulatedFrazilIceSalinityOld, 1) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceSalinity', accumulatedFrazilIceSalinityNew, 2) + call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxSubglacialRunoff', activeTracersSurfaceFluxSubglacialRunoff) call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_salinitySurfaceFlux', index_salinity_flux) From 7c404b94c7c25415fac6d8e54b10257b4a20c58c Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 28 Nov 2023 18:19:49 -0700 Subject: [PATCH 226/904] typo in print sgheatflux --- .../src/analysis_members/mpas_ocn_conservation_check.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index 1278342ceb2d..881b53efd5b5 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -769,7 +769,7 @@ subroutine energy_conservation(domain, err) v=accumulatedEvapTemperatureFlux *rho_sw*cp_sw; write(m,"('EvapTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedSeaIceTemperatureFlux *rho_sw*cp_sw; write(m,"('SeaIceTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedRiverRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('RiverRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v -v=accumulatedSubglacialRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('RiverRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v +v=accumulatedSubglacialRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('SubglacialRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedIcebergTemperatureFlux*rho_sw*cp_sw; write(m,"('IcebergTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v write(m,"('SUM IMPLICIT HEAT FLUXES ',es16.8,' hh20temp ',f16.8,es16.8)") s, s/A; call mpas_log_write(m) s = s + explicitHeatFluxSum From e89e8544f1ef2740cca4e105fff2a3ea98153e82 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 30 Nov 2023 11:52:31 -0700 Subject: [PATCH 227/904] add subglacial to kpp in diagnostics --- .../src/shared/mpas_ocn_diagnostics.F | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index f65749af25fc..f04893b671c0 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3289,16 +3289,18 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & !----------------------------------------------------------------- integer :: & - iCell, iEdge, i, &! loop indices for cell, edge, neighbors - kmin, &! topmost active cell index - nCells, &! number of cells - err, &! local error code - timeLevel ! time level for state variables (default 1) + iCell, iEdge, i, k, &! loop indices for cell, edge, neighbors + kmin, &! topmost active cell index + nCells, &! number of cells + err, &! local error code + timeLevel ! time level for state variables (default 1) real (kind=RKIND) :: & fracAbsorbed, &! fraction of sfc flux absorbed fracAbsorbedRunoff, &! same for runoff fracAbsorbedSubglacialRunoff, &! same for subglacial runoff + zTop,zBot, &! temporary variables + transmissionCoeffTop,transmissionCoeffBot, &! temporary variables sumSurfaceStressSquared ! sum of sfc stress squared ! pointers for variable/pool retrievals @@ -3488,10 +3490,31 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & -layerThickness(kmin, iCell)/ & config_flux_attenuation_coefficient_runoff)) if (config_use_sgr_opt_kpp == 1) then - fracAbsorbedSubglacialRunoff = 1.0_RKIND & + if ( trim(config_sgr_flux_vertical_location) == 'top' ) then + fracAbsorbedSubglacialRunoff = 1.0_RKIND & - exp( max(-100.0_RKIND, & -layerThickness(kmin, iCell)/ & config_flux_attenuation_coefficient_subglacial_runoff)) + else if ( trim(config_sgr_flux_vertical_location) == 'uniform' ) then + ! calculate total thickness into variable zTop + zTop = 0.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + zTop = zTop + layerThickness(k,iCell) + end do + ! distribute flux evenly throughout water column + fracAbsorbedSubglacialRunoff = layerThickness(kmin, iCell) / zTop + else if ( trim(config_sgr_flux_vertical_location) == 'bottom' ) then + zTop = 0.0_RKIND + do k = maxLevelCell(iCell), minLevelCell(iCell), -1 + zBot = zTop - layerThickness(k,iCell) + if (k == minLevelCell(iCell)) then + transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) + transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) + fracAbsorbedSubglacialRunoff = transmissionCoeffTop - transmissionCoeffBot + end if + zTop = zBot + end do + end if else fracAbsorbedSubglacialRunoff = 0.0_RKIND end if From 52a47584d0f553a65f0f386516cac415c83e2eee Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 12:36:10 -0600 Subject: [PATCH 228/904] resolved: fris meshes --- components/mpas-ocean/cime_config/buildnml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 725c26d28181..f5a05e03aade 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -281,6 +281,8 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_adusumilli2020.SOwISC12to60E2r4.230516.nc' if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.SOwISC12to60E2r4.20210114.nc' + if ocn_sgr == 'data': + data_sgr_file = 'SOwISC12to60E2r4_DSGR.nc' elif ocn_grid == 'FRISwISC08to60E3r1': decomp_date = '20230913' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC08to60E3r1/partitions From c6a0104624162cf57cf49241d3655446cb697885 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 1 Feb 2024 10:53:19 -0700 Subject: [PATCH 229/904] cleanup --- components/mpas-ocean/bld/build-namelist | 2 +- .../bld/namelist_files/namelist_definition_mpaso.xml | 4 ++-- components/mpas-ocean/cime_config/config_component.xml | 2 +- components/mpas-ocean/src/Registry.xml | 4 ++-- components/mpas-ocean/src/shared/mpas_ocn_forcing.F | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index e1d2548dffeb..e33f4dc1c796 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -56,7 +56,7 @@ OPTIONS Options are: none, data, internal, coupled -ocn_sgr variable for defining how the ocn model will handle subglacial runoff - Options are: none, data, coupled + Options are: none, data -decomp_prefix decomp_prefix variable -date_stamp date_stamp variable -cfg_grid Directory containing MPASO configuration scripts. diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index 8152b1fec724..97bd989f06c5 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1216,7 +1216,7 @@ Default: Defined in namelist_defaults.xml category="forcing" group="forcing"> Option to choose sgr temperature -Valid values: 'sgr','current', 'prescribed'. +Valid values: 'sgr', 'prescribed'. Default: Defined in namelist_defaults.xml @@ -1224,7 +1224,7 @@ Default: Defined in namelist_defaults.xml category="forcing" group="forcing"> Option to choose sgr salinity -Valid values: 'sgr','current', 'prescribed'. +Valid values: 'sgr', 'prescribed'. Default: Defined in namelist_defaults.xml diff --git a/components/mpas-ocean/cime_config/config_component.xml b/components/mpas-ocean/cime_config/config_component.xml index eb8e09519829..83b4501ee5e5 100644 --- a/components/mpas-ocean/cime_config/config_component.xml +++ b/components/mpas-ocean/cime_config/config_component.xml @@ -41,7 +41,7 @@ char - none,data,coupled + none,data none none diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 7e3b1d125dae..1ba2259da87f 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -716,11 +716,11 @@ /> Date: Tue, 9 Jul 2024 12:37:17 -0600 Subject: [PATCH 230/904] resolved: fris meshes #2 --- .../mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml | 2 +- components/mpas-ocean/cime_config/buildnml | 3 ++- components/mpas-ocean/src/Registry.xml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index a2c6465c6d1c..9e14fbf04d1b 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -343,7 +343,7 @@ 0.001 10.0 0.001 -'top' +'uniform' 0 'sgr' 'sgr' diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index f5a05e03aade..4d5029555afc 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -379,6 +379,8 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_paolo2023.IcoswISC30E3r5.20240227.nc' if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.IcoswISC30E3r5.20231120.nc' + if ocn_sgr == 'data': + data_sgr_file = 'IcoswISC30E3r5_DSGR.nc' elif ocn_grid == 'IcosXISC30E3r7': decomp_date = '20240314' @@ -404,7 +406,6 @@ def buildnml(case, caseroot, compname): if ocn_ismf == 'data': data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' - #-------------------------------------------------------------------- # Set OCN_FORCING = datm_forced_restoring if restoring file is available #-------------------------------------------------------------------- diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 1ba2259da87f..e48a84d2437c 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -706,7 +706,7 @@ description="The length scale of exponential decay of subglacial runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$." possible_values="Any positive real number." /> - From cdc60afb77c6129f2ece53ee85b4d49e7c92f095 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 7 Feb 2024 16:13:31 -0700 Subject: [PATCH 231/904] changed sgr kpp default to 1 = use --- .../mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml | 2 +- components/mpas-ocean/src/Registry.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 9e14fbf04d1b..40a3b3ca86f5 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -344,7 +344,7 @@ 10.0 0.001 'uniform' -0 +1 'sgr' 'sgr' 0.0 diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index e48a84d2437c..506dd5772c09 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -710,7 +710,7 @@ description="Selects the vertical location where subglacial runoff is fluxed." possible_values="'top','uniform', 'bottom'" /> - From 1664002c0aba6072e624795f62bb72fa1b185298 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 8 Jul 2024 16:04:08 -0600 Subject: [PATCH 232/904] change config_sgr_flux_vertical_location default to bottom --- components/mpas-ocean/src/Registry.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 506dd5772c09..1ceddac39fcb 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -706,7 +706,7 @@ description="The length scale of exponential decay of subglacial runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$." possible_values="Any positive real number." /> - From f0e802d4a6d447c9dcadcced6bcb02fc06770438 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 13:07:55 -0600 Subject: [PATCH 233/904] rmv conservation check on sgr --- .../Registry_conservation_check.xml | 22 ----- .../mpas_ocn_conservation_check.F | 96 +++++-------------- 2 files changed, 23 insertions(+), 95 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml b/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml index 8a79ac12629e..0fd2d28d1c6e 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml @@ -106,9 +106,6 @@ - @@ -151,10 +148,6 @@ - - @@ -205,9 +194,6 @@ - - - - @@ -319,7 +302,6 @@ - @@ -361,17 +343,14 @@ - - - @@ -379,7 +358,6 @@ - diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index 881b53efd5b5..5551d5758db2 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -28,7 +28,6 @@ module ocn_conservation_check use ocn_constants use ocn_config - use ocn_mesh implicit none private @@ -428,7 +427,6 @@ subroutine energy_conservation(domain, err) accumulatedEvapTemperatureFlux, & accumulatedSeaIceTemperatureFlux, & accumulatedRiverRunoffTemperatureFlux, & - accumulatedSubglacialRunoffTemperatureFlux, & accumulatedIcebergTemperatureFlux real(kind=RKIND), dimension(:), allocatable :: & @@ -465,8 +463,7 @@ subroutine energy_conservation(domain, err) real(kind=RKIND), dimension(:,:), pointer :: & - activeTracersSurfaceFluxRunoff, & - activeTracersSurfaceFluxSubglacialRunoff + activeTracersSurfaceFluxRunoff type (MPAS_timeInterval_type) :: & timeStepESMF @@ -483,7 +480,7 @@ subroutine energy_conservation(domain, err) ierr integer, parameter :: & - nSums = 20 + nSums = 19 character(len=160) :: & m @@ -516,7 +513,6 @@ subroutine energy_conservation(domain, err) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedEvapTemperatureFlux", accumulatedEvapTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSeaIceTemperatureFlux", accumulatedSeaIceTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedRiverRunoffTemperatureFlux", accumulatedRiverRunoffTemperatureFlux) - call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSubglacialRunoffTemperatureFlux", accumulatedSubglacialRunoffTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedIcebergTemperatureFlux", accumulatedIcebergTemperatureFlux) !------------------------------------------------------------- @@ -560,7 +556,6 @@ subroutine energy_conservation(domain, err) call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_temperatureSurfaceFlux', index_temperature_flux) call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxRunoff', activeTracersSurfaceFluxRunoff) - call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxSubglacialRunoff', activeTracersSurfaceFluxSubglacialRunoff) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMassNew, 2) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMassOld, 1) @@ -585,7 +580,6 @@ subroutine energy_conservation(domain, err) ! river runoff temperature flux sumArray(14) = sumArray(14) + areaCell(iCell) * activeTracersSurfaceFluxRunoff(index_temperature_flux,iCell) sumArray(15) = sumArray(15) + areaCell(iCell) * icebergTemperatureFlux(iCell) - sumArray(20) = sumArray(20) + areaCell(iCell) * activeTracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) enddo ! iCell @@ -640,7 +634,6 @@ subroutine energy_conservation(domain, err) accumulatedLandIceHeatFlux = accumulatedLandIceHeatFlux + sumArrayOut(17) accumulatedLandIceFrazilHeatFlux = accumulatedLandIceFrazilHeatFlux + sumArrayOut(18) accumulatedRemovedIceRunoffHeatFlux = accumulatedRemovedIceRunoffHeatFlux + sumArrayOut(19) - accumulatedSubglacialRunoffTemperatureFlux = accumulatedSubglacialRunoffTemperatureFlux + sumArrayOut(20) ! cleanup deallocate(sumArray) @@ -670,7 +663,6 @@ subroutine energy_conservation(domain, err) accumulatedEvapTemperatureFlux = accumulatedEvapTemperatureFlux /accumulatedFluxCounter accumulatedSeaIceTemperatureFlux = accumulatedSeaIceTemperatureFlux /accumulatedFluxCounter accumulatedRiverRunoffTemperatureFlux = accumulatedRiverRunoffTemperatureFlux /accumulatedFluxCounter - accumulatedSubglacialRunoffTemperatureFlux = accumulatedSubglacialRunoffTemperatureFlux /accumulatedFluxCounter accumulatedIcebergTemperatureFlux = accumulatedIcebergTemperatureFlux /accumulatedFluxCounter accumulatedLandIceFrazilHeatFlux = accumulatedLandIceFrazilHeatFlux /accumulatedFluxCounter accumulatedRemovedIceRunoffHeatFlux = accumulatedRemovedIceRunoffHeatFlux /accumulatedFluxCounter @@ -705,7 +697,6 @@ subroutine energy_conservation(domain, err) + accumulatedEvapTemperatureFlux *rho_sw*cp_sw & + accumulatedSeaIceTemperatureFlux *rho_sw*cp_sw & + accumulatedRiverRunoffTemperatureFlux *rho_sw*cp_sw & - + accumulatedSubglacialRunoffTemperatureFlux *rho_sw*cp_sw & + accumulatedIcebergTemperatureFlux*rho_sw*cp_sw ! note, accumulatedLandIceFrazilHeatFlux not added because already in accumulatedFrazilHeatFlux @@ -769,7 +760,6 @@ subroutine energy_conservation(domain, err) v=accumulatedEvapTemperatureFlux *rho_sw*cp_sw; write(m,"('EvapTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedSeaIceTemperatureFlux *rho_sw*cp_sw; write(m,"('SeaIceTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedRiverRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('RiverRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v -v=accumulatedSubglacialRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('SubglacialRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedIcebergTemperatureFlux*rho_sw*cp_sw; write(m,"('IcebergTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v write(m,"('SUM IMPLICIT HEAT FLUXES ',es16.8,' hh20temp ',f16.8,es16.8)") s, s/A; call mpas_log_write(m) s = s + explicitHeatFluxSum @@ -838,7 +828,6 @@ subroutine mass_conservation(domain, err) accumulatedEvaporationFlux, & accumulatedSeaIceFlux, & accumulatedRiverRunoffFlux, & - accumulatedSubglacialRunoffFlux, & accumulatedIceRunoffFlux, & accumulatedRemovedRiverRunoffFlux, & accumulatedRemovedIceRunoffFlux, & @@ -863,7 +852,6 @@ subroutine mass_conservation(domain, err) evaporationFlux, & seaIceFreshwaterFlux, & riverRunoffFlux, & - subglacialRunoffFlux, & iceRunoffFlux, & removedRiverRunoffFlux, & removedIceRunoffFlux, & @@ -889,7 +877,7 @@ subroutine mass_conservation(domain, err) iCell, ierr, k integer, parameter :: & - nSums = 13 + nSums = 12 integer, dimension(:), pointer :: minLevelCell, maxLevelCell @@ -912,7 +900,6 @@ subroutine mass_conservation(domain, err) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedEvaporationFlux", accumulatedEvaporationFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSeaIceFlux", accumulatedSeaIceFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRiverRunoffFlux", accumulatedRiverRunoffFlux) - call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSubglacialRunoffFlux", accumulatedSubglacialRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedIceRunoffFlux", accumulatedIceRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedRiverRunoffFlux",accumulatedRemovedRiverRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedIceRunoffFlux", accumulatedRemovedIceRunoffFlux) @@ -950,7 +937,6 @@ subroutine mass_conservation(domain, err) call mpas_pool_get_array(forcingPool, 'evaporationFlux', evaporationFlux) call mpas_pool_get_array(forcingPool, 'seaIceFreshWaterFlux', seaIceFreshwaterFlux) call mpas_pool_get_array(forcingPool, 'riverRunoffFlux', riverRunoffFlux) - call mpas_pool_get_array(forcingPool, 'subglacialRunoffFlux', subglacialRunoffFlux) call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) call mpas_pool_get_array(forcingPool, 'removedRiverRunoffFlux', removedRiverRunoffFlux) call mpas_pool_get_array(forcingPool, 'removedIceRunoffFlux', removedIceRunoffFlux) @@ -970,7 +956,6 @@ subroutine mass_conservation(domain, err) sumArray( 7) = sumArray( 7) + areaCell(iCell) * removedRiverRunoffFlux(iCell) sumArray( 8) = sumArray( 8) + areaCell(iCell) * removedIceRunoffFlux(iCell) sumArray( 9) = sumArray( 9) + areaCell(iCell) * icebergFreshwaterFlux(iCell) - sumArray(13) = sumArray(13) + areaCell(iCell) * subglacialRunoffFlux(iCell) / areaCell(iCell) * rho_sw enddo if (config_use_frazil_ice_formation) then @@ -1017,7 +1002,6 @@ subroutine mass_conservation(domain, err) accumulatedFrazilFlux = accumulatedFrazilFlux + sumArrayOut(10) accumulatedLandIceFlux = accumulatedLandIceFlux + sumArrayOut(11) accumulatedLandIceFrazilFlux = accumulatedLandIceFrazilFlux + sumArrayOut(12) - accumulatedSubglacialRunoffFlux = accumulatedSubglacialRunoffFlux + sumArrayOut(13) ! cleanup deallocate(sumArray) @@ -1037,7 +1021,6 @@ subroutine mass_conservation(domain, err) accumulatedEvaporationFlux = accumulatedEvaporationFlux /accumulatedFluxCounter accumulatedSeaIceFlux = accumulatedSeaIceFlux /accumulatedFluxCounter accumulatedRiverRunoffFlux = accumulatedRiverRunoffFlux /accumulatedFluxCounter - accumulatedSubglacialRunoffFlux = accumulatedSubglacialRunoffFlux /accumulatedFluxCounter accumulatedIceRunoffFlux = accumulatedIceRunoffFlux /accumulatedFluxCounter accumulatedRemovedRiverRunoffFlux = accumulatedRemovedRiverRunoffFlux /accumulatedFluxCounter accumulatedRemovedIceRunoffFlux = accumulatedRemovedIceRunoffFlux /accumulatedFluxCounter @@ -1066,7 +1049,6 @@ subroutine mass_conservation(domain, err) + accumulatedEvaporationFlux & + accumulatedSeaIceFlux & + accumulatedRiverRunoffFlux & - + accumulatedSubglacialRunoffFlux & + accumulatedIceRunoffFlux & + accumulatedIcebergFlux & + accumulatedFrazilFlux & @@ -1101,8 +1083,6 @@ subroutine mass_conservation(domain, err) v=accumulatedIcebergFlux ; write(m,"('icebergFreshwaterFlux ',es16.8,' x2o_Fioi_bergw wberg ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v v=accumulatedEvaporationFlux ; write(m,"('evaporationFlux ',es16.8,' x2o_Foxx_evap wevap ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v v=accumulatedRiverRunoffFlux ; write(m,"('riverRunoffFlux ',es16.8,' x2o_Foxx_rofl wrunoff ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v -v=accumulatedSubglacialRunoffFlux ; write(m,"('subglacialRunoffFlux ',es16.8,' x2o_Foxx_rofl wsgr ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v - if (landIceFreshwaterFluxesOn) then v=accumulatedRemovedRiverRunoffFlux; write(m,"('removedRiverRunoffFlux ',es16.8,' wrunoff ',f16.8)") v,v*c; call mpas_log_write(m); v=accumulatedRiverRunoffFlux+accumulatedRemovedRiverRunoffFlux; @@ -1170,8 +1150,7 @@ subroutine salt_conservation(domain, err) conservationCheckSaltAMPool, & meshPool, & statePool, & - forcingPool, & - tracersSurfaceFluxPool + forcingPool real(kind=RKIND), pointer :: & initialSalt, & @@ -1181,14 +1160,10 @@ subroutine salt_conservation(domain, err) absoluteSaltError, & relativeSaltError - real(kind=RKIND), dimension(:,:), pointer :: & - activeTracersSurfaceFluxSubglacialRunoff - real(kind=RKIND), pointer :: & accumulatedSeaIceSalinityFlux, & + accumulatedFrazilSalinityFlux ! accumulatedLandIceFrazilSalinityFlux is not present because it is always 0 - accumulatedFrazilSalinityFlux, & - accumulatedSubglacialRunoffSalinityFlux real(kind=RKIND), dimension(:), allocatable :: & sumArray, & @@ -1209,15 +1184,14 @@ subroutine salt_conservation(domain, err) dt, dtAvg, v, A, s, c integer, pointer :: & - nCellsSolve, & - index_salinity_flux + nCellsSolve integer :: & iCell, & ierr integer, parameter :: & - nSums = 4 + nSums = 3 logical, pointer :: & activeTracersBulkRestoringPKG @@ -1235,7 +1209,6 @@ subroutine salt_conservation(domain, err) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSeaIceSalinityFlux", accumulatedSeaIceSalinityFlux) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedFrazilSalinityFlux", accumulatedFrazilSalinityFlux) - call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSubglacialRunoffSalinityFlux", accumulatedSubglacialRunoffSalinityFlux) !------------------------------------------------------------- ! Net salt flux to ice @@ -1261,18 +1234,11 @@ subroutine salt_conservation(domain, err) call mpas_pool_get_array(forcingPool, 'seaIceSalinityFlux', seaIceSalinityFlux) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceSalinity', accumulatedFrazilIceSalinityOld, 1) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceSalinity', accumulatedFrazilIceSalinityNew, 2) - call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) - call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxSubglacialRunoff', activeTracersSurfaceFluxSubglacialRunoff) - call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_salinitySurfaceFlux', index_salinity_flux) - - do iCell = 1, nCellsSolve ! salt flux to ocean sumArray(1) = sumArray(1) + areaCell(iCell) * seaIceSalinityFlux(iCell) - ! subglacial runoff - sumArray(4) = sumArray(4) + areaCell(iCell) * activeTracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) enddo ! iCell if (config_use_frazil_ice_formation) then @@ -1299,7 +1265,6 @@ subroutine salt_conservation(domain, err) ! accumulate fluxes accumulatedSeaIceSalinityFlux = accumulatedSeaIceSalinityFlux + sumArrayOut(1) accumulatedFrazilSalinityFlux = accumulatedFrazilSalinityFlux + sumArrayOut(2) - accumulatedSubglacialRunoffSalinityFlux = accumulatedSubglacialRunoffSalinityFlux + sumArrayOut(4) ! cleanup deallocate(sumArray) @@ -1316,7 +1281,6 @@ subroutine salt_conservation(domain, err) ! Average the fluxes accumulatedSeaIceSalinityFlux = accumulatedSeaIceSalinityFlux /accumulatedFluxCounter accumulatedFrazilSalinityFlux = accumulatedFrazilSalinityFlux /accumulatedFluxCounter - accumulatedSubglacialRunoffSalinityFlux = accumulatedSubglacialRunoffSalinityFlux /accumulatedFluxCounter ! get initial salt content call MPAS_pool_get_array(conservationCheckSaltAMPool, "initialSalt", initialSalt) @@ -1333,8 +1297,7 @@ subroutine salt_conservation(domain, err) call MPAS_pool_get_array(conservationCheckSaltAMPool, "netSaltFlux", netSaltFlux) netSaltFlux = accumulatedSeaIceSalinityFlux & - + accumulatedFrazilSalinityFlux & - + accumulatedSubglacialRunoffSalinityFlux + + accumulatedFrazilSalinityFlux ! compute the final salt error call MPAS_pool_get_array(conservationCheckSaltAMPool, "absoluteSaltError", absoluteSaltError) @@ -1364,8 +1327,6 @@ subroutine salt_conservation(domain, err) .and.config_frazil_under_land_ice) then v=0; write(m,"('LandIceFrazilSalinityFlux',es16.8,' (already in wmelt, do not sum) ',f16.8)") v,v*c; call mpas_log_write(m); !no sum: s=s+v end if -v=accumulatedSubglacialRunoffSalinityFlux ; write(m,"('SubglacialRunoffSalinityFlux ',es16.8,' x2o_Fioi_salt salt ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v - write(m,"('SUM VOLUME FLUXES ',es16.8,' ',f16.8,es16.8)") s, s*c; call mpas_log_write(m) call mpas_log_write(' ') @@ -1562,11 +1523,11 @@ subroutine carbon_conservation(domain, err) block => domain % blocklist do while (associated(block)) call MPAS_pool_get_dimension(block % dimensions, "nCellsSolve", nCellsSolve) - + call MPAS_pool_get_subpool(block % structs, "mesh", meshPool) call MPAS_pool_get_subpool(block % structs, "forcing", forcingPool) call MPAS_pool_get_subpool(block % structs, "state", statePool) - + call MPAS_pool_get_array(meshPool, "areaCell", areaCell) call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) @@ -1633,7 +1594,7 @@ subroutine carbon_conservation(domain, err) + ecosysTracersSurfaceFlux(index_spCFlux, iCell) & + ecosysTracersSurfaceFlux(index_diatCFlux, iCell) & + ecosysTracersSurfaceFlux(index_diazCFlux, iCell)) - + do k = 1, maxLevelCell(iCell) sumArray(4) = sumArray(4) + areaCell(iCell) * ( & ecosysTracersTend(index_DICTend, k, iCell) & @@ -1684,7 +1645,7 @@ subroutine carbon_conservation(domain, err) !------------------------------------------------------------- if (MPAS_stream_mgr_ringing_alarms(domain % streamManager, "conservationCheckOutput", ierr=ierr)) then - + ! Average the fluxes accumulatedCarbonSourceSink = accumulatedCarbonSourceSink /accumulatedFluxCounter accumulatedCarbonSedimentFlux = accumulatedCarbonSedimentFlux /accumulatedFluxCounter @@ -1696,7 +1657,7 @@ subroutine carbon_conservation(domain, err) ! get initial carbon call MPAS_pool_get_array(conservationCheckCarbonAMPool, "initialCarbon", initialCarbon) - + ! get final carbon call MPAS_pool_get_array(conservationCheckCarbonAMPool, "finalCarbon", finalCarbon) call compute_total_carbon(domain, finalCarbon) @@ -1733,13 +1694,13 @@ subroutine carbon_conservation(domain, err) relativeCarbonErrorBounds = & relativeCarbonErrorStepBounds*relativeCarbonErrorBoundsFac*totalTimeSteps relativeCarbonErrorPerTimeStep = relativeCarbonError/accumulatedFluxCounter - + !------------------------------------------------------------- ! Output to log file !------------------------------------------------------------- if (config_AM_conservationCheck_write_to_logfile) then - + call mpas_log_write('') call mpas_log_write('----------------------------------------------------------') call mpas_log_write('CARBON CONSERVATION CHECK') @@ -1770,7 +1731,7 @@ subroutine carbon_conservation(domain, err) v=accumulatedIceOceanInorganicCarbonFlux*mmol_to_kg_C write(m,"('Ice-Ocean Inorganic Flux ',es16.8,' ',f16.8)") v,v*c call mpas_log_write(m) - write(m,"('SUM FLUXES (surf + sed) ',es16.8,' ',f16.8,es16.8)") s, s*c; + write(m,"('SUM FLUXES (surf + sed) ',es16.8,' ',f16.8,es16.8)") s, s*c; call mpas_log_write(m) call mpas_log_write(' ') @@ -1790,24 +1751,24 @@ subroutine carbon_conservation(domain, err) write(m,"('Carbon change ', 3es16.8)") & carbonChange*mmol_to_kg_C, & carbonChange*mmol_to_kg_C/dtAvg, & - carbonChange*mmol_to_kg_C/dtAvg*c + carbonChange*mmol_to_kg_C/dtAvg*c call mpas_log_write(m) write(m,"('Net carbon flux ', 3es16.8)") & netCarbonFlux*mmol_to_kg_C*dtAvg, & netCarbonFlux*mmol_to_kg_C, & - netCarbonFlux*mmol_to_kg_C*c + netCarbonFlux*mmol_to_kg_C*c call mpas_log_write(m) write(m,"('Absolute carbon error ', 3es16.8)") & absoluteCarbonError*mmol_to_kg_C, & absoluteCarbonError*mmol_to_kg_C/dtAvg, & - absoluteCarbonError*mmol_to_kg_C/dtAvg*c + absoluteCarbonError*mmol_to_kg_C/dtAvg*c call mpas_log_write(m) call mpas_log_write(' ') write(m,"('RELATIVE CARBON ERROR =', es16.8)") & relativeCarbonError call mpas_log_write(m) call mpas_log_write(' ') - + write(m,"('Relative carbon error per timestep = ', es16.8)") & relativeCarbonErrorPerTimeStep call mpas_log_write(m) @@ -1853,7 +1814,7 @@ subroutine carbon_conservation(domain, err) endif end subroutine carbon_conservation - + !*********************************************************************** ! ! routine compute_total_energy @@ -2229,7 +2190,7 @@ subroutine compute_total_carbon(domain, totalCarbon) call MPAS_dmpar_sum_real(domain % dminfo, carbon, totalCarbon) end subroutine compute_total_carbon - + !*********************************************************************** ! ! routine reset_accumulated_variables @@ -2271,7 +2232,6 @@ subroutine reset_accumulated_variables(domain) accumulatedEvapTemperatureFlux, & accumulatedSeaIceTemperatureFlux, & accumulatedRiverRunoffTemperatureFlux, & - accumulatedSubglacialRunoffTemperatureFlux, & accumulatedIcebergTemperatureFlux real(kind=RKIND), pointer :: & @@ -2280,7 +2240,6 @@ subroutine reset_accumulated_variables(domain) accumulatedEvaporationFlux, & accumulatedSeaIceFlux, & accumulatedRiverRunoffFlux, & - accumulatedSubglacialRunoffFlux, & accumulatedIceRunoffFlux, & accumulatedRemovedRiverRunoffFlux, & accumulatedRemovedIceRunoffFlux, & @@ -2291,8 +2250,7 @@ subroutine reset_accumulated_variables(domain) real(kind=RKIND), pointer :: & accumulatedSeaIceSalinityFlux, & - accumulatedFrazilSalinityFlux, & - accumulatedSubglacialRunoffSalinityFlux + accumulatedFrazilSalinityFlux real(kind=RKIND), pointer :: & accumulatedCarbonSourceSink, & @@ -2322,7 +2280,6 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedEvapTemperatureFlux", accumulatedEvapTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSeaIceTemperatureFlux", accumulatedSeaIceTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedRiverRunoffTemperatureFlux", accumulatedRiverRunoffTemperatureFlux) - call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSubglacialRunoffTemperatureFlux", accumulatedSubglacialRunoffTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedIcebergTemperatureFlux", accumulatedIcebergTemperatureFlux) accumulatedFluxCounter = 0 @@ -2341,7 +2298,6 @@ subroutine reset_accumulated_variables(domain) accumulatedEvapTemperatureFlux = 0.0_RKIND accumulatedSeaIceTemperatureFlux = 0.0_RKIND accumulatedRiverRunoffTemperatureFlux = 0.0_RKIND - accumulatedSubglacialRunoffTemperatureFlux = 0.0_RKIND accumulatedIcebergTemperatureFlux = 0.0_RKIND accumulatedLandIceFrazilHeatFlux = 0.0_RKIND accumulatedRemovedIceRunoffHeatFlux = 0.0_RKIND @@ -2354,7 +2310,6 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedEvaporationFlux", accumulatedEvaporationFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSeaIceFlux", accumulatedSeaIceFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRiverRunoffFlux", accumulatedRiverRunoffFlux) - call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSubglacialRunoffFlux", accumulatedSubglacialRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedIceRunoffFlux", accumulatedIceRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedRiverRunoffFlux",accumulatedRemovedRiverRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedIceRunoffFlux", accumulatedRemovedIceRunoffFlux) @@ -2368,7 +2323,6 @@ subroutine reset_accumulated_variables(domain) accumulatedEvaporationFlux = 0.0_RKIND accumulatedSeaIceFlux = 0.0_RKIND accumulatedRiverRunoffFlux = 0.0_RKIND - accumulatedSubglacialRunoffFlux = 0.0_RKIND accumulatedIceRunoffFlux = 0.0_RKIND accumulatedRemovedRiverRunoffFlux = 0.0_RKIND accumulatedRemovedIceRunoffFlux = 0.0_RKIND @@ -2382,13 +2336,9 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSeaIceSalinityFlux", accumulatedSeaIceSalinityFlux) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedFrazilSalinityFlux", accumulatedFrazilSalinityFlux) - call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedLandIceFrazilSalinityFlux", accumulatedLandIceFrazilSalinityFlux) - call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSubglacialRunoffSalinityFlux", accumulatedSubglacialRunoffSalinityFlux) - accumulatedSeaIceSalinityFlux = 0.0_RKIND accumulatedFrazilSalinityFlux = 0.0_RKIND - accumulatedSubglacialRunoffSalinityFlux = 0.0_RKIND call MPAS_pool_get_subpool(domain % blocklist % structs, "conservationCheckCarbonAM", conservationCheckCarbonAMPool) From ea8bb59c4710c99561d0993a57cdfc505ac39216 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 13:32:31 -0600 Subject: [PATCH 234/904] rmv conservation check on sgr buildnml --- components/mpas-ocean/cime_config/buildnml | 1 - 1 file changed, 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 4d5029555afc..91c481ad8f5c 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -1195,7 +1195,6 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') - lines.append('') lines.append('') lines.append('') lines.append('') From e969c31a02875303c5e0f0435195fb7a0ff1edcb Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 13:56:39 -0600 Subject: [PATCH 235/904] rmv conservation check on sgr buildnml #2 --- components/mpas-ocean/cime_config/buildnml | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 91c481ad8f5c..d6d40491baa6 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -1201,7 +1201,6 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') - lines.append('') lines.append('') lines.append('') lines.append('') @@ -1210,7 +1209,6 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') - lines.append('') lines.append('') lines.append('') lines.append('') From 800961c2e6f6efa310da7a00018b91fa51f3ba93 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 8 Jul 2024 12:14:43 -0600 Subject: [PATCH 236/904] pkg change in bld --- components/mpas-ocean/bld/build-namelist | 5 +++++ components/mpas-ocean/bld/build-namelist-section | 1 + .../bld/namelist_files/namelist_defaults_mpaso.xml | 2 +- .../bld/namelist_files/namelist_definition_mpaso.xml | 8 ++++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index e33f4dc1c796..2c5ee97b0df2 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -715,6 +715,11 @@ add_default($nl, 'config_use_sgr_opt_temp'); add_default($nl, 'config_use_sgr_opt_salt'); add_default($nl, 'config_sgr_temperature'); add_default($nl, 'config_sgr_salinity'); +if ($OCN_SGR eq 'data') { + add_default($nl, 'config_subglacial_runoff_mode', 'val'=>"data"); +} else { + add_default($nl, 'config_subglacial_runoff_mode'); +} ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index 0d44d3f328b6..3f2d4ab4839d 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -231,6 +231,7 @@ add_default($nl, 'config_use_sgr_opt_temp'); add_default($nl, 'config_use_sgr_opt_salt'); add_default($nl, 'config_sgr_temperature'); add_default($nl, 'config_sgr_salinity'); +add_default($nl, 'config_subglacial_runoff_mode'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 40a3b3ca86f5..0514704b9514 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -349,7 +349,7 @@ 'sgr' 0.0 0.0 - +'off' .false. diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index 97bd989f06c5..616ec67d24cd 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1244,6 +1244,14 @@ Valid values: Any real number. Default: Defined in namelist_defaults.xml + +Selects the mode in which subglacial runoff fluxes are computed. + +Valid values: 'off', 'data' +Default: Defined in namelist_defaults.xml + + Date: Mon, 8 Jul 2024 12:32:42 -0600 Subject: [PATCH 237/904] pkg in driver and registry --- components/mpas-ocean/src/Registry.xml | 5 +++++ .../mpas-ocean/src/driver/mpas_ocn_core_interface.F | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 1ceddac39fcb..bab430733500 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -702,6 +702,10 @@ description="The length scale of exponential decay of river runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$." possible_values="Any positive real number." /> + + diff --git a/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F b/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F index 7d9eb083f907..6d542fb0c3f3 100644 --- a/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F +++ b/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F @@ -123,6 +123,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ logical, pointer :: dataLandIceFluxesPKGActive logical, pointer :: landIceFluxesPKGActive logical, pointer :: landIceCouplingPKGActive + logical, pointer :: dataSubglacialRunoffFluxPKGActive logical, pointer :: thicknessBulkPKGActive logical, pointer :: frazilIceActive logical, pointer :: tidalForcingActive @@ -186,6 +187,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ logical, pointer :: config_use_bulk_thickness_flux logical, pointer :: config_compute_active_tracer_budgets character (len=StrKIND), pointer :: config_land_ice_flux_mode + character (len=StrKIND), pointer :: config_subglacial_runoff_mode type (mpas_pool_iterator_type) :: groupItr character (len=StrKIND) :: tracerGroupName, configName, packageName @@ -319,6 +321,15 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ landIceCouplingPKGActive = .true. end if + ! + ! test for use of subglacial runoff flux, dataSubglacialRunoffFluxPKGActive + ! + call mpas_pool_get_package(packagePool, 'dataSubglacialRunoffFluxPKGActive', dataSubglacialRunoffFluxPKGActive) + call mpas_pool_get_config(configPool, 'config_subglacial_runoff_mode', config_subglacial_runoff_mode) + if ( trim(config_subglacial_runoff_mode) == 'data' ) then + dataSubglacialRunoffFluxPKGActive = .true. + end if + ! ! test for use of frazil ice formation, frazilIceActive ! From b07587b10a5d5f9c767dcbd07ee58a384c5aa09c Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 8 Jul 2024 13:05:43 -0600 Subject: [PATCH 238/904] fix group for sgr in namelist_definition_mpaso --- .../mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index 616ec67d24cd..43ba139ab65f 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1245,7 +1245,7 @@ Default: Defined in namelist_defaults.xml + category="forcing" group="forcing"> Selects the mode in which subglacial runoff fluxes are computed. Valid values: 'off', 'data' From f1b13ba8657dd579a5aae5e2fa18c8fd8a3ccb5a Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 8 Jul 2024 14:10:11 -0600 Subject: [PATCH 239/904] sgr pkg in shared folder, add loops --- .../src/shared/mpas_ocn_diagnostics.F | 100 ++++++++++-------- .../mpas-ocean/src/shared/mpas_ocn_forcing.F | 77 +++++++------- .../shared/mpas_ocn_surface_bulk_forcing.F | 55 ++++++---- .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 12 ++- .../src/shared/mpas_ocn_thick_surface_flux.F | 21 ++-- .../mpas_ocn_tracer_surface_flux_to_tend.F | 2 +- 6 files changed, 153 insertions(+), 114 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index f04893b671c0..daf9bfcab6d5 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -1904,9 +1904,13 @@ subroutine ocn_diagnostic_solve_surfaceLayer(layerThickness, & config_flux_attenuation_coefficient surfaceFluxAttenuationCoefficientRunoff(iCell) = & config_flux_attenuation_coefficient_runoff - surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell) = & - config_flux_attenuation_coefficient_subglacial_runoff end do + if (trim(config_subglacial_runoff_mode) == 'data') then + do iCell = 1, nCells + surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell) = & + config_flux_attenuation_coefficient_subglacial_runoff + end do + end if #ifndef MPAS_OPENACC !$omp end do !$omp end parallel @@ -3489,34 +3493,36 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & - exp( max(-100.0_RKIND, & -layerThickness(kmin, iCell)/ & config_flux_attenuation_coefficient_runoff)) - if (config_use_sgr_opt_kpp == 1) then - if ( trim(config_sgr_flux_vertical_location) == 'top' ) then - fracAbsorbedSubglacialRunoff = 1.0_RKIND & - - exp( max(-100.0_RKIND, & - -layerThickness(kmin, iCell)/ & - config_flux_attenuation_coefficient_subglacial_runoff)) - else if ( trim(config_sgr_flux_vertical_location) == 'uniform' ) then - ! calculate total thickness into variable zTop - zTop = 0.0_RKIND - do k = minLevelCell(iCell), maxLevelCell(iCell) - zTop = zTop + layerThickness(k,iCell) - end do - ! distribute flux evenly throughout water column - fracAbsorbedSubglacialRunoff = layerThickness(kmin, iCell) / zTop - else if ( trim(config_sgr_flux_vertical_location) == 'bottom' ) then - zTop = 0.0_RKIND - do k = maxLevelCell(iCell), minLevelCell(iCell), -1 - zBot = zTop - layerThickness(k,iCell) - if (k == minLevelCell(iCell)) then - transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) - transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) - fracAbsorbedSubglacialRunoff = transmissionCoeffTop - transmissionCoeffBot - end if - zTop = zBot - end do - end if - else - fracAbsorbedSubglacialRunoff = 0.0_RKIND + if (trim(config_subglacial_runoff_mode) == 'data') then + if (config_use_sgr_opt_kpp == 1) then + if ( trim(config_sgr_flux_vertical_location) == 'top' ) then + fracAbsorbedSubglacialRunoff = 1.0_RKIND & + - exp( max(-100.0_RKIND, & + -layerThickness(kmin, iCell)/ & + config_flux_attenuation_coefficient_subglacial_runoff)) + else if ( trim(config_sgr_flux_vertical_location) == 'uniform' ) then + ! calculate total thickness into variable zTop + zTop = 0.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + zTop = zTop + layerThickness(k,iCell) + end do + ! distribute flux evenly throughout water column + fracAbsorbedSubglacialRunoff = layerThickness(kmin, iCell) / zTop + else if ( trim(config_sgr_flux_vertical_location) == 'bottom' ) then + zTop = 0.0_RKIND + do k = maxLevelCell(iCell), minLevelCell(iCell), -1 + zBot = zTop - layerThickness(k,iCell) + if (k == minLevelCell(iCell)) then + transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) + transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) + fracAbsorbedSubglacialRunoff = transmissionCoeffTop - transmissionCoeffBot + end if + zTop = zBot + end do + end if + else + fracAbsorbedSubglacialRunoff = 0.0_RKIND + end if end if ! Store the total tracer flux below in @@ -3532,18 +3538,24 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & seaIceTemperatureFlux(iCell) + & icebergTemperatureFlux(iCell)) & - fracAbsorbedRunoff* & - activeTracersSurfaceFluxRunoff(indexTempFlux,iCell) & + activeTracersSurfaceFluxRunoff(indexTempFlux,iCell) + if (trim(config_subglacial_runoff_mode) == 'data') then + nonLocalSurfaceTracerFlux(indexTempFlux, iCell) = nonLocalSurfaceTracerFlux(indexTempFlux, iCell) & - fracAbsorbedSubglacialRunoff* & activeTracersSurfaceFluxSubglacialRunoff(indexTempFlux,iCell) + end if nonLocalSurfaceTracerFlux(indexSaltFlux,iCell) = & activeTracersSurfaceFlux(indexSaltFlux,iCell) & - fracAbsorbed*surfaceThicknessFlux(iCell)* & activeTracers(indexSaltFlux,kmin,iCell) & - fracAbsorbedRunoff*surfaceThicknessFluxRunoff(iCell)* & - activeTracers(indexSaltFlux,kmin,iCell) & + activeTracers(indexSaltFlux,kmin,iCell) + if (trim(config_subglacial_runoff_mode) == 'data') then + nonLocalSurfaceTracerFlux(indexSaltFlux,iCell) = nonLocalSurfaceTracerFlux(indexSaltFlux,iCell) & - fracAbsorbedSubglacialRunoff*surfaceThicknessFluxSubglacialRunoff(iCell)* & activeTracers(indexSaltFlux,kmin,iCell) + end if surfaceBuoyancyForcing(iCell) = & thermalExpansionCoeff(kmin,iCell)* & @@ -4485,17 +4497,19 @@ subroutine ocn_validate_state(domain, timeLevel)!{{{ call ocn_write_field_statistics(debugUnit, fieldName, minValue, maxValue) end if - ! Test subglacialRunoffFlux - fieldName = 'subglacialRunoffFlux' - minValue = HUGE(minValue) - maxValue = -HUGE(maxValue) - call mpas_pool_get_array(forcingPool, fieldName, real1DArr) - if ( associated(real1DArr) ) then - do iCell = 1, nCellsSolve - minValue = min( minValue, real1DArr(iCell) ) - maxValue = max( maxValue, real1DArr(iCell) ) - end do - call ocn_write_field_statistics(debugUnit, fieldName, minValue, maxValue) + if (trim(config_subglacial_runoff_mode) == 'data') then + ! Test subglacialRunoffFlux + fieldName = 'subglacialRunoffFlux' + minValue = HUGE(minValue) + maxValue = -HUGE(maxValue) + call mpas_pool_get_array(forcingPool, fieldName, real1DArr) + if ( associated(real1DArr) ) then + do iCell = 1, nCellsSolve + minValue = min( minValue, real1DArr(iCell) ) + maxValue = max( maxValue, real1DArr(iCell) ) + end do + call ocn_write_field_statistics(debugUnit, fieldName, minValue, maxValue) + end if end if ! Test seaIceSalinityFlux diff --git a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F index 75fc20c8d7ed..5cfce4c4e690 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F @@ -183,44 +183,45 @@ subroutine ocn_forcing_build_fraction_absorbed_array(meshPool, statePool, forcin end do ! now do subglacial runoff separately - - if ( trim(config_sgr_flux_vertical_location) == 'top' ) then - do iCell = 1, nCells - zTop = 0.0_RKIND - transmissionCoeffTop = ocn_forcing_transmission(zTop, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) - do k = minLevelCell(iCell), maxLevelCell(iCell) - zBot = zTop - layerThickness(k,iCell) - transmissionCoeffBot = ocn_forcing_transmission(zBot, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) - fractionAbsorbedSubglacialRunoff(k, iCell) = transmissionCoeffTop - transmissionCoeffBot - zTop = zBot - transmissionCoeffTop = transmissionCoeffBot - end do - end do - else if ( trim(config_sgr_flux_vertical_location) == 'uniform' ) then - do iCell = 1, nCells - ! calculate total thickness - zTop = 0.0_RKIND - do k = minLevelCell(iCell), maxLevelCell(iCell) - zTop = zTop + layerThickness(k,iCell) - end do - ! distribute flux evenly throughout water column - zBot = 0.0_RKIND - do k = minLevelCell(iCell), maxLevelCell(iCell) - fractionAbsorbedSubglacialRunoff(k, iCell) = layerThickness(k,iCell) / zTop - end do - end do - else if ( trim(config_sgr_flux_vertical_location) == 'bottom' ) then - do iCell = 1, nCells - zTop = 0.0_RKIND - transmissionCoeffTop = ocn_forcing_transmission(zTop, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) - do k = maxLevelCell(iCell), minLevelCell(iCell), -1 - zBot = zTop - layerThickness(k,iCell) - transmissionCoeffBot = ocn_forcing_transmission(zBot, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) - fractionAbsorbedSubglacialRunoff(k, iCell) = transmissionCoeffTop - transmissionCoeffBot - zTop = zBot - transmissionCoeffTop = transmissionCoeffBot - end do - end do + if ( trim(config_subglacial_runoff_mode) == 'data' ) then + if ( trim(config_sgr_flux_vertical_location) == 'top' ) then + do iCell = 1, nCells + zTop = 0.0_RKIND + transmissionCoeffTop = ocn_forcing_transmission(zTop, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) + do k = minLevelCell(iCell), maxLevelCell(iCell) + zBot = zTop - layerThickness(k,iCell) + transmissionCoeffBot = ocn_forcing_transmission(zBot, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) + fractionAbsorbedSubglacialRunoff(k, iCell) = transmissionCoeffTop - transmissionCoeffBot + zTop = zBot + transmissionCoeffTop = transmissionCoeffBot + end do + end do + else if ( trim(config_sgr_flux_vertical_location) == 'uniform' ) then + do iCell = 1, nCells + ! calculate total thickness + zTop = 0.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + zTop = zTop + layerThickness(k,iCell) + end do + ! distribute flux evenly throughout water column + zBot = 0.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + fractionAbsorbedSubglacialRunoff(k, iCell) = layerThickness(k,iCell) / zTop + end do + end do + else if ( trim(config_sgr_flux_vertical_location) == 'bottom' ) then + do iCell = 1, nCells + zTop = 0.0_RKIND + transmissionCoeffTop = ocn_forcing_transmission(zTop, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) + do k = maxLevelCell(iCell), minLevelCell(iCell), -1 + zBot = zTop - layerThickness(k,iCell) + transmissionCoeffBot = ocn_forcing_transmission(zBot, surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell)) + fractionAbsorbedSubglacialRunoff(k, iCell) = transmissionCoeffTop - transmissionCoeffBot + zTop = zBot + transmissionCoeffTop = transmissionCoeffBot + end do + end do + end if end if end subroutine ocn_forcing_build_fraction_absorbed_array!}}} diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index fe4f2cac5abf..0547f73319dd 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -351,8 +351,12 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur surfaceThicknessFlux(iCell) = surfaceThicknessFlux(iCell) + ( snowFlux(iCell) + rainFlux(iCell) + evaporationFlux(iCell) & + seaIceFreshWaterFlux(iCell) + icebergFreshWaterFlux(iCell) + iceRunoffFlux(iCell) ) / rho_sw surfaceThicknessFluxRunoff(iCell) = riverRunoffFlux(iCell) / rho_sw - surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / areaCell(iCell) end do + if (trim(config_subglacial_runoff_mode) == 'data') then + do iCell = 1, nCells + surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / areaCell(iCell) + end do + end if #ifndef MPAS_OPENACC !$omp end do !$omp end parallel @@ -578,28 +582,30 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer tracersSurfaceFluxRunoff(index_temperature_flux,iCell) = riverRunoffFlux(iCell) & * max(tracerGroup(index_temperature_flux,minLevelCell(iCell),iCell), 0.0_RKIND) / rho_sw - if ( trim(config_use_sgr_opt_temp) == 'prescribed' ) then - !sgr with fixed prescribed temperature (for debugging only) - tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & - config_sgr_temperature / areaCell(iCell) - else if ( trim(config_use_sgr_opt_temp) == 'sgr' ) then - !sgr with temperature equal to the local freezing point of freshwater - tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & - ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & - inLandIceCavity=.true.) / areaCell(iCell) - end if - - if ( trim(config_use_sgr_opt_salt) == 'prescribed' ) then - !sgr with fixed prescribed temperature (for debugging only) - tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = subglacialRunoffFlux(iCell) * & - config_sgr_salinity / areaCell(iCell) - else if ( trim(config_use_sgr_opt_salt) == 'sgr' ) then - !sgr with temperature equal to the local freezing point of freshwater - tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = 0.0_RKIND + if (trim(config_subglacial_runoff_mode) == 'data') then + if ( trim(config_use_sgr_opt_temp) == 'prescribed' ) then + !sgr with fixed prescribed temperature (for debugging only) + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & + config_sgr_temperature / areaCell(iCell) + else if ( trim(config_use_sgr_opt_temp) == 'sgr' ) then + !sgr with temperature equal to the local freezing point of freshwater + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & + ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & + inLandIceCavity=.true.) / areaCell(iCell) + end if + + if ( trim(config_use_sgr_opt_salt) == 'prescribed' ) then + !sgr with fixed prescribed temperature (for debugging only) + tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = subglacialRunoffFlux(iCell) * & + config_sgr_salinity / areaCell(iCell) + else if ( trim(config_use_sgr_opt_salt) == 'sgr' ) then + !sgr with temperature equal to the local freezing point of freshwater + tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = 0.0_RKIND + end if end if ! Accumulate fluxes that use the freezing point -! mrp performance note: should call ocn_freezing_temperature just once here +! mrp performance note: should call ocn_freezing_temperature just once here seaIceTemperatureFlux(iCell) = seaIceFreshWaterFlux(iCell) * & ocn_freezing_temperature( tracerGroup(index_salinity_flux, minLevelCell(iCell), iCell), pressure=0.0_RKIND, & inLandIceCavity=.false.) / rho_sw @@ -613,10 +619,15 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer tracersSurfaceFlux(index_temperature_flux, iCell) = tracersSurfaceFlux(index_temperature_flux, iCell) & + surfaceTemperatureFluxWithoutRunoff - ! add river and subglacial runoff contribution for sending through coupler + ! add river runoff contribution for sending through coupler totalFreshWaterTemperatureFlux(iCell) = surfaceTemperatureFluxWithoutRunoff & - + tracersSurfaceFluxRunoff(index_temperature_flux,iCell) & + + tracersSurfaceFluxRunoff(index_temperature_flux,iCell) + + ! add subglacial runoff contribution for sending through coupler + if (trim(config_subglacial_runoff_mode) == 'data') then + totalFreshWaterTemperatureFlux(iCell) = totalFreshWaterTemperatureFlux(iCell) & + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) + end if ! Fields with zero temperature are not accumulated. These include: ! snowFlux diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index 2d3dd66cbdb9..30584386666c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -186,11 +186,15 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ do iCell = 1, nCellsAll surfaceThicknessFlux(iCell) = 0.0_RKIND surfaceThicknessFluxRunoff(iCell) = 0.0_RKIND - surfaceThicknessFluxSubglacialRunoff(iCell) = 0.0_RKIND do k=1,nVertLevels tendThick(k, iCell) = 0.0_RKIND end do end do + if (trim(config_subglacial_runoff_mode) == 'data') then + do iCell = 1, nCellsAll + surfaceThicknessFluxSubglacialRunoff(iCell) = 0.0_RKIND + end do + end if #ifndef MPAS_OPENACC !$omp end do !$omp end parallel @@ -841,9 +845,13 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & do n=1,nTracersGroup tracerGroupSurfaceFlux (n,iCell) = 0.0_RKIND tracerGroupSurfaceFluxRunoff (n,iCell) = 0.0_RKIND - tracerGroupSurfaceFluxSubglacialRunoff (n,iCell) = 0.0_RKIND tracerGroupSurfaceFluxRemoved(n,iCell) = 0.0_RKIND end do + if (trim(config_subglacial_runoff_mode) == 'data') then + do n=1,nTracersGroup + tracerGroupSurfaceFluxSubglacialRunoff (n,iCell) = 0.0_RKIND + end do + end if end do !$omp end do !$omp end parallel diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F index 1f7c67b88e09..eb531a999582 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F @@ -141,15 +141,18 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe do iCell = 1, nCellsOwned remainingFlux = 1.0_RKIND remainingFluxRunoff = 1.0_RKIND - remainingFluxSubglacialRunoff = 1.0_RKIND + if (trim(config_subglacial_runoff_mode) == 'data') then + remainingFluxSubglacialRunoff = 1.0_RKIND + end if do k = minLevelCell(iCell), maxLevelCell(iCell) remainingFlux = remainingFlux - transmissionCoefficients(k, iCell) remainingFluxRunoff = remainingFluxRunoff - transmissionCoefficientsRunoff(k, iCell) - remainingFluxSubglacialRunoff = remainingFluxSubglacialRunoff - transmissionCoefficientsSubglacialRunoff(k, iCell) - tend(k, iCell) = tend(k, iCell) + surfaceThicknessFlux(iCell) * transmissionCoefficients(k, iCell) & - + surfaceThicknessFluxRunoff(iCell) * transmissionCoefficientsRunoff(k, iCell) & - + surfaceThicknessFluxSubglacialRunoff(iCell) * transmissionCoefficientsSubglacialRunoff(k, iCell) + + surfaceThicknessFluxRunoff(iCell) * transmissionCoefficientsRunoff(k, iCell) + if (trim(config_subglacial_runoff_mode) == 'data') then + remainingFluxSubglacialRunoff = remainingFluxSubglacialRunoff - transmissionCoefficientsSubglacialRunoff(k, iCell) + tend(k, iCell) = tend(k, iCell) + surfaceThicknessFluxSubglacialRunoff(iCell) * transmissionCoefficientsSubglacialRunoff(k, iCell) + end if end do if(maxLevelCell(iCell) > 0 .and. remainingFlux > 0.0_RKIND) then @@ -161,9 +164,11 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe + remainingFluxRunoff * surfaceThicknessFluxRunoff(iCell) end if - if(maxLevelCell(iCell) > 0 .and. remainingFluxSubglacialRunoff > 0.0_RKIND) then - tend(maxLevelCell(iCell), iCell) = tend(maxLevelCell(iCell), iCell) & - + remainingFluxSubglacialRunoff * surfaceThicknessFluxSubglacialRunoff(iCell) + if (trim(config_subglacial_runoff_mode) == 'data') then + if(maxLevelCell(iCell) > 0 .and. remainingFluxSubglacialRunoff > 0.0_RKIND) then + tend(maxLevelCell(iCell), iCell) = tend(maxLevelCell(iCell), iCell) & + + remainingFluxSubglacialRunoff * surfaceThicknessFluxSubglacialRunoff(iCell) + end if end if end do #ifndef MPAS_OPENACC diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F index 4303d1b78ff2..4a1dc5fce098 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F @@ -206,7 +206,7 @@ subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbso ! now do subglacial runoff component - if (associated(surfaceTracerFluxSubglacialRunoff)) then + if (trim(config_subglacial_runoff_mode) == 'data') then call mpas_timer_start("surface_tracer_subglacial_runoff_flux") !$omp parallel From 9b3e339867dfbf0f564be21ef25deb71ce089ffe Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 8 Jul 2024 14:46:56 -0600 Subject: [PATCH 240/904] pkg dataSubglacialRunoffFluxPKG into registry variables --- components/mpas-ocean/src/Registry.xml | 7 ++++--- .../src/tracer_groups/Registry_activeTracers.xml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index bab430733500..79ef88b33181 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -3384,6 +3384,7 @@ /> @@ -3807,7 +3808,7 @@ /> - + From 5ea51b28153984ba3c484d453a55cd48181b8edb Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 16:30:12 -0600 Subject: [PATCH 241/904] add conservation registry and streams --- components/mpas-ocean/cime_config/buildnml | 3 +++ .../Registry_conservation_check.xml | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index d6d40491baa6..4d5029555afc 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -1195,12 +1195,14 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') + lines.append('') lines.append('') lines.append('') lines.append('') lines.append('') lines.append('') lines.append('') + lines.append('') lines.append('') lines.append('') lines.append('') @@ -1209,6 +1211,7 @@ def buildnml(case, caseroot, compname): lines.append('') lines.append('') lines.append('') + lines.append('') lines.append('') lines.append('') lines.append('') diff --git a/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml b/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml index 0fd2d28d1c6e..7870e6fc2178 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_conservation_check.xml @@ -106,6 +106,10 @@ + @@ -145,6 +149,10 @@ description="Fresh water flux from river runoff from coupler. Positive into the ocean." packages="thicknessBulkPKG" /> + @@ -194,6 +202,11 @@ + + + + @@ -302,6 +317,7 @@ + @@ -343,12 +359,14 @@ + + @@ -358,6 +376,7 @@ + From 361900104fd6e9fa284639f73395d5ca04732ffc Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 9 Jul 2024 17:04:01 -0600 Subject: [PATCH 242/904] try copy mpas_ocn_conservation_check.F --- .../mpas_ocn_conservation_check.F | 105 ++++++++++++++++-- 1 file changed, 97 insertions(+), 8 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index 5551d5758db2..756328a3d5dd 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -28,6 +28,7 @@ module ocn_conservation_check use ocn_constants use ocn_config + use ocn_mesh implicit none private @@ -427,6 +428,7 @@ subroutine energy_conservation(domain, err) accumulatedEvapTemperatureFlux, & accumulatedSeaIceTemperatureFlux, & accumulatedRiverRunoffTemperatureFlux, & + accumulatedSubglacialRunoffTemperatureFlux, & accumulatedIcebergTemperatureFlux real(kind=RKIND), dimension(:), allocatable :: & @@ -463,7 +465,8 @@ subroutine energy_conservation(domain, err) real(kind=RKIND), dimension(:,:), pointer :: & - activeTracersSurfaceFluxRunoff + activeTracersSurfaceFluxRunoff, & + activeTracersSurfaceFluxSubglacialRunoff type (MPAS_timeInterval_type) :: & timeStepESMF @@ -480,7 +483,7 @@ subroutine energy_conservation(domain, err) ierr integer, parameter :: & - nSums = 19 + nSums = 20 character(len=160) :: & m @@ -513,6 +516,7 @@ subroutine energy_conservation(domain, err) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedEvapTemperatureFlux", accumulatedEvapTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSeaIceTemperatureFlux", accumulatedSeaIceTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedRiverRunoffTemperatureFlux", accumulatedRiverRunoffTemperatureFlux) + call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSubglacialRunoffTemperatureFlux", accumulatedSubglacialRunoffTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedIcebergTemperatureFlux", accumulatedIcebergTemperatureFlux) !------------------------------------------------------------- @@ -556,6 +560,7 @@ subroutine energy_conservation(domain, err) call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_temperatureSurfaceFlux', index_temperature_flux) call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxRunoff', activeTracersSurfaceFluxRunoff) + call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxSubglacialRunoff', activeTracersSurfaceFluxSubglacialRunoff) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMassNew, 2) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMassOld, 1) @@ -581,6 +586,11 @@ subroutine energy_conservation(domain, err) sumArray(14) = sumArray(14) + areaCell(iCell) * activeTracersSurfaceFluxRunoff(index_temperature_flux,iCell) sumArray(15) = sumArray(15) + areaCell(iCell) * icebergTemperatureFlux(iCell) + ! subglacial river runoff temperature flux + if (trim(config_subglacial_runoff_mode) == 'data') then + sumArray(20) = sumArray(20) + areaCell(iCell) * activeTracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) + end if + enddo ! iCell if (config_use_frazil_ice_formation) then @@ -634,6 +644,9 @@ subroutine energy_conservation(domain, err) accumulatedLandIceHeatFlux = accumulatedLandIceHeatFlux + sumArrayOut(17) accumulatedLandIceFrazilHeatFlux = accumulatedLandIceFrazilHeatFlux + sumArrayOut(18) accumulatedRemovedIceRunoffHeatFlux = accumulatedRemovedIceRunoffHeatFlux + sumArrayOut(19) + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffTemperatureFlux = accumulatedSubglacialRunoffTemperatureFlux + sumArrayOut(20) + end if ! cleanup deallocate(sumArray) @@ -663,6 +676,9 @@ subroutine energy_conservation(domain, err) accumulatedEvapTemperatureFlux = accumulatedEvapTemperatureFlux /accumulatedFluxCounter accumulatedSeaIceTemperatureFlux = accumulatedSeaIceTemperatureFlux /accumulatedFluxCounter accumulatedRiverRunoffTemperatureFlux = accumulatedRiverRunoffTemperatureFlux /accumulatedFluxCounter + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffTemperatureFlux = accumulatedSubglacialRunoffTemperatureFlux /accumulatedFluxCounter + end if accumulatedIcebergTemperatureFlux = accumulatedIcebergTemperatureFlux /accumulatedFluxCounter accumulatedLandIceFrazilHeatFlux = accumulatedLandIceFrazilHeatFlux /accumulatedFluxCounter accumulatedRemovedIceRunoffHeatFlux = accumulatedRemovedIceRunoffHeatFlux /accumulatedFluxCounter @@ -699,6 +715,9 @@ subroutine energy_conservation(domain, err) + accumulatedRiverRunoffTemperatureFlux *rho_sw*cp_sw & + accumulatedIcebergTemperatureFlux*rho_sw*cp_sw ! note, accumulatedLandIceFrazilHeatFlux not added because already in accumulatedFrazilHeatFlux + if (trim(config_subglacial_runoff_mode) == 'data') then + netEnergyFlux = netEnergyFlux + accumulatedSubglacialRunoffTemperatureFlux * rho_sw*cp_sw + end if ! compute the final energy error call MPAS_pool_get_array(conservationCheckEnergyAMPool, "absoluteEnergyError", absoluteEnergyError) @@ -760,6 +779,9 @@ subroutine energy_conservation(domain, err) v=accumulatedEvapTemperatureFlux *rho_sw*cp_sw; write(m,"('EvapTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedSeaIceTemperatureFlux *rho_sw*cp_sw; write(m,"('SeaIceTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v v=accumulatedRiverRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('RiverRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v +if (trim(config_subglacial_runoff_mode) == 'data') then + v=accumulatedSubglacialRunoffTemperatureFlux*rho_sw*cp_sw; write(m,"('SubglacialRunoffTempFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v +end if v=accumulatedIcebergTemperatureFlux*rho_sw*cp_sw; write(m,"('IcebergTemperatureFlux ',es16.8,' ',f16.8)") v,v/A; call mpas_log_write(m); s=s+v write(m,"('SUM IMPLICIT HEAT FLUXES ',es16.8,' hh20temp ',f16.8,es16.8)") s, s/A; call mpas_log_write(m) s = s + explicitHeatFluxSum @@ -828,6 +850,7 @@ subroutine mass_conservation(domain, err) accumulatedEvaporationFlux, & accumulatedSeaIceFlux, & accumulatedRiverRunoffFlux, & + accumulatedSubglacialRunoffFlux, & accumulatedIceRunoffFlux, & accumulatedRemovedRiverRunoffFlux, & accumulatedRemovedIceRunoffFlux, & @@ -852,6 +875,7 @@ subroutine mass_conservation(domain, err) evaporationFlux, & seaIceFreshwaterFlux, & riverRunoffFlux, & + subglacialRunoffFlux, & iceRunoffFlux, & removedRiverRunoffFlux, & removedIceRunoffFlux, & @@ -877,7 +901,7 @@ subroutine mass_conservation(domain, err) iCell, ierr, k integer, parameter :: & - nSums = 12 + nSums = 13 integer, dimension(:), pointer :: minLevelCell, maxLevelCell @@ -900,6 +924,7 @@ subroutine mass_conservation(domain, err) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedEvaporationFlux", accumulatedEvaporationFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSeaIceFlux", accumulatedSeaIceFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRiverRunoffFlux", accumulatedRiverRunoffFlux) + call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSubglacialRunoffFlux", accumulatedSubglacialRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedIceRunoffFlux", accumulatedIceRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedRiverRunoffFlux",accumulatedRemovedRiverRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedIceRunoffFlux", accumulatedRemovedIceRunoffFlux) @@ -937,6 +962,7 @@ subroutine mass_conservation(domain, err) call mpas_pool_get_array(forcingPool, 'evaporationFlux', evaporationFlux) call mpas_pool_get_array(forcingPool, 'seaIceFreshWaterFlux', seaIceFreshwaterFlux) call mpas_pool_get_array(forcingPool, 'riverRunoffFlux', riverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'subglacialRunoffFlux', subglacialRunoffFlux) call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) call mpas_pool_get_array(forcingPool, 'removedRiverRunoffFlux', removedRiverRunoffFlux) call mpas_pool_get_array(forcingPool, 'removedIceRunoffFlux', removedIceRunoffFlux) @@ -956,6 +982,9 @@ subroutine mass_conservation(domain, err) sumArray( 7) = sumArray( 7) + areaCell(iCell) * removedRiverRunoffFlux(iCell) sumArray( 8) = sumArray( 8) + areaCell(iCell) * removedIceRunoffFlux(iCell) sumArray( 9) = sumArray( 9) + areaCell(iCell) * icebergFreshwaterFlux(iCell) + if (trim(config_subglacial_runoff_mode) == 'data') then + sumArray(13) = sumArray(13) + areaCell(iCell) * subglacialRunoffFlux(iCell) / areaCell(iCell) * rho_sw + end if enddo if (config_use_frazil_ice_formation) then @@ -1002,6 +1031,9 @@ subroutine mass_conservation(domain, err) accumulatedFrazilFlux = accumulatedFrazilFlux + sumArrayOut(10) accumulatedLandIceFlux = accumulatedLandIceFlux + sumArrayOut(11) accumulatedLandIceFrazilFlux = accumulatedLandIceFrazilFlux + sumArrayOut(12) + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffFlux = accumulatedSubglacialRunoffFlux + sumArrayOut(13) + end if ! cleanup deallocate(sumArray) @@ -1021,6 +1053,9 @@ subroutine mass_conservation(domain, err) accumulatedEvaporationFlux = accumulatedEvaporationFlux /accumulatedFluxCounter accumulatedSeaIceFlux = accumulatedSeaIceFlux /accumulatedFluxCounter accumulatedRiverRunoffFlux = accumulatedRiverRunoffFlux /accumulatedFluxCounter + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffFlux = accumulatedSubglacialRunoffFlux /accumulatedFluxCounter + end if accumulatedIceRunoffFlux = accumulatedIceRunoffFlux /accumulatedFluxCounter accumulatedRemovedRiverRunoffFlux = accumulatedRemovedRiverRunoffFlux /accumulatedFluxCounter accumulatedRemovedIceRunoffFlux = accumulatedRemovedIceRunoffFlux /accumulatedFluxCounter @@ -1054,6 +1089,9 @@ subroutine mass_conservation(domain, err) + accumulatedFrazilFlux & + accumulatedLandIceFlux ! note, accumulatedLandIceFrazilFlux not added because already in accumulatedFrazilFlux + if (trim(config_subglacial_runoff_mode) == 'data') then + netMassFlux = netMassFlux + accumulatedSubglacialRunoffFlux + end if ! compute the final mass error call MPAS_pool_get_array(conservationCheckMassAMPool, "absoluteMassError", absoluteMassError) @@ -1083,6 +1121,10 @@ subroutine mass_conservation(domain, err) v=accumulatedIcebergFlux ; write(m,"('icebergFreshwaterFlux ',es16.8,' x2o_Fioi_bergw wberg ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v v=accumulatedEvaporationFlux ; write(m,"('evaporationFlux ',es16.8,' x2o_Foxx_evap wevap ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v v=accumulatedRiverRunoffFlux ; write(m,"('riverRunoffFlux ',es16.8,' x2o_Foxx_rofl wrunoff ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v +if (trim(config_subglacial_runoff_mode) == 'data') then + v=accumulatedSubglacialRunoffFlux ; write(m,"('subglacialRunoffFlux ',es16.8,' x2o_Foxx_rofl wsgr ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v +end if + if (landIceFreshwaterFluxesOn) then v=accumulatedRemovedRiverRunoffFlux; write(m,"('removedRiverRunoffFlux ',es16.8,' wrunoff ',f16.8)") v,v*c; call mpas_log_write(m); v=accumulatedRiverRunoffFlux+accumulatedRemovedRiverRunoffFlux; @@ -1150,7 +1192,8 @@ subroutine salt_conservation(domain, err) conservationCheckSaltAMPool, & meshPool, & statePool, & - forcingPool + forcingPool, & + tracersSurfaceFluxPool real(kind=RKIND), pointer :: & initialSalt, & @@ -1160,9 +1203,13 @@ subroutine salt_conservation(domain, err) absoluteSaltError, & relativeSaltError + real(kind=RKIND), dimension(:,:), pointer :: & + activeTracersSurfaceFluxSubglacialRunoff + real(kind=RKIND), pointer :: & accumulatedSeaIceSalinityFlux, & - accumulatedFrazilSalinityFlux + accumulatedFrazilSalinityFlux, & + accumulatedSubglacialRunoffSalinityFlux ! accumulatedLandIceFrazilSalinityFlux is not present because it is always 0 real(kind=RKIND), dimension(:), allocatable :: & @@ -1184,14 +1231,15 @@ subroutine salt_conservation(domain, err) dt, dtAvg, v, A, s, c integer, pointer :: & - nCellsSolve + nCellsSolve, & + index_salinity_flux integer :: & iCell, & ierr integer, parameter :: & - nSums = 3 + nSums = 4 logical, pointer :: & activeTracersBulkRestoringPKG @@ -1209,6 +1257,7 @@ subroutine salt_conservation(domain, err) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSeaIceSalinityFlux", accumulatedSeaIceSalinityFlux) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedFrazilSalinityFlux", accumulatedFrazilSalinityFlux) + call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSubglacialRunoffSalinityFlux", accumulatedSubglacialRunoffSalinityFlux) !------------------------------------------------------------- ! Net salt flux to ice @@ -1234,12 +1283,23 @@ subroutine salt_conservation(domain, err) call mpas_pool_get_array(forcingPool, 'seaIceSalinityFlux', seaIceSalinityFlux) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceSalinity', accumulatedFrazilIceSalinityOld, 1) call mpas_pool_get_array(statePool, 'accumulatedFrazilIceSalinity', accumulatedFrazilIceSalinityNew, 2) + call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) + call mpas_pool_get_array(tracersSurfaceFluxPool, 'activeTracersSurfaceFluxSubglacialRunoff', activeTracersSurfaceFluxSubglacialRunoff) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_salinitySurfaceFlux', index_salinity_flux) + + do iCell = 1, nCellsSolve ! salt flux to ocean sumArray(1) = sumArray(1) + areaCell(iCell) * seaIceSalinityFlux(iCell) enddo ! iCell + ! subglacial runoff + if (trim(config_subglacial_runoff_mode) == 'data') then + do iCell = 1, nCellsSolve + sumArray(4) = sumArray(4) + areaCell(iCell) * activeTracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) + enddo ! iCell + end if if (config_use_frazil_ice_formation) then do iCell = 1, nCellsSolve @@ -1265,6 +1325,9 @@ subroutine salt_conservation(domain, err) ! accumulate fluxes accumulatedSeaIceSalinityFlux = accumulatedSeaIceSalinityFlux + sumArrayOut(1) accumulatedFrazilSalinityFlux = accumulatedFrazilSalinityFlux + sumArrayOut(2) + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffSalinityFlux = accumulatedSubglacialRunoffSalinityFlux + sumArrayOut(4) + end if ! cleanup deallocate(sumArray) @@ -1281,6 +1344,9 @@ subroutine salt_conservation(domain, err) ! Average the fluxes accumulatedSeaIceSalinityFlux = accumulatedSeaIceSalinityFlux /accumulatedFluxCounter accumulatedFrazilSalinityFlux = accumulatedFrazilSalinityFlux /accumulatedFluxCounter + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffSalinityFlux = accumulatedSubglacialRunoffSalinityFlux /accumulatedFluxCounter + end if ! get initial salt content call MPAS_pool_get_array(conservationCheckSaltAMPool, "initialSalt", initialSalt) @@ -1299,6 +1365,10 @@ subroutine salt_conservation(domain, err) netSaltFlux = accumulatedSeaIceSalinityFlux & + accumulatedFrazilSalinityFlux + if (trim(config_subglacial_runoff_mode) == 'data') then + netSaltFlux = netSaltFlux + accumulatedSubglacialRunoffSalinityFlux + end if + ! compute the final salt error call MPAS_pool_get_array(conservationCheckSaltAMPool, "absoluteSaltError", absoluteSaltError) call MPAS_pool_get_array(conservationCheckSaltAMPool, "relativeSaltError", relativeSaltError) @@ -1327,6 +1397,10 @@ subroutine salt_conservation(domain, err) .and.config_frazil_under_land_ice) then v=0; write(m,"('LandIceFrazilSalinityFlux',es16.8,' (already in wmelt, do not sum) ',f16.8)") v,v*c; call mpas_log_write(m); !no sum: s=s+v end if +if (trim(config_subglacial_runoff_mode) == 'data') then + v=accumulatedSubglacialRunoffSalinityFlux ; write(m,"('SubglacialRunoffSalinityFlux ',es16.8,' x2o_Fioi_salt salt ',f16.8)") v,v*c; call mpas_log_write(m); s=s+v +end if + write(m,"('SUM VOLUME FLUXES ',es16.8,' ',f16.8,es16.8)") s, s*c; call mpas_log_write(m) call mpas_log_write(' ') @@ -2232,6 +2306,7 @@ subroutine reset_accumulated_variables(domain) accumulatedEvapTemperatureFlux, & accumulatedSeaIceTemperatureFlux, & accumulatedRiverRunoffTemperatureFlux, & + accumulatedSubglacialRunoffTemperatureFlux, & accumulatedIcebergTemperatureFlux real(kind=RKIND), pointer :: & @@ -2240,6 +2315,7 @@ subroutine reset_accumulated_variables(domain) accumulatedEvaporationFlux, & accumulatedSeaIceFlux, & accumulatedRiverRunoffFlux, & + accumulatedSubglacialRunoffFlux, & accumulatedIceRunoffFlux, & accumulatedRemovedRiverRunoffFlux, & accumulatedRemovedIceRunoffFlux, & @@ -2250,7 +2326,8 @@ subroutine reset_accumulated_variables(domain) real(kind=RKIND), pointer :: & accumulatedSeaIceSalinityFlux, & - accumulatedFrazilSalinityFlux + accumulatedFrazilSalinityFlux, & + accumulatedSubglacialRunoffSalinityFlux real(kind=RKIND), pointer :: & accumulatedCarbonSourceSink, & @@ -2280,6 +2357,7 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedEvapTemperatureFlux", accumulatedEvapTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSeaIceTemperatureFlux", accumulatedSeaIceTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedRiverRunoffTemperatureFlux", accumulatedRiverRunoffTemperatureFlux) + call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedSubglacialRunoffTemperatureFlux", accumulatedSubglacialRunoffTemperatureFlux) call MPAS_pool_get_array(conservationCheckEnergyAMPool, "accumulatedIcebergTemperatureFlux", accumulatedIcebergTemperatureFlux) accumulatedFluxCounter = 0 @@ -2298,6 +2376,9 @@ subroutine reset_accumulated_variables(domain) accumulatedEvapTemperatureFlux = 0.0_RKIND accumulatedSeaIceTemperatureFlux = 0.0_RKIND accumulatedRiverRunoffTemperatureFlux = 0.0_RKIND + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffTemperatureFlux = 0.0_RKIND + end if accumulatedIcebergTemperatureFlux = 0.0_RKIND accumulatedLandIceFrazilHeatFlux = 0.0_RKIND accumulatedRemovedIceRunoffHeatFlux = 0.0_RKIND @@ -2310,6 +2391,7 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedEvaporationFlux", accumulatedEvaporationFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSeaIceFlux", accumulatedSeaIceFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRiverRunoffFlux", accumulatedRiverRunoffFlux) + call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedSubglacialRunoffFlux", accumulatedSubglacialRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedIceRunoffFlux", accumulatedIceRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedRiverRunoffFlux",accumulatedRemovedRiverRunoffFlux) call MPAS_pool_get_array(conservationCheckMassAMPool, "accumulatedRemovedIceRunoffFlux", accumulatedRemovedIceRunoffFlux) @@ -2323,6 +2405,9 @@ subroutine reset_accumulated_variables(domain) accumulatedEvaporationFlux = 0.0_RKIND accumulatedSeaIceFlux = 0.0_RKIND accumulatedRiverRunoffFlux = 0.0_RKIND + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffFlux = 0.0_RKIND + end if accumulatedIceRunoffFlux = 0.0_RKIND accumulatedRemovedRiverRunoffFlux = 0.0_RKIND accumulatedRemovedIceRunoffFlux = 0.0_RKIND @@ -2336,9 +2421,13 @@ subroutine reset_accumulated_variables(domain) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSeaIceSalinityFlux", accumulatedSeaIceSalinityFlux) call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedFrazilSalinityFlux", accumulatedFrazilSalinityFlux) + call MPAS_pool_get_array(conservationCheckSaltAMPool, "accumulatedSubglacialRunoffSalinityFlux", accumulatedSubglacialRunoffSalinityFlux) accumulatedSeaIceSalinityFlux = 0.0_RKIND accumulatedFrazilSalinityFlux = 0.0_RKIND + if (trim(config_subglacial_runoff_mode) == 'data') then + accumulatedSubglacialRunoffSalinityFlux = 0.0_RKIND + end if call MPAS_pool_get_subpool(domain % blocklist % structs, "conservationCheckCarbonAM", conservationCheckCarbonAMPool) From 26720b56bd801322af5b5cb92bc9f2e491be035a Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 10 Jul 2024 15:45:14 -0600 Subject: [PATCH 243/904] update dsgr files --- components/mpas-ocean/cime_config/buildnml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 4d5029555afc..6ac2b043aaf3 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -143,7 +143,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.oQU240wLI.20240221.nc' if ocn_sgr == 'data': - data_sgr_file = 'oQU240wLI_DSGR.nc' + data_sgr_file = 'DSGR.MALI.out2055.oQU240wLI.20240328.nc' elif ocn_grid == 'oQU120': decomp_date = '230424' @@ -282,7 +282,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.SOwISC12to60E2r4.20210114.nc' if ocn_sgr == 'data': - data_sgr_file = 'SOwISC12to60E2r4_DSGR.nc' + data_sgr_file = 'DSGR.MALI.out2055.SOwISC12to60E2r4.20240328.nc' elif ocn_grid == 'FRISwISC08to60E3r1': decomp_date = '20230913' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC08to60E3r1/partitions @@ -359,7 +359,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.ECwISC30to60E2r1.20240221.nc' if ocn_sgr == 'data': - data_sgr_file = 'ECwISC30to60E2r1_DSGR.nc' + data_sgr_file = 'DSGR.MALI.out2055.ECwISC30to60E2r1.20240328.nc' elif ocn_grid == 'IcoswISC30E3r5': decomp_date = '20231120' @@ -380,7 +380,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.IcoswISC30E3r5.20231120.nc' if ocn_sgr == 'data': - data_sgr_file = 'IcoswISC30E3r5_DSGR.nc' + data_sgr_file = 'DSGR.MALI.out2055.IcoswISC30E3r5.20240328.nc.nc' elif ocn_grid == 'IcosXISC30E3r7': decomp_date = '20240314' From 21a593e20d77166c25ea6b09ec02d0789c9139e7 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 10 Jul 2024 15:51:33 -0600 Subject: [PATCH 244/904] remove redundant compsets --- .../cime_config/config_compsets.xml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index 6950fd8c2179..be36f0103472 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -42,11 +42,6 @@ 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYFAIS45_SGLC_SWAV - - GMPAS-NYF-DSGR - 2000_DATM%NYF_SLND_MPASSI_MPASO%DATMFORCEDDSGR_DROF%NYFAIS45_SGLC_SWAV - - GMPAS-NYF-DISMF 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV @@ -103,8 +98,8 @@ - GMPAS-JRA1p5-DSGR - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%DATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + GMPAS-JRA1p5-DIB-PISMF-DSGR-TMIX + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGRTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV @@ -117,11 +112,6 @@ 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV - - GMPAS-JRA1p5-DIB-DISMF-DSGR - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV - - GMPAS-JRA1p4 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV @@ -157,11 +147,6 @@ 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV - - GMPAS-IAF-PISMF-DSGR - 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%IAFAIS45_SGLC_SWAV - - GMPAS-IAF-DISMF 2000_DATM%IAF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV From fe4380c621bd7c5ea21c6fe15337c56ed9572c0a Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 10 Jul 2024 16:25:01 -0600 Subject: [PATCH 245/904] fix typo in dsgr filename --- components/mpas-ocean/cime_config/buildnml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 6ac2b043aaf3..192fd70bc6ae 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -380,7 +380,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.IcoswISC30E3r5.20231120.nc' if ocn_sgr == 'data': - data_sgr_file = 'DSGR.MALI.out2055.IcoswISC30E3r5.20240328.nc.nc' + data_sgr_file = 'DSGR.MALI.out2055.IcoswISC30E3r5.20240328.nc' elif ocn_grid == 'IcosXISC30E3r7': decomp_date = '20240314' From 476a38c098fbf34c9828fe55476bf60a4dd3a40d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 06:27:56 -0700 Subject: [PATCH 246/904] Moves around code to match previous mam4xx processes --- ...x_mam_dry_deposition_process_interface.cpp | 460 ++++++++++-------- ...x_mam_dry_deposition_process_interface.hpp | 29 +- .../single-process/mam/drydep/input.yaml | 1 + 3 files changed, 273 insertions(+), 217 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index eced2cf49643..aa0f643b974a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -35,8 +35,10 @@ Diagnostic Scalar Parameters, one per column: Diagnostics: d_tracer_mixing_ratio_dt "d_qtracers_dt" - deposition_flux_of_cloud_borne_aerosols "deposition_flux_of_cloud_borne_aerosols" - deposition_flux_of_interstitial_aerosols "deposition_flux_of_interstitial_aerosols" + deposition_flux_of_cloud_borne_aerosols +"deposition_flux_of_cloud_borne_aerosols" + deposition_flux_of_interstitial_aerosols +"deposition_flux_of_interstitial_aerosols" Computed internally, could be exposed if needed by another process vlc_grv(nlev) : dep velocity of gravitational settling [m/s] @@ -51,7 +53,6 @@ Computed internally, could be exposed if needed by another process namespace scream { -// ========================================================================================= MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be @@ -59,22 +60,18 @@ MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) */ } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMDryDep::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - - auto m3 = m * m * m; // meter cubed - - auto nondim = ekat::units::Units::nondimensional(); + // set grid for all the inputs and outputs + // use physics grid + grid_ = grids_manager->get_grid("Physics"); - grid_ = grids_manager->get_grid("Physics"); + // Name of the grid const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -83,90 +80,139 @@ void MAMDryDep::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; - + // Layout for 2D (2d horiz) variable + // FIXME: Fix layouts based on new format + const FieldLayout scalar2d{{COL}, {ncol_}}; + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + const FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; + const FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ - {NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; + FieldLayout scalar4d_mid{{NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; // Layout for tracers. const int pcnst = mam4::aero_model::pcnst; - FieldLayout scalar4d_layout_q{ - {COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; - FieldLayout scalar4d_layout_qqcw_tends{ - {COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; + FieldLayout scalar4d_q{{COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; + FieldLayout scalar4d_qqcw_tends{{COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; + + using namespace ekat::units; + + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + auto nondim = ekat::units::Units::nondimensional(); + + auto m3 = m * m * m; // meter cubed + + // -------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg] + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_mid, Pa / s, grid_name); - // Tempary memory to format the wet_aero.int_aero_nmr and wet_aero.int_aero_mmr - // views into the order expected by mam4xx. - add_field("qtracers", scalar4d_layout_q, kg / kg, grid_name); - add_field("d_qtracers_dt", scalar4d_layout_q, kg / kg / s, grid_name); + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + // Planetary boundary layer height [m] + add_field("pbl_height", scalar2d, m, grid_name); + + static constexpr auto m2 = m * m; + static constexpr auto s2 = s * s; + + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Total cloud fraction [fraction] + // FIXME: Is is cldfrac_liq instead? find out + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + + // Obukhov length [m] + add_field("Obukhov_length", scalar2d, m, grid_name); + + // Surface friction velocty [m] + add_field("surface_friction_velocty", scalar2d, m / s, grid_name); + + // Land fraction [fraction] + add_field("land_fraction", scalar2d, nondim, grid_name); + + // Ice fraction [unitless] + add_field("ice_fraction", scalar2d, nondim, grid_name); + + // Ocean fraction [unitless] + add_field("ocean_fraction", scalar2d, nondim, grid_name); + + // Friction velocity from land model [m/s] + add_field("friction_velocity", scalar2d, m / s, grid_name); + + // Aerodynamical resistance from land model [s/m] + add_field("aerodynamical_resistance", scalar2d, s / m, grid_name); + + // Tempary memory to format the wet_aero.int_aero_nmr and + // wet_aero.int_aero_mmr views into the order expected by mam4xx. + add_field("qtracers", scalar4d_q, kg / kg, grid_name); + add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); + add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, + 1 / (m * m) / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_interstitial_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); - add_field("Tendencies", scalar4d_layout_qqcw_tends, kg / kg / s, grid_name); + add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, + 1 / (m * m) / s, grid_name); + add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, + grid_name); - // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio - - add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); - - // TODO: The following are not used by drydep but to create a dry atmosphere object. - add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - - add_field("Obukhov_length", scalar2d_layout, m, grid_name); - add_field("surface_friction_velocty", scalar2d_layout, m / s, grid_name); - add_field("land_fraction", scalar2d_layout, nondim, grid_name); - add_field("ice_fraction", scalar2d_layout, nondim, grid_name); - add_field("ocean_fraction", scalar2d_layout, nondim, grid_name); - add_field("friction_velocity", scalar2d_layout, m / s, grid_name); - add_field("aerodynamical_resistance", scalar2d_layout, s / m, grid_name); + add_field("wetdens", scalar4d_mid, kg / m3, grid_name); + add_field("dgncur_awet", scalar4d_mid, m, grid_name); + + // TODO: The following are not used by drydep but to create a dry atmosphere + // object. // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for(int m = 0; m < num_aero_modes; ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); } } } @@ -174,15 +220,13 @@ void MAMDryDep::set_grids( for(int m = 0; m < num_aero_modes; ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } } @@ -190,9 +234,9 @@ void MAMDryDep::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } } // ========================================================================================= @@ -222,47 +266,53 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? - dry_atm_.pblh = get_field_in("pbl_height").get_view(); - - obukhov_length_ = get_field_in("Obukhov_length").get_view(); - surface_friction_velocty_ = get_field_in("surface_friction_velocty").get_view(); - land_fraction_ = get_field_in("land_fraction").get_view(); - ice_fraction_ = get_field_in("ice_fraction").get_view(); - ocean_fraction_ = get_field_in("ocean_fraction").get_view(); - friction_velocity_ = get_field_in("friction_velocity").get_view(); - aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); - - dry_atm_.p_int = get_field_in("p_int").get_view(); - dry_atm_.z_mid = buffer_.z_mid; - dry_atm_.z_iface = buffer_.z_iface; - dry_atm_.dz = buffer_.dz; - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot") + .get_view(); // FIXME: tot or liq? + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + + obukhov_length_ = get_field_in("Obukhov_length").get_view(); + surface_friction_velocty_ = + get_field_in("surface_friction_velocty").get_view(); + land_fraction_ = get_field_in("land_fraction").get_view(); + ice_fraction_ = get_field_in("ice_fraction").get_view(); + ocean_fraction_ = get_field_in("ocean_fraction").get_view(); + friction_velocity_ = + get_field_in("friction_velocity").get_view(); + aerodynamical_resistance_ = + get_field_in("aerodynamical_resistance").get_view(); + + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; dry_atm_.w_updraft = buffer_.w_updraft; - dry_atm_.z_surf = 0.0; // FIXME: for now + dry_atm_.z_surf = 0.0; // FIXME: for now dgncur_awet_ = get_field_out("dgncur_awet").get_view(); - wet_dens_ = get_field_out("wetdens").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); - qtracers_ = get_field_out("qtracers").get_view(); - d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + qtracers_ = get_field_out("qtracers").get_view(); + d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); - aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols").get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols").get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + .get_view(); + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + .get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -306,99 +356,101 @@ void MAMDryDep::initialize_impl(const RunType run_type) { } // set up our preprocess functor - preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, - dry_atm_, dry_aero_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } namespace { void compute_tendencies( - const int ncol, - const int nlev, - const mam4::DryDeposition dry_deposition, - const double dt, - const MAMDryDep::const_view_1d obklen, + const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, + const double dt, const MAMDryDep::const_view_1d obklen, const MAMDryDep::const_view_1d surfric, const MAMDryDep::const_view_1d landfrac, const MAMDryDep::const_view_1d icefrac, const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, - MAMDryDep::view_3d d_qtracers_dt, - const MAMDryDep::view_3d dgncur_awet_, - const MAMDryDep::view_3d wet_dens_, + MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, - const mam_coupling::AerosolState wet_aero, - MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, - MAMDryDep::view_3d tendencies) -{ - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlev); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { - const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); - const int icol = team.league_rank(); - const Real t = 0; - - compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); - team.team_barrier(); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int lev) { - for (int mode = 0; mode < num_aero_modes; ++mode) { - int icnst = mam4::ConvProc::numptrcw_amode(mode); - qtracers(icol, lev, icnst) = wet_aero.int_aero_nmr[mode](icol,lev); - for (int species = 0; species < num_aero_species; ++species) { - icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); - if (-1 < icnst) { - qtracers(icol, lev, icnst) = wet_aero.int_aero_mmr[mode][species](icol,lev); - } - } - } - }); - team.team_barrier(); - - mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); - mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::Surface surf; - mam4::Diagnostics diags; - mam4::Tendencies tends; - - for (int i=0; i::get_default_team_policy( + ncol, nlev); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); + const int icol = team.league_rank(); + const Real t = 0; + + compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); + team.team_barrier(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { + for(int mode = 0; mode < num_aero_modes; ++mode) { + int icnst = mam4::ConvProc::numptrcw_amode(mode); + qtracers(icol, lev, icnst) = + wet_aero.int_aero_nmr[mode](icol, lev); + for(int species = 0; species < num_aero_species; ++species) { + icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); + if(-1 < icnst) { + qtracers(icol, lev, icnst) = + wet_aero.int_aero_mmr[mode][species](icol, lev); + } + } + } + }); + team.team_barrier(); + + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + mam4::Surface surf; + mam4::Diagnostics diags; + mam4::Tendencies tends; + + for(int i = 0; i < num_aero_modes; ++i) { + diags.wet_geometric_mean_diameter_i[i] = + ekat::subview(dgncur_awet_, i, icol); + diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); + } + diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); + diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); + diags.deposition_flux_of_cloud_borne_aerosols = + ekat::subview(aerdepdrycw, icol); + diags.deposition_flux_of_interstitial_aerosols = + ekat::subview(aerdepdryis, icol); + + diags.Obukhov_length = obklen[icol]; + diags.surface_friction_velocty = surfric[icol]; + diags.land_fraction = landfrac[icol]; + diags.ice_fraction = icefrac[icol]; + diags.ocean_fraction = ocnfrac[icol]; + diags.friction_velocity = friction_velocity[icol]; + diags.aerodynamical_resistance = aerodynamical_resistance[icol]; + + // Fill Tendency views + for(int m = 0; m < num_aero_modes; ++m) { + int iconv = mam4::ConvProc::numptrcw_amode(m); + tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); + for(int a = 0; a < num_aero_species; ++a) { + iconv = mam4::ConvProc::lmassptrcw_amode(a, m); + if(-1 < iconv) + tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); + } + } + + const mam4::AeroConfig aero_config; + dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, + progs, diags, tends); + }); } +} // namespace // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - using DryDep = mam4::DryDeposition; + using DryDep = mam4::DryDeposition; const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -414,27 +466,27 @@ void MAMDryDep::run_impl(const double dt) { // for (int i=0; i::get_default_team_policy(ncol_, nlev_); + // const auto policy = + // ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); // Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { // dry_deposition.validate( aero_config, team, atm, progs); // } // Kokkos::fence(); - compute_tendencies(ncol_, nlev_, dry_deposition, dt, - obukhov_length_, surface_friction_velocty_, - land_fraction_, ice_fraction_, ocean_fraction_, - friction_velocity_, aerodynamical_resistance_, - qtracers_, d_qtracers_dt_, // d_qtracers_dt_ is an output - dgncur_awet_, wet_dens_, - dry_atm_, dry_aero_, wet_aero_, - // Outputs: - aerdepdrycw_, aerdepdryis_, tendencies_); + compute_tendencies(ncol_, nlev_, dry_deposition, dt, obukhov_length_, + surface_friction_velocty_, land_fraction_, ice_fraction_, + ocean_fraction_, friction_velocity_, + aerodynamical_resistance_, qtracers_, + d_qtracers_dt_, // d_qtracers_dt_ is an output + dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, + // Outputs: + aerdepdrycw_, aerdepdryis_, tendencies_); Kokkos::fence(); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 418bea8fe973..1144bd79ff68 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -15,18 +15,19 @@ namespace scream { // The process responsible for handling MAM4 dry deposition. The AD // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { -public: - using view_2d = Field::view_dev_t; - using view_3d = Field::view_dev_t; - using const_view_1d = Field::view_dev_t; - using const_view_2d = Field::view_dev_t; - using const_view_3d = Field::view_dev_t; + public: + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); -private: + + private: // number of horizontal columns and vertical levels int ncol_, nlev_; - //Wet and dry states of atmosphere + // Wet and dry states of atmosphere mam_coupling::WetAtmosphere wet_atm_; mam_coupling::DryAtmosphere dry_atm_; @@ -56,7 +57,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { const_view_1d friction_velocity_; const_view_1d aerodynamical_resistance_; -public: + public: using KT = ekat::KokkosTypes; // Constructor @@ -67,10 +68,12 @@ class MAMDryDep final : public scream::AtmosphereProcess { // -------------------------------------------------------------------------- // The type of subcomponent - AtmosphereProcessType type() const override { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type() const override { + return AtmosphereProcessType::Physics; + } // The name of the subcomponent - std::string name() const override { return "mam_dry_deposition"; } + std::string name() const override { return "mam_dry_deposition"; } // grid void set_grids( @@ -87,9 +90,9 @@ class MAMDryDep final : public scream::AtmosphereProcess { void run_impl(const double dt) override; // Finalize - void finalize_impl() override {/*Do nothing*/}; + void finalize_impl() override{/*Do nothing*/}; - // Atmosphere processes often have a pre-processing step that constructs + // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. struct Preprocess { diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 8b110dc1634d..517ed4a53afd 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -36,6 +36,7 @@ initial_conditions: aerodynamical_resistance: 1e-5 w_updraft: 1e-5 pbl_height: 0.0 + phis: 0.1 # The parameters for I/O control Scorpio: From d0c6d2c873a35bc01aefe875fc0acfbe75ae975f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 06:56:59 -0700 Subject: [PATCH 247/904] Moves variables to match I/O --- ...x_mam_dry_deposition_process_interface.cpp | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index aa0f643b974a..6e492b8983a8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -158,6 +158,7 @@ void MAMDryDep::set_grids( // FIXME: Is is cldfrac_liq instead? find out add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + //----------- Variables from coupler (land component)--------- // Obukhov length [m] add_field("Obukhov_length", scalar2d, m, grid_name); @@ -167,17 +168,27 @@ void MAMDryDep::set_grids( // Land fraction [fraction] add_field("land_fraction", scalar2d, nondim, grid_name); + // Friction velocity from land model [m/s] + add_field("friction_velocity", scalar2d, m / s, grid_name); + + // Aerodynamical resistance from land model [s/m] + add_field("aerodynamical_resistance", scalar2d, s / m, grid_name); + + //----------- Variables from coupler (ice component)--------- + // Ice fraction [unitless] add_field("ice_fraction", scalar2d, nondim, grid_name); + //----------- Variables from coupler (ocean component)--------- // Ocean fraction [unitless] add_field("ocean_fraction", scalar2d, nondim, grid_name); - // Friction velocity from land model [m/s] - add_field("friction_velocity", scalar2d, m / s, grid_name); + // --------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // --------------------------------------------------------------------- - // Aerodynamical resistance from land model [s/m] - add_field("aerodynamical_resistance", scalar2d, s / m, grid_name); + add_field("wetdens", scalar4d_mid, kg / m3, grid_name); + add_field("dgncur_awet", scalar4d_mid, m, grid_name); // Tempary memory to format the wet_aero.int_aero_nmr and // wet_aero.int_aero_mmr views into the order expected by mam4xx. @@ -193,9 +204,6 @@ void MAMDryDep::set_grids( add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, grid_name); - add_field("wetdens", scalar4d_mid, kg / m3, grid_name); - add_field("dgncur_awet", scalar4d_mid, m, grid_name); - // TODO: The following are not used by drydep but to create a dry atmosphere // object. From 880ce7d94a9a4a52a63e99ee3ce801a0386f5ab8 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 06:59:23 -0700 Subject: [PATCH 248/904] Moves Updated vars above computed --- ...x_mam_dry_deposition_process_interface.cpp | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 6e492b8983a8..b56336f96cb8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -190,23 +190,6 @@ void MAMDryDep::set_grids( add_field("wetdens", scalar4d_mid, kg / m3, grid_name); add_field("dgncur_awet", scalar4d_mid, m, grid_name); - // Tempary memory to format the wet_aero.int_aero_nmr and - // wet_aero.int_aero_mmr views into the order expected by mam4xx. - add_field("qtracers", scalar4d_q, kg / kg, grid_name); - add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); - - // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, - 1 / (m * m) / s, grid_name); - // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, - 1 / (m * m) / s, grid_name); - add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, - grid_name); - - // TODO: The following are not used by drydep but to create a dry atmosphere - // object. - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for(int m = 0; m < num_aero_modes; ++m) { @@ -245,6 +228,20 @@ void MAMDryDep::set_grids( add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } + + // Tempary memory to format the wet_aero.int_aero_nmr and + // wet_aero.int_aero_mmr views into the order expected by mam4xx. + add_field("qtracers", scalar4d_q, kg / kg, grid_name); + add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); + + // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, + 1 / (m * m) / s, grid_name); + // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, + 1 / (m * m) / s, grid_name); + add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, + grid_name); } // ========================================================================================= From 1e76a7d0b58ec336449ab66722ea20a7f164d692 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 07:28:09 -0700 Subject: [PATCH 249/904] Some more i/o variable movements in initialize impl --- ...x_mam_dry_deposition_process_interface.cpp | 92 ++++++++++++------- .../eamxx/src/physics/mam/mam_coupling.hpp | 1 + 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index b56336f96cb8..6040a534fa05 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -229,9 +229,14 @@ void MAMDryDep::set_grids( "tracers"); } - // Tempary memory to format the wet_aero.int_aero_nmr and - // wet_aero.int_aero_mmr views into the order expected by mam4xx. + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- + add_field("qtracers", scalar4d_q, kg / kg, grid_name); + + // FIXME: what is the diff between d_qtracers_dt and tendencies???? + // FIXME: we might not need this in FM add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] @@ -240,21 +245,28 @@ void MAMDryDep::set_grids( // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, 1 / (m * m) / s, grid_name); + // FIXME: we might not need this in FM add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, grid_name); } -// ========================================================================================= -// ON HOST, returns the number of bytes of device memory needed by the above -// Buffer type given the number of columns and vertical levels +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by +// the above. Buffer type given the number of columns and vertical +// levels size_t MAMDryDep::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -// ========================================================================================= -// ON HOST, initializeŃ• the Buffer type with sufficient memory to store -// intermediate (dry) quantities on the given number of columns with the given -// number of vertical levels. Returns the number of bytes allocated. +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializeŃ• the Buffer type with sufficient memory to +// store intermediate (dry) quantities on the given number of +// columns with the given number of vertical levels. Returns the +// number of bytes allocated. void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -266,23 +278,30 @@ void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMDryDep."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMDryDep::initialize_impl(const RunType run_type) { - // Gather runtime options - //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + // Populate the wet atmosphere state with views from fields + // FIMXE: specifically look which among these are actually used by the process wet_atm_.qv = get_field_in("qv").get_view(); wet_atm_.qc = get_field_in("qc").get_view(); wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.cldfrac = get_field_in("cldfrac_tot") - .get_view(); // FIXME: tot or liq? - dry_atm_.pblh = get_field_in("pbl_height").get_view(); + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + // FIXME: tot or liq? + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); obukhov_length_ = get_field_in("Obukhov_length").get_view(); surface_friction_velocty_ = @@ -295,7 +314,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); + // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; dry_atm_.z_iface = buffer_.z_iface; dry_atm_.dz = buffer_.dz; @@ -307,19 +326,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.w_updraft = buffer_.w_updraft; dry_atm_.z_surf = 0.0; // FIXME: for now - dgncur_awet_ = get_field_out("dgncur_awet").get_view(); - wet_dens_ = get_field_out("wetdens").get_view(); - - qtracers_ = get_field_out("qtracers").get_view(); - d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); - - aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") - .get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") - .get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios + // ---- set wet/dry aerosol-related gas state data for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); @@ -360,9 +367,28 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } - // set up our preprocess functor + // ------------------------------------------------------------- + // Output fields for the process + // ------------------------------------------------------------- + + // FIXME: We might need to get rid of few of these fields + // as we do not need them in FM + dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); + qtracers_ = get_field_out("qtracers").get_view(); + d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + .get_view(); + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + .get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); + + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + // FIXME: Where is post processing functor???? } namespace { void compute_tendencies( diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 2bc76cec3311..189b78b044d2 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -610,6 +610,7 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); + //FIXME : add an assert statement to check id z_surf is zero or not PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output team.team_barrier(); // likely necessary to have z_iface up to date From 34a7099b040d7ee28f0c4f2c17580f5ffcfff572 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 11 Jul 2024 09:35:41 -0600 Subject: [PATCH 250/904] Use TeamVectorRange Also, only call kernel if one of iop_nudge_tq or iop_nudge_uv is true --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 37e6c9de8adc..74579df552bd 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -577,32 +577,34 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { - if (iop_nudge_tq) { - // Restrict nudging of T and qv to certain levels if requested by user - // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high - // is in units of [hPa], thus convert iop_nudge_tq_low/high - Mask nudge_level(false); - int max_size = hyam.size(); - for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { - const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; - nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + if (iop_nudge_tq or iop_nudge_uv) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { + if (iop_nudge_tq) { + // Restrict nudging of T and qv to certain levels if requested by user + // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high + // is in units of [hPa], thus convert iop_nudge_tq_low/high + Mask nudge_level(false); + int max_size = hyam.size(); + for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { + const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; + nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 + and + pressure_from_iop >= iop_nudge_tq_high*100); + } + + qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); + temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); + + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); } - - qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); - temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - } - if (iop_nudge_uv) { - u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); - v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); - } - }); + if (iop_nudge_uv) { + u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); + v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); + } + }); + } // Release WS views ws.release_many_contiguous<1>({&pmid}); From 8f147f9101c196801cfcb0663cce9df37cee0043 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 11 Jul 2024 10:44:22 -0600 Subject: [PATCH 251/904] fix omp and acc added loops --- .../src/shared/mpas_ocn_diagnostics.F | 17 ++++++++++--- .../shared/mpas_ocn_surface_bulk_forcing.F | 22 +++++++++++++---- .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 24 +++++++++++++++++-- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index daf9bfcab6d5..7dc43284a755 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -1893,8 +1893,7 @@ subroutine ocn_diagnostic_solve_surfaceLayer(layerThickness, & #ifdef MPAS_OPENACC !$acc parallel loop & !$acc present(surfaceFluxAttenuationCoefficient, & - !$acc surfaceFluxAttenuationCoefficientRunoff, & - !$acc surfaceFluxAttenuationCoefficientSubglacialRunoff) + !$acc surfaceFluxAttenuationCoefficientRunoff) #else !$omp parallel !$omp do schedule(runtime) @@ -1905,16 +1904,28 @@ subroutine ocn_diagnostic_solve_surfaceLayer(layerThickness, & surfaceFluxAttenuationCoefficientRunoff(iCell) = & config_flux_attenuation_coefficient_runoff end do +#ifndef MPAS_OPENACC + !$omp end do + !$omp end parallel +#endif + if (trim(config_subglacial_runoff_mode) == 'data') then +#ifdef MPAS_OPENACC + !$acc parallel loop & + !$acc present(surfaceFluxAttenuationCoefficientSubglacialRunoff) +#else + !$omp parallel + !$omp do schedule(runtime) +#endif do iCell = 1, nCells surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell) = & config_flux_attenuation_coefficient_subglacial_runoff end do - end if #ifndef MPAS_OPENACC !$omp end do !$omp end parallel #endif + end if !-------------------------------------------------------------------- diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 0547f73319dd..4375ae8f9eaf 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -336,13 +336,13 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur ! Build surface fluxes at cell centers #ifdef MPAS_OPENACC !$acc enter data copyin(evaporationFlux, snowFlux, seaIceFreshWaterFlux, icebergFreshWaterFlux, & - !$acc riverRunoffFlux, subglacialRunoffFlux, iceRunoffFlux, rainFlux) + !$acc riverRunoffFlux, iceRunoffFlux, rainFlux) !$acc parallel loop & !$acc present(surfaceThicknessFlux, surfaceThicknessFluxRunoff, & - !$acc surfaceThicknessFluxSubglacialRunoff, evaporationFlux, snowFlux, & + !$acc evaporationFlux, snowFlux, & !$acc seaIceFreshWaterFlux, icebergFreshWaterFlux, riverRunoffFlux, & - !$acc subglacialRunoffFlux, iceRunoffFlux, rainFlux) + !$acc iceRunoffFlux, rainFlux) #else !$omp parallel !$omp do schedule(runtime) @@ -352,15 +352,29 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur + seaIceFreshWaterFlux(iCell) + icebergFreshWaterFlux(iCell) + iceRunoffFlux(iCell) ) / rho_sw surfaceThicknessFluxRunoff(iCell) = riverRunoffFlux(iCell) / rho_sw end do +#ifndef MPAS_OPENACC + !$omp end do + !$omp end parallel +#endif + if (trim(config_subglacial_runoff_mode) == 'data') then +#ifdef MPAS_OPENACC + !$acc enter data copyin(subglacialRunoffFlux) + + !$acc parallel loop & + !$acc present(surfaceThicknessFluxSubglacialRunoff, subglacialRunoffFlux, areaCell) +#else + !$omp parallel + !$omp do schedule(runtime) +#endif do iCell = 1, nCells surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / areaCell(iCell) end do - end if #ifndef MPAS_OPENACC !$omp end do !$omp end parallel #endif + end if #ifdef MPAS_OPENACC !$acc exit data delete(evaporationFlux, snowFlux, seaIceFreshWaterFlux, icebergFreshWaterFlux, & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index 30584386666c..727b55f4e5b8 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -190,15 +190,29 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ tendThick(k, iCell) = 0.0_RKIND end do end do +#ifndef MPAS_OPENACC + !$omp end do + !$omp end parallel +#endif + if (trim(config_subglacial_runoff_mode) == 'data') then +#ifdef MPAS_OPENACC + !$acc enter data create(surfaceThicknessFluxSubglacialRunoff) + + !$acc parallel loop & + !$acc present(surfaceThicknessFluxSubglacialRunoff) +#else + !$omp parallel + !$omp do schedule(runtime) +#endif do iCell = 1, nCellsAll surfaceThicknessFluxSubglacialRunoff(iCell) = 0.0_RKIND end do - end if #ifndef MPAS_OPENACC !$omp end do !$omp end parallel #endif + end if ! If turned off, return with zero fluxes, tendencies ! Otherwise, start time and call routines to accumulate @@ -847,14 +861,20 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & tracerGroupSurfaceFluxRunoff (n,iCell) = 0.0_RKIND tracerGroupSurfaceFluxRemoved(n,iCell) = 0.0_RKIND end do + end do + !$omp end do + !$omp end parallel + if (trim(config_subglacial_runoff_mode) == 'data') then + !$omp do schedule(runtime) private(n) + do iCell = 1, nCellsAll do n=1,nTracersGroup tracerGroupSurfaceFluxSubglacialRunoff (n,iCell) = 0.0_RKIND end do - end if end do !$omp end do !$omp end parallel + end if ! ! compute surface tracer flux from bulk forcing From 2443c697855c1bdc4ab695eae21f573ae3803070 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 09:46:27 -0700 Subject: [PATCH 252/904] Adds a new file and remove that code the the cpp file --- .../eamxx_mam_dry_deposition_functions.hpp | 99 +++++++++++++++++++ ...x_mam_dry_deposition_process_interface.cpp | 16 +-- 2 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp new file mode 100644 index 000000000000..f62d092523e4 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -0,0 +1,99 @@ +#ifndef EAMXX_MAM_DRY_DEPOSITION_FUNCTIONS_HPP +#define EAMXX_MAM_DRY_DEPOSITION_FUNCTIONS_HPP + +#include +#include + +namespace scream { + +namespace { +void compute_tendencies( + const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, + const double dt, const MAMDryDep::const_view_1d obklen, + const MAMDryDep::const_view_1d surfric, + const MAMDryDep::const_view_1d landfrac, + const MAMDryDep::const_view_1d icefrac, + const MAMDryDep::const_view_1d ocnfrac, + const MAMDryDep::const_view_1d friction_velocity, + const MAMDryDep::const_view_1d aerodynamical_resistance, + MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, + const mam_coupling::DryAtmosphere dry_atm, + const mam_coupling::AerosolState dry_aero, + const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies) { + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol, nlev); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); + const int icol = team.league_rank(); + const Real t = 0; + + compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); + team.team_barrier(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { + for(int mode = 0; mode < num_aero_modes; ++mode) { + int icnst = mam4::ConvProc::numptrcw_amode(mode); + qtracers(icol, lev, icnst) = + wet_aero.int_aero_nmr[mode](icol, lev); + for(int species = 0; species < num_aero_species; ++species) { + icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); + if(-1 < icnst) { + qtracers(icol, lev, icnst) = + wet_aero.int_aero_mmr[mode][species](icol, lev); + } + } + } + }); + team.team_barrier(); + + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + mam4::Surface surf; + mam4::Diagnostics diags; + mam4::Tendencies tends; + + for(int i = 0; i < num_aero_modes; ++i) { + diags.wet_geometric_mean_diameter_i[i] = + ekat::subview(dgncur_awet_, i, icol); + diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); + } + diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); + diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); + diags.deposition_flux_of_cloud_borne_aerosols = + ekat::subview(aerdepdrycw, icol); + diags.deposition_flux_of_interstitial_aerosols = + ekat::subview(aerdepdryis, icol); + + diags.Obukhov_length = obklen[icol]; + diags.surface_friction_velocty = surfric[icol]; + diags.land_fraction = landfrac[icol]; + diags.ice_fraction = icefrac[icol]; + diags.ocean_fraction = ocnfrac[icol]; + diags.friction_velocity = friction_velocity[icol]; + diags.aerodynamical_resistance = aerodynamical_resistance[icol]; + + // Fill Tendency views + for(int m = 0; m < num_aero_modes; ++m) { + int iconv = mam4::ConvProc::numptrcw_amode(m); + tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); + for(int a = 0; a < num_aero_species; ++a) { + iconv = mam4::ConvProc::lmassptrcw_amode(a, m); + if(-1 < iconv) + tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); + } + } + + const mam4::AeroConfig aero_config; + dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, + progs, diags, tends); + }); +} +} // namespace +} // namespace scream + +#endif \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 6040a534fa05..d2336d814f28 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -1,6 +1,9 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" -#include "mam4xx/drydep.hpp" +// ACI functions are stored in the following hpp file +#include + +#include "mam4xx/drydep.hpp"//FIXME: Do we need it here??? /* Inputs: @@ -241,10 +244,10 @@ void MAMDryDep::set_grids( // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, - 1 / (m * m) / s, grid_name); + 1 / m2 / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, - 1 / (m * m) / s, grid_name); + 1 / m2 / s, grid_name); // FIXME: we might not need this in FM add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, grid_name); @@ -379,9 +382,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -390,6 +393,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_aero_); // FIXME: Where is post processing functor???? } +/* namespace { void compute_tendencies( const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, @@ -478,7 +482,7 @@ void compute_tendencies( }); } } // namespace - +*/ // ========================================================================================= void MAMDryDep::run_impl(const double dt) { using DryDep = mam4::DryDeposition; From 43c3190dd1f0ee672f5e7aba701a914fa452b323 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 11 Jul 2024 11:14:30 -0600 Subject: [PATCH 253/904] fix omp in tendency --- components/mpas-ocean/src/shared/mpas_ocn_tendency.F | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index 727b55f4e5b8..c6d9e72f83a9 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -172,12 +172,10 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ ! initialize to zero and start accumulating tendency terms ! #ifdef MPAS_OPENACC - !$acc enter data create(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & - !$acc surfaceThicknessFluxSubglacialRunoff) + !$acc enter data create(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff) !$acc parallel loop & - !$acc present(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & - !$acc surfaceThicknessFluxSubglacialRunoff) & + !$acc present(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff) & !$acc private(k) #else !$omp parallel @@ -866,6 +864,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & !$omp end parallel if (trim(config_subglacial_runoff_mode) == 'data') then + !$omp parallel !$omp do schedule(runtime) private(n) do iCell = 1, nCellsAll do n=1,nTracersGroup From 89c14b8540fcf1916d92356738361f00fd010950 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 11 Jul 2024 11:42:50 -0600 Subject: [PATCH 254/904] add sgr fail if bad string option --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 2 ++ components/mpas-ocean/src/shared/mpas_ocn_forcing.F | 2 ++ 2 files changed, 4 insertions(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 7dc43284a755..3e7f432a415b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3530,6 +3530,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & end if zTop = zBot end do + else + call mpas_log_write("config_sgr_flux_vertical_location not one of 'bottom', 'uniform', 'top'.", MPAS_LOG_CRIT) end if else fracAbsorbedSubglacialRunoff = 0.0_RKIND diff --git a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F index 5cfce4c4e690..afdacc6c942c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F @@ -221,6 +221,8 @@ subroutine ocn_forcing_build_fraction_absorbed_array(meshPool, statePool, forcin transmissionCoeffTop = transmissionCoeffBot end do end do + else + call mpas_log_write("config_sgr_flux_vertical_location not one of 'bottom', 'uniform', 'top'.", MPAS_LOG_CRIT) end if end if From 518ab8061cf97d4f9893a27eb1b41ef3bc783c42 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 10:51:21 -0700 Subject: [PATCH 255/904] Removes the commented out code that is moved to a new file --- ...x_mam_dry_deposition_process_interface.cpp | 97 +------------------ 1 file changed, 4 insertions(+), 93 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index d2336d814f28..7fd56f490981 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -3,7 +3,7 @@ // ACI functions are stored in the following hpp file #include -#include "mam4xx/drydep.hpp"//FIXME: Do we need it here??? +#include "mam4xx/drydep.hpp" //FIXME: Do we need it here??? /* Inputs: @@ -382,9 +382,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -393,96 +393,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_aero_); // FIXME: Where is post processing functor???? } -/* -namespace { -void compute_tendencies( - const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, - const double dt, const MAMDryDep::const_view_1d obklen, - const MAMDryDep::const_view_1d surfric, - const MAMDryDep::const_view_1d landfrac, - const MAMDryDep::const_view_1d icefrac, - const MAMDryDep::const_view_1d ocnfrac, - const MAMDryDep::const_view_1d friction_velocity, - const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, - const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, - const mam_coupling::DryAtmosphere dry_atm, - const mam_coupling::AerosolState dry_aero, - const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies) { - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy( - ncol, nlev); - Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { - const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); - const int icol = team.league_rank(); - const Real t = 0; - - compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); - team.team_barrier(); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { - for(int mode = 0; mode < num_aero_modes; ++mode) { - int icnst = mam4::ConvProc::numptrcw_amode(mode); - qtracers(icol, lev, icnst) = - wet_aero.int_aero_nmr[mode](icol, lev); - for(int species = 0; species < num_aero_species; ++species) { - icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); - if(-1 < icnst) { - qtracers(icol, lev, icnst) = - wet_aero.int_aero_mmr[mode][species](icol, lev); - } - } - } - }); - team.team_barrier(); - - mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); - mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::Surface surf; - mam4::Diagnostics diags; - mam4::Tendencies tends; - - for(int i = 0; i < num_aero_modes; ++i) { - diags.wet_geometric_mean_diameter_i[i] = - ekat::subview(dgncur_awet_, i, icol); - diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); - } - diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); - diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - diags.deposition_flux_of_cloud_borne_aerosols = - ekat::subview(aerdepdrycw, icol); - diags.deposition_flux_of_interstitial_aerosols = - ekat::subview(aerdepdryis, icol); - - diags.Obukhov_length = obklen[icol]; - diags.surface_friction_velocty = surfric[icol]; - diags.land_fraction = landfrac[icol]; - diags.ice_fraction = icefrac[icol]; - diags.ocean_fraction = ocnfrac[icol]; - diags.friction_velocity = friction_velocity[icol]; - diags.aerodynamical_resistance = aerodynamical_resistance[icol]; - - // Fill Tendency views - for(int m = 0; m < num_aero_modes; ++m) { - int iconv = mam4::ConvProc::numptrcw_amode(m); - tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); - for(int a = 0; a < num_aero_species; ++a) { - iconv = mam4::ConvProc::lmassptrcw_amode(a, m); - if(-1 < iconv) - tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); - } - } - - const mam4::AeroConfig aero_config; - dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, - progs, diags, tends); - }); -} -} // namespace -*/ + // ========================================================================================= void MAMDryDep::run_impl(const double dt) { using DryDep = mam4::DryDeposition; From 7491a836e534a13d4ef1d59ae98b6f36b4f92870 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 11 Jul 2024 12:12:34 -0600 Subject: [PATCH 256/904] add line --- components/mpas-ocean/src/Registry.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 79ef88b33181..01e96c635c1d 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -3740,6 +3740,7 @@ description="Divergence of transmission through interfaces of surface fluxes below the surface layer at cell centers. These are applied only to subglacial runoff." packages="dataSubglacialRunoffFluxPKG" /> + Date: Thu, 11 Jul 2024 12:14:07 -0700 Subject: [PATCH 257/904] Removes qtracer from the output and allocate it locally --- .../mam/eamxx_mam_dry_deposition_process_interface.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 7fd56f490981..1a600bd34921 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -236,8 +236,6 @@ void MAMDryDep::set_grids( // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- - add_field("qtracers", scalar4d_q, kg / kg, grid_name); - // FIXME: what is the diff between d_qtracers_dt and tendencies???? // FIXME: we might not need this in FM add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); @@ -378,7 +376,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // as we do not need them in FM dgncur_awet_ = get_field_out("dgncur_awet").get_view(); wet_dens_ = get_field_out("wetdens").get_view(); - qtracers_ = get_field_out("qtracers").get_view(); d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); @@ -386,6 +383,12 @@ void MAMDryDep::initialize_impl(const RunType run_type) { .get_view(); tendencies_ = get_field_out("Tendencies").get_view(); + //----------------------------------------------------------------- + // Allocate memory + //----------------------------------------------------------------- + const int pcnst = mam4::aero_model::pcnst; + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- From bd91719612f63033b308e5eaba63e05768f1c3d6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 12:28:29 -0700 Subject: [PATCH 258/904] Removes d_qtracers_dt_ as output and allocate it as a local var --- ...x_mam_dry_deposition_process_interface.cpp | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 1a600bd34921..1c5527820736 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -249,7 +249,7 @@ void MAMDryDep::set_grids( // FIXME: we might not need this in FM add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, grid_name); -} +} // set_grids // ================================================================ // REQUEST_BUFFER_SIZE_IN_BYTES @@ -259,7 +259,7 @@ void MAMDryDep::set_grids( // levels size_t MAMDryDep::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); -} +} // requested_buffer_size_in_bytes // ================================================================ // INIT_BUFFERS @@ -277,7 +277,7 @@ void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMDryDep."); -} +} // init_buffers // ================================================================ // INITIALIZE_IMPL @@ -374,10 +374,10 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: We might need to get rid of few of these fields // as we do not need them in FM - dgncur_awet_ = get_field_out("dgncur_awet").get_view(); - wet_dens_ = get_field_out("wetdens").get_view(); - d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); - aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); + // d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); @@ -387,7 +387,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Allocate memory //----------------------------------------------------------------- const int pcnst = mam4::aero_model::pcnst; - qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + // FIXME: comment what they are and units..... + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + d_qtracers_dt_ = view_3d("d_qtracers_dt", ncol_, nlev_, pcnst); //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -395,7 +397,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); // FIXME: Where is post processing functor???? -} +} // initialize_impl // ========================================================================================= void MAMDryDep::run_impl(const double dt) { @@ -437,6 +439,5 @@ void MAMDryDep::run_impl(const double dt) { // Outputs: aerdepdrycw_, aerdepdryis_, tendencies_); Kokkos::fence(); -} -// ========================================================================================= +} // run_impl } // namespace scream From de33553985085fc53324d87fb31825abee7fbe80 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 11 Jul 2024 12:45:16 -0700 Subject: [PATCH 259/904] read in IOP lev data before surface pressure adjustment is applied --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 657f7ad742a0..70f1c41b603a 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -541,6 +541,15 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); + // Read in IOP lev data + auto data = iop_file_pressure.get_view().data(); + scorpio::read_var(iop_file,"lev",data); + + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilev(); From 25464521f07859148a91e62b0b7a30eb18ee1ac8 Mon Sep 17 00:00:00 2001 From: Luke Van Roekel Date: Thu, 11 Jul 2024 16:11:22 -0500 Subject: [PATCH 260/904] Fixes gotm interface for vertical mixing if GOTM is enabled surface friction velocity is not calculated at present. This fixes that issue Addresses #6507 --- components/mpas-ocean/src/shared/mpas_ocn_tendency.F | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index a16e52f14499..99a3bf49844b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -1243,12 +1243,12 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & ! ! Compute tracer tendency due to non-local flux computed in KPP ! - if (config_use_cvmix_kpp) then - call mpas_timer_start("non-local flux from KPP") - if (.not. config_cvmix_kpp_nonlocal_with_implicit_mix) then - call ocn_compute_KPP_input_fields(statePool, forcingPool,& + if (config_use_cvmix_kpp .or. config_use_gotm) then + call ocn_compute_KPP_input_fields(statePool, forcingPool,& meshPool, timeLevel) + if (.not. config_cvmix_kpp_nonlocal_with_implicit_mix) then + call mpas_timer_start("non-local flux from KPP") if (computeBudgets) then !$omp parallel !$omp do schedule(runtime) private(k,n) @@ -1291,8 +1291,8 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & !$omp end do !$omp end parallel endif ! compute budgets + call mpas_timer_stop("non-local flux from KPP") end if ! not non-local with implicit mix - call mpas_timer_stop("non-local flux from KPP") end if ! KPP ! Compute tracer tendency due to production/destruction of From b019f2e2f5b1c2343786ae7fcd138d45638b4539 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Thu, 11 Jul 2024 15:42:50 -0700 Subject: [PATCH 261/904] Fix bug in io with registering avg_cnt vars in Append mode. This commit addresses a bug where average count tracking variables may be registered again when a file is in Append mode. There is logic in the registration of typical output variables that was missing for average count variables. Addresses #2898 --- .../eamxx/src/share/io/scorpio_output.cpp | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 4cb3f50403ac..a105e244d76f 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,11 +1038,42 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { + std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - scorpio::define_var(filename, name, "unitless", vec_of_dims, - "real",fp_precision, m_add_time_dim); + if (mode==scorpio::FileMode::Append) { + // Similar to the regular fields above, check that the var is in the file, and has the right properties + EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), + "Error! Cannot append, due to variable missing from the file.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n"); + const auto& var = scorpio::get_var(filename,name); + EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, + "Error! Cannot append, due to variable dimensions mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var dims : " + ekat::join(vec_of_dims,",") + "\n" + " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + EKAT_REQUIRE_MSG (var.units==unitless, + "Error! Cannot append, due to variable units mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var units: " + unitless + "\n" + " - var units from file: " + var.units + "\n"); + EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, + "Error! Cannot append, due to time dependency mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" + " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); + } else { + // Note, unlike with regular output variables, for the average counting + // variables we don't need to add all of the extra metadata. So we simply + // define the variable. + scorpio::define_var(filename, name, unitless, vec_of_dims, + "real",fp_precision, m_add_time_dim); + } } } } // register_variables From 0f86fe3c757f4abcb9e3ccbe6cab97258e5e5502 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 11 Jul 2024 16:21:55 -0700 Subject: [PATCH 262/904] add minimum threshold when checking lat and lon relative error for IOP file to prevent NaN --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 70f1c41b603a..42ea7d0ec03f 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - m_params.get("target_latitude"); + ekat::impl::max(m_params.get("target_latitude"),0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - m_params.get("target_longitude"); + ekat::impl::max(m_params.get("target_longitude"),0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), From 2f2319675b4aae88a6e3b873090760ee4216b1b3 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Fri, 12 Jul 2024 02:43:08 -0700 Subject: [PATCH 263/904] add a general DPxx compset and a DPxx RCE compset --- .../eamxx/cime_config/config_component.xml | 12 +++++++ .../eamxx/cime_config/config_compsets.xml | 18 ++++++++-- .../cime_config/namelist_defaults_scream.xml | 8 ++--- .../usermods_dirs/rcemip/user_nl_cpl | 34 +++++++++++++++++++ 4 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl diff --git a/components/eamxx/cime_config/config_component.xml b/components/eamxx/cime_config/config_component.xml index 0aeccd1e6d24..d3b6e34744d0 100644 --- a/components/eamxx/cime_config/config_component.xml +++ b/components/eamxx/cime_config/config_component.xml @@ -60,6 +60,18 @@ + + + char + + + + $SRCROOT/components/eamxx/cime_config/usermods_dirs/rcemip + + run_component_cam + env_case.xml + User mods to apply to specific compset matches. + scream default diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index 6b7035e9c813..d33b865bde93 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -80,6 +80,18 @@ Experimental, under development + + F2010-SCREAMv1-DP + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx + Experimental, under development + + + + F2000-SCREAMv1-RCE-DP + 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx + Experimental, under development + + @@ -93,19 +105,19 @@ - 864 + 864 - TRUE + TRUE - TRUE + TRUE diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50188236e4e0..36b72df87ff1 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -363,7 +363,7 @@ be lost if SCREAM_HACK_XML is not enabled. 3 3 3 - 3 + 3 4 true false @@ -597,10 +597,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0 - 2 + 2 False 2 - 1 + 1 1 6 0 @@ -614,7 +614,7 @@ be lost if SCREAM_HACK_XML is not enabled. 1 1 3.4e-08 - 0.216784 + 0.216784 UNSET 250000.0 250000.0 diff --git a/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl new file mode 100644 index 000000000000..3ecd465f7a14 --- /dev/null +++ b/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -0,0 +1,34 @@ +!------------------------------------------------------------------------ +! Users should ONLY USE user_nl_cpl to change namelists variables +! for namelist variables in drv_in (except for the ones below) and +! any keyword/values in seq_maps.rc +! Users should add ALL user specific namelist and seq_maps.rc changes below +! using the following syntax +! namelist_var = new_namelist_value +! or +! mapname = new_map_name +! For example to change the default value of ocn2atm_fmapname to 'foo' use +! ocn2atm_fmapname = 'foo' +! +! Note that some namelist variables MAY NOT be changed in user_nl_cpl - +! they are defined in a $CASEROOT xml file and must be changed with +! xmlchange. +! +! For example, rather than set username to 'foo' in user_nl_cpl, call +! ./xmlchange USER=foo +!------------------------------------------------------------------------ +! +! SPECIAL NOTE FOR AQUAPLANET +! Do not modify any of the following orb_ entries +! co2vmr is not specified via the namelist but rather in the env_run.xml +! +orb_eccen = 0 +orb_obliq = 0 +orb_mvelp = 0 +orb_mode = "fixed_parameters" + +seq_flux_mct_albdif = 0.07 +seq_flux_mct_albdir = 0.07 +seq_flux_atmocn_minwind = 1 + +constant_zenith_deg = 42.05 \ No newline at end of file From a4209e476757684a82da1d395383db01d24b315a Mon Sep 17 00:00:00 2001 From: irenavankova Date: Fri, 12 Jul 2024 10:11:39 -0600 Subject: [PATCH 264/904] Update components/mpas-ocean/src/Registry.xml description Co-authored-by: Carolyn Begeman --- components/mpas-ocean/src/Registry.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 01e96c635c1d..8dbd4870dc65 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -707,7 +707,7 @@ possible_values="'off', 'data'" /> Date: Fri, 12 Jul 2024 09:31:44 -0700 Subject: [PATCH 265/904] add default gas emissions for RCE --- components/eamxx/cime_config/namelist_defaults_scream.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 36b72df87ff1..bc09ce40952d 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -343,6 +343,11 @@ be lost if SCREAM_HACK_XML is not enabled. 531.2820e-12 0.7906 1.0e-7 + 1650.e-9 + 348.0e-6 + 306.0e-9 + 0.0 + 0.0 From b00a709d4a9da196e40c19dac0349854521cc70a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 10:37:48 -0700 Subject: [PATCH 266/904] Adds missing call to update the state (int and cldbrn aerosols) --- .../physics/mam/eamxx_mam_wetscav_process_interface.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c6153794257a..c62dcd073b09 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -391,8 +391,8 @@ void MAMWetscav::run_impl(const double dt) { //----------- Variables from convective scheme ------------- - // TODO: Following variables are from convective parameterization (not implemented - // yet in EAMxx), so should be zero for now + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now auto sh_frac = view_2d("sh_frac", ncol_, nlev_); Kokkos::deep_copy(sh_frac, 0); @@ -527,6 +527,11 @@ void MAMWetscav::run_impl(const double dt) { } }); }); // icol parallel_for loop + + // call post processing to convert dry mixing ratios to wet mixing ratios + // and update the state + Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + Kokkos::fence(); // wait before returning to calling function } // ========================================================================================= From 4734cac912b4f51fb25f04957400299df8c47187 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 11:30:04 -0700 Subject: [PATCH 267/904] Updates number mixing ratios for state --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c62dcd073b09..4f1af79a057b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -519,6 +519,9 @@ void MAMWetscav::run_impl(const double dt) { // update interstitial aerosol state Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const auto n_mode_i = progs.n_mode_i[m]; + const auto tends_n_mode_i = tends.n_mode_i[m]; + n_mode_i(kk) += tends_n_mode_i(kk) * dt; for(int a = 0; a < mam4::num_species_mode(m); ++a) { const auto q_aero_i = progs.q_aero_i[m][a]; const auto tends_q_aero_i = tends.q_aero_i[m][a]; From e69425bba8efa25131e6a86cf7e0b8e196aba00d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 16:22:05 -0700 Subject: [PATCH 268/904] Commented out code with some new variables-code compiles fine --- .../eamxx_mam_dry_deposition_functions.hpp | 38 ++++++++++++- ...x_mam_dry_deposition_process_interface.cpp | 55 +++++++++++-------- ...x_mam_dry_deposition_process_interface.hpp | 4 ++ .../single-process/mam/drydep/input.yaml | 1 + 4 files changed, 72 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index f62d092523e4..17e6885c842d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -2,6 +2,8 @@ #define EAMXX_MAM_DRY_DEPOSITION_FUNCTIONS_HPP #include +#include +#include #include namespace scream { @@ -17,11 +19,13 @@ void compute_tendencies( const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::const_view_2d fraction_landuse, const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies) { + MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies, + MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst]) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( ncol, nlev); @@ -89,8 +93,36 @@ void compute_tendencies( } const mam4::AeroConfig aero_config; - dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, - progs, diags, tends); + /*dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, + progs, diags, tends);*/ + mam4::ColumnView qqcw_tends[mam4::aero_model::pcnst]; + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { + qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); + } + // Extract Prognostics + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + for(int m = 0; m < mam4::AeroConfig::num_modes(); ++m) { + // qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = + // progs.n_mode_c[m][kk]; + qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = + progs.n_mode_c[m][kk]; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + qqcw_tends[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = + progs.q_aero_c[m][a][kk]; + } + }); + /*mam4::aero_model_drydep( + team, fraction_landuse, atm.temperature, atm.pressure, + atm.interface_pressure , atm.hydrostatic_dp, ekat::subview(qtracers, + icol), ekat::subview(dgncur_awet_, i, icol), ekat::subview(wet_dens_, + i, icol), qqcw_tends, obklen[icol], surfric[icol], landfrac[icol], + icefrac[icol], ocnfrac[icol], friction_velocity[icol], + aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), + ptend_lq, dt, ekat::subview(aerdepdrycw, icol), + ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, + dqdt_tmp);*/ }); } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 1c5527820736..ae6b0819c10f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -102,6 +102,16 @@ void MAMDryDep::set_grids( FieldLayout scalar4d_q{{COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; FieldLayout scalar4d_qqcw_tends{{COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; + auto make_layout = [](const std::vector &extents, + const std::vector &names) { + std::vector tags(extents.size(), CMP); + return FieldLayout(tags, extents, names); + }; + + static constexpr int n_land_type = 11; + FieldLayout scalar3d_landtype = + make_layout({ncol_, n_land_type}, {"COL", "N_LAND_TYPE"}); + using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers @@ -186,12 +196,19 @@ void MAMDryDep::set_grids( // Ocean fraction [unitless] add_field("ocean_fraction", scalar2d, nondim, grid_name); + //----------- Variables from other mam4xx processes ------------ + // geometric mean wet diameter for number distribution [m] + add_field("dgncur_awet", scalar4d_mid, m, grid_name); + + //----------- FIXME:Variables to revisit------------------------------- + // FIXME:[TEMPORARY] These variables should come from data files + add_field("fraction_landuse", scalar4d_mid, m, grid_name); + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- add_field("wetdens", scalar4d_mid, kg / m3, grid_name); - add_field("dgncur_awet", scalar4d_mid, m, grid_name); // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -305,15 +322,17 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.pblh = get_field_in("pbl_height").get_view(); obukhov_length_ = get_field_in("Obukhov_length").get_view(); - surface_friction_velocty_ = - get_field_in("surface_friction_velocty").get_view(); land_fraction_ = get_field_in("land_fraction").get_view(); ice_fraction_ = get_field_in("ice_fraction").get_view(); ocean_fraction_ = get_field_in("ocean_fraction").get_view(); + // geometric mean wet diameter for number distribution [m] + dgncur_awet_ = get_field_out("dgncur_awet").get_view(); friction_velocity_ = get_field_in("friction_velocity").get_view(); aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); + surface_friction_velocty_ = + get_field_in("surface_friction_velocty").get_view(); // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; @@ -374,9 +393,8 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: We might need to get rid of few of these fields // as we do not need them in FM - dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); - // d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") @@ -389,7 +407,10 @@ void MAMDryDep::initialize_impl(const RunType run_type) { const int pcnst = mam4::aero_model::pcnst; // FIXME: comment what they are and units..... qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - d_qtracers_dt_ = view_3d("d_qtracers_dt", ncol_, nlev_, pcnst); + d_qtracers_dt_ = view_3d("d_qtracers_d_t", ncol_, nlev_, pcnst); + for(int i = 0; i < pcnst; ++i) { + Kokkos::resize(qqcw_tends_[i], ncol_, nlev_); + } //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -410,34 +431,22 @@ void MAMDryDep::run_impl(const double dt) { Kokkos::fence(); const DryDep::Config process_config; - // Future enhancement: - // This is where we can set the fraction of land use parameters: - // if (do_soilw) { - // const Real *landuse = get_landuse_from_file(do_soilw); - // for (int i=0; i::get_default_team_policy(ncol_, nlev_); - // Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - // dry_deposition.validate( aero_config, team, atm, progs); - // } - // Kokkos::fence(); + auto fraction_landuse = + get_field_in("fraction_landuse").get_view(); compute_tendencies(ncol_, nlev_, dry_deposition, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, - aerodynamical_resistance_, qtracers_, - d_qtracers_dt_, // d_qtracers_dt_ is an output + aerodynamical_resistance_, qtracers_, d_qtracers_dt_, + fraction_landuse, // d_qtracers_dt_ is an output dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, // Outputs: - aerdepdrycw_, aerdepdryis_, tendencies_); + aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_); Kokkos::fence(); } // run_impl } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 1144bd79ff68..ae764e0baf5a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -42,12 +42,16 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_3d qtracers_; view_3d d_qtracers_dt_; + + // inputs + // FIXME: collect all inputs and outputs together view_3d dgncur_awet_; view_3d wet_dens_; view_3d tendencies_; view_2d aerdepdrycw_; view_2d aerdepdryis_; + view_2d qqcw_tends_[mam4::aero_model::pcnst]; const_view_1d obukhov_length_; const_view_1d surface_friction_velocty_; diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 517ed4a53afd..5ebfbe4acfbc 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -34,6 +34,7 @@ initial_conditions: ocean_fraction: 0.0 friction_velocity: 1e-5 aerodynamical_resistance: 1e-5 + fraction_landuse: 0.2 w_updraft: 1e-5 pbl_height: 0.0 phis: 0.1 From c1bbdf5598927f8d3072847f99b30a2f9b69166f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 19:23:52 -0700 Subject: [PATCH 269/904] Adds new vars and allocate --- .../eamxx_mam_dry_deposition_process_interface.cpp | 12 ++++++++++++ .../eamxx_mam_dry_deposition_process_interface.hpp | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index ae6b0819c10f..2eb42f668264 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -408,8 +408,20 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: comment what they are and units..... qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); d_qtracers_dt_ = view_3d("d_qtracers_d_t", ncol_, nlev_, pcnst); + + rho_ = view_2d("rho", ncol_, nlev_); + + for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { + for(int j = 0; j < aerosol_categories_; ++j) { + Kokkos::resize(vlc_dry_[i][j], ncol_, nlev_); + Kokkos::resize(vlc_grv_[i][j], ncol_, nlev_); + } + } + + vlc_dry_, vlc_trb_, vlc_grv_; for(int i = 0; i < pcnst; ++i) { Kokkos::resize(qqcw_tends_[i], ncol_, nlev_); + Kokkos::resize(dqdt_tmp_[i], ncol_, nlev_); } //----------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index ae764e0baf5a..08dcf4dfe3c1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -47,6 +47,12 @@ class MAMDryDep final : public scream::AtmosphereProcess { // FIXME: collect all inputs and outputs together view_3d dgncur_awet_; view_3d wet_dens_; + view_2d rho_; + static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx + view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; + view_2d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; + view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; + view_2d dqdt_tmp_[mam4::aero_model::pcnst]; view_3d tendencies_; view_2d aerdepdrycw_; From 0532e27701342a45bdfb092f5a73c9524fa00c7c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 19:41:25 -0700 Subject: [PATCH 270/904] added as args --- .../src/physics/mam/eamxx_mam_dry_deposition_functions.hpp | 7 ++++++- .../mam/eamxx_mam_dry_deposition_process_interface.cpp | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 17e6885c842d..e328cc4036c0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -25,7 +25,12 @@ void compute_tendencies( const mam_coupling::AerosolState dry_aero, const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies, - MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst]) { + MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst], + MAMDryDep::view_2d rho_, + MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()][4], + MAMDryDep::view_2d vlc_trb_[mam4::AeroConfig::num_modes()][4], + MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()][4], + MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( ncol, nlev); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 2eb42f668264..004a5eab08f9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -458,7 +458,8 @@ void MAMDryDep::run_impl(const double dt) { fraction_landuse, // d_qtracers_dt_ is an output dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, // Outputs: - aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_); + aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_, rho_, + vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); } // run_impl } // namespace scream From 33202bec2246a1340cfc5b75c765ec5280f5c381 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 10:05:48 -0700 Subject: [PATCH 271/904] Submodule change, commented out code but compiles --- .../eamxx_mam_dry_deposition_functions.hpp | 33 +++++++++++++------ ...x_mam_dry_deposition_process_interface.cpp | 25 +++++++------- ...x_mam_dry_deposition_process_interface.hpp | 3 +- externals/mam4xx | 2 +- 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index e328cc4036c0..58a263f723e8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,7 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, - const MAMDryDep::const_view_2d fraction_landuse, + MAMDryDep::view_1d fraction_landuse_[mam4::DryDeposition::n_land_type], const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, @@ -65,11 +65,14 @@ void compute_tendencies( mam4::Surface surf; mam4::Diagnostics diags; mam4::Tendencies tends; + mam4::ColumnView dgncur_awet[num_aero_modes], wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { diags.wet_geometric_mean_diameter_i[i] = ekat::subview(dgncur_awet_, i, icol); + dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); + wet_dens[i] = ekat::subview(wet_dens_, i, icol); } diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); @@ -100,6 +103,15 @@ void compute_tendencies( const mam4::AeroConfig aero_config; /*dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, progs, diags, tends);*/ + + mam4::ColumnView rho; + rho = ekat::subview(rho_, icol); + + Real fraction_landuse[mam4::DryDeposition::n_land_type]; + for(int i = 0; i < mam4::DryDeposition::n_land_type; ++i) { + fraction_landuse[i] = fraction_landuse_[i](icol); + } + mam4::ColumnView qqcw_tends[mam4::aero_model::pcnst]; for(int i = 0; i < mam4::aero_model::pcnst; ++i) { qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); @@ -118,16 +130,17 @@ void compute_tendencies( progs.q_aero_c[m][a][kk]; } }); - /*mam4::aero_model_drydep( + bool ptend_lq[mam4::aero_model::pcnst]; + mam4::aero_model_drydep( team, fraction_landuse, atm.temperature, atm.pressure, - atm.interface_pressure , atm.hydrostatic_dp, ekat::subview(qtracers, - icol), ekat::subview(dgncur_awet_, i, icol), ekat::subview(wet_dens_, - i, icol), qqcw_tends, obklen[icol], surfric[icol], landfrac[icol], - icefrac[icol], ocnfrac[icol], friction_velocity[icol], - aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), - ptend_lq, dt, ekat::subview(aerdepdrycw, icol), - ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, - dqdt_tmp);*/ + atm.interface_pressure, atm.hydrostatic_dp, + ekat::subview(qtracers, icol), dgncur_awet, wet_dens, qqcw_tends, + obklen[icol], surfric[icol], landfrac[icol], icefrac[icol], + ocnfrac[icol], friction_velocity[icol], + aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), + ptend_lq, dt, ekat::subview(aerdepdrycw, icol), + ekat::subview(aerdepdryis, icol)); /*, rho, vlc_dry, vlc_trb, + vlc_grv, dqdt_tmp);*/ }); } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 004a5eab08f9..1a3d0850f182 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -108,10 +108,6 @@ void MAMDryDep::set_grids( return FieldLayout(tags, extents, names); }; - static constexpr int n_land_type = 11; - FieldLayout scalar3d_landtype = - make_layout({ncol_, n_land_type}, {"COL", "N_LAND_TYPE"}); - using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers @@ -200,10 +196,6 @@ void MAMDryDep::set_grids( // geometric mean wet diameter for number distribution [m] add_field("dgncur_awet", scalar4d_mid, m, grid_name); - //----------- FIXME:Variables to revisit------------------------------- - // FIXME:[TEMPORARY] These variables should come from data files - add_field("fraction_landuse", scalar4d_mid, m, grid_name); - // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- @@ -325,8 +317,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { land_fraction_ = get_field_in("land_fraction").get_view(); ice_fraction_ = get_field_in("ice_fraction").get_view(); ocean_fraction_ = get_field_in("ocean_fraction").get_view(); - // geometric mean wet diameter for number distribution [m] - dgncur_awet_ = get_field_out("dgncur_awet").get_view(); friction_velocity_ = get_field_in("friction_velocity").get_view(); aerodynamical_resistance_ = @@ -418,12 +408,16 @@ void MAMDryDep::initialize_impl(const RunType run_type) { } } - vlc_dry_, vlc_trb_, vlc_grv_; for(int i = 0; i < pcnst; ++i) { Kokkos::resize(qqcw_tends_[i], ncol_, nlev_); Kokkos::resize(dqdt_tmp_[i], ncol_, nlev_); } + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; + for(int i = 0; i < n_land_type; ++i) { + Kokkos::resize(fraction_landuse_[i], ncol_); + } + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- @@ -448,14 +442,17 @@ void MAMDryDep::run_impl(const double dt) { const mam4::AeroConfig aero_config; dry_deposition.init(aero_config, process_config); - auto fraction_landuse = - get_field_in("fraction_landuse").get_view(); + // FIXME: There are some vars that are not declared "REQUIRED" etc. but still + // used!!! + + // geometric mean wet diameter for number distribution [m] + auto dgncur_awet_ = get_field_out("dgncur_awet").get_view(); compute_tendencies(ncol_, nlev_, dry_deposition, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, aerodynamical_resistance_, qtracers_, d_qtracers_dt_, - fraction_landuse, // d_qtracers_dt_ is an output + fraction_landuse_, // d_qtracers_dt_ is an output dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, // Outputs: aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_, rho_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 08dcf4dfe3c1..c423a20ebc89 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -16,6 +16,7 @@ namespace scream { // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { public: + using view_1d = Field::view_dev_t; using view_2d = Field::view_dev_t; using view_3d = Field::view_dev_t; using const_view_1d = Field::view_dev_t; @@ -45,7 +46,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { // inputs // FIXME: collect all inputs and outputs together - view_3d dgncur_awet_; + view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; view_3d wet_dens_; view_2d rho_; static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx diff --git a/externals/mam4xx b/externals/mam4xx index a8aa56ba915c..eb2d728412b4 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e +Subproject commit eb2d728412b43081afbd579230450da352e716fb From fba5bd77e7677a2b692d078aaf67d2574fd54711 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 13 Jul 2024 13:32:02 -0400 Subject: [PATCH 272/904] fix warning due to format type --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 9fb5726084f4..c406c1ebd9b4 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } From 309f2d28ef770610f4054f1ebe7697d59032f47f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 11:30:24 -0700 Subject: [PATCH 273/904] subm changes, All func args are in, compiles fine --- .../eamxx_mam_dry_deposition_functions.hpp | 36 +++++++++++++++---- ...x_mam_dry_deposition_process_interface.cpp | 1 + ...x_mam_dry_deposition_process_interface.hpp | 17 ++++----- externals/mam4xx | 2 +- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 58a263f723e8..806c98cdf128 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -27,9 +27,12 @@ void compute_tendencies( MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies, MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst], MAMDryDep::view_2d rho_, - MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()][4], - MAMDryDep::view_2d vlc_trb_[mam4::AeroConfig::num_modes()][4], - MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()][4], + MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], + MAMDryDep::view_2d vlc_trb_[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], + MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( @@ -112,9 +115,30 @@ void compute_tendencies( fraction_landuse[i] = fraction_landuse_[i](icol); } + // FIXME: why mam4::ColumnView didn;t work here, why use + // Kokkos::View. Solution: Use ColumnView in drydep.hpp as well. + Kokkos::View vlc_dry[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], + vlc_trb[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], + vlc_grv[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_]; + + for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { + for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { + vlc_dry[i][j] = ekat::subview(vlc_dry_[i][j], icol); + vlc_trb[i][j] = ekat::subview(vlc_trb_[i][j], icol); + vlc_grv[i][j] = ekat::subview(vlc_grv_[i][j], icol); + } + } + mam4::ColumnView qqcw_tends[mam4::aero_model::pcnst]; + Kokkos::View dqdt_tmp[mam4::aero_model::pcnst]; for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); + qqcw_tends[i] = ekat::subview( + qqcw_tends_[i], icol); // FIXME: Do we need qqcw_tends_, why + // can't we just use qqcw_tends + dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); } // Extract Prognostics Kokkos::parallel_for( @@ -139,8 +163,8 @@ void compute_tendencies( ocnfrac[icol], friction_velocity[icol], aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), ptend_lq, dt, ekat::subview(aerdepdrycw, icol), - ekat::subview(aerdepdryis, icol)); /*, rho, vlc_dry, vlc_trb, - vlc_grv, dqdt_tmp);*/ + ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, + dqdt_tmp); }); } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 1a3d0850f182..b19d32f41694 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -405,6 +405,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { for(int j = 0; j < aerosol_categories_; ++j) { Kokkos::resize(vlc_dry_[i][j], ncol_, nlev_); Kokkos::resize(vlc_grv_[i][j], ncol_, nlev_); + Kokkos::resize(vlc_trb_[i][j], ncol_, nlev_); } } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index c423a20ebc89..13d006005aa9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -16,13 +16,14 @@ namespace scream { // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { public: - using view_1d = Field::view_dev_t; - using view_2d = Field::view_dev_t; - using view_3d = Field::view_dev_t; - using const_view_1d = Field::view_dev_t; - using const_view_2d = Field::view_dev_t; - using const_view_3d = Field::view_dev_t; - static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + using view_1d = Field::view_dev_t; + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx private: // number of horizontal columns and vertical levels @@ -49,7 +50,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; view_3d wet_dens_; view_2d rho_; - static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx + view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; diff --git a/externals/mam4xx b/externals/mam4xx index eb2d728412b4..df7c3c451ccc 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit eb2d728412b43081afbd579230450da352e716fb +Subproject commit df7c3c451ccc4c0c6f7f2c779cac6ba8b598e059 From 8dca5fab793d6735f75fb5147c1bd81c1fb0552a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 12:40:59 -0700 Subject: [PATCH 274/904] Subm change; a working aero_model_drydep call --- .../eamxx_mam_dry_deposition_functions.hpp | 50 +++++++++---------- ...x_mam_dry_deposition_process_interface.cpp | 2 + ...x_mam_dry_deposition_process_interface.hpp | 42 ++++++++++++++++ .../eamxx/src/physics/mam/mam_coupling.hpp | 6 ++- externals/mam4xx | 2 +- 5 files changed, 74 insertions(+), 28 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 806c98cdf128..4ee76a54470b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -44,8 +44,6 @@ void compute_tendencies( const int icol = team.league_rank(); const Real t = 0; - compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); - team.team_barrier(); Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { for(int mode = 0; mode < num_aero_modes; ++mode) { @@ -65,35 +63,35 @@ void compute_tendencies( mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::Surface surf; - mam4::Diagnostics diags; - mam4::Tendencies tends; + // mam4::Surface surf; + // mam4::Diagnostics diags; + // mam4::Tendencies tends; mam4::ColumnView dgncur_awet[num_aero_modes], wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { - diags.wet_geometric_mean_diameter_i[i] = - ekat::subview(dgncur_awet_, i, icol); - dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); - diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); - wet_dens[i] = ekat::subview(wet_dens_, i, icol); + // diags.wet_geometric_mean_diameter_i[i] = + // ekat::subview(dgncur_awet_, i, icol); + dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); + // diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); + wet_dens[i] = ekat::subview(wet_dens_, i, icol); } - diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); - diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - diags.deposition_flux_of_cloud_borne_aerosols = - ekat::subview(aerdepdrycw, icol); - diags.deposition_flux_of_interstitial_aerosols = - ekat::subview(aerdepdryis, icol); + // diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); + // diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); + // diags.deposition_flux_of_cloud_borne_aerosols = + // ekat::subview(aerdepdrycw, icol); + // diags.deposition_flux_of_interstitial_aerosols = + // ekat::subview(aerdepdryis, icol); - diags.Obukhov_length = obklen[icol]; - diags.surface_friction_velocty = surfric[icol]; - diags.land_fraction = landfrac[icol]; - diags.ice_fraction = icefrac[icol]; - diags.ocean_fraction = ocnfrac[icol]; - diags.friction_velocity = friction_velocity[icol]; - diags.aerodynamical_resistance = aerodynamical_resistance[icol]; + // diags.Obukhov_length = obklen[icol]; + // diags.surface_friction_velocty = surfric[icol]; + // diags.land_fraction = landfrac[icol]; + // diags.ice_fraction = icefrac[icol]; + // diags.ocean_fraction = ocnfrac[icol]; + // diags.friction_velocity = friction_velocity[icol]; + // diags.aerodynamical_resistance = aerodynamical_resistance[icol]; // Fill Tendency views - for(int m = 0; m < num_aero_modes; ++m) { + /*for(int m = 0; m < num_aero_modes; ++m) { int iconv = mam4::ConvProc::numptrcw_amode(m); tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); for(int a = 0; a < num_aero_species; ++a) { @@ -101,9 +99,9 @@ void compute_tendencies( if(-1 < iconv) tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); } - } + }*/ - const mam4::AeroConfig aero_config; + // const mam4::AeroConfig aero_config; /*dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, progs, diags, tends);*/ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index b19d32f41694..ff29f71e89d8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -312,6 +312,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: tot or liq? dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); obukhov_length_ = get_field_in("Obukhov_length").get_view(); land_fraction_ = get_field_in("land_fraction").get_view(); @@ -459,5 +460,6 @@ void MAMDryDep::run_impl(const double dt) { aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_, rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); + // FIXME: Where is update tends and post processing???? } // run_impl } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 13d006005aa9..11ec5f044102 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -132,6 +132,10 @@ class MAMDryDep final : public scream::AtmosphereProcess { compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); } // operator() // local variables for preprocess struct @@ -144,6 +148,44 @@ class MAMDryDep final : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess + // Atmosphere processes often have a post-processing step prepares output + // from this process for the Field Manager. This functor implements this + // step, which is called during run_impl. + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_post_, nlev_post_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // Postprocess + private: // pre- and postprocessing scratch pads Preprocess preprocess_; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 189b78b044d2..030ffea88789 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -599,13 +599,16 @@ void compute_vertical_layer_heights(const Team& team, EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), "Given column index does not correspond to given team!"); + //outputs const auto dz = ekat::subview(dry_atm.dz, column_index); const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); + //inputs const auto qv = ekat::subview(dry_atm.qv, column_index); const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + // NOTE: we are using dry qv. Does calculate_dz require dry or wet? PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output @@ -614,7 +617,8 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output team.team_barrier(); // likely necessary to have z_iface up to date - PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); + PF::calculate_z_mid(team, mam4::nlev, z_iface, //input + z_mid); //output } // Given a thread team and wet and dry atmospheres, dispatches threads from the diff --git a/externals/mam4xx b/externals/mam4xx index df7c3c451ccc..1c8b5b6e0ff5 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit df7c3c451ccc4c0c6f7f2c779cac6ba8b598e059 +Subproject commit 1c8b5b6e0ff5ac20bc58907f3593f816762a77f2 From 17c09a0c78d415a7191a7095c071d8aa46ba7f10 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 12:47:11 -0700 Subject: [PATCH 275/904] Remove compute tends to drydep.hpp from drydep interface --- .../eamxx_mam_dry_deposition_functions.hpp | 45 ++----------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 4ee76a54470b..0204461f2bcd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -58,52 +58,17 @@ void compute_tendencies( } } } - }); + }); // parallel_for for nlevs team.team_barrier(); mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - // mam4::Surface surf; - // mam4::Diagnostics diags; - // mam4::Tendencies tends; mam4::ColumnView dgncur_awet[num_aero_modes], wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { - // diags.wet_geometric_mean_diameter_i[i] = - // ekat::subview(dgncur_awet_, i, icol); dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); - // diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); - wet_dens[i] = ekat::subview(wet_dens_, i, icol); + wet_dens[i] = ekat::subview(wet_dens_, i, icol); } - // diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); - // diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - // diags.deposition_flux_of_cloud_borne_aerosols = - // ekat::subview(aerdepdrycw, icol); - // diags.deposition_flux_of_interstitial_aerosols = - // ekat::subview(aerdepdryis, icol); - - // diags.Obukhov_length = obklen[icol]; - // diags.surface_friction_velocty = surfric[icol]; - // diags.land_fraction = landfrac[icol]; - // diags.ice_fraction = icefrac[icol]; - // diags.ocean_fraction = ocnfrac[icol]; - // diags.friction_velocity = friction_velocity[icol]; - // diags.aerodynamical_resistance = aerodynamical_resistance[icol]; - - // Fill Tendency views - /*for(int m = 0; m < num_aero_modes; ++m) { - int iconv = mam4::ConvProc::numptrcw_amode(m); - tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); - for(int a = 0; a < num_aero_species; ++a) { - iconv = mam4::ConvProc::lmassptrcw_amode(a, m); - if(-1 < iconv) - tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); - } - }*/ - - // const mam4::AeroConfig aero_config; - /*dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, - progs, diags, tends);*/ mam4::ColumnView rho; rho = ekat::subview(rho_, icol); @@ -142,8 +107,6 @@ void compute_tendencies( Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { for(int m = 0; m < mam4::AeroConfig::num_modes(); ++m) { - // qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = - // progs.n_mode_c[m][kk]; qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) @@ -151,7 +114,7 @@ void compute_tendencies( qqcw_tends[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = progs.q_aero_c[m][a][kk]; } - }); + }); // parallel_for nlevs bool ptend_lq[mam4::aero_model::pcnst]; mam4::aero_model_drydep( team, fraction_landuse, atm.temperature, atm.pressure, @@ -163,7 +126,7 @@ void compute_tendencies( ptend_lq, dt, ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); - }); + }); // parallel_for for ncols } } // namespace } // namespace scream From 1323ea6a4b5a1324f6b4a9d10299d6a2ae0d7e2a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 13:26:47 -0700 Subject: [PATCH 276/904] Zero out tends before computing tends as they had junk values --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 4f1af79a057b..91e69443b588 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -506,6 +506,14 @@ void MAMWetscav::run_impl(const double dt) { auto wetdens_icol = ekat::subview(wetdens, icol); const auto prain_icol = ekat::subview(prain, icol); + // Zero out tendencies otherwise, they are initialized to junk values + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(tends.n_mode_i[m], 0); + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + Kokkos::deep_copy(tends.q_aero_i[m][a], 0); + } + } + mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, // inputs @@ -528,8 +536,8 @@ void MAMWetscav::run_impl(const double dt) { q_aero_i(kk) += tends_q_aero_i(kk) * dt; } } - }); - }); // icol parallel_for loop + }); // parallel_for for update interstitial aerosol state + }); // icol parallel_for loop // call post processing to convert dry mixing ratios to wet mixing ratios // and update the state From 0dd177a071afec7b82a82d783c36efe40f2b172d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 13:29:57 -0700 Subject: [PATCH 277/904] Cleanup, subm changed, Use ColumnView for some views --- .../eamxx_mam_dry_deposition_functions.hpp | 50 +++++++++---------- ...x_mam_dry_deposition_process_interface.hpp | 14 +++--- externals/mam4xx | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 0204461f2bcd..0dc2caf63baa 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,7 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, - MAMDryDep::view_1d fraction_landuse_[mam4::DryDeposition::n_land_type], + MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, @@ -39,22 +39,24 @@ void compute_tendencies( ncol, nlev); Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { - const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); - const int icol = team.league_rank(); - const Real t = 0; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int num_aero_species = + mam_coupling::num_aero_species(); + static constexpr int n_land_type = MAMDryDep::n_land_type; + + const int icol = team.league_rank(); Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { for(int mode = 0; mode < num_aero_modes; ++mode) { int icnst = mam4::ConvProc::numptrcw_amode(mode); qtracers(icol, lev, icnst) = - wet_aero.int_aero_nmr[mode](icol, lev); + dry_aero.int_aero_nmr[mode](icol, lev); for(int species = 0; species < num_aero_species; ++species) { icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); if(-1 < icnst) { qtracers(icol, lev, icnst) = - wet_aero.int_aero_mmr[mode][species](icol, lev); + dry_aero.int_aero_mmr[mode][species](icol, lev); } } } @@ -73,40 +75,36 @@ void compute_tendencies( mam4::ColumnView rho; rho = ekat::subview(rho_, icol); - Real fraction_landuse[mam4::DryDeposition::n_land_type]; - for(int i = 0; i < mam4::DryDeposition::n_land_type; ++i) { + Real fraction_landuse[n_land_type]; + for(int i = 0; i < n_land_type; ++i) { fraction_landuse[i] = fraction_landuse_[i](icol); } // FIXME: why mam4::ColumnView didn;t work here, why use // Kokkos::View. Solution: Use ColumnView in drydep.hpp as well. - Kokkos::View vlc_dry[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - vlc_trb[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - vlc_grv[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_]; + static constexpr int nmodes = mam4::AeroConfig::num_modes(); + mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_], + vlc_trb[nmodes][MAMDryDep::aerosol_categories_], + vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; - for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { + for(int i = 0; i < nmodes; ++i) { for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { vlc_dry[i][j] = ekat::subview(vlc_dry_[i][j], icol); vlc_trb[i][j] = ekat::subview(vlc_trb_[i][j], icol); vlc_grv[i][j] = ekat::subview(vlc_grv_[i][j], icol); } } - - mam4::ColumnView qqcw_tends[mam4::aero_model::pcnst]; - Kokkos::View dqdt_tmp[mam4::aero_model::pcnst]; - for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - qqcw_tends[i] = ekat::subview( - qqcw_tends_[i], icol); // FIXME: Do we need qqcw_tends_, why - // can't we just use qqcw_tends - dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); + static constexpr int pcnst = mam4::aero_model::pcnst; + mam4::ColumnView qqcw_tends[pcnst]; + mam4::ColumnView dqdt_tmp[pcnst]; + for(int i = 0; i < pcnst; ++i) { + qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); + dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); } // Extract Prognostics Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { - for(int m = 0; m < mam4::AeroConfig::num_modes(); ++m) { + for(int m = 0; m < nmodes; ++m) { qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) @@ -115,7 +113,7 @@ void compute_tendencies( progs.q_aero_c[m][a][kk]; } }); // parallel_for nlevs - bool ptend_lq[mam4::aero_model::pcnst]; + bool ptend_lq[pcnst]; mam4::aero_model_drydep( team, fraction_landuse, atm.temperature, atm.pressure, atm.interface_pressure, atm.hydrostatic_dp, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 11ec5f044102..fdfb387524d8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -16,14 +16,16 @@ namespace scream { // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { public: - using view_1d = Field::view_dev_t; - using view_2d = Field::view_dev_t; - using view_3d = Field::view_dev_t; - using const_view_1d = Field::view_dev_t; - using const_view_2d = Field::view_dev_t; - using const_view_3d = Field::view_dev_t; + using view_1d = Field::view_dev_t; + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; private: // number of horizontal columns and vertical levels diff --git a/externals/mam4xx b/externals/mam4xx index 1c8b5b6e0ff5..d0ea0c4f68f4 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 1c8b5b6e0ff5ac20bc58907f3593f816762a77f2 +Subproject commit d0ea0c4f68f4a939ee73f92bdea797e597eafc3f From e9f52c0147e908f311ca6e8a07a680472367d75f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 15:40:34 -0700 Subject: [PATCH 278/904] Cleanup-makes dgn wet size a input and other cleanup --- .../eamxx_mam_dry_deposition_functions.hpp | 23 ++++++----- ...x_mam_dry_deposition_process_interface.cpp | 41 +++++++------------ .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- externals/mam4xx | 2 +- 4 files changed, 28 insertions(+), 40 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 0dc2caf63baa..3fb8b19abe0c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -10,8 +10,8 @@ namespace scream { namespace { void compute_tendencies( - const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, - const double dt, const MAMDryDep::const_view_1d obklen, + const int ncol, const int nlev, const double dt, + const MAMDryDep::const_view_1d obklen, const MAMDryDep::const_view_1d surfric, const MAMDryDep::const_view_1d landfrac, const MAMDryDep::const_view_1d icefrac, @@ -20,11 +20,11 @@ void compute_tendencies( const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], - const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, + const MAMDryDep::const_view_3d dgncur_awet_, + const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, - const mam_coupling::AerosolState dry_aero, - const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies, + const mam_coupling::AerosolState dry_aero, MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst], MAMDryDep::view_2d rho_, MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] @@ -34,15 +34,14 @@ void compute_tendencies( MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( ncol, nlev); Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { - static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); static constexpr int num_aero_species = mam_coupling::num_aero_species(); - static constexpr int n_land_type = MAMDryDep::n_land_type; const int icol = team.league_rank(); @@ -65,16 +64,18 @@ void compute_tendencies( mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::ColumnView dgncur_awet[num_aero_modes], wet_dens[num_aero_modes]; + mam4::ConstColumnView dgncur_awet[num_aero_modes]; + mam4::ColumnView wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { - dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); - wet_dens[i] = ekat::subview(wet_dens_, i, icol); + dgncur_awet[i] = ekat::subview(dgncur_awet_, icol, i); + wet_dens[i] = ekat::subview(wet_dens_, icol, i); } mam4::ColumnView rho; rho = ekat::subview(rho_, icol); + static constexpr int n_land_type = MAMDryDep::n_land_type; Real fraction_landuse[n_land_type]; for(int i = 0; i < n_land_type; ++i) { fraction_landuse[i] = fraction_landuse_[i](icol); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index ff29f71e89d8..70260f3ef97f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -94,19 +94,14 @@ void MAMDryDep::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points - const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_mid{{NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; - - // Layout for tracers. - const int pcnst = mam4::aero_model::pcnst; - FieldLayout scalar4d_q{{COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; - FieldLayout scalar4d_qqcw_tends{{COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; - auto make_layout = [](const std::vector &extents, const std::vector &names) { std::vector tags(extents.size(), CMP); return FieldLayout(tags, extents, names); }; + const int num_aero_modes = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_mid = + make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); using namespace ekat::units; @@ -194,7 +189,7 @@ void MAMDryDep::set_grids( //----------- Variables from other mam4xx processes ------------ // geometric mean wet diameter for number distribution [m] - add_field("dgncur_awet", scalar4d_mid, m, grid_name); + add_field("dgncur_awet", scalar4d_mid, m, grid_name); // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process @@ -244,20 +239,12 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- - - // FIXME: what is the diff between d_qtracers_dt and tendencies???? - // FIXME: we might not need this in FM - add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); - // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, 1 / m2 / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, 1 / m2 / s, grid_name); - // FIXME: we might not need this in FM - add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, - grid_name); } // set_grids // ================================================================ @@ -309,7 +296,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); - // FIXME: tot or liq? + // FIXME: tot or liq? make notes about it to ensure it is the right one dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); @@ -390,17 +377,16 @@ void MAMDryDep::initialize_impl(const RunType run_type) { .get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); //----------------------------------------------------------------- // Allocate memory //----------------------------------------------------------------- const int pcnst = mam4::aero_model::pcnst; // FIXME: comment what they are and units..... - qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - d_qtracers_dt_ = view_3d("d_qtracers_d_t", ncol_, nlev_, pcnst); + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - rho_ = view_2d("rho", ncol_, nlev_); + rho_ = view_2d("rho", ncol_, nlev_); + d_qtracers_dt_ = view_3d("d_qtracers_dt_", ncol_, nlev_, pcnst); for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { for(int j = 0; j < aerosol_categories_; ++j) { @@ -447,18 +433,19 @@ void MAMDryDep::run_impl(const double dt) { // FIXME: There are some vars that are not declared "REQUIRED" etc. but still // used!!! + // Inputs: // geometric mean wet diameter for number distribution [m] - auto dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + auto dgncur_awet_ = get_field_in("dgncur_awet").get_view(); - compute_tendencies(ncol_, nlev_, dry_deposition, dt, obukhov_length_, + compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, aerodynamical_resistance_, qtracers_, d_qtracers_dt_, fraction_landuse_, // d_qtracers_dt_ is an output - dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, + dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, // Outputs: - aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_, rho_, - vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); + aerdepdrycw_, aerdepdryis_, qqcw_tends_, rho_, vlc_dry_, + vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); // FIXME: Where is update tends and post processing???? } // run_impl diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 030ffea88789..096311451ace 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -613,7 +613,7 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); - //FIXME : add an assert statement to check id z_surf is zero or not + EKAT_KERNEL_ASSERT_MSG(dry_atm.z_surf == 0, "dry_atm.z_surf must be zero"); PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output team.team_barrier(); // likely necessary to have z_iface up to date diff --git a/externals/mam4xx b/externals/mam4xx index d0ea0c4f68f4..4d16c226c62c 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit d0ea0c4f68f4a939ee73f92bdea797e597eafc3f +Subproject commit 4d16c226c62cc412497fac425359b90a63821efc From 57b4c890e20275440d23bedd411a0747913e6ba8 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 13 Jul 2024 19:05:30 -0400 Subject: [PATCH 279/904] fix second instance of format mismatch --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index c406c1ebd9b4..37cb251d7796 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } From 67200f97ae6744dbdf1bf5f5d19c4b4fcbae6b6a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 16:19:17 -0700 Subject: [PATCH 280/904] Teases out i/o for func --- .../eamxx_mam_dry_deposition_functions.hpp | 46 ++++++++++++------- ...x_mam_dry_deposition_process_interface.cpp | 24 +++++----- ...x_mam_dry_deposition_process_interface.hpp | 4 +- externals/mam4xx | 2 +- 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 3fb8b19abe0c..44742044a0a6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -10,6 +10,7 @@ namespace scream { namespace { void compute_tendencies( + // inputs const int ncol, const int nlev, const double dt, const MAMDryDep::const_view_1d obklen, const MAMDryDep::const_view_1d surfric, @@ -18,14 +19,21 @@ void compute_tendencies( const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, + MAMDryDep::view_3d qtracers, // FIXME: Make it a constant view MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], const MAMDryDep::const_view_3d dgncur_awet_, - const MAMDryDep::view_3d wet_dens_, + const MAMDryDep::const_view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, - const mam_coupling::AerosolState dry_aero, MAMDryDep::view_2d aerdepdrycw, + const mam_coupling::AerosolState dry_aero, + + // input-outputs + MAMDryDep::view_2d qqcw_[mam4::aero_model::pcnst], + + // outputs + MAMDryDep::view_3d d_qtracers_dt, MAMDryDep::view_2d aerdepdrycw, MAMDryDep::view_2d aerdepdryis, - MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst], + + // work arrays MAMDryDep::view_2d rho_, MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], @@ -65,7 +73,7 @@ void compute_tendencies( mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); mam4::ConstColumnView dgncur_awet[num_aero_modes]; - mam4::ColumnView wet_dens[num_aero_modes]; + mam4::ConstColumnView wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { dgncur_awet[i] = ekat::subview(dgncur_awet_, icol, i); @@ -96,35 +104,39 @@ void compute_tendencies( } } static constexpr int pcnst = mam4::aero_model::pcnst; - mam4::ColumnView qqcw_tends[pcnst]; + mam4::ColumnView qqcw[pcnst]; mam4::ColumnView dqdt_tmp[pcnst]; for(int i = 0; i < pcnst; ++i) { - qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); - dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); + qqcw[i] = ekat::subview(qqcw_[i], icol); + dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); } // Extract Prognostics Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { for(int m = 0; m < nmodes; ++m) { - qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = + qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) - qqcw_tends[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = + qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = progs.q_aero_c[m][a][kk]; } }); // parallel_for nlevs bool ptend_lq[pcnst]; mam4::aero_model_drydep( + // inputs team, fraction_landuse, atm.temperature, atm.pressure, atm.interface_pressure, atm.hydrostatic_dp, - ekat::subview(qtracers, icol), dgncur_awet, wet_dens, qqcw_tends, - obklen[icol], surfric[icol], landfrac[icol], icefrac[icol], - ocnfrac[icol], friction_velocity[icol], - aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), - ptend_lq, dt, ekat::subview(aerdepdrycw, icol), - ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, - dqdt_tmp); + ekat::subview(qtracers, icol), dgncur_awet, wet_dens, obklen[icol], + surfric[icol], landfrac[icol], icefrac[icol], ocnfrac[icol], + friction_velocity[icol], aerodynamical_resistance[icol], dt, + // input-outputs + qqcw, + // outputs + ekat::subview(d_qtracers_dt, icol), ptend_lq, + ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), + // work arrays + rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); }); // parallel_for for ncols } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 70260f3ef97f..e8218fbe5c8a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -188,15 +188,16 @@ void MAMDryDep::set_grids( add_field("ocean_fraction", scalar2d, nondim, grid_name); //----------- Variables from other mam4xx processes ------------ - // geometric mean wet diameter for number distribution [m] + // Geometric mean wet diameter for number distribution [m] add_field("dgncur_awet", scalar4d_mid, m, grid_name); + // Wet density of interstitial aerosol [kg/m3] + add_field("wetdens", scalar4d_mid, kg / m3, grid_name); + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- - add_field("wetdens", scalar4d_mid, kg / m3, grid_name); - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for(int m = 0; m < num_aero_modes; ++m) { @@ -239,6 +240,7 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- + // FIXME: These are diagnostics, remove them from FM after initial evaluation // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, 1 / m2 / s, grid_name); @@ -372,7 +374,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: We might need to get rid of few of these fields // as we do not need them in FM - wet_dens_ = get_field_out("wetdens").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") @@ -397,7 +398,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { } for(int i = 0; i < pcnst; ++i) { - Kokkos::resize(qqcw_tends_[i], ncol_, nlev_); + Kokkos::resize(qqcw_[i], ncol_, nlev_); Kokkos::resize(dqdt_tmp_[i], ncol_, nlev_); } @@ -424,27 +425,24 @@ void MAMDryDep::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - const DryDep::Config process_config; - - DryDep dry_deposition; - const mam4::AeroConfig aero_config; - dry_deposition.init(aero_config, process_config); - // FIXME: There are some vars that are not declared "REQUIRED" etc. but still // used!!! // Inputs: // geometric mean wet diameter for number distribution [m] auto dgncur_awet_ = get_field_in("dgncur_awet").get_view(); + auto wet_dens_ = get_field_in("wetdens").get_view(); compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, - aerodynamical_resistance_, qtracers_, d_qtracers_dt_, + aerodynamical_resistance_, qtracers_, fraction_landuse_, // d_qtracers_dt_ is an output dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, + // Inouts-outputs + qqcw_, // Outputs: - aerdepdrycw_, aerdepdryis_, qqcw_tends_, rho_, vlc_dry_, + d_qtracers_dt_, aerdepdrycw_, aerdepdryis_, rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); // FIXME: Where is update tends and post processing???? diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index fdfb387524d8..d7ca643b9bbe 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -50,7 +50,6 @@ class MAMDryDep final : public scream::AtmosphereProcess { // inputs // FIXME: collect all inputs and outputs together view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; - view_3d wet_dens_; view_2d rho_; view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; @@ -58,10 +57,9 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d dqdt_tmp_[mam4::aero_model::pcnst]; - view_3d tendencies_; view_2d aerdepdrycw_; view_2d aerdepdryis_; - view_2d qqcw_tends_[mam4::aero_model::pcnst]; + view_2d qqcw_[mam4::aero_model::pcnst]; const_view_1d obukhov_length_; const_view_1d surface_friction_velocty_; diff --git a/externals/mam4xx b/externals/mam4xx index 4d16c226c62c..036b4c83b289 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 4d16c226c62cc412497fac425359b90a63821efc +Subproject commit 036b4c83b289ff6c4258018a92c87e4defa628a9 From 90eaa5848947fc620dbf1141b5694c2472b0b8db Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 16:35:44 -0700 Subject: [PATCH 281/904] A lot of cleanup-Removes vars from hpp, cleans i/o for functions --- ...x_mam_dry_deposition_process_interface.cpp | 72 +++++++++++-------- ...x_mam_dry_deposition_process_interface.hpp | 18 ++--- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index e8218fbe5c8a..9b99e6fe78a4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -303,17 +303,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); - obukhov_length_ = get_field_in("Obukhov_length").get_view(); - land_fraction_ = get_field_in("land_fraction").get_view(); - ice_fraction_ = get_field_in("ice_fraction").get_view(); - ocean_fraction_ = get_field_in("ocean_fraction").get_view(); - friction_velocity_ = - get_field_in("friction_velocity").get_view(); - aerodynamical_resistance_ = - get_field_in("aerodynamical_resistance").get_view(); - surface_friction_velocty_ = - get_field_in("surface_friction_velocty").get_view(); - // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; dry_atm_.z_iface = buffer_.z_iface; @@ -367,18 +356,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } - // ------------------------------------------------------------- - // Output fields for the process - // ------------------------------------------------------------- - - // FIXME: We might need to get rid of few of these fields - // as we do not need them in FM - - aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") - .get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") - .get_view(); - //----------------------------------------------------------------- // Allocate memory //----------------------------------------------------------------- @@ -428,22 +405,55 @@ void MAMDryDep::run_impl(const double dt) { // FIXME: There are some vars that are not declared "REQUIRED" etc. but still // used!!! - // Inputs: - // geometric mean wet diameter for number distribution [m] + // ------------------------------------------------------------- + // Inputs fields for the process + // ------------------------------------------------------------- + + // Geometric mean wet diameter for number distribution [m] auto dgncur_awet_ = get_field_in("dgncur_awet").get_view(); - auto wet_dens_ = get_field_in("wetdens").get_view(); + // Wet density of interstitial aerosol [kg/m3] + auto wet_dens_ = get_field_in("wetdens").get_view(); + // Obukhov length [m] + auto obukhov_length_ = + get_field_in("Obukhov_length").get_view(); + // Land fraction [unitless] + auto land_fraction_ = get_field_in("land_fraction").get_view(); + // Ice fraction [unitless] + auto ice_fraction_ = get_field_in("ice_fraction").get_view(); + // Ocean fraction [unitless] + auto ocean_fraction_ = + get_field_in("ocean_fraction").get_view(); + // Friction velocity from land model [m/s] + auto friction_velocity_ = + get_field_in("friction_velocity").get_view(); + // Aerodynamical resistance from land model [s/m] + auto aerodynamical_resistance_ = + get_field_in("aerodynamical_resistance").get_view(); + // Sfc friction velocity [m/s] + auto surface_friction_velocty_ = + get_field_in("surface_friction_velocty").get_view(); + + // ------------------------------------------------------------- + // Output fields for the process + // ------------------------------------------------------------- + // Surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] + auto aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + .get_view(); + // Surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] + auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + .get_view(); compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, - aerodynamical_resistance_, qtracers_, - fraction_landuse_, // d_qtracers_dt_ is an output + aerodynamical_resistance_, qtracers_, fraction_landuse_, dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, // Inouts-outputs qqcw_, - // Outputs: - d_qtracers_dt_, aerdepdrycw_, aerdepdryis_, rho_, vlc_dry_, - vlc_trb_, vlc_grv_, dqdt_tmp_); + // Outputs + d_qtracers_dt_, aerdepdrycw_, aerdepdryis_, + // work arrays + rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); // FIXME: Where is update tends and post processing???? } // run_impl diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index d7ca643b9bbe..58267cb919a7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -20,12 +20,12 @@ class MAMDryDep final : public scream::AtmosphereProcess { using view_2d = Field::view_dev_t; using view_3d = Field::view_dev_t; using const_view_1d = Field::view_dev_t; - using const_view_2d = Field::view_dev_t; using const_view_3d = Field::view_dev_t; - static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); - static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx - static constexpr int n_land_type = mam4::DryDeposition::n_land_type; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int aerosol_categories_ = + mam4::DryDeposition::aerosol_categories; + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; private: // number of horizontal columns and vertical levels @@ -57,18 +57,8 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d dqdt_tmp_[mam4::aero_model::pcnst]; - view_2d aerdepdrycw_; - view_2d aerdepdryis_; view_2d qqcw_[mam4::aero_model::pcnst]; - const_view_1d obukhov_length_; - const_view_1d surface_friction_velocty_; - const_view_1d land_fraction_; - const_view_1d ice_fraction_; - const_view_1d ocean_fraction_; - const_view_1d friction_velocity_; - const_view_1d aerodynamical_resistance_; - public: using KT = ekat::KokkosTypes; From 811f80b6671d65f147a1e8db644ecb8efa5b8e9c Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 12 Jul 2024 16:42:47 -0500 Subject: [PATCH 282/904] rename pyeamxx to pyscream --- components/eamxx/src/python/CMakeLists.txt | 4 ++-- components/eamxx/src/python/pyatmproc.hpp | 2 +- components/eamxx/src/python/pyeamxx.cpp | 2 +- components/eamxx/src/python/pyeamxx.hpp | 6 +++--- components/eamxx/src/python/pyeamxx/__init__.py | 16 ++++++++-------- components/eamxx/src/python/pygrid.hpp | 4 ++-- components/eamxx/src/python/pyproject.toml | 6 +++--- components/eamxx/src/python/readme | 6 +++--- .../eamxx/tests/python/pyp3/CMakeLists.txt | 2 +- .../eamxx/tests/python/pyp3/p3_standalone_py | 12 ++++++------ 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 43e2db73e533..41eae6d65463 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,8 +1,8 @@ find_package(pybind11 REQUIRED) find_package(mpi4py REQUIRED) -pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC +pybind11_add_module(pyscream pyscream.cpp) +target_link_libraries(pyscream PUBLIC mpi4py scream_share scream_io diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index 98d7b9a525ed..9e45bc513f1b 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -8,7 +8,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" -#include "pyeamxx.hpp" +#include "pyscream.hpp" #include diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index c4ec360d80a1..2efb8b6f8df9 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -45,7 +45,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyeamxx,m) { +PYBIND11_MODULE (pyscream,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/pyeamxx.hpp index 59371844a763..781c195919dd 100644 --- a/components/eamxx/src/python/pyeamxx.hpp +++ b/components/eamxx/src/python/pyeamxx.hpp @@ -1,5 +1,5 @@ -#ifndef PYEAMXX_HPP -#define PYEAMXX_HPP +#ifndef PYSCREAM_HPP +#define PYSCREAM_HPP #include "physics/register_physics.hpp" #include "diagnostics/register_diagnostics.hpp" @@ -27,4 +27,4 @@ struct PySession { } // namespace scream -#endif // PYEAMXX_HPP +#endif // PYSCREAM_HPP diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py index 75a36e148291..98c31744eb9f 100644 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ b/components/eamxx/src/python/pyeamxx/__init__.py @@ -1,15 +1,15 @@ """ This file will serve as a way to organize and expose - libpyeamxx internals to the rest of pyeamxx + libpyscream internals to the rest of pyscream """ -from libpyeamxx.libpyeamxx_ext import AtmProc -from libpyeamxx.libpyeamxx_ext import Grid -from libpyeamxx.libpyeamxx_ext import ParameterList -from libpyeamxx.libpyeamxx_ext import init -from libpyeamxx.libpyeamxx_ext import Field -from libpyeamxx.libpyeamxx_ext import P3 -from libpyeamxx.libpyeamxx_ext import finalize +from libpyscream.libpyscream_ext import AtmProc +from libpyscream.libpyscream_ext import Grid +from libpyscream.libpyscream_ext import ParameterList +from libpyscream.libpyscream_ext import init +from libpyscream.libpyscream_ext import Field +from libpyscream.libpyscream_ext import P3 +from libpyscream.libpyscream_ext import finalize __all__ = [ diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp index 102cf6037c62..e49a0443a345 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/pygrid.hpp @@ -3,7 +3,7 @@ #include "share/grid/mesh_free_grids_manager.hpp" -#include "pyeamxx.hpp" +#include "pyscream.hpp" #include @@ -14,7 +14,7 @@ namespace scream { inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) { EKAT_REQUIRE_MSG (PySession::get().inited, - "Error! You did not initialize pyeamxx, or you already finalized it!\n"); + "Error! You did not initialize pyscream, or you already finalized it!\n"); auto& comm = PySession::get().comm; ekat::ParameterList gm_params; std::vector grids_names = {"Physics"}; diff --git a/components/eamxx/src/python/pyproject.toml b/components/eamxx/src/python/pyproject.toml index fa5d6fc44817..490ea99c143e 100644 --- a/components/eamxx/src/python/pyproject.toml +++ b/components/eamxx/src/python/pyproject.toml @@ -3,15 +3,15 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "pyeamxx" +name = "pyscream" version = "0.0.2" dependencies = ["numpy", "mpi4py"] [tool.setuptools.packages.find] where = ["."] -include = ["pyeamxx", "libpyeamxx"] +include = ["pyscream", "libpyscream"] exclude = ["build_src", "tests"] namespaces = true [tool.setuptools.package-data] -"libpyeamxx" = ["*.so*"] +"libpyscream" = ["*.so*"] diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme index f5ed48f058ee..c16c32a5d217 100644 --- a/components/eamxx/src/python/readme +++ b/components/eamxx/src/python/readme @@ -1,7 +1,7 @@ INFO: - EAMxx python bindings -- pyeamxx is where we will house the python code -- libpyeamxx is where we will house the extensions +- pyscream is where we will house the python code +- libpyscream is where we will house the extensions - packaging moved to https://github.com/mahf708/experimental-scream-feedstock TODO: @@ -11,5 +11,5 @@ TODO: - decide archs/pythons/mpis to tgt USER: -- conda install pyeamxx -c mahf708/label/$mac (mac is chrysalis or pm-cpu) +- conda install pyscream -c mahf708/label/$mac (mac is chrysalis or pm-cpu) - see example in components/eamxx/tests/python/pyp3 diff --git a/components/eamxx/tests/python/pyp3/CMakeLists.txt b/components/eamxx/tests/python/pyp3/CMakeLists.txt index 7142be8dd8f8..945f15faaf65 100644 --- a/components/eamxx/tests/python/pyp3/CMakeLists.txt +++ b/components/eamxx/tests/python/pyp3/CMakeLists.txt @@ -53,6 +53,6 @@ foreach (rank IN LISTS MpiRanks) SRC_FILE p3_standalone_cxx.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc TGT_FILE p3_standalone_py.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_cxx${suffix} ${FIXTURES_BASE_NAME}_py${suffix} - LABELS pyeamxx + LABELS pyscream ) endforeach() diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index 0db6bcfcef61..34747943d081 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -5,7 +5,7 @@ import sys # Add path to scream libs sys.path.append('@SCREAM_BASE_DIR@/scripts') -# Add path to pyeamxx libs +# Add path to pyscream libs sys.path.append('@CMAKE_BINARY_DIR@/src/python') # Without these, and manual init/finalize, on my laptop I get @@ -15,7 +15,7 @@ mpi4py.rc.initialize = False # do not initialize MPI automatically mpi4py.rc.finalize = False # do not finalize MPI automatically from mpi4py import MPI -import pyeamxx +import pyscream from pathlib import Path from utils import ensure_yaml @@ -38,9 +38,9 @@ def main (): # Create the grid ncols = 218 nlevs = 72 - pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) + pyscream.create_grids_manager(ncols,nlevs,str(ic_file)) - p3 = pyeamxx.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') + p3 = pyscream.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') params = p3.get_params() old = params.get_dbl('max_total_ni') print (f"max_total_ni: {params.get_dbl('max_total_ni')}") @@ -64,7 +64,7 @@ if __name__ == "__main__": # This level of indirection ensures all pybind structs are destroyed # before we finalize eamxx (and hence kokkos) MPI.Init() - pyeamxx.init() + pyscream.init() main () - pyeamxx.finalize() + pyscream.finalize() MPI.Finalize() From c9af8dc8f11f853740303ed2cfddac25cd66d2d0 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 12 Jul 2024 16:54:10 -0500 Subject: [PATCH 283/904] more renaming --- components/eamxx/src/python/CMakeLists.txt | 13 +---------- .../src/python/libpyscream/CMakeLists.txt | 12 ++++++++++ .../python/{ => libpyscream}/pyatmproc.hpp | 2 +- .../src/python/{ => libpyscream}/pyfield.hpp | 0 .../src/python/{ => libpyscream}/pygrid.hpp | 2 +- .../python/{ => libpyscream}/pyparamlist.hpp | 0 .../pyscream_ext.cpp} | 2 +- .../pyscream_ext.hpp} | 0 .../src/python/{ => libpyscream}/pyutils.hpp | 0 .../eamxx/src/python/pyeamxx/__init__.py | 23 ------------------- .../eamxx/src/python/pyscream/__init__.py | 10 ++++++++ 11 files changed, 26 insertions(+), 38 deletions(-) create mode 100644 components/eamxx/src/python/libpyscream/CMakeLists.txt rename components/eamxx/src/python/{ => libpyscream}/pyatmproc.hpp (99%) rename components/eamxx/src/python/{ => libpyscream}/pyfield.hpp (100%) rename components/eamxx/src/python/{ => libpyscream}/pygrid.hpp (97%) rename components/eamxx/src/python/{ => libpyscream}/pyparamlist.hpp (100%) rename components/eamxx/src/python/{pyeamxx.cpp => libpyscream/pyscream_ext.cpp} (97%) rename components/eamxx/src/python/{pyeamxx.hpp => libpyscream/pyscream_ext.hpp} (100%) rename components/eamxx/src/python/{ => libpyscream}/pyutils.hpp (100%) delete mode 100644 components/eamxx/src/python/pyeamxx/__init__.py create mode 100644 components/eamxx/src/python/pyscream/__init__.py diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 41eae6d65463..4c8adf40fca7 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,12 +1 @@ -find_package(pybind11 REQUIRED) -find_package(mpi4py REQUIRED) - -pybind11_add_module(pyscream pyscream.cpp) -target_link_libraries(pyscream PUBLIC - mpi4py - scream_share - scream_io - diagnostics - eamxx_physics - scream_test_support -) +add_subdirectory(libpyscream) \ No newline at end of file diff --git a/components/eamxx/src/python/libpyscream/CMakeLists.txt b/components/eamxx/src/python/libpyscream/CMakeLists.txt new file mode 100644 index 000000000000..103c6a80b888 --- /dev/null +++ b/components/eamxx/src/python/libpyscream/CMakeLists.txt @@ -0,0 +1,12 @@ +find_package(pybind11 REQUIRED) +find_package(mpi4py REQUIRED) + +pybind11_add_module(pyscream_ext pyscream_ext.cpp) +target_link_libraries(pyscream_ext PUBLIC + mpi4py + scream_share + scream_io + diagnostics + eamxx_physics + scream_test_support +) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp similarity index 99% rename from components/eamxx/src/python/pyatmproc.hpp rename to components/eamxx/src/python/libpyscream/pyatmproc.hpp index 9e45bc513f1b..0acc371c1f71 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -8,7 +8,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" -#include "pyscream.hpp" +#include "pyscream_ext.hpp" #include diff --git a/components/eamxx/src/python/pyfield.hpp b/components/eamxx/src/python/libpyscream/pyfield.hpp similarity index 100% rename from components/eamxx/src/python/pyfield.hpp rename to components/eamxx/src/python/libpyscream/pyfield.hpp diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp similarity index 97% rename from components/eamxx/src/python/pygrid.hpp rename to components/eamxx/src/python/libpyscream/pygrid.hpp index e49a0443a345..e8adfa3bbc49 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -3,7 +3,7 @@ #include "share/grid/mesh_free_grids_manager.hpp" -#include "pyscream.hpp" +#include "pyscream_ext.hpp" #include diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/libpyscream/pyparamlist.hpp similarity index 100% rename from components/eamxx/src/python/pyparamlist.hpp rename to components/eamxx/src/python/libpyscream/pyparamlist.hpp diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp similarity index 97% rename from components/eamxx/src/python/pyeamxx.cpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.cpp index 2efb8b6f8df9..e0ad9ceec260 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp @@ -45,7 +45,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyscream,m) { +PYBIND11_MODULE (pyscream_ext,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/libpyscream/pyscream_ext.hpp similarity index 100% rename from components/eamxx/src/python/pyeamxx.hpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.hpp diff --git a/components/eamxx/src/python/pyutils.hpp b/components/eamxx/src/python/libpyscream/pyutils.hpp similarity index 100% rename from components/eamxx/src/python/pyutils.hpp rename to components/eamxx/src/python/libpyscream/pyutils.hpp diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py deleted file mode 100644 index 98c31744eb9f..000000000000 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" - This file will serve as a way to organize and expose - libpyscream internals to the rest of pyscream -""" - -from libpyscream.libpyscream_ext import AtmProc -from libpyscream.libpyscream_ext import Grid -from libpyscream.libpyscream_ext import ParameterList -from libpyscream.libpyscream_ext import init -from libpyscream.libpyscream_ext import Field -from libpyscream.libpyscream_ext import P3 -from libpyscream.libpyscream_ext import finalize - - -__all__ = [ - 'init', - 'finalize', - 'AtmProc', - 'Grid', - 'ParameterList', - 'Field', - 'P3', -] diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py new file mode 100644 index 000000000000..8159f5da47c7 --- /dev/null +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -0,0 +1,10 @@ +""" + This file will serve as a way to organize and expose + libpyscream internals to the rest of pyscream +""" + +from libpyscream import pyscream_ext as pyscream + +__all__ = [ + "pyscream" +] From 98ee753002733fa2e377ad5b38b8d92b4d666eaa Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 12:06:01 -0500 Subject: [PATCH 284/904] skip cosp when building python --- components/eamxx/src/physics/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e0e89e60f80d..e5bce027bcc4 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -7,7 +7,11 @@ add_subdirectory(share) add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) - add_subdirectory(cosp) + if (EAMXX_ENABLE_PYBIND) + message(STATUS "WARNING: skipping COSP when buidling Python") + else() + add_subdirectory(cosp) + endif() else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() From 86d53719be0079009b6b02637915ecf321fb93af Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 14:02:39 -0500 Subject: [PATCH 285/904] Add geo_data_source --- components/eamxx/src/python/libpyscream/pygrid.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index e8adfa3bbc49..93a8a5395dba 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -25,6 +25,7 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo gm_params.set("grids_names",grids_names); if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } From 628e401b45f4e395c28dec843742ae17da1b23a7 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 23:04:05 -0500 Subject: [PATCH 286/904] dump fields and reorg things --- components/eamxx/src/physics/CMakeLists.txt | 6 +----- components/eamxx/src/python/CMakeLists.txt | 2 +- .../eamxx/src/python/libpyscream/pyatmproc.hpp | 9 +++++++++ components/eamxx/src/python/libpyscream/pygrid.hpp | 1 - components/eamxx/src/python/pyscream/__init__.py | 14 ++++++++++++-- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e5bce027bcc4..e0e89e60f80d 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -7,11 +7,7 @@ add_subdirectory(share) add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) - if (EAMXX_ENABLE_PYBIND) - message(STATUS "WARNING: skipping COSP when buidling Python") - else() - add_subdirectory(cosp) - endif() + add_subdirectory(cosp) else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 4c8adf40fca7..d19bc05c240e 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1 +1 @@ -add_subdirectory(libpyscream) \ No newline at end of file +add_subdirectory(libpyscream) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 0acc371c1f71..5248ccf1f965 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -131,6 +131,14 @@ struct PyAtmProc { return pybind11::cast(missing); } + pybind11::list dump_fields() { + std::vector all_fields; + for (auto it : fields) { + all_fields.push_back(it.first); + } + return pybind11::cast(all_fields); + } + void setup_output (const std::string& yaml_file) { auto comm = PySession::get().comm; @@ -176,6 +184,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) .def("read_ic",&PyAtmProc::read_ic); + .def("dump_fields",&PyAtmProc::dump_fields); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index 93a8a5395dba..e8adfa3bbc49 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -25,7 +25,6 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo gm_params.set("grids_names",grids_names); if (latlon_nc_file!="") { - gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py index 8159f5da47c7..e76f52170fb3 100644 --- a/components/eamxx/src/python/pyscream/__init__.py +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -3,8 +3,18 @@ libpyscream internals to the rest of pyscream """ -from libpyscream import pyscream_ext as pyscream +from libpyscream.pyscream_ext import init +from libpyscream.pyscream_ext import finalize +from libpyscream.pyscream_ext import Field +from libpyscream.pyscream_ext import AtmProc +from libpyscream.pyscream_ext import ParameterList +from libpyscream.pyscream_ext import create_grids_manager __all__ = [ - "pyscream" + "init", + "finalize", + "Field", + "AtmProc", + "ParameterList", + "create_grids_manager", ] From 2599ce5eedbabcd12ef6ec78d3b2ba14c2097bef Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 14 Jul 2024 09:56:44 -0500 Subject: [PATCH 287/904] fix minor typo in def dumpfields --- components/eamxx/src/python/libpyscream/pyatmproc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 5248ccf1f965..000a7d64c9a9 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -183,7 +183,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("get_params",&PyAtmProc::get_params) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) - .def("read_ic",&PyAtmProc::read_ic); + .def("read_ic",&PyAtmProc::read_ic) .def("dump_fields",&PyAtmProc::dump_fields); } } // namespace scream From 3e0bb221b1c5308b5567a268e2cfe5b565a031fd Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 14 Jul 2024 11:48:48 -0500 Subject: [PATCH 288/904] add geo data source --- components/eamxx/src/python/libpyscream/pygrid.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index e8adfa3bbc49..1c0d56eb08b3 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -23,8 +23,10 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo pl.set("number_of_global_columns",ncols); pl.set("number_of_vertical_levels",nlevs); gm_params.set("grids_names",grids_names); + gm_params.set("geo_data_source",std::string("CREATE_EMPTY_DATA")); if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } From f12a60023f23ab9b5387798b991d9b33f540ad9d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 14 Jul 2024 15:53:13 -0700 Subject: [PATCH 289/904] Reduces vlc_trb to real from view_1d and other cleanup --- .../eamxx_mam_dry_deposition_functions.hpp | 11 ++-- ...x_mam_dry_deposition_process_interface.cpp | 52 +++++++++---------- ...x_mam_dry_deposition_process_interface.hpp | 9 ++-- .../single-process/mam/aci/CMakeLists.txt | 1 + .../single-process/mam/drydep/CMakeLists.txt | 3 +- externals/mam4xx | 2 +- 6 files changed, 42 insertions(+), 36 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 44742044a0a6..ad3c023e3878 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -37,12 +37,14 @@ void compute_tendencies( MAMDryDep::view_2d rho_, MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], - MAMDryDep::view_2d vlc_trb_[mam4::AeroConfig::num_modes()] + MAMDryDep::view_1d vlc_trb_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + // FIXME: WHY we are using a new policy here?? can't we get it from the + // run_impl?? const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( ncol, nlev); @@ -93,13 +95,14 @@ void compute_tendencies( // Kokkos::View. Solution: Use ColumnView in drydep.hpp as well. static constexpr int nmodes = mam4::AeroConfig::num_modes(); mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_], - vlc_trb[nmodes][MAMDryDep::aerosol_categories_], vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; + Real vlc_trb[nmodes][MAMDryDep::aerosol_categories_]; + for(int i = 0; i < nmodes; ++i) { for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { vlc_dry[i][j] = ekat::subview(vlc_dry_[i][j], icol); - vlc_trb[i][j] = ekat::subview(vlc_trb_[i][j], icol); + vlc_trb[i][j] = vlc_trb_[i][j](icol); vlc_grv[i][j] = ekat::subview(vlc_grv_[i][j], icol); } } @@ -142,4 +145,4 @@ void compute_tendencies( } // namespace } // namespace scream -#endif \ No newline at end of file +#endif diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 9b99e6fe78a4..4160b467f340 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -1,6 +1,6 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" -// ACI functions are stored in the following hpp file +// Drydep functions are stored in the following hpp file #include #include "mam4xx/drydep.hpp" //FIXME: Do we need it here??? @@ -84,7 +84,6 @@ void MAMDryDep::set_grids( using namespace ShortFieldTagsNames; // Layout for 2D (2d horiz) variable - // FIXME: Fix layouts based on new format const FieldLayout scalar2d{{COL}, {ncol_}}; // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and @@ -113,29 +112,30 @@ void MAMDryDep::set_grids( auto m3 = m * m * m; // meter cubed // -------------------------------------------------------------------------- - // These variables are "required" or pure inputs for the process + // These variables are "Required" or pure inputs for the process // -------------------------------------------------------------------------- // ----------- Atmospheric quantities ------------- - // Specific humidity [kg/kg] + // Specific humidity [kg/kg](Require only for building DS) add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); - // Cloud liquid mass mixing ratio [kg/kg] + // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); - // Cloud ice mass mixing ratio [kg/kg] + // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); - // Cloud liquid number mixing ratio [1/kg] + // Cloud liquid number mixing ratio [1/kg](Require only for building DS) add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); - // Cloud ice number mixing ratio [1/kg] + // Cloud ice number mixing ratio [1/kg](Require only for building DS) add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); - // Vertical pressure velocity [Pa/s] at midpoints + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) add_field("omega", scalar3d_mid, Pa / s, grid_name); // Total pressure [Pa] at midpoints @@ -147,26 +147,25 @@ void MAMDryDep::set_grids( // Layer thickness(pdel) [Pa] at midpoints add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - // Planetary boundary layer height [m] + // Planetary boundary layer height [m] (Require only for building DS) add_field("pbl_height", scalar2d, m, grid_name); static constexpr auto m2 = m * m; static constexpr auto s2 = s * s; - // Surface geopotential [m2/s2] + // Surface geopotential [m2/s2] (Require only for building DS) add_field("phis", scalar2d, m2 / s2, grid_name); //----------- Variables from microphysics scheme ------------- - // Total cloud fraction [fraction] - // FIXME: Is is cldfrac_liq instead? find out + // Total cloud fraction [fraction] (Require only for building DS) add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); //----------- Variables from coupler (land component)--------- // Obukhov length [m] add_field("Obukhov_length", scalar2d, m, grid_name); - // Surface friction velocty [m] + // Surface friction velocty or ustar[m/s] add_field("surface_friction_velocty", scalar2d, m / s, grid_name); // Land fraction [fraction] @@ -294,11 +293,10 @@ void MAMDryDep::initialize_impl(const RunType run_type) { wet_atm_.ni = get_field_in("ni").get_view(); // Populate the dry atmosphere state with views from fields - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); - // FIXME: tot or liq? make notes about it to ensure it is the right one + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); @@ -370,7 +368,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { for(int j = 0; j < aerosol_categories_; ++j) { Kokkos::resize(vlc_dry_[i][j], ncol_, nlev_); Kokkos::resize(vlc_grv_[i][j], ncol_, nlev_); - Kokkos::resize(vlc_trb_[i][j], ncol_, nlev_); + Kokkos::resize(vlc_trb_[i][j], ncol_); } } @@ -381,6 +379,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { static constexpr int n_land_type = mam4::DryDeposition::n_land_type; for(int i = 0; i < n_land_type; ++i) { + // FIXME: This should come from a file reading Kokkos::resize(fraction_landuse_[i], ncol_); } @@ -389,7 +388,8 @@ void MAMDryDep::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - // FIXME: Where is post processing functor???? + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // initialize_impl // ========================================================================================= @@ -402,9 +402,6 @@ void MAMDryDep::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // FIXME: There are some vars that are not declared "REQUIRED" etc. but still - // used!!! - // ------------------------------------------------------------- // Inputs fields for the process // ------------------------------------------------------------- @@ -429,7 +426,7 @@ void MAMDryDep::run_impl(const double dt) { // Aerodynamical resistance from land model [s/m] auto aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); - // Sfc friction velocity [m/s] + // Sfc friction velocity or ustar [m/s] auto surface_friction_velocty_ = get_field_in("surface_friction_velocty").get_view(); @@ -455,6 +452,9 @@ void MAMDryDep::run_impl(const double dt) { // work arrays rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); - // FIXME: Where is update tends and post processing???? + // call post processing to convert dry mixing ratios to wet mixing ratios + // and update the state + // Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + // Kokkos::fence(); // wait before returning to calling function } // run_impl } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 58267cb919a7..867a81d9fb40 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -50,10 +50,10 @@ class MAMDryDep final : public scream::AtmosphereProcess { // inputs // FIXME: collect all inputs and outputs together view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; + view_1d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d rho_; view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; - view_2d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d dqdt_tmp_[mam4::aero_model::pcnst]; @@ -126,7 +126,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { // for atmosphere compute_vertical_layer_heights(team, dry_atm_pre_, i); compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - } // operator() + } // Preprocess operator() // local variables for preprocess struct // number of horizontal columns and vertical levels @@ -136,7 +136,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // Preprocess // Atmosphere processes often have a post-processing step prepares output // from this process for the Field Manager. This functor implements this @@ -165,7 +165,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { const int i = team.league_rank(); // column index compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, wet_aero_post_, i); - } // operator() + } // operator() Postprocess // number of horizontal columns and vertical levels int ncol_post_, nlev_post_; @@ -179,6 +179,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { private: // pre- and postprocessing scratch pads Preprocess preprocess_; + Postprocess postprocess_; }; // MAMDryDep } // namespace scream diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 0d5d60ee7d5a..490dc389d952 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -24,6 +24,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) diff --git a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt index 50d030cdc2cf..7bc5010370a4 100644 --- a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt @@ -21,4 +21,5 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) # Ensure test input files are present in the data dir -GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) diff --git a/externals/mam4xx b/externals/mam4xx index 036b4c83b289..f7f445dcb646 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 036b4c83b289ff6c4258018a92c87e4defa628a9 +Subproject commit f7f445dcb6464105a1719e16b68928e583cc8ec6 From ef6e76653875f7e7c49c1f6649b8e41f39a3682d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:48:25 +0000 Subject: [PATCH 290/904] Bump actions/setup-python from 5.1.0 to 5.1.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ccca0c479f26..9ea25ae1864e 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index abd8b92e4f6c..e6fb53ba29f5 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 5ecdf6dec00c..e0a8e19f9c01 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" - name: Run unit tests From 8fc5d8f338eb3d3c9d501ec2f17e644c31e4bb4f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 15 Jul 2024 12:02:37 -0700 Subject: [PATCH 291/904] All the printouts for thevalidation --- .../eamxx_mam_dry_deposition_functions.hpp | 23 ++++++++++++++++++- ...x_mam_dry_deposition_process_interface.cpp | 6 +++-- .../eamxx/src/physics/mam/mam_coupling.hpp | 20 ++++++++-------- .../src/share/io/scream_scorpio_interface.cpp | 4 ++-- .../single-process/mam/drydep/CMakeLists.txt | 3 ++- .../single-process/mam/drydep/input.yaml | 14 ++--------- externals/mam4xx | 2 +- 7 files changed, 43 insertions(+), 29 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index ad3c023e3878..ff876cdcf425 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -42,12 +42,15 @@ void compute_tendencies( MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { + auto printb = [](const std::string &name, const double &val) { + std::cout << name << ":" << std::setprecision(15) << val << std::endl; + }; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); // FIXME: WHY we are using a new policy here?? can't we get it from the // run_impl?? const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( - ncol, nlev); + 1, nlev); Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { static constexpr int num_aero_species = @@ -126,6 +129,7 @@ void compute_tendencies( } }); // parallel_for nlevs bool ptend_lq[pcnst]; + printb("bef:qqcw:", qqcw[23](63)); mam4::aero_model_drydep( // inputs team, fraction_landuse, atm.temperature, atm.pressure, @@ -140,8 +144,25 @@ void compute_tendencies( ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), // work arrays rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); + printb("aft:qqcw:", qqcw[23](63)); }); // parallel_for for ncols } + +void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { + Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, + 0.16803558403621365E-001, 0.18076055155371872E-001, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.91803784897907303E+000, 0.17186036997038400E-002, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.18448503115578840E-001}; + + for(int icol = 0; icol < ncol; ++icol) { + for(int kk = 0; kk < 11; ++kk) { + flu[kk](icol) = temp[kk]; + } + } +} + } // namespace } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 4160b467f340..a38ad080a3d4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -100,7 +100,7 @@ void MAMDryDep::set_grids( }; const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); + make_layout({ncol_, num_aero_modes, nlev_}, {"ncol", "num_modes", "lev"}); using namespace ekat::units; @@ -396,7 +396,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { void MAMDryDep::run_impl(const double dt) { using DryDep = mam4::DryDeposition; const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -440,6 +440,8 @@ void MAMDryDep::run_impl(const double dt) { auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); + populated_fraction_landuse(fraction_landuse_, ncol_); + // std::cout<<"fraction_landuse_:"< Date: Mon, 15 Jul 2024 16:02:16 -0600 Subject: [PATCH 292/904] Ensure that Kokkos::deep_copy is called outside of parallel_for. --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 91e69443b588..bbd99a66cfb0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -461,6 +461,14 @@ void MAMWetscav::run_impl(const double dt) { // inside a parallel_for. const int nlev = nlev_; + // Zero out tendencies otherwise, they are initialized to junk values + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(tends.n_mode_i[m], 0); + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + Kokkos::deep_copy(tends.q_aero_i[m][a], 0); + } + } + // Loop over atmosphere columns Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { @@ -506,14 +514,6 @@ void MAMWetscav::run_impl(const double dt) { auto wetdens_icol = ekat::subview(wetdens, icol); const auto prain_icol = ekat::subview(prain, icol); - // Zero out tendencies otherwise, they are initialized to junk values - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - Kokkos::deep_copy(tends.n_mode_i[m], 0); - for(int a = 0; a < mam4::num_species_mode(m); ++a) { - Kokkos::deep_copy(tends.q_aero_i[m][a], 0); - } - } - mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, // inputs From 9cb990cfd6976cb989a34dc49a8a1bb33f61637e Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 15 Jul 2024 16:16:33 -0600 Subject: [PATCH 293/904] wet_scav - Fixing compilation errors. --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index bbd99a66cfb0..b598fa6d7483 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -463,9 +463,9 @@ void MAMWetscav::run_impl(const double dt) { // Zero out tendencies otherwise, they are initialized to junk values for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - Kokkos::deep_copy(tends.n_mode_i[m], 0); + Kokkos::deep_copy(dry_aero_tends_.int_aero_nmr[m], 0); for(int a = 0; a < mam4::num_species_mode(m); ++a) { - Kokkos::deep_copy(tends.q_aero_i[m][a], 0); + Kokkos::deep_copy(dry_aero_tends_.int_aero_mmr[m][a], 0); } } From d3a1ebc9f670db2c74fbf150183423f4bec173a7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 15 Jul 2024 22:19:19 -0700 Subject: [PATCH 294/904] More prints for validation --- .../eamxx_mam_dry_deposition_functions.hpp | 51 ++++++++++- ...x_mam_dry_deposition_process_interface.cpp | 86 +++++++++++++++++-- ...x_mam_dry_deposition_process_interface.hpp | 2 +- externals/mam4xx | 2 +- 4 files changed, 128 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index ff876cdcf425..034471f047ea 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -30,7 +30,7 @@ void compute_tendencies( MAMDryDep::view_2d qqcw_[mam4::aero_model::pcnst], // outputs - MAMDryDep::view_3d d_qtracers_dt, MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_3d ptend_q, MAMDryDep::view_2d aerdepdrycw, MAMDryDep::view_2d aerdepdryis, // work arrays @@ -43,7 +43,7 @@ void compute_tendencies( [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { auto printb = [](const std::string &name, const double &val) { - std::cout << name << ":" << std::setprecision(15) << val << std::endl; + // std::cout << name << ":" << std::setprecision(15) << val << std::endl; }; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); // FIXME: WHY we are using a new policy here?? can't we get it from the @@ -140,14 +140,59 @@ void compute_tendencies( // input-outputs qqcw, // outputs - ekat::subview(d_qtracers_dt, icol), ptend_lq, + ekat::subview(ptend_q, icol), ptend_lq, ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), // work arrays rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); printb("aft:qqcw:", qqcw[23](63)); + printb("aft:ptend:", ptend_q(0, 63, 23)); }); // parallel_for for ncols } +void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, + const int ncol, const int nlev, + // output + const mam_coupling::AerosolState dry_aero) { + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + 1, nlev); + static constexpr int nmodes = mam4::AeroConfig::num_modes(); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + for(int m = 0; m < nmodes; ++m) { + dry_aero.int_aero_nmr[m](icol, kk) += + ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + dry_aero.int_aero_mmr[m][a](icol, kk) += + ptend_q(icol, kk, + mam4::ConvProc::lmassptrcw_amode(a, m)) * + dt; + } + }); // parallel_for nlevs + }); +} + +void update_cloudborne_mmrs( + const MAMDryDep::view_2d qqcw[mam4::aero_model::pcnst], const double dt, + const int ncol, const int nlev_, + // output + const mam_coupling::AerosolState dry_aero) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], + qqcw[mam4::ConvProc::numptrcw_amode(m)]); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], + qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)]); + } + } + } +} + void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, 0.16803558403621365E-001, 0.18076055155371872E-001, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index a38ad080a3d4..a40845b8c0b1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -361,8 +361,8 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: comment what they are and units..... qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - rho_ = view_2d("rho", ncol_, nlev_); - d_qtracers_dt_ = view_3d("d_qtracers_dt_", ncol_, nlev_, pcnst); + rho_ = view_2d("rho", ncol_, nlev_); + ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { for(int j = 0; j < aerosol_categories_; ++j) { @@ -388,12 +388,16 @@ void MAMDryDep::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, - dry_aero_); + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // initialize_impl // ========================================================================================= void MAMDryDep::run_impl(const double dt) { + auto printb = [](const std::string &name, const double &val) { + std::cout << name << ":" << std::setprecision(15) << val << std::endl; + }; + using DryDep = mam4::DryDeposition; const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); @@ -441,7 +445,33 @@ void MAMDryDep::run_impl(const double dt) { .get_view(); populated_fraction_landuse(fraction_landuse_, ncol_); - // std::cout<<"fraction_landuse_:"< 0) { + printb("inter_before:", dry_aero_.int_aero_mmr[m][a](0, 63)); + } + } + } + + for(int m = 0; m < num_aero_modes; ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + printb("cld_before:", dry_aero_.cld_aero_nmr[m](0, 63)); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + printb("cld_before:", dry_aero_.cld_aero_mmr[m][a](0, 63)); + } + } + } + compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, @@ -450,13 +480,53 @@ void MAMDryDep::run_impl(const double dt) { // Inouts-outputs qqcw_, // Outputs - d_qtracers_dt_, aerdepdrycw_, aerdepdryis_, + ptend_q_, aerdepdrycw_, aerdepdryis_, // work arrays rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); + + // Update the interstitial aerosols using ptend. + update_interstitial_mmrs(ptend_q_, dt, ncol_, nlev_, // inputs + dry_aero_); // output + + // Update the interstitial aerosols + update_cloudborne_mmrs(qqcw_, dt, ncol_, nlev_, // inputs + dry_aero_); // output + + for(int m = 0; m < num_aero_modes; ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + printb("inter_after:", dry_aero_.int_aero_nmr[m](0, 63)); + std::cout << int_nmr_field_name << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + printb("inter_after:", dry_aero_.int_aero_mmr[m][a](0, 63)); + std::cout << int_mmr_field_name << std::endl; + } + } + } + + for(int m = 0; m < num_aero_modes; ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + printb("cld_after:", dry_aero_.cld_aero_nmr[m](0, 63)); + std::cout << cld_nmr_field_name << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + printb("cld_after:", dry_aero_.cld_aero_mmr[m][a](0, 63)); + std::cout << cld_mmr_field_name << std::endl; + } + } + } + // call post processing to convert dry mixing ratios to wet mixing ratios // and update the state - // Kokkos::parallel_for("postprocess", scan_policy, postprocess_); - // Kokkos::fence(); // wait before returning to calling function + Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + Kokkos::fence(); // wait before returning to calling function } // run_impl } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 867a81d9fb40..8a28b940ee15 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -45,7 +45,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { std::shared_ptr grid_; view_3d qtracers_; - view_3d d_qtracers_dt_; + view_3d ptend_q_; // inputs // FIXME: collect all inputs and outputs together diff --git a/externals/mam4xx b/externals/mam4xx index 535f67b3dd20..89fd96c4a81f 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 535f67b3dd20e5df714149eeefdb7b50ba9862ed +Subproject commit 89fd96c4a81f9b22b0f24fc35282a04599443efc From 123ec08857e58715e6d8b8f8bca47d4ba9540b4f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 16 Jul 2024 06:51:26 -0700 Subject: [PATCH 295/904] Cleaned some prints and adds new comments --- .../eamxx_mam_dry_deposition_functions.hpp | 42 +++++++------- ...x_mam_dry_deposition_process_interface.cpp | 58 +++---------------- externals/mam4xx | 2 +- 3 files changed, 29 insertions(+), 73 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 034471f047ea..d7d1542eeced 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -42,22 +42,20 @@ void compute_tendencies( MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { - auto printb = [](const std::string &name, const double &val) { - // std::cout << name << ":" << std::setprecision(15) << val << std::endl; - }; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); - // FIXME: WHY we are using a new policy here?? can't we get it from the - // run_impl?? const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( - 1, nlev); + 1, nlev); // FIXME: change 1 to ncol + + // Parallel loop over all the columns Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { static constexpr int num_aero_species = mam_coupling::num_aero_species(); const int icol = team.league_rank(); - + // Parallel loop over all the levels to populate qtracers array using + // dry_aero Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { for(int mode = 0; mode < num_aero_modes; ++mode) { @@ -75,8 +73,11 @@ void compute_tendencies( }); // parallel_for for nlevs team.team_barrier(); + // Create atm and progs objects mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + + // Extract column data (or 1d view) from 2d views of data mam4::ConstColumnView dgncur_awet[num_aero_modes]; mam4::ConstColumnView wet_dens[num_aero_modes]; @@ -94,11 +95,9 @@ void compute_tendencies( fraction_landuse[i] = fraction_landuse_[i](icol); } - // FIXME: why mam4::ColumnView didn;t work here, why use - // Kokkos::View. Solution: Use ColumnView in drydep.hpp as well. static constexpr int nmodes = mam4::AeroConfig::num_modes(); - mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_], - vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; + mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_]; + mam4::ColumnView vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; Real vlc_trb[nmodes][MAMDryDep::aerosol_categories_]; @@ -116,7 +115,7 @@ void compute_tendencies( qqcw[i] = ekat::subview(qqcw_[i], icol); dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); } - // Extract Prognostics + // Extract qqcw from Prognostics Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { for(int m = 0; m < nmodes; ++m) { @@ -128,8 +127,8 @@ void compute_tendencies( progs.q_aero_c[m][a][kk]; } }); // parallel_for nlevs - bool ptend_lq[pcnst]; - printb("bef:qqcw:", qqcw[23](63)); + + bool ptend_lq[pcnst]; // currently unused mam4::aero_model_drydep( // inputs team, fraction_landuse, atm.temperature, atm.pressure, @@ -144,11 +143,10 @@ void compute_tendencies( ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), // work arrays rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); - printb("aft:qqcw:", qqcw[23](63)); - printb("aft:ptend:", ptend_q(0, 63, 23)); }); // parallel_for for ncols -} +} // Compute_tendencies ends +// Update interstitial aerosols using ptend_q tendencies void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, const int ncol, const int nlev, // output @@ -173,12 +171,13 @@ void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, dt; } }); // parallel_for nlevs - }); -} + }); // parallel_for icol +} // Update interstitial aerosols ends +// Update cloud borne aerosols using qqcw void update_cloudborne_mmrs( const MAMDryDep::view_2d qqcw[mam4::aero_model::pcnst], const double dt, - const int ncol, const int nlev_, + const int nlev_, // output const mam_coupling::AerosolState dry_aero) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -191,8 +190,9 @@ void update_cloudborne_mmrs( } } } -} +} // Update cloud borne aerosols ends +// FIXME: remove the following function void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, 0.16803558403621365E-001, 0.18076055155371872E-001, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index a40845b8c0b1..7599324db1a2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -3,57 +3,13 @@ // Drydep functions are stored in the following hpp file #include -#include "mam4xx/drydep.hpp" //FIXME: Do we need it here??? - /* -Inputs: - -Atmosphere: - temperature "T_mid" - pressure "p_mid" - interface_pressure "p_int" - hydrostatic_dp "pseudo_density" - -Diagnostics: - tracer_mixing_ratio "qtracers" - wet_geometric_mean_diameter_i "dgncur_awet" - wet_density "wetdens" - -Diagnostic Scalar Parameters, one per column: - "Obukhov_length" - "surface_friction_velocty" - "land_fraction" - "ice_fraction" - "ocean_fraction" - "friction_velocity" - "aerodynamical_resistance" - -Prognostics: - n_mode_c "mam_coupling::cld_aero_nmr_field_name(m)" - q_aero_c "mam_coupling::int_aero_nmr_field_name(m)" - - - -Outputs: - -Diagnostics: - d_tracer_mixing_ratio_dt "d_qtracers_dt" - deposition_flux_of_cloud_borne_aerosols -"deposition_flux_of_cloud_borne_aerosols" - deposition_flux_of_interstitial_aerosols -"deposition_flux_of_interstitial_aerosols" - -Computed internally, could be exposed if needed by another process - vlc_grv(nlev) : dep velocity of gravitational settling [m/s] - vlc_trb(nlev) : dep velocity of turbulent dry deposition [m/s] - vlc_dry(nlev) : dep velocity, sum of vlc_grv and vlc_trb [m/s] - -Tendencies: - n_mode_c "Tendencies" - q_aero_c "Tendencies" - +----------------------------------------------------------------- +NOTES: +1. Add a CIME test and multi-process tests +2. Ensure that the submodule for MAM4xx is the main branch +----------------------------------------------------------------- */ - namespace scream { MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) @@ -490,8 +446,8 @@ void MAMDryDep::run_impl(const double dt) { dry_aero_); // output // Update the interstitial aerosols - update_cloudborne_mmrs(qqcw_, dt, ncol_, nlev_, // inputs - dry_aero_); // output + update_cloudborne_mmrs(qqcw_, dt, nlev_, // inputs + dry_aero_); // output for(int m = 0; m < num_aero_modes; ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); diff --git a/externals/mam4xx b/externals/mam4xx index 89fd96c4a81f..5e7ff7200b49 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 89fd96c4a81f9b22b0f24fc35282a04599443efc +Subproject commit 5e7ff7200b4910a43fbd254951c6ff61b6463e30 From c8f15f52dfed06a7de14e5a2d983059cad817fbc Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 16 Jul 2024 09:36:47 -0600 Subject: [PATCH 296/904] rename config_sgr_salinity to config_sgr_salinity_prescribed --- components/mpas-ocean/bld/build-namelist | 4 ++-- components/mpas-ocean/bld/build-namelist-section | 4 ++-- .../bld/namelist_files/namelist_defaults_mpaso.xml | 4 ++-- .../bld/namelist_files/namelist_definition_mpaso.xml | 4 ++-- components/mpas-ocean/src/Registry.xml | 8 ++++---- .../mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 2c5ee97b0df2..f891c5d089a8 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -713,8 +713,8 @@ add_default($nl, 'config_sgr_flux_vertical_location'); add_default($nl, 'config_use_sgr_opt_kpp'); add_default($nl, 'config_use_sgr_opt_temp'); add_default($nl, 'config_use_sgr_opt_salt'); -add_default($nl, 'config_sgr_temperature'); -add_default($nl, 'config_sgr_salinity'); +add_default($nl, 'config_sgr_temperature_prescribed'); +add_default($nl, 'config_sgr_salinity_prescribed'); if ($OCN_SGR eq 'data') { add_default($nl, 'config_subglacial_runoff_mode', 'val'=>"data"); } else { diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index 3f2d4ab4839d..9091c1e961c0 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -229,8 +229,8 @@ add_default($nl, 'config_sgr_flux_vertical_location'); add_default($nl, 'config_use_sgr_opt_kpp'); add_default($nl, 'config_use_sgr_opt_temp'); add_default($nl, 'config_use_sgr_opt_salt'); -add_default($nl, 'config_sgr_temperature'); -add_default($nl, 'config_sgr_salinity'); +add_default($nl, 'config_sgr_temperature_prescribed'); +add_default($nl, 'config_sgr_salinity_prescribed'); add_default($nl, 'config_subglacial_runoff_mode'); ############################ diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 0514704b9514..4957f2ce60f9 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -347,8 +347,8 @@ 1 'sgr' 'sgr' -0.0 -0.0 +0.0 +0.0 'off' diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index 43ba139ab65f..725308006f53 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1228,7 +1228,7 @@ Valid values: 'sgr', 'prescribed'. Default: Defined in namelist_defaults.xml - Option to choose sgr temperature, applied when config_use_sgr_opt_temp = 'prescribed' @@ -1236,7 +1236,7 @@ Valid values: Any real number. Default: Defined in namelist_defaults.xml - Option to choose sgr salinity, applied when config_use_sgr_opt_temp = 'prescribed' diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 8dbd4870dc65..07afdb0f9af4 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -720,17 +720,17 @@ /> - - diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 4375ae8f9eaf..1c7cba2f39cf 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -600,7 +600,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer if ( trim(config_use_sgr_opt_temp) == 'prescribed' ) then !sgr with fixed prescribed temperature (for debugging only) tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & - config_sgr_temperature / areaCell(iCell) + config_sgr_temperature_prescribed / areaCell(iCell) else if ( trim(config_use_sgr_opt_temp) == 'sgr' ) then !sgr with temperature equal to the local freezing point of freshwater tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & @@ -611,7 +611,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer if ( trim(config_use_sgr_opt_salt) == 'prescribed' ) then !sgr with fixed prescribed temperature (for debugging only) tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = subglacialRunoffFlux(iCell) * & - config_sgr_salinity / areaCell(iCell) + config_sgr_salinity_prescribed / areaCell(iCell) else if ( trim(config_use_sgr_opt_salt) == 'sgr' ) then !sgr with temperature equal to the local freezing point of freshwater tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = 0.0_RKIND From 307b4d0e3f5abe341fada74494c86a4d2e1a232a Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 16 Jul 2024 10:51:53 -0600 Subject: [PATCH 297/904] change config_use_sgr_opt_property from character to logical --- components/mpas-ocean/bld/build-namelist | 4 ++-- components/mpas-ocean/bld/build-namelist-section | 4 ++-- .../namelist_files/namelist_defaults_mpaso.xml | 4 ++-- .../namelist_files/namelist_definition_mpaso.xml | 16 ++++++++-------- components/mpas-ocean/src/Registry.xml | 16 ++++++++-------- .../src/shared/mpas_ocn_surface_bulk_forcing.F | 12 ++++++------ 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index f891c5d089a8..490100040c00 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -711,8 +711,8 @@ add_default($nl, 'config_flux_attenuation_coefficient_runoff'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); add_default($nl, 'config_sgr_flux_vertical_location'); add_default($nl, 'config_use_sgr_opt_kpp'); -add_default($nl, 'config_use_sgr_opt_temp'); -add_default($nl, 'config_use_sgr_opt_salt'); +add_default($nl, 'config_use_sgr_opt_temp_prescribed'); +add_default($nl, 'config_use_sgr_opt_salt_prescribed'); add_default($nl, 'config_sgr_temperature_prescribed'); add_default($nl, 'config_sgr_salinity_prescribed'); if ($OCN_SGR eq 'data') { diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index 9091c1e961c0..189d32fc7b52 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -227,8 +227,8 @@ add_default($nl, 'config_flux_attenuation_coefficient_runoff'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); add_default($nl, 'config_sgr_flux_vertical_location'); add_default($nl, 'config_use_sgr_opt_kpp'); -add_default($nl, 'config_use_sgr_opt_temp'); -add_default($nl, 'config_use_sgr_opt_salt'); +add_default($nl, 'config_use_sgr_opt_temp_prescribed'); +add_default($nl, 'config_use_sgr_opt_salt_prescribed'); add_default($nl, 'config_sgr_temperature_prescribed'); add_default($nl, 'config_sgr_salinity_prescribed'); add_default($nl, 'config_subglacial_runoff_mode'); diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 4957f2ce60f9..3147d1cc92d4 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -345,8 +345,8 @@ 0.001 'uniform' 1 -'sgr' -'sgr' +.false. +.false. 0.0 0.0 'off' diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index 725308006f53..803b280a1241 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1212,25 +1212,25 @@ Valid values: 0 - don't use, 1 - use Default: Defined in namelist_defaults.xml - -Option to choose sgr temperature +If true, subglacial runoff temperature is set to config_sgr_temperature_prescribed. If false, the temperature is set to local freezing point. -Valid values: 'sgr', 'prescribed'. +Valid values: .true. or .false. Default: Defined in namelist_defaults.xml - -Option to choose sgr salinity +If true, subglacial runoff salinity is set to config_sgr_salinity_prescribed. If false, the salinity is set to 0 PSU. -Valid values: 'sgr', 'prescribed'. +Valid values: .true. or .false. Default: Defined in namelist_defaults.xml -Option to choose sgr temperature, applied when config_use_sgr_opt_temp = 'prescribed' +Prescribed subglacial runoff temperature value, applied when config_use_sgr_opt_temp_prescribed = .true. Valid values: Any real number. Default: Defined in namelist_defaults.xml @@ -1238,7 +1238,7 @@ Default: Defined in namelist_defaults.xml -Option to choose sgr salinity, applied when config_use_sgr_opt_temp = 'prescribed' +Prescribed subglacial runoff salinity value, applied when config_use_sgr_opt_salt_prescribed = .true. Valid values: Any real number. Default: Defined in namelist_defaults.xml diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 07afdb0f9af4..3fdfd58ca61d 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -718,20 +718,20 @@ description="option to use or not subglacial runoff (sgr) in kpp calculation" possible_values="0 - don't use, 1 - use" /> - - diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 1c7cba2f39cf..ce019053fc7c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -597,22 +597,22 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer * max(tracerGroup(index_temperature_flux,minLevelCell(iCell),iCell), 0.0_RKIND) / rho_sw if (trim(config_subglacial_runoff_mode) == 'data') then - if ( trim(config_use_sgr_opt_temp) == 'prescribed' ) then - !sgr with fixed prescribed temperature (for debugging only) + if ( config_use_sgr_opt_temp_prescribed ) then + !sgr with fixed prescribed temperature tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & config_sgr_temperature_prescribed / areaCell(iCell) - else if ( trim(config_use_sgr_opt_temp) == 'sgr' ) then + else !sgr with temperature equal to the local freezing point of freshwater tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & inLandIceCavity=.true.) / areaCell(iCell) end if - if ( trim(config_use_sgr_opt_salt) == 'prescribed' ) then - !sgr with fixed prescribed temperature (for debugging only) + if ( config_use_sgr_opt_salt_prescribed ) then + !sgr with fixed prescribed temperature tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = subglacialRunoffFlux(iCell) * & config_sgr_salinity_prescribed / areaCell(iCell) - else if ( trim(config_use_sgr_opt_salt) == 'sgr' ) then + else !sgr with temperature equal to the local freezing point of freshwater tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = 0.0_RKIND end if From f6cbc38e7ef811504d6b90bd45d813d00659aea6 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 16 Jul 2024 11:01:42 -0600 Subject: [PATCH 298/904] change config_use_sgr_opt_kpp to logical variable --- .../bld/namelist_files/namelist_defaults_mpaso.xml | 2 +- .../bld/namelist_files/namelist_definition_mpaso.xml | 6 +++--- components/mpas-ocean/src/Registry.xml | 10 +++++----- .../mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 3147d1cc92d4..dce88f0127e3 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -344,7 +344,7 @@ 10.0 0.001 'uniform' -1 +.true. .false. .false. 0.0 diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index 803b280a1241..fed7828e6a8b 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1204,11 +1204,11 @@ Valid values: 'top','uniform', 'bottom'. Default: Defined in namelist_defaults.xml - -Option to use or not subglacial runoff (sgr) in kpp calculation. +If true, include subglacial runoff (sgr) contribution in kpp calculation. -Valid values: 0 - don't use, 1 - use +Valid values: .true. or .false. Default: Defined in namelist_defaults.xml diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 3fdfd58ca61d..3e339aefe1a3 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -714,16 +714,16 @@ description="Selects the vertical location where subglacial runoff is fluxed." possible_values="'top','uniform', 'bottom'" /> - Date: Tue, 16 Jul 2024 11:18:15 -0600 Subject: [PATCH 299/904] Move checking for valid string in config_sgr_flux_vertical_location to an init routine --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 9 +++++++-- components/mpas-ocean/src/shared/mpas_ocn_forcing.F | 2 -- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 368e4e9a50da..a6539f520f1b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3530,8 +3530,6 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & end if zTop = zBot end do - else - call mpas_log_write("config_sgr_flux_vertical_location not one of 'bottom', 'uniform', 'top'.", MPAS_LOG_CRIT) end if else fracAbsorbedSubglacialRunoff = 0.0_RKIND @@ -4695,6 +4693,13 @@ subroutine ocn_diagnostics_init(domain, err)!{{{ call ocn_diagnostics_variables_init(domain, jenkinsOn, & hollandJenkinsOn, err) + if ( trim(config_sgr_flux_vertical_location) == 'top' ) then + else if ( trim(config_sgr_flux_vertical_location) == 'uniform' ) then + else if ( trim(config_sgr_flux_vertical_location) == 'bottom' ) then + else + call mpas_log_write("config_sgr_flux_vertical_location not one of 'bottom', 'uniform', 'top'.", MPAS_LOG_CRIT) + end if + end subroutine ocn_diagnostics_init!}}} !*********************************************************************** diff --git a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F index afdacc6c942c..5cfce4c4e690 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_forcing.F @@ -221,8 +221,6 @@ subroutine ocn_forcing_build_fraction_absorbed_array(meshPool, statePool, forcin transmissionCoeffTop = transmissionCoeffBot end do end do - else - call mpas_log_write("config_sgr_flux_vertical_location not one of 'bottom', 'uniform', 'top'.", MPAS_LOG_CRIT) end if end if From a5aec91cf445c5314bd476cb5e192a04a7f4c92b Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 16 Jul 2024 10:28:54 -0700 Subject: [PATCH 300/904] change ekat::impl::max instances to std::max and remove unneeded m_helper_fields line --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 42ea7d0ec03f..cf20435fd2ce 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - ekat::impl::max(m_params.get("target_latitude"),0.1); + std::max(m_params.get("target_latitude"),0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - ekat::impl::max(m_params.get("target_longitude"),0.1); + std::max(m_params.get("target_longitude"),0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -548,7 +548,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) // Convert to pressure to millibar (file gives pressure in Pa) for (int ilev=0; ilev Date: Tue, 16 Jul 2024 11:31:26 -0600 Subject: [PATCH 301/904] set config_sgr_flux_vertical_location to bottom in namelist_defaults_mpaso.xml for consistency with Registry --- .../mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index dce88f0127e3..c57920e5c388 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -343,7 +343,7 @@ 0.001 10.0 0.001 -'uniform' +'bottom' .true. .false. .false. From b45249581205070d1f39e2400cc574907a4ab6d8 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 16 Jul 2024 12:35:25 -0600 Subject: [PATCH 302/904] Change subglacialRunoffFlux into mass flux from volume flux for consistency with riverRunoffFlux and reference density, update DSGR files --- components/mpas-ocean/cime_config/buildnml | 8 ++++---- components/mpas-ocean/src/Registry.xml | 2 +- .../src/shared/mpas_ocn_surface_bulk_forcing.F | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 192fd70bc6ae..f3eb22881683 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -143,7 +143,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.oQU240wLI.20240221.nc' if ocn_sgr == 'data': - data_sgr_file = 'DSGR.MALI.out2055.oQU240wLI.20240328.nc' + data_sgr_file = 'DSGR.massFlux.MALI.out2055.oQU240wLI.20240328.nc' elif ocn_grid == 'oQU120': decomp_date = '230424' @@ -282,7 +282,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.SOwISC12to60E2r4.20210114.nc' if ocn_sgr == 'data': - data_sgr_file = 'DSGR.MALI.out2055.SOwISC12to60E2r4.20240328.nc' + data_sgr_file = 'DSGR.massFlux.MALI.out2055.SOwISC12to60E2r4.20240328.nc' elif ocn_grid == 'FRISwISC08to60E3r1': decomp_date = '20230913' # changed to date of partiotions in ../files_for_e3sm/assembled_files/inputdata/ocn/mpas-o/FRISwISC08to60E3r1/partitions @@ -359,7 +359,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.ECwISC30to60E2r1.20240221.nc' if ocn_sgr == 'data': - data_sgr_file = 'DSGR.MALI.out2055.ECwISC30to60E2r1.20240328.nc' + data_sgr_file = 'DSGR.massFlux.MALI.out2055.ECwISC30to60E2r1.20240328.nc' elif ocn_grid == 'IcoswISC30E3r5': decomp_date = '20231120' @@ -380,7 +380,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.IcoswISC30E3r5.20231120.nc' if ocn_sgr == 'data': - data_sgr_file = 'DSGR.MALI.out2055.IcoswISC30E3r5.20240328.nc' + data_sgr_file = 'DSGR.massFlux.massFlux.MALI.out2055.IcoswISC30E3r5.20240328.nc' elif ocn_grid == 'IcosXISC30E3r7': decomp_date = '20240314' diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 3e339aefe1a3..bf02edf96b51 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -3807,7 +3807,7 @@ description="Fresh water flux from river runoff at cell centers from coupler. Positive into the ocean." packages="thicknessBulkPKG" /> - diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index ce019053fc7c..2cf79881a7f8 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -362,13 +362,13 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur !$acc enter data copyin(subglacialRunoffFlux) !$acc parallel loop & - !$acc present(surfaceThicknessFluxSubglacialRunoff, subglacialRunoffFlux, areaCell) + !$acc present(surfaceThicknessFluxSubglacialRunoff, subglacialRunoffFlux) #else !$omp parallel !$omp do schedule(runtime) #endif do iCell = 1, nCells - surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / areaCell(iCell) + surfaceThicknessFluxSubglacialRunoff(iCell) = subglacialRunoffFlux(iCell) / rho_sw end do #ifndef MPAS_OPENACC !$omp end do @@ -600,18 +600,18 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer if ( config_use_sgr_opt_temp_prescribed ) then !sgr with fixed prescribed temperature tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & - config_sgr_temperature_prescribed / areaCell(iCell) + config_sgr_temperature_prescribed / rho_sw else !sgr with temperature equal to the local freezing point of freshwater tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & - inLandIceCavity=.true.) / areaCell(iCell) + inLandIceCavity=.true.) / rho_sw end if if ( config_use_sgr_opt_salt_prescribed ) then !sgr with fixed prescribed temperature tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = subglacialRunoffFlux(iCell) * & - config_sgr_salinity_prescribed / areaCell(iCell) + config_sgr_salinity_prescribed / rho_sw else !sgr with temperature equal to the local freezing point of freshwater tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = 0.0_RKIND From fe9eaa0a60066fc19370d2fca9646801d6bc2bf5 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 16 Jul 2024 16:34:51 -0700 Subject: [PATCH 303/904] create a general compset that is tested with the dycoms case, an ARM97 compset and test, and add test case for RCE --- cime_config/tests.py | 5 ++- .../eamxx/cime_config/config_component.xml | 2 +- .../eamxx/cime_config/config_compsets.xml | 45 +++++++++++-------- .../cime_config/namelist_defaults_scream.xml | 36 ++++++++++----- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 06c16ca176c9..275754e26a40 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -655,8 +655,11 @@ "e3sm_scream_v1_dp-eamxx" : { "time" : "01:00:00", + # each test 225 phys cols, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-DYCOMSrf01", # 225 phys cols, roughly size of ne2 + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-ARM97", + "ERS_P16_Ln22.ne30_ne30.F2000-SCREAMv1-DP-RCE", ) }, diff --git a/components/eamxx/cime_config/config_component.xml b/components/eamxx/cime_config/config_component.xml index d3b6e34744d0..7380b371401d 100644 --- a/components/eamxx/cime_config/config_component.xml +++ b/components/eamxx/cime_config/config_component.xml @@ -60,7 +60,7 @@ - + char diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index d33b865bde93..504c8dfbf848 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -75,19 +75,19 @@ - F2010-SCREAMv1-DP-DYCOMSrf01 - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%DYCOMSrf01 + F2010-SCREAMv1-DP-General + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%General Experimental, under development - F2010-SCREAMv1-DP - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx + F2010-SCREAMv1-DP-ARM97 + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%ARM97 Experimental, under development - F2000-SCREAMv1-RCE-DP + F2000-SCREAMv1-DP-RCE 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx Experimental, under development @@ -98,11 +98,13 @@ 2016-08-01 2020-01-20 - 1999-07-10 + 1999-07-10 + 1999-06-23 + 2000-01-01 - + 864 @@ -121,39 +123,44 @@ - + - 31.5 + 225 - + - 238.5 + 1 - + - 225 + 225 - + - 1 + 1 - + + - 225 + 31.5 + 36.605 + 0.0 - + - 1 + 238.5 + 262.515 + 0.0 diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index bc09ce40952d..3a0722220b2d 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -578,25 +578,41 @@ be lost if SCREAM_HACK_XML is not enabled. ${CASE} - + true UNSET - ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -999 - 31.5 -999 - 238.5 false - true false - true false false 1100 0 10800 false + + + + ${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc + 36.605 + 262.515 + true + true + + + ${DIN_LOC_ROOT}/atm/cam/scam/iop/RCE_iopfile_4scam.nc + 0.0 + 0.0 + + + ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc + 31.5 + 238.5 + true + true + @@ -649,13 +665,13 @@ be lost if SCREAM_HACK_XML is not enabled. 1024 0 0 - 5 + 5 0 - 5 + 5 0 - 50000 + 50000 0 - 50000 + 50000 -1 4 cube From 9f3d4930c75a439e3a2ceecfbc5b5a3b707ad706 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 16 Jul 2024 16:44:29 -0700 Subject: [PATCH 304/904] update some comments to make them more clear --- components/eamxx/cime_config/config_compsets.xml | 2 +- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index 504c8dfbf848..77c2de92ed32 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -104,7 +104,7 @@ - + 864 diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 3a0722220b2d..f1264b8b4e08 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -593,7 +593,7 @@ be lost if SCREAM_HACK_XML is not enabled. 10800 false - + ${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc 36.605 From 8b9b3f01684235591fdccdae385db58cd168d98a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 17 Jul 2024 06:58:00 -0700 Subject: [PATCH 305/904] Adds a CIME test and fixed FIXME comments and removed prinouts --- .../cime_config/namelist_defaults_scream.xml | 4 + .../scream/mam4xx/drydep/shell_commands | 13 +++ .../eamxx_mam_dry_deposition_functions.hpp | 6 +- ...x_mam_dry_deposition_process_interface.cpp | 92 ++++--------------- 4 files changed, 39 insertions(+), 76 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50188236e4e0..e586c6ce2151 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -241,6 +241,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0 + + + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands new file mode 100644 index 000000000000..d219b2a489e0 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands @@ -0,0 +1,13 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +#modify initial condition file to get aerosol species ICs +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b + +# Add drydep process +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_drydep" -b + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index d7d1542eeced..98a59afab79e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,7 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, // FIXME: Make it a constant view + MAMDryDep::view_3d qtracers, MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], const MAMDryDep::const_view_3d dgncur_awet_, const MAMDryDep::const_view_3d wet_dens_, @@ -45,7 +45,7 @@ void compute_tendencies( static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( - 1, nlev); // FIXME: change 1 to ncol + ncol, nlev); // Parallel loop over all the columns Kokkos::parallel_for( @@ -192,7 +192,7 @@ void update_cloudborne_mmrs( } } // Update cloud borne aerosols ends -// FIXME: remove the following function +// FIXME: remove the following function after implementing file read for landuse void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, 0.16803558403621365E-001, 0.18076055155371872E-001, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 7599324db1a2..c7f2471c447f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -8,6 +8,7 @@ NOTES: 1. Add a CIME test and multi-process tests 2. Ensure that the submodule for MAM4xx is the main branch +3. Read file for fractional landuse ----------------------------------------------------------------- */ namespace scream { @@ -56,7 +57,7 @@ void MAMDryDep::set_grids( }; const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"ncol", "num_modes", "lev"}); + make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "num_modes", "lev"}); using namespace ekat::units; @@ -243,16 +244,20 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Populate the wet atmosphere state with views from fields // FIMXE: specifically look which among these are actually used by the process wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS wet_atm_.qc = get_field_in("qc").get_view(); wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); // Populate the dry atmosphere state with views from fields - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); @@ -322,21 +327,21 @@ void MAMDryDep::initialize_impl(const RunType run_type) { for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { for(int j = 0; j < aerosol_categories_; ++j) { - Kokkos::resize(vlc_dry_[i][j], ncol_, nlev_); - Kokkos::resize(vlc_grv_[i][j], ncol_, nlev_); - Kokkos::resize(vlc_trb_[i][j], ncol_); + vlc_dry_[i][j] = view_2d("vlc_dry_[i][j]", ncol_, nlev_); + vlc_grv_[i][j] = view_2d("vlc_grv_[i][j]", ncol_, nlev_); + vlc_trb_[i][j] = view_1d("vlc_trb_[i][j]", ncol_); } } for(int i = 0; i < pcnst; ++i) { - Kokkos::resize(qqcw_[i], ncol_, nlev_); - Kokkos::resize(dqdt_tmp_[i], ncol_, nlev_); + qqcw_[i] = view_2d("qqcw_[i]", ncol_, nlev_); + dqdt_tmp_[i] = view_2d("dqdt_tmp_[i]", ncol_, nlev_); } static constexpr int n_land_type = mam4::DryDeposition::n_land_type; for(int i = 0; i < n_land_type; ++i) { // FIXME: This should come from a file reading - Kokkos::resize(fraction_landuse_[i], ncol_); + fraction_landuse_[i] = view_1d("fraction_landuse_[i]", ncol_); } //----------------------------------------------------------------- @@ -350,13 +355,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - auto printb = [](const std::string &name, const double &val) { - std::cout << name << ":" << std::setprecision(15) << val << std::endl; - }; - using DryDep = mam4::DryDeposition; const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -400,34 +401,10 @@ void MAMDryDep::run_impl(const double dt) { auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); + //FIXME: remove it if it read from a file populated_fraction_landuse(fraction_landuse_, ncol_); - for(int m = 0; m < num_aero_modes; ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - printb("inter_before:", dry_aero_.int_aero_nmr[m](0, 63)); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - - if(strlen(int_mmr_field_name) > 0) { - printb("inter_before:", dry_aero_.int_aero_mmr[m][a](0, 63)); - } - } - } - - for(int m = 0; m < num_aero_modes; ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - printb("cld_before:", dry_aero_.cld_aero_nmr[m](0, 63)); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - - if(strlen(cld_mmr_field_name) > 0) { - printb("cld_before:", dry_aero_.cld_aero_mmr[m][a](0, 63)); - } - } - } + //Call drydeposition and get tendencies compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, @@ -449,37 +426,6 @@ void MAMDryDep::run_impl(const double dt) { update_cloudborne_mmrs(qqcw_, dt, nlev_, // inputs dry_aero_); // output - for(int m = 0; m < num_aero_modes; ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - printb("inter_after:", dry_aero_.int_aero_nmr[m](0, 63)); - std::cout << int_nmr_field_name << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - - if(strlen(int_mmr_field_name) > 0) { - printb("inter_after:", dry_aero_.int_aero_mmr[m][a](0, 63)); - std::cout << int_mmr_field_name << std::endl; - } - } - } - - for(int m = 0; m < num_aero_modes; ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - printb("cld_after:", dry_aero_.cld_aero_nmr[m](0, 63)); - std::cout << cld_nmr_field_name << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - - if(strlen(cld_mmr_field_name) > 0) { - printb("cld_after:", dry_aero_.cld_aero_mmr[m][a](0, 63)); - std::cout << cld_mmr_field_name << std::endl; - } - } - } - // call post processing to convert dry mixing ratios to wet mixing ratios // and update the state Kokkos::parallel_for("postprocess", scan_policy, postprocess_); From f2a7b976fce6f5750518773349f0f4351f227796 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 17 Jul 2024 09:42:48 -0700 Subject: [PATCH 306/904] cast minimum lat/lon threshold as a real --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index cf20435fd2ce..6f8474f8d85e 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),0.1); + std::max(m_params.get("target_latitude"),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - std::max(m_params.get("target_longitude"),0.1); + std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), From 3b0dca4de256cc4c14c0b907a8371e7074d6aa48 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 17 Jul 2024 10:53:17 -0600 Subject: [PATCH 307/904] introduce separate subroutine ocn_diagnostic_solve_surfaceLayer_subglacialRunoff --- .../src/shared/mpas_ocn_diagnostics.F | 88 ++++++++++++++----- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index a6539f520f1b..82fe1d9a950d 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -452,13 +452,16 @@ subroutine ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, verticalMe ! ! inputs: layerThickness, activeTracers, layerThicknessEdgeFlux, normalVelocity ! outputs: tracersSurfaceLayerValue, indexSurfaceLayerDepth, normalVelocitySurfaceLayer, - ! surfaceFluxAttenuationCoefficient, surfaceFluxAttenuationCoefficientRunoff, - ! surfaceFluxAttenuationCoefficientSubglacialRunoff + ! surfaceFluxAttenuationCoefficient, surfaceFluxAttenuationCoefficientRunoff call ocn_diagnostic_solve_surfaceLayer(layerThickness, activeTracers, & layerThickEdgeFlux, normalVelocity, tracersSurfaceLayerValue, & indexSurfaceLayerDepth, normalVelocitySurfaceLayer, & - sfcFlxAttCoeff, surfaceFluxAttenuationCoefficientRunoff, & - surfaceFluxAttenuationCoefficientSubglacialRunoff) + sfcFlxAttCoeff, surfaceFluxAttenuationCoefficientRunoff) + + if (trim(config_subglacial_runoff_mode) == 'data') then + ! outputs: surfaceFluxAttenuationCoefficientSubglacialRunoff + call ocn_diagnostic_solve_surfaceLayer_subglacialRunoff(surfaceFluxAttenuationCoefficientSubglacialRunoff) + end if end if ! full_compute @@ -565,8 +568,11 @@ subroutine ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, verticalMe ! !$acc indexSurfaceLayerDepth, & ! !$acc normalVelocitySurfaceLayer, & ! !$acc sfcFlxAttCoeff, & -! !$acc surfaceFluxAttenuationCoefficientRunoff, & -! !$acc surfaceFluxAttenuationCoefficientSubglacialRunoff) +! !$acc surfaceFluxAttenuationCoefficientRunoff) +! if (trim(config_subglacial_runoff_mode) == 'data') then +! !$acc update host(surfaceFluxAttenuationCoefficientSubglacialRunoff) +! end if + ! end if ! full_compute ! !$acc exit data delete (atmosphericPressure, seaIcePressure) @@ -1701,9 +1707,7 @@ subroutine ocn_diagnostic_solve_surfaceLayer(layerThickness, & activeTracers, layerThicknessEdgeFlux, normalVelocity, & tracersSurfaceLayerValue, indexSurfaceLayerDepth, & normalVelocitySurfaceLayer, & - surfaceFluxAttenuationCoefficient, & - surfaceFluxAttenuationCoefficientRunoff, & - surfaceFluxAttenuationCoefficientSubglacialRunoff)!{{{ + surfaceFluxAttenuationCoefficient)!{{{ !----------------------------------------------------------------- ! input variables @@ -1728,8 +1732,7 @@ subroutine ocn_diagnostic_solve_surfaceLayer(layerThickness, & indexSurfaceLayerDepth, &!< [out] layer index at sfc lyr depth normalVelocitySurfaceLayer, &!< [out] avg velocity over sfc layer surfaceFluxAttenuationCoefficient, &!< [out] sfc flx attenuation - surfaceFluxAttenuationCoefficientRunoff, &!< [out] sfc flx attenuation - surfaceFluxAttenuationCoefficientSubglacialRunoff !< [out] sfc flx attenuation + surfaceFluxAttenuationCoefficientRunoff &!< [out] sfc flx attenuation !----------------------------------------------------------------- ! local variables @@ -1909,27 +1912,72 @@ subroutine ocn_diagnostic_solve_surfaceLayer(layerThickness, & !$omp end parallel #endif - if (trim(config_subglacial_runoff_mode) == 'data') then + !-------------------------------------------------------------------- + + end subroutine ocn_diagnostic_solve_surfaceLayer !}}} + +!*********************************************************************** +! +! routine ocn_diagnostic_solve_surfaceLayer_subglacialRunoff +! +!> \brief Computes diagnostic subglacialRunoff variables for surface layer +!> \author Irena Vankova +!> \date July 2024 +!> \details +! +!----------------------------------------------------------------------- + + subroutine ocn_diagnostic_solve_surfaceLayer_subglacialRunoff(surfaceFluxAttenuationCoefficientSubglacialRunoff)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:), intent(out) :: & + surfaceFluxAttenuationCoefficientSubglacialRunoff !< [out] sfc flx attenuation + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + + integer :: & + nCells, &! num of cells + iCell ! loop indices for cell + + ! End preamble + !----------------------------------------------------------------- + ! Begin code + + ! compute the attenuation coefficient for subglacial runoff surface fluxes + + nCells = nCellsHalo( 1 ) + #ifdef MPAS_OPENACC - !$acc parallel loop & - !$acc present(surfaceFluxAttenuationCoefficientSubglacialRunoff) + !$acc parallel loop & + !$acc present(surfaceFluxAttenuationCoefficientSubglacialRunoff) #else - !$omp parallel - !$omp do schedule(runtime) + !$omp parallel + !$omp do schedule(runtime) #endif - do iCell = 1, nCells + do iCell = 1, nCells surfaceFluxAttenuationCoefficientSubglacialRunoff(iCell) = & config_flux_attenuation_coefficient_subglacial_runoff - end do + end do #ifndef MPAS_OPENACC !$omp end do !$omp end parallel #endif - end if !-------------------------------------------------------------------- - end subroutine ocn_diagnostic_solve_surfaceLayer !}}} + end subroutine ocn_diagnostic_solve_surfaceLayer_subglacialRunoff !}}} + +!*********************************************************************** + !*********************************************************************** ! From 7fa3a50ebc4d682f52d125ac85874da692665676 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 17 Jul 2024 11:14:07 -0600 Subject: [PATCH 308/904] correct extra delete --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 82fe1d9a950d..f50d6bb697a3 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -1707,7 +1707,8 @@ subroutine ocn_diagnostic_solve_surfaceLayer(layerThickness, & activeTracers, layerThicknessEdgeFlux, normalVelocity, & tracersSurfaceLayerValue, indexSurfaceLayerDepth, & normalVelocitySurfaceLayer, & - surfaceFluxAttenuationCoefficient)!{{{ + surfaceFluxAttenuationCoefficient, & + surfaceFluxAttenuationCoefficientRunoff)!{{{ !----------------------------------------------------------------- ! input variables From a4a5798d449547de67b1414b12541b8f2772e971 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 17 Jul 2024 12:12:26 -0700 Subject: [PATCH 309/904] Fixes single process test and some merge master fixes --- .../eamxx/src/physics/mam/CMakeLists.txt | 1 - ...x_mam_dry_deposition_process_interface.cpp | 9 ++- .../mam/impl/compute_water_content.cpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 33 ++++------- .../eamxx/src/share/field/field_tag.hpp | 43 -------------- components/eamxx/tests/CMakeLists.txt | 1 - .../single-process/mam/drydep/CMakeLists.txt | 26 +++++++- .../single-process/mam/drydep/input.yaml | 11 +++- .../single-process/mam/drydep/output.yaml | 59 ++++++++++++++++++- 9 files changed, 107 insertions(+), 78 deletions(-) diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 6339c85aef0b..e41517077538 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -45,7 +45,6 @@ add_library(mam eamxx_mam_optics_process_interface.cpp eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp) - target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index c7f2471c447f..ebbab7ab7571 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -145,7 +145,7 @@ void MAMDryDep::set_grids( //----------- Variables from other mam4xx processes ------------ // Geometric mean wet diameter for number distribution [m] - add_field("dgncur_awet", scalar4d_mid, m, grid_name); + add_field("dgnumwet", scalar4d_mid, m, grid_name); // Wet density of interstitial aerosol [kg/m3] add_field("wetdens", scalar4d_mid, kg / m3, grid_name); @@ -355,7 +355,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -368,7 +367,7 @@ void MAMDryDep::run_impl(const double dt) { // ------------------------------------------------------------- // Geometric mean wet diameter for number distribution [m] - auto dgncur_awet_ = get_field_in("dgncur_awet").get_view(); + auto dgncur_awet_ = get_field_in("dgnumwet").get_view(); // Wet density of interstitial aerosol [kg/m3] auto wet_dens_ = get_field_in("wetdens").get_view(); // Obukhov length [m] @@ -401,10 +400,10 @@ void MAMDryDep::run_impl(const double dt) { auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - //FIXME: remove it if it read from a file + // FIXME: remove it if it read from a file populated_fraction_landuse(fraction_landuse_, ncol_); - //Call drydeposition and get tendencies + // Call drydeposition and get tendencies compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, diff --git a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp index 673fe06292e1..ea7190afff91 100644 --- a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp +++ b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp @@ -23,7 +23,7 @@ void compute_water_content(const mam4::Prognostics &progs, int k, // extract aerosol tracers for this level into state_q, which is needed // for computing dry aerosol properties below // FIXME: we should eliminate this index translation stuff - constexpr int nvars = mam4::aero_model::pcnst; + constexpr int nvars = aero_model::pcnst; Real state_q[nvars]; // aerosol tracers for level k for (int imode = 0; imode < num_modes; ++imode) { int la, lc; // interstitial and cloudborne indices within state_q diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 8e77ae2484b8..1e13cece98d9 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -251,10 +251,7 @@ const char* cld_aero_mmr_field_name(const int mode, const int species) { // mixing ratio field in EAMxx KOKKOS_INLINE_FUNCTION const char* gas_mmr_field_name(const int gas) { - if (!gas_mmr_names(gas)[0]) { - concat_2_strings("", gas_species_name(gas), gas_mmr_names(gas)); - } - return gas_mmr_names(gas); + return const_cast(gas_species_name(gas)); } // This type stores multi-column views related specifically to the wet @@ -351,9 +348,8 @@ struct Buffer { // ======================= // number of local fields stored at column interfaces - static constexpr int num_2d_iface = 2; + static constexpr int num_2d_iface = 1; - uview_2d p_int; // pressure at interfaces uview_2d z_iface; // height at interfaces // storage @@ -469,7 +465,6 @@ inline size_t init_buffer(const ATMBufferManager &buffer_manager, // set view pointers for interface fields uview_2d* view_2d_iface_ptrs[Buffer::num_2d_iface] = { - &buffer.p_int, &buffer.z_iface }; for (int i = 0; i < Buffer::num_2d_iface; ++i) { @@ -524,8 +519,6 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, "cldfrac not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, "w_updraft not defined for dry atmosphere state!"); - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_int.data() != nullptr, - "p_int not defined for dry atmosphere state!"); return haero::Atmosphere(mam4::nlev, ekat::subview(dry_atm.T_mid, column_index), ekat::subview(dry_atm.p_mid, column_index), @@ -656,11 +649,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -681,21 +674,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);// PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } @@ -766,7 +759,7 @@ void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs const auto PC = mam4::ModeIndex::PrimaryCarbon; \ const auto NoMode = mam4::ModeIndex::None; \ static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index e500250b80f2..673208d6ae86 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -34,15 +34,6 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, - // Added for RRTMGP, TODO: Revisit this approach, is there a better way than adding more field tags? - Gases, - ShortWaveBand, - ShortWaveGpoint, - LongWaveBand, - LongWaveGpoint, - IsccpTau, - IsccpPrs, - num_modes }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -59,15 +50,6 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; - // Added for rrtmgp - see TODO item above - constexpr auto NGAS = FieldTag::Gases; - constexpr auto SWBND = FieldTag::ShortWaveBand; - constexpr auto LWBND = FieldTag::LongWaveBand; - constexpr auto SWGPT = FieldTag::ShortWaveGpoint; - constexpr auto LWGPT = FieldTag::LongWaveGpoint; - constexpr auto ISCCPTAU = FieldTag::IsccpTau; - constexpr auto ISCCPPRS = FieldTag::IsccpPrs; - constexpr auto NMODES = FieldTag::num_modes; } inline std::string e2str (const FieldTag ft) { @@ -98,31 +80,6 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; - // Added for rrtmgp - see TODO item above - case FieldTag::Gases: - name = "ngas"; - break; - case FieldTag::ShortWaveBand: - name = "swband"; - break; - case FieldTag::ShortWaveGpoint: - name = "swgpt"; - break; - case FieldTag::LongWaveBand: - name = "lwband"; - break; - case FieldTag::LongWaveGpoint: - name = "lwgpt"; - break; - case FieldTag::IsccpTau: - name = "ISCCPTAU"; - break; - case FieldTag::IsccpPrs: - name = "ISCCPPRS"; - break; - case FieldTag::num_modes: - name = "num_modes"; - break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 635f5de80a87..5fd23a41d9d8 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -63,7 +63,6 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") - set(EAMxx_tests_IC_FILE_MAM "scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc") set(EAMxx_tests_IC_FILE_MAM4xx_72lev "screami_unit_tests_mam4xx_ne2np4L72_20240329.nc") # Testing individual atm processes diff --git a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt index 698b9cb7b542..d05e6823642f 100644 --- a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt @@ -1,18 +1,19 @@ include (ScreamUtils) set (TEST_BASE_NAME mam4_drydep_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LABELS mam4_drydep physics LIBS mam MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) # Set AD configurable options -set (ATM_TIME_STEP 3600) -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h set (RUN_T0 2021-10-12-45000) ## Copy (and configure) yaml files needed by tests @@ -24,3 +25,22 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_drydep physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() + diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index a95faf9047a4..1de57ad7e682 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -23,11 +23,20 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/3src/screami_unit_tests_mam4xx_ne2np4L72_LAT_m38.0068938875434_LON_16p9323465700125_DRYDEP.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} fraction_landuse: 0.2 w_updraft: 1e-5 pbl_height: 0.0 + Obukhov_length: -0.48167567011091336E+002 + friction_velocity: 0.47569674433601039E+000 + land_fraction: 0.14675684171817760E+000 + ice_fraction: 0.00000000000000000E+000 + ocean_fraction: 0.85324315828182240E+000 + surface_friction_velocty: 0.27482164092221828E+000 + aerodynamical_resistance: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/single-process/mam/drydep/output.yaml b/components/eamxx/tests/single-process/mam/drydep/output.yaml index e900bb804340..6d8a79e8fe00 100644 --- a/components/eamxx/tests/single-process/mam/drydep/output.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/output.yaml @@ -2,10 +2,63 @@ --- filename_prefix: mam4_drydep_standalone_output Averaging Type: Instant -Field Names: - - T_mid +Fields: + Physics: + Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols + output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... From f508702d342902a4e9faeaf5611b618f71e765e1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 17 Jul 2024 12:19:22 -0700 Subject: [PATCH 310/904] Removes commented-out scorpio error message code --- components/eamxx/src/share/io/scream_scorpio_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 68a432819ec3..cb2b501f2b5c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - /*for (auto& it : s.files) { + for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - }*/ + } s.files.clear(); for (auto& it : s.decomps) { From 579e3dd8eab64098afb47109bd6175302eba3fc0 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Wed, 17 Jul 2024 13:59:02 -0600 Subject: [PATCH 311/904] remove & --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index f50d6bb697a3..5046e19d5d59 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -1733,7 +1733,7 @@ subroutine ocn_diagnostic_solve_surfaceLayer(layerThickness, & indexSurfaceLayerDepth, &!< [out] layer index at sfc lyr depth normalVelocitySurfaceLayer, &!< [out] avg velocity over sfc layer surfaceFluxAttenuationCoefficient, &!< [out] sfc flx attenuation - surfaceFluxAttenuationCoefficientRunoff &!< [out] sfc flx attenuation + surfaceFluxAttenuationCoefficientRunoff !< [out] sfc flx attenuation !----------------------------------------------------------------- ! local variables From 9e4978bb3ccc4bf0353109c8671430ba0a3c1d39 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 18 Jul 2024 10:18:19 -0600 Subject: [PATCH 312/904] Fix GPU tests: Do not use host pointers on device. --- .../eamxx_mam_dry_deposition_functions.hpp | 68 +++++++++---------- ...x_mam_dry_deposition_process_interface.cpp | 22 ++---- ...x_mam_dry_deposition_process_interface.hpp | 23 ++++--- 3 files changed, 52 insertions(+), 61 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 98a59afab79e..1fdd8fd1243e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -20,14 +20,14 @@ void compute_tendencies( const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, - MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], + MAMDryDep::view_2d fraction_landuse_, const MAMDryDep::const_view_3d dgncur_awet_, const MAMDryDep::const_view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, // input-outputs - MAMDryDep::view_2d qqcw_[mam4::aero_model::pcnst], + MAMDryDep::view_3d qqcw_, // outputs MAMDryDep::view_3d ptend_q, MAMDryDep::view_2d aerdepdrycw, @@ -35,13 +35,10 @@ void compute_tendencies( // work arrays MAMDryDep::view_2d rho_, - MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - MAMDryDep::view_1d vlc_trb_[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { + MAMDryDep::view_4d vlc_dry_, + MAMDryDep::view_3d vlc_trb_, + MAMDryDep::view_4d vlc_grv_, + MAMDryDep::view_3d dqdt_tmp_) { static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( @@ -92,32 +89,31 @@ void compute_tendencies( static constexpr int n_land_type = MAMDryDep::n_land_type; Real fraction_landuse[n_land_type]; for(int i = 0; i < n_land_type; ++i) { - fraction_landuse[i] = fraction_landuse_[i](icol); + fraction_landuse[i] = fraction_landuse_(i, icol); } static constexpr int nmodes = mam4::AeroConfig::num_modes(); mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_]; mam4::ColumnView vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; - Real vlc_trb[nmodes][MAMDryDep::aerosol_categories_]; for(int i = 0; i < nmodes; ++i) { for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { - vlc_dry[i][j] = ekat::subview(vlc_dry_[i][j], icol); - vlc_trb[i][j] = vlc_trb_[i][j](icol); - vlc_grv[i][j] = ekat::subview(vlc_grv_[i][j], icol); + vlc_dry[i][j] = ekat::subview(vlc_dry_, i, j, icol); + vlc_trb[i][j] = vlc_trb_(i, j, icol); + vlc_grv[i][j] = ekat::subview(vlc_grv_, i, j, icol); } } static constexpr int pcnst = mam4::aero_model::pcnst; mam4::ColumnView qqcw[pcnst]; mam4::ColumnView dqdt_tmp[pcnst]; for(int i = 0; i < pcnst; ++i) { - qqcw[i] = ekat::subview(qqcw_[i], icol); - dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); + qqcw[i] = ekat::subview(qqcw_, i, icol); + dqdt_tmp[i] = ekat::subview(dqdt_tmp_, i, icol); } // Extract qqcw from Prognostics Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, nlev), [&](int kk) { for(int m = 0; m < nmodes; ++m) { qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; @@ -127,7 +123,7 @@ void compute_tendencies( progs.q_aero_c[m][a][kk]; } }); // parallel_for nlevs - + team.team_barrier(); bool ptend_lq[pcnst]; // currently unused mam4::aero_model_drydep( // inputs @@ -159,7 +155,7 @@ void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { const int icol = team.league_rank(); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, nlev), [&](int kk) { for(int m = 0; m < nmodes; ++m) { dry_aero.int_aero_nmr[m](icol, kk) += ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; @@ -176,36 +172,38 @@ void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, // Update cloud borne aerosols using qqcw void update_cloudborne_mmrs( - const MAMDryDep::view_2d qqcw[mam4::aero_model::pcnst], const double dt, + const MAMDryDep::view_3d qqcw, const double dt, const int nlev_, // output const mam_coupling::AerosolState dry_aero) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], - qqcw[mam4::ConvProc::numptrcw_amode(m)]); + ekat::subview(qqcw, mam4::ConvProc::numptrcw_amode(m))); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { if(dry_aero.cld_aero_mmr[m][a].data()) { Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], - qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)]); + ekat::subview(qqcw, mam4::ConvProc::lmassptrcw_amode(a, m))); } } } } // Update cloud borne aerosols ends // FIXME: remove the following function after implementing file read for landuse -void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { - Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, - 0.16803558403621365E-001, 0.18076055155371872E-001, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.91803784897907303E+000, 0.17186036997038400E-002, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.18448503115578840E-001}; - - for(int icol = 0; icol < ncol; ++icol) { - for(int kk = 0; kk < 11; ++kk) { - flu[kk](icol) = temp[kk]; - } - } +void populated_fraction_landuse(MAMDryDep::view_2d flu, const int ncol) { + Kokkos::parallel_for("populated_fraction_landuse", 1, KOKKOS_LAMBDA(int ) { + static constexpr int n_land_type = MAMDryDep::n_land_type; + const Real temp[n_land_type] = + {0.28044346587077795E-003, 0.26634987180780171E-001, + 0.16803558403621365E-001, 0.18076055155371872E-001, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.91803784897907303E+000, 0.17186036997038400E-002, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.18448503115578840E-001}; + for(int i = 0; i < n_land_type; ++i) + for(int j = 0; j < ncol; ++j) + flu(i, j) = temp[i]; + }); + Kokkos::fence(); } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index ebbab7ab7571..b4625387352f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -325,24 +325,16 @@ void MAMDryDep::initialize_impl(const RunType run_type) { rho_ = view_2d("rho", ncol_, nlev_); ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); - for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { - for(int j = 0; j < aerosol_categories_; ++j) { - vlc_dry_[i][j] = view_2d("vlc_dry_[i][j]", ncol_, nlev_); - vlc_grv_[i][j] = view_2d("vlc_grv_[i][j]", ncol_, nlev_); - vlc_trb_[i][j] = view_1d("vlc_trb_[i][j]", ncol_); - } - } + vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_, nlev_); + vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_, nlev_); + vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_); - for(int i = 0; i < pcnst; ++i) { - qqcw_[i] = view_2d("qqcw_[i]", ncol_, nlev_); - dqdt_tmp_[i] = view_2d("dqdt_tmp_[i]", ncol_, nlev_); - } + qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); + dqdt_tmp_ = view_3d("dqdt_tmp_", pcnst, ncol_, nlev_); static constexpr int n_land_type = mam4::DryDeposition::n_land_type; - for(int i = 0; i < n_land_type; ++i) { - // FIXME: This should come from a file reading - fraction_landuse_[i] = view_1d("fraction_landuse_[i]", ncol_); - } + // FIXME: This should come from a file reading + fraction_landuse_ = view_2d("fraction_landuse_", n_land_type, ncol_); //----------------------------------------------------------------- // Setup preprocessing and post processing diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 8a28b940ee15..8feb71e04693 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -16,17 +16,18 @@ namespace scream { // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { public: + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int aerosol_categories_ = + mam4::DryDeposition::aerosol_categories; + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; + using view_1d = Field::view_dev_t; using view_2d = Field::view_dev_t; using view_3d = Field::view_dev_t; + using view_4d = Field::view_dev_t; using const_view_1d = Field::view_dev_t; using const_view_3d = Field::view_dev_t; - static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); - static constexpr int aerosol_categories_ = - mam4::DryDeposition::aerosol_categories; - static constexpr int n_land_type = mam4::DryDeposition::n_land_type; - private: // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -49,15 +50,15 @@ class MAMDryDep final : public scream::AtmosphereProcess { // inputs // FIXME: collect all inputs and outputs together - view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; - view_1d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; + view_2d fraction_landuse_; + view_3d vlc_trb_; view_2d rho_; - view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; - view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; - view_2d dqdt_tmp_[mam4::aero_model::pcnst]; + view_4d vlc_dry_; + view_4d vlc_grv_; + view_3d dqdt_tmp_; - view_2d qqcw_[mam4::aero_model::pcnst]; + view_3d qqcw_; public: using KT = ekat::KokkosTypes; From 344ce4ea429db569be902812b01e1ec0cb1e24c9 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 18 Jul 2024 09:58:00 -0700 Subject: [PATCH 313/904] fix start date for ARM97 test case --- components/eamxx/cime_config/config_compsets.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index 77c2de92ed32..e8c94534b919 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -99,7 +99,7 @@ 2016-08-01 2020-01-20 1999-07-10 - 1999-06-23 + 1997-06-23 2000-01-01 From 6c384867d50e6d858cf5a19d081c572924edddb8 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 18 Jul 2024 15:46:35 -0600 Subject: [PATCH 314/904] Add ustar and obklen to FM --- .../shoc/eamxx_shoc_process_interface.cpp | 9 ++++++--- .../shoc/eamxx_shoc_process_interface.hpp | 4 +--- .../src/physics/shoc/impl/shoc_main_impl.hpp | 20 +++++++++++-------- .../eamxx/src/physics/shoc/shoc_functions.hpp | 12 +++++++---- .../src/physics/shoc/shoc_functions_f90.cpp | 10 +++++----- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 15d43c82f7c6..80485bf67130 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -88,6 +88,8 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); + add_field("ustar", scalar2d, m/s, grid_name, ps); + add_field("obklen", scalar2d, m, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -157,7 +159,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) #ifdef SCREAM_SMALL_KERNELS , &m_buffer.se_b, &m_buffer.ke_b, &m_buffer.wv_b, &m_buffer.wl_b , &m_buffer.se_a, &m_buffer.ke_a, &m_buffer.wv_a, &m_buffer.wl_a - , &m_buffer.ustar, &m_buffer.kbfs, &m_buffer.obklen, &m_buffer.ustar2, &m_buffer.wstar + , &m_buffer.kbfs, &m_buffer.ustar2, &m_buffer.wstar #endif }; for (int i = 0; i < Buffer::num_1d_scalar_ncol; ++i) { @@ -332,6 +334,8 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) output.pblh = get_field_out("pbl_height").get_view(); output.shoc_ql2 = shoc_ql2; output.tkh = get_field_out("eddy_diff_heat").get_view(); + output.ustar = get_field_out("ustar").get_view(); + output.obklen = get_field_out("obklen").get_view(); // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; @@ -358,9 +362,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) temporaries.ke_a = m_buffer.ke_a; temporaries.wv_a = m_buffer.wv_a; temporaries.wl_a = m_buffer.wl_a; - temporaries.ustar = m_buffer.ustar; temporaries.kbfs = m_buffer.kbfs; - temporaries.obklen = m_buffer.obklen; temporaries.ustar2 = m_buffer.ustar2; temporaries.wstar = m_buffer.wstar; @@ -393,6 +395,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); + add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 6dfadb1eee7b..1695e60dfbc1 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -388,7 +388,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess #ifndef SCREAM_SMALL_KERNELS static constexpr int num_1d_scalar_ncol = 4; #else - static constexpr int num_1d_scalar_ncol = 17; + static constexpr int num_1d_scalar_ncol = 15; #endif static constexpr int num_1d_scalar_nlev = 1; #ifndef SCREAM_SMALL_KERNELS @@ -413,9 +413,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_1d ke_a; uview_1d wv_a; uview_1d wl_a; - uview_1d ustar; uview_1d kbfs; - uview_1d obklen; uview_1d ustar2; uview_1d wstar; #endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 120ed505f261..1e9b4746701f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -118,6 +118,8 @@ void Functions::shoc_main_internal( const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -144,9 +146,9 @@ void Functions::shoc_main_internal( {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); // Local scalars - Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, - se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, - ustar{0}, kbfs{0}, obklen{0}, ustar2{0}, wstar{0}; + Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, + se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, + kbfs{0}, ustar2{0}, wstar{0}; // Scalarize some views for single entry access const auto s_thetal = ekat::scalarize(thetal); @@ -371,6 +373,8 @@ void Functions::shoc_main_internal( const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -397,9 +401,7 @@ void Functions::shoc_main_internal( const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, @@ -625,6 +627,8 @@ Int Functions::shoc_main( const Scalar vw_sfc_s{shoc_input.vw_sfc(i)}; const Scalar phis_s{shoc_input.phis(i)}; Scalar pblh_s{0}; + Scalar ustar_s{0}; + Scalar obklen_s{0}; const auto zt_grid_s = ekat::subview(shoc_input.zt_grid, i); const auto zi_grid_s = ekat::subview(shoc_input.zi_grid, i); @@ -676,7 +680,7 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_s, // Output + pblh_s, ustar_s, obklen_s, shoc_ql2_s, tkh_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables @@ -700,14 +704,14 @@ Int Functions::shoc_main( shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output - shoc_output.pblh, shoc_output.shoc_ql2, shoc_output.tkh, // Output + shoc_output.pblh, shoc_output.ustar, shoc_output.obklen, shoc_output.shoc_ql2, shoc_output.tkh, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables shoc_history_output.wthl_sec, shoc_history_output.wqw_sec, shoc_history_output.wtke_sec, shoc_history_output.uw_sec, shoc_history_output.vw_sec, // Diagnostic Output Variables shoc_history_output.w3, shoc_history_output.wqls_sec, shoc_history_output.brunt, shoc_history_output.isotropy, // Diagnostic Output Variables // Temporaries shoc_temporaries.se_b, shoc_temporaries.ke_b, shoc_temporaries.wv_b, shoc_temporaries.wl_b, shoc_temporaries.se_a, shoc_temporaries.ke_a, shoc_temporaries.wv_a, shoc_temporaries.wl_a, - shoc_temporaries.ustar, shoc_temporaries.kbfs, shoc_temporaries.obklen, shoc_temporaries.ustar2, + shoc_temporaries.kbfs, shoc_temporaries.ustar2, shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi); #endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 74602c688c23..afb9f618d52b 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -159,6 +159,10 @@ struct Functions // planetary boundary layer depth [m] view_1d pblh; + // surface friction velocity [m/s] + view_1d ustar; + // Monin Obukhov length [m] + view_1d obklen; // cloud liquid mixing ratio variance [kg^2/kg^2] view_2d shoc_ql2; // eddy coefficient for heat [m2/s] @@ -211,9 +215,7 @@ struct Functions view_1d ke_a; view_1d wv_a; view_1d wl_a; - view_1d ustar; view_1d kbfs; - view_1d obklen; view_1d ustar2; view_1d wstar; @@ -889,6 +891,8 @@ struct Functions const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -961,6 +965,8 @@ struct Functions const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -987,9 +993,7 @@ struct Functions const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index dc45e2456aa8..b5a2a00d27c8 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2801,7 +2801,9 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, uw_sfc_d (temp_1d_d[index_counter++]), vw_sfc_d (temp_1d_d[index_counter++]), phis_d (temp_1d_d[index_counter++]), - pblh_d ("pblh",shcol); + pblh_d ("pblh",shcol), + ustar_d ("ustar",shcol), + obklen_d ("obklen",shcol); index_counter = 0; view_2d @@ -2878,7 +2880,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, SHF::SHOCInputOutput shoc_input_output{host_dse_d, tke_d, thetal_d, qw_d, horiz_wind_d, wthv_sec_d, qtracers_cxx_d, tk_d, shoc_cldfrac_d, shoc_ql_d}; - SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d, tkh_d}; + SHF::SHOCOutput shoc_output{pblh_d, ustar_d, obklen_d, shoc_ql2_d, tkh_d}; SHF::SHOCHistoryOutput shoc_history_output{shoc_mix_d, w_sec_d, thl_sec_d, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, @@ -2897,9 +2899,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, ke_a ("ke_a", shcol), wv_a ("wv_a", shcol), wl_a ("wl_a", shcol), - ustar ("ustar", shcol), kbfs ("kbfs", shcol), - obklen ("obklen", shcol), ustar2 ("ustar2", shcol), wstar ("wstar", shcol); @@ -2911,7 +2911,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, dz_zi ("dz_zi", shcol, nlevi_packs); SHF::SHOCTemporaries shoc_temporaries{ - se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, + se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, kbfs, ustar2, wstar, rho_zt, shoc_qv, tabs, dz_zt, dz_zi}; #endif From 4a0e4d99cd5ce242c3218605d1104eeb98fe33fa Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 18 Jul 2024 23:29:52 -0700 Subject: [PATCH 315/904] Fixes np1 vs npX tests and clang format --- .../eamxx_mam_dry_deposition_functions.hpp | 97 +++++++++---------- ...x_mam_dry_deposition_process_interface.cpp | 9 +- 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 1fdd8fd1243e..f5423e8eb1cf 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,8 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, - MAMDryDep::view_2d fraction_landuse_, + MAMDryDep::view_3d qtracers, MAMDryDep::view_2d fraction_landuse_, const MAMDryDep::const_view_3d dgncur_awet_, const MAMDryDep::const_view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, @@ -34,10 +33,8 @@ void compute_tendencies( MAMDryDep::view_2d aerdepdryis, // work arrays - MAMDryDep::view_2d rho_, - MAMDryDep::view_4d vlc_dry_, - MAMDryDep::view_3d vlc_trb_, - MAMDryDep::view_4d vlc_grv_, + MAMDryDep::view_2d rho_, MAMDryDep::view_4d vlc_dry_, + MAMDryDep::view_3d vlc_trb_, MAMDryDep::view_4d vlc_grv_, MAMDryDep::view_3d dqdt_tmp_) { static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = @@ -112,17 +109,15 @@ void compute_tendencies( dqdt_tmp[i] = ekat::subview(dqdt_tmp_, i, icol); } // Extract qqcw from Prognostics - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](int kk) { - for(int m = 0; m < nmodes; ++m) { - qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = - progs.n_mode_c[m][kk]; - for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) - if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) - qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = - progs.q_aero_c[m][a][kk]; - } - }); // parallel_for nlevs + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < nmodes; ++m) { + qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = + progs.q_aero_c[m][a][kk]; + } + }); // parallel_for nlevs team.team_barrier(); bool ptend_lq[pcnst]; // currently unused mam4::aero_model_drydep( @@ -149,40 +144,38 @@ void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, const mam_coupling::AerosolState dry_aero) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( - 1, nlev); + ncol, nlev); static constexpr int nmodes = mam4::AeroConfig::num_modes(); Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { const int icol = team.league_rank(); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](int kk) { - for(int m = 0; m < nmodes; ++m) { - dry_aero.int_aero_nmr[m](icol, kk) += - ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; - for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) - if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) - dry_aero.int_aero_mmr[m][a](icol, kk) += - ptend_q(icol, kk, - mam4::ConvProc::lmassptrcw_amode(a, m)) * - dt; - } - }); // parallel_for nlevs - }); // parallel_for icol + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < nmodes; ++m) { + dry_aero.int_aero_nmr[m](icol, kk) += + ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + dry_aero.int_aero_mmr[m][a](icol, kk) += + ptend_q(icol, kk, mam4::ConvProc::lmassptrcw_amode(a, m)) * + dt; + } + }); // parallel_for nlevs + }); // parallel_for icol } // Update interstitial aerosols ends // Update cloud borne aerosols using qqcw -void update_cloudborne_mmrs( - const MAMDryDep::view_3d qqcw, const double dt, - const int nlev_, - // output - const mam_coupling::AerosolState dry_aero) { +void update_cloudborne_mmrs(const MAMDryDep::view_3d qqcw, const double dt, + const int nlev_, + // output + const mam_coupling::AerosolState dry_aero) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], ekat::subview(qqcw, mam4::ConvProc::numptrcw_amode(m))); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { if(dry_aero.cld_aero_mmr[m][a].data()) { - Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], - ekat::subview(qqcw, mam4::ConvProc::lmassptrcw_amode(a, m))); + Kokkos::deep_copy( + dry_aero.cld_aero_mmr[m][a], + ekat::subview(qqcw, mam4::ConvProc::lmassptrcw_amode(a, m))); } } } @@ -190,19 +183,19 @@ void update_cloudborne_mmrs( // FIXME: remove the following function after implementing file read for landuse void populated_fraction_landuse(MAMDryDep::view_2d flu, const int ncol) { - Kokkos::parallel_for("populated_fraction_landuse", 1, KOKKOS_LAMBDA(int ) { - static constexpr int n_land_type = MAMDryDep::n_land_type; - const Real temp[n_land_type] = - {0.28044346587077795E-003, 0.26634987180780171E-001, - 0.16803558403621365E-001, 0.18076055155371872E-001, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.91803784897907303E+000, 0.17186036997038400E-002, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.18448503115578840E-001}; - for(int i = 0; i < n_land_type; ++i) - for(int j = 0; j < ncol; ++j) - flu(i, j) = temp[i]; - }); + Kokkos::parallel_for( + "populated_fraction_landuse", 1, KOKKOS_LAMBDA(int) { + static constexpr int n_land_type = MAMDryDep::n_land_type; + const Real temp[n_land_type] = { + 0.28044346587077795E-003, 0.26634987180780171E-001, + 0.16803558403621365E-001, 0.18076055155371872E-001, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.91803784897907303E+000, 0.17186036997038400E-002, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.18448503115578840E-001}; + for(int i = 0; i < n_land_type; ++i) + for(int j = 0; j < ncol; ++j) flu(i, j) = temp[i]; + }); Kokkos::fence(); } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index b4625387352f..da4e393e749b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -325,9 +325,12 @@ void MAMDryDep::initialize_impl(const RunType run_type) { rho_ = view_2d("rho", ncol_, nlev_); ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); - vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_, nlev_); - vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_, nlev_); - vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_); + vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_); qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); dqdt_tmp_ = view_3d("dqdt_tmp_", pcnst, ncol_, nlev_); From 047802bf4eec3cdd341c9ac7bc1f7b6869ce8950 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 06:29:57 -0700 Subject: [PATCH 316/904] Updates mam4xx submodule-pointed to main for wetscav qqcw update --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index cafdbc1b8813..316817f6925d 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit cafdbc1b88134038aff12c1cb00110c6397a4544 +Subproject commit 316817f6925d5d0380f624402c0c5ede3070e334 From 77fcad25dd09d878265857b5f5d69fb139c2e17c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 07:02:19 -0700 Subject: [PATCH 317/904] point mam4xx submodule to main to get Thread Vector changes among others --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index a8aa56ba915c..316817f6925d 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e +Subproject commit 316817f6925d5d0380f624402c0c5ede3070e334 From 63175dfbd18a2453f679752b8b4ccd80c60873de Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 19 Jul 2024 15:04:44 -0400 Subject: [PATCH 318/904] fix capture this warning in mam micro --- .../physics/mam/eamxx_mam_microphysics_process_interface.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7ea2adfddb63..8b93936148ce 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,6 +317,8 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index @@ -448,7 +450,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, some_step, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, From 98c8ca1a7be2e864992f8ef894f3dfc46d0ee163 Mon Sep 17 00:00:00 2001 From: Hassan Beydoun Date: Fri, 19 Jul 2024 13:05:12 -0700 Subject: [PATCH 319/904] add output variable for rain effective diameter --- components/eam/src/physics/p3/scream/micro_p3_interface.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 index edea27959d25..11d13c71a7a7 100644 --- a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 @@ -834,6 +834,8 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) real(rtype) :: icwmrst(pcols,pver) ! stratus water mixing ratio - on grid real(rtype) :: rho(pcols,pver) real(rtype) :: reff_rain(pcols,pver) + real(rtype) :: deff_rain(pcols,pver) + real(rtype) :: col_location(pcols,3),tmp_loc(pcols) ! Array of column lon (index 1) and lat (index 2) integer :: tmpi_loc(pcols) ! Global column index temp array @@ -1358,6 +1360,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) aqrain = 0._rtype anrain = 0._rtype freqr = 0._rtype + deff_rain = reff_rain*2._rtype*1.e6_rtype !rain effective diameter in microns ! Prognostic precipitation where (rain(:ncol,top_lev:) >= 1.e-7_rtype) aqrain(:ncol,top_lev:) = rain(:ncol,top_lev:) * cld_frac_r(:ncol,top_lev:) @@ -1412,6 +1415,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) call outfld('FREQI', freqi, pcols, lchnk) call outfld('FREQR', freqr, pcols, lchnk) call outfld('CDNUMC', cdnumc, pcols, lchnk) + call outfld('ADRAIN', deff_rain, pcols, lchnk) call outfld('CLOUDFRAC_LIQ_MICRO', cld_frac_l, pcols, lchnk) call outfld('CLOUDFRAC_ICE_MICRO', cld_frac_i, pcols, lchnk) From d92ea27914b7b87571b6d53355d12dac0e42606a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 13:56:26 -0700 Subject: [PATCH 320/904] Declared local variables for device (recent fixes from Naser) --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 316817f6925d..5495c27a6df7 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 316817f6925d5d0380f624402c0c5ede3070e334 +Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d From c75615f181d95850610e80c7f839f6ee4150da29 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 14:11:49 -0700 Subject: [PATCH 321/904] Submodule update: declare local vrs for device code --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 316817f6925d..5495c27a6df7 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 316817f6925d5d0380f624402c0c5ede3070e334 +Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d From b86b4df708fc6cb91d760177162c6183dfd0a0d2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 14:27:06 -0700 Subject: [PATCH 322/904] Adds 2d fields from land model and updates mam4xx sub to point to main --- .../atmosphere_surface_coupling_importer.cpp | 5 +++++ .../src/mct_coupling/scream_cpl_indices.F90 | 10 +++++++-- ...x_mam_dry_deposition_process_interface.cpp | 22 +++++++++---------- .../single-process/mam/drydep/input.yaml | 10 ++++----- externals/mam4xx | 2 +- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 52deba16e8d3..7573993a3189 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -52,6 +52,11 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("snow_depth_land", scalar2d_layout, m, grid_name); add_field("ocnfrac", scalar2d_layout, nondim, grid_name); add_field("landfrac", scalar2d_layout, nondim, grid_name); + add_field("icefrac", scalar2d_layout, nondim, grid_name); + // Friction velocity [m/s] + add_field("fv", scalar2d_layout, m/s, grid_name); + // Aerodynamical resistance + add_field("ram1", scalar2d_layout, s/m, grid_name); } // ========================================================================================= void SurfaceCouplingImporter::setup_surface_coupling_data(const SCDataManager &sc_data_manager) diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index df96edab0f06..e754d2fef1b0 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -6,7 +6,7 @@ module scream_cpl_indices private ! Focus only on the ones that scream imports/exports (subsets of x2a and a2x) - integer, parameter, public :: num_scream_imports = 16 + integer, parameter, public :: num_scream_imports = 19 integer, parameter, public :: num_scream_exports = 17 integer, public :: num_cpl_imports, num_cpl_exports, import_field_size, export_field_size @@ -89,6 +89,9 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_field_names(14) = 'surf_evap' import_field_names(15) = 'ocnfrac' import_field_names(16) = 'landfrac' + import_field_names(17) = 'icefrac' + import_field_names(18) = 'fv' + import_field_names(19) = 'ram1' ! CPL indices import_cpl_indices(1) = mct_avect_indexra(x2a,'Sx_avsdr') @@ -106,7 +109,10 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(13) = mct_avect_indexra(x2a,'Faxx_sen') import_cpl_indices(14) = mct_avect_indexra(x2a,'Faxx_evap') import_cpl_indices(15) = mct_avect_indexra(x2a,'Sf_ofrac') - import_cpl_indices(16) = mct_avect_indexra(x2a,'Sf_lfrac') + import_cpl_indices(16) = mct_avect_indexra(x2a,'Sf_lfrac') + import_cpl_indices(17) = mct_avect_indexra(x2a,'Sf_ifrac') + import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_fv') + import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_ram1') ! Vector components import_vector_components(11) = 0 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index da4e393e749b..70f69761d18b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -126,22 +126,22 @@ void MAMDryDep::set_grids( add_field("surface_friction_velocty", scalar2d, m / s, grid_name); // Land fraction [fraction] - add_field("land_fraction", scalar2d, nondim, grid_name); + add_field("landfrac", scalar2d, nondim, grid_name); // Friction velocity from land model [m/s] - add_field("friction_velocity", scalar2d, m / s, grid_name); + add_field("fv", scalar2d, m / s, grid_name); // Aerodynamical resistance from land model [s/m] - add_field("aerodynamical_resistance", scalar2d, s / m, grid_name); + add_field("ram1", scalar2d, s / m, grid_name); //----------- Variables from coupler (ice component)--------- // Ice fraction [unitless] - add_field("ice_fraction", scalar2d, nondim, grid_name); + add_field("icefrac", scalar2d, nondim, grid_name); //----------- Variables from coupler (ocean component)--------- // Ocean fraction [unitless] - add_field("ocean_fraction", scalar2d, nondim, grid_name); + add_field("ocnfrac", scalar2d, nondim, grid_name); //----------- Variables from other mam4xx processes ------------ // Geometric mean wet diameter for number distribution [m] @@ -369,18 +369,16 @@ void MAMDryDep::run_impl(const double dt) { auto obukhov_length_ = get_field_in("Obukhov_length").get_view(); // Land fraction [unitless] - auto land_fraction_ = get_field_in("land_fraction").get_view(); + auto land_fraction_ = get_field_in("landfrac").get_view(); // Ice fraction [unitless] - auto ice_fraction_ = get_field_in("ice_fraction").get_view(); + auto ice_fraction_ = get_field_in("icefrac").get_view(); // Ocean fraction [unitless] - auto ocean_fraction_ = - get_field_in("ocean_fraction").get_view(); + auto ocean_fraction_ = get_field_in("ocnfrac").get_view(); // Friction velocity from land model [m/s] - auto friction_velocity_ = - get_field_in("friction_velocity").get_view(); + auto friction_velocity_ = get_field_in("fv").get_view(); // Aerodynamical resistance from land model [s/m] auto aerodynamical_resistance_ = - get_field_in("aerodynamical_resistance").get_view(); + get_field_in("ram1").get_view(); // Sfc friction velocity or ustar [m/s] auto surface_friction_velocty_ = get_field_in("surface_friction_velocty").get_view(); diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 1de57ad7e682..685a68fcea04 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -29,12 +29,12 @@ initial_conditions: w_updraft: 1e-5 pbl_height: 0.0 Obukhov_length: -0.48167567011091336E+002 - friction_velocity: 0.47569674433601039E+000 - land_fraction: 0.14675684171817760E+000 - ice_fraction: 0.00000000000000000E+000 - ocean_fraction: 0.85324315828182240E+000 + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 surface_friction_velocty: 0.27482164092221828E+000 - aerodynamical_resistance: 0.45506166067091662E+002 + ram1: 0.45506166067091662E+002 dgnumwet: 0.41417721820867320E-007 wetdens: 0.15100083211582764E+004 diff --git a/externals/mam4xx b/externals/mam4xx index 5e7ff7200b49..5495c27a6df7 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 5e7ff7200b4910a43fbd254951c6ff61b6463e30 +Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d From ed2e0ea624c4e479cef454b9e5e40e0cff1148d6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 15:02:51 -0700 Subject: [PATCH 323/904] Adds shoc+drydep unit test --- .../multi-process/physics_only/CMakeLists.txt | 1 + .../mam/shoc_mam4_drydep/CMakeLists.txt | 41 +++++++++++ .../mam/shoc_mam4_drydep/input.yaml | 69 +++++++++++++++++++ .../mam/shoc_mam4_drydep/output.yaml | 69 +++++++++++++++++++ 4 files changed, 180 insertions(+) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index b9c13234796b..716152324e57 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -5,6 +5,7 @@ if (SCREAM_DOUBLE_PRECISION) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) add_subdirectory(mam/shoc_mam4_aci) + add_subdirectory(mam/shoc_mam4_drydep) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt new file mode 100644 index 000000000000..9f6fb8733c2c --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_mam4_drydep) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc mam + LABELS shoc physics mam4_drydep + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc physics mam4_drydep + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml new file mode 100644 index 000000000000..035c90c3a6a6 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml @@ -0,0 +1,69 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic,mam4_drydep] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables needed for SHOC + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variables needed for mam4-drydep + fraction_landuse: 0.2 + w_updraft: 1e-5 + pbl_height: 0.0 + Obukhov_length: -0.48167567011091336E+002 + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 + surface_friction_velocty: 0.27482164092221828E+000 + ram1: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml new file mode 100644 index 000000000000..d8c1291bc3c9 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml @@ -0,0 +1,69 @@ +%YAML 1.1 +--- +filename_prefix: shoc_mam4_drydep_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 9b937f91e22bbcb3abce0cbad927001913ac4299 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 18 Jul 2024 15:46:35 -0600 Subject: [PATCH 324/904] Add ustar and obklen to FM --- .../shoc/eamxx_shoc_process_interface.cpp | 9 ++++++--- .../shoc/eamxx_shoc_process_interface.hpp | 4 +--- .../src/physics/shoc/impl/shoc_main_impl.hpp | 20 +++++++++++-------- .../eamxx/src/physics/shoc/shoc_functions.hpp | 12 +++++++---- .../src/physics/shoc/shoc_functions_f90.cpp | 10 +++++----- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 15d43c82f7c6..80485bf67130 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -88,6 +88,8 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); + add_field("ustar", scalar2d, m/s, grid_name, ps); + add_field("obklen", scalar2d, m, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -157,7 +159,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) #ifdef SCREAM_SMALL_KERNELS , &m_buffer.se_b, &m_buffer.ke_b, &m_buffer.wv_b, &m_buffer.wl_b , &m_buffer.se_a, &m_buffer.ke_a, &m_buffer.wv_a, &m_buffer.wl_a - , &m_buffer.ustar, &m_buffer.kbfs, &m_buffer.obklen, &m_buffer.ustar2, &m_buffer.wstar + , &m_buffer.kbfs, &m_buffer.ustar2, &m_buffer.wstar #endif }; for (int i = 0; i < Buffer::num_1d_scalar_ncol; ++i) { @@ -332,6 +334,8 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) output.pblh = get_field_out("pbl_height").get_view(); output.shoc_ql2 = shoc_ql2; output.tkh = get_field_out("eddy_diff_heat").get_view(); + output.ustar = get_field_out("ustar").get_view(); + output.obklen = get_field_out("obklen").get_view(); // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; @@ -358,9 +362,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) temporaries.ke_a = m_buffer.ke_a; temporaries.wv_a = m_buffer.wv_a; temporaries.wl_a = m_buffer.wl_a; - temporaries.ustar = m_buffer.ustar; temporaries.kbfs = m_buffer.kbfs; - temporaries.obklen = m_buffer.obklen; temporaries.ustar2 = m_buffer.ustar2; temporaries.wstar = m_buffer.wstar; @@ -393,6 +395,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); + add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 6dfadb1eee7b..1695e60dfbc1 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -388,7 +388,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess #ifndef SCREAM_SMALL_KERNELS static constexpr int num_1d_scalar_ncol = 4; #else - static constexpr int num_1d_scalar_ncol = 17; + static constexpr int num_1d_scalar_ncol = 15; #endif static constexpr int num_1d_scalar_nlev = 1; #ifndef SCREAM_SMALL_KERNELS @@ -413,9 +413,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_1d ke_a; uview_1d wv_a; uview_1d wl_a; - uview_1d ustar; uview_1d kbfs; - uview_1d obklen; uview_1d ustar2; uview_1d wstar; #endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 120ed505f261..1e9b4746701f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -118,6 +118,8 @@ void Functions::shoc_main_internal( const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -144,9 +146,9 @@ void Functions::shoc_main_internal( {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); // Local scalars - Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, - se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, - ustar{0}, kbfs{0}, obklen{0}, ustar2{0}, wstar{0}; + Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, + se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, + kbfs{0}, ustar2{0}, wstar{0}; // Scalarize some views for single entry access const auto s_thetal = ekat::scalarize(thetal); @@ -371,6 +373,8 @@ void Functions::shoc_main_internal( const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -397,9 +401,7 @@ void Functions::shoc_main_internal( const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, @@ -625,6 +627,8 @@ Int Functions::shoc_main( const Scalar vw_sfc_s{shoc_input.vw_sfc(i)}; const Scalar phis_s{shoc_input.phis(i)}; Scalar pblh_s{0}; + Scalar ustar_s{0}; + Scalar obklen_s{0}; const auto zt_grid_s = ekat::subview(shoc_input.zt_grid, i); const auto zi_grid_s = ekat::subview(shoc_input.zi_grid, i); @@ -676,7 +680,7 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_s, // Output + pblh_s, ustar_s, obklen_s, shoc_ql2_s, tkh_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables @@ -700,14 +704,14 @@ Int Functions::shoc_main( shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output - shoc_output.pblh, shoc_output.shoc_ql2, shoc_output.tkh, // Output + shoc_output.pblh, shoc_output.ustar, shoc_output.obklen, shoc_output.shoc_ql2, shoc_output.tkh, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables shoc_history_output.wthl_sec, shoc_history_output.wqw_sec, shoc_history_output.wtke_sec, shoc_history_output.uw_sec, shoc_history_output.vw_sec, // Diagnostic Output Variables shoc_history_output.w3, shoc_history_output.wqls_sec, shoc_history_output.brunt, shoc_history_output.isotropy, // Diagnostic Output Variables // Temporaries shoc_temporaries.se_b, shoc_temporaries.ke_b, shoc_temporaries.wv_b, shoc_temporaries.wl_b, shoc_temporaries.se_a, shoc_temporaries.ke_a, shoc_temporaries.wv_a, shoc_temporaries.wl_a, - shoc_temporaries.ustar, shoc_temporaries.kbfs, shoc_temporaries.obklen, shoc_temporaries.ustar2, + shoc_temporaries.kbfs, shoc_temporaries.ustar2, shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi); #endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 74602c688c23..afb9f618d52b 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -159,6 +159,10 @@ struct Functions // planetary boundary layer depth [m] view_1d pblh; + // surface friction velocity [m/s] + view_1d ustar; + // Monin Obukhov length [m] + view_1d obklen; // cloud liquid mixing ratio variance [kg^2/kg^2] view_2d shoc_ql2; // eddy coefficient for heat [m2/s] @@ -211,9 +215,7 @@ struct Functions view_1d ke_a; view_1d wv_a; view_1d wl_a; - view_1d ustar; view_1d kbfs; - view_1d obklen; view_1d ustar2; view_1d wstar; @@ -889,6 +891,8 @@ struct Functions const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -961,6 +965,8 @@ struct Functions const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -987,9 +993,7 @@ struct Functions const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index dc45e2456aa8..b5a2a00d27c8 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2801,7 +2801,9 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, uw_sfc_d (temp_1d_d[index_counter++]), vw_sfc_d (temp_1d_d[index_counter++]), phis_d (temp_1d_d[index_counter++]), - pblh_d ("pblh",shcol); + pblh_d ("pblh",shcol), + ustar_d ("ustar",shcol), + obklen_d ("obklen",shcol); index_counter = 0; view_2d @@ -2878,7 +2880,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, SHF::SHOCInputOutput shoc_input_output{host_dse_d, tke_d, thetal_d, qw_d, horiz_wind_d, wthv_sec_d, qtracers_cxx_d, tk_d, shoc_cldfrac_d, shoc_ql_d}; - SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d, tkh_d}; + SHF::SHOCOutput shoc_output{pblh_d, ustar_d, obklen_d, shoc_ql2_d, tkh_d}; SHF::SHOCHistoryOutput shoc_history_output{shoc_mix_d, w_sec_d, thl_sec_d, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, @@ -2897,9 +2899,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, ke_a ("ke_a", shcol), wv_a ("wv_a", shcol), wl_a ("wl_a", shcol), - ustar ("ustar", shcol), kbfs ("kbfs", shcol), - obklen ("obklen", shcol), ustar2 ("ustar2", shcol), wstar ("wstar", shcol); @@ -2911,7 +2911,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, dz_zi ("dz_zi", shcol, nlevi_packs); SHF::SHOCTemporaries shoc_temporaries{ - se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, + se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, kbfs, ustar2, wstar, rho_zt, shoc_qv, tabs, dz_zt, dz_zi}; #endif From 155ec3c142e54ac65ce080e953715b8f5d6ece36 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:10:21 -0500 Subject: [PATCH 325/904] Add ums and ums002 to load libunwind 1.6.2 After recent Frontier maintenance, ums/default and ums002/default are required to load libunwind/1.6.2. --- cime_config/machines/config_machines.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index c04dff5976cd..3617bb88f9ec 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1437,6 +1437,8 @@ PrgEnv-cray craype-accel-amd-gfx90a rocm/5.4.0 + ums/default + ums002/default libunwind/1.6.2 From 555aa19b0967bb2dd6a570f8bf74aadaa6eb8f67 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:12:27 -0500 Subject: [PATCH 326/904] Add to scream_scorpio_types.hpp to use std::int64_t This fixes a known build error after recent Frontier maintenance: no type named 'int64_t' in namespace 'std'; did you mean simply 'int64_t'? --- components/eamxx/src/share/io/scream_scorpio_types.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/share/io/scream_scorpio_types.hpp b/components/eamxx/src/share/io/scream_scorpio_types.hpp index 645b1b26937b..3b16cebc06c8 100644 --- a/components/eamxx/src/share/io/scream_scorpio_types.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_types.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace scream { namespace scorpio { From 805ab1f68d5dd2599a1eaf90e16c26fc8cf47d08 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:16:28 -0500 Subject: [PATCH 327/904] Explicitly add gcc/12.2.0 lib path to CMAKE_EXE_LINKER_FLAGS This is a simple workaround to fix scream linking errors after recent Frontier maintenance before we find better solutions. --- .../cmake_macros/crayclang-scream_frontier-scream-gpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index afcca8f479e5..cc90b369244a 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -19,7 +19,7 @@ endif() string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64 -L/opt/gcc/12.2.0/snos/lib64") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{ROCM_PATH}/include") # Crusher: this resolves a crash in mct in docn init From 2f4352dd38a5b93fd9dd17c8d814d6ca7cb1e99d Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 19 Jul 2024 15:48:20 -0700 Subject: [PATCH 328/904] filter aodvis by day indices --- components/eamxx/src/diagnostics/aodvis.cpp | 19 ++++++-- .../src/diagnostics/tests/aodvis_test.cpp | 46 ++++++++++++++++--- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index efb9b5ba9510..ac16c0033730 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -2,6 +2,8 @@ #include +#include "share/util/scream_universal_constants.hpp" + namespace scream { AODVis::AODVis(const ekat::Comm &comm, const ekat::ParameterList ¶ms) @@ -23,11 +25,13 @@ void AODVis::set_grids( m_nlevs = grid->get_num_vertical_levels(); // Define layouts we need (both inputs and outputs) - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,m_swbands,"swband"); - FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, m_swbands, "swband"); + FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); // The fields required for this diagnostic to be computed add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + add_field("sunlit", scalar1d_layout, nondim, grid_name); // Construct and allocate the aodvis field FieldIdentifier fid("AerosolOpticalDepth550nm", scalar1d_layout, nondim, @@ -41,18 +45,25 @@ void AODVis::compute_diagnostic_impl() { using MT = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; + Real var_fill_value = constants::DefaultFillValue().value; + const auto aod = m_diagnostic_output.get_view(); const auto tau_vis = get_field_in("aero_tau_sw") .subfield(1, m_vis_bnd) .get_view(); + const auto sunlit = get_field_in("sunlit").get_view(); const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); - auto tau_icol = ekat::subview(tau_vis, icol); - aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + if(sunlit(icol) == 0.0) { + aod(icol) = var_fill_value; + } else { + auto tau_icol = ekat::subview(tau_vis, icol); + aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + } }); } diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 140114812215..15de175da67d 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -1,10 +1,11 @@ +#include + #include "catch2/catch.hpp" #include "diagnostics/register_diagnostics.hpp" #include "share/field/field_utils.hpp" #include "share/grid/mesh_free_grids_manager.hpp" #include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_utils.hpp" - +#include "share/util/scream_universal_constants.hpp" namespace scream { std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, @@ -30,6 +31,10 @@ TEST_CASE("aodvis") { using namespace ShortFieldTagsNames; using namespace ekat::units; + Real var_fill_value = constants::DefaultFillValue().value; + + Real some_limit = 0.0025; + // A world comm ekat::Comm comm(MPI_COMM_WORLD); @@ -40,7 +45,7 @@ TEST_CASE("aodvis") { // Create a grids manager - single column for these tests constexpr int nlevs = 33; - const int ngcols = 2 * comm.size(); + const int ngcols = 10 * comm.size(); int nbnds = eamxx_swbands(); int swvis = eamxx_vis_swband_idx(); @@ -49,16 +54,23 @@ TEST_CASE("aodvis") { auto grid = gm->get_grid("Physics"); // Input (randomized) tau - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,nbnds,"swband"); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, nbnds, "swband"); FieldIdentifier tau_fid("aero_tau_sw", scalar3d_swband_layout, nondim, grid->name()); Field tau(tau_fid); tau.allocate_view(); tau.get_header().get_tracking().update_time_stamp(t0); + // Input (randomized) sunlit + FieldLayout scalar2d_layout = grid->get_2d_scalar_layout(); + FieldIdentifier sunlit_fid("sunlit", scalar2d_layout, nondim, grid->name()); + Field sunlit(sunlit_fid); + sunlit.allocate_view(); + sunlit.get_header().get_tracking().update_time_stamp(t0); // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(0, 0.05); + RPDF pdf(0, 0.005); auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -71,18 +83,33 @@ TEST_CASE("aodvis") { // Randomize tau randomize(tau, engine, pdf); + // Randomize sunlit + randomize(sunlit, engine, pdf); + // Create and set up the diagnostic ekat::ParameterList params; auto diag = diag_factory.create("AerosolOpticalDepth550nm", comm, params); diag->set_grids(gm); diag->set_required_field(tau); + diag->set_required_field(sunlit); diag->initialize(t0, RunType::Initial); + auto sun_h = sunlit.get_view(); + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + // zero out all sun_h if below 0.05 + if(sun_h(icol) < some_limit) { + sun_h(icol) = 0.0; + } + } + // sync to device + sunlit.sync_to_dev(); + // Run diag diag->compute_diagnostic(); // Check result tau.sync_to_host(); + sunlit.sync_to_host(); diag->get_diagnostic().sync_to_host(); const auto tau_h = tau.get_view(); @@ -93,12 +120,17 @@ TEST_CASE("aodvis") { auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - for(int ilev = 0; ilev < nlevs; ++ilev) { - aod_t(icol) += tau_h(icol, swvis, ilev); + if(sun_h(icol) < some_limit) { + aod_t(icol) = var_fill_value; + } else { + for(int ilev = 0; ilev < nlevs; ++ilev) { + aod_t(icol) += tau_h(icol, swvis, ilev); + } } } aod_hf.sync_to_dev(); aod_tf.sync_to_dev(); + // Workaround for non-bfb behavior of view_reduction() in release builds if(SCREAM_BFB_TESTING) { REQUIRE(views_are_equal(aod_hf, aod_tf)); From 3baa1b1827c72dc8c36b006b49017a8136d4a70b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 16:01:44 -0700 Subject: [PATCH 329/904] Uses shoc output variables for obklen and ustar in drydep --- .../mam/eamxx_mam_dry_deposition_process_interface.cpp | 9 ++++----- .../physics_only/mam/shoc_mam4_drydep/input.yaml | 2 -- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 70f69761d18b..047878796d9a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -120,10 +120,10 @@ void MAMDryDep::set_grids( //----------- Variables from coupler (land component)--------- // Obukhov length [m] - add_field("Obukhov_length", scalar2d, m, grid_name); + add_field("obklen", scalar2d, m, grid_name); // Surface friction velocty or ustar[m/s] - add_field("surface_friction_velocty", scalar2d, m / s, grid_name); + add_field("ustar", scalar2d, m / s, grid_name); // Land fraction [fraction] add_field("landfrac", scalar2d, nondim, grid_name); @@ -366,8 +366,7 @@ void MAMDryDep::run_impl(const double dt) { // Wet density of interstitial aerosol [kg/m3] auto wet_dens_ = get_field_in("wetdens").get_view(); // Obukhov length [m] - auto obukhov_length_ = - get_field_in("Obukhov_length").get_view(); + auto obukhov_length_ = get_field_in("obklen").get_view(); // Land fraction [unitless] auto land_fraction_ = get_field_in("landfrac").get_view(); // Ice fraction [unitless] @@ -381,7 +380,7 @@ void MAMDryDep::run_impl(const double dt) { get_field_in("ram1").get_view(); // Sfc friction velocity or ustar [m/s] auto surface_friction_velocty_ = - get_field_in("surface_friction_velocty").get_view(); + get_field_in("ustar").get_view(); // ------------------------------------------------------------- // Output fields for the process diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml index 035c90c3a6a6..c451008adc76 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml @@ -53,12 +53,10 @@ initial_conditions: fraction_landuse: 0.2 w_updraft: 1e-5 pbl_height: 0.0 - Obukhov_length: -0.48167567011091336E+002 fv: 0.47569674433601039E+000 landfrac: 0.14675684171817760E+000 icefrac: 0.00000000000000000E+000 ocnfrac: 0.85324315828182240E+000 - surface_friction_velocty: 0.27482164092221828E+000 ram1: 0.45506166067091662E+002 dgnumwet: 0.41417721820867320E-007 wetdens: 0.15100083211582764E+004 From f78062d1a993acd389681c7f4445e45cc258f4bf Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Sun, 21 Jul 2024 17:56:52 -0600 Subject: [PATCH 330/904] make separate subroutines whenever subglacial needs to be in subroutine argument --- .../shared/mpas_ocn_surface_bulk_forcing.F | 79 ++++++++-- .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 20 ++- .../src/shared/mpas_ocn_thick_surface_flux.F | 139 ++++++++++++++---- .../mpas_ocn_tracer_surface_flux_to_tend.F | 116 ++++++++++++--- 4 files changed, 291 insertions(+), 63 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 2cf79881a7f8..168f8c31e279 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -49,7 +49,8 @@ module ocn_surface_bulk_forcing public :: ocn_surface_bulk_forcing_tracers, & ocn_surface_bulk_forcing_vel, & ocn_surface_bulk_forcing_thick, & - ocn_surface_bulk_forcing_init + ocn_surface_bulk_forcing_init, & + ocn_surface_bulk_forcing_thick_subglacial_runoff !-------------------------------------------------------------------- ! @@ -281,7 +282,7 @@ end subroutine ocn_surface_bulk_forcing_vel!}}} ! !----------------------------------------------------------------------- - subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, surfaceThicknessFluxRunoff, surfaceThicknessFluxSubglacialRunoff, err)!{{{ + subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, surfaceThicknessFluxRunoff, err)!{{{ !----------------------------------------------------------------- ! @@ -292,8 +293,6 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur real (kind=RKIND), dimension(:), intent(inout) :: surfaceThicknessFlux !< Input/Output: Array for surface thickness flux real (kind=RKIND), dimension(:), intent(inout) :: & surfaceThicknessFluxRunoff !< Input/Output: Array for surface thickness flux due to river runoff - real (kind=RKIND), dimension(:), intent(inout) :: & - surfaceThicknessFluxSubglacialRunoff !< Input/Output: Array for surface thickness flux due to subglacial runoff !----------------------------------------------------------------- ! @@ -313,7 +312,6 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur real (kind=RKIND), dimension(:), pointer :: evaporationFlux, snowFlux real (kind=RKIND), dimension(:), pointer :: seaIceFreshWaterFlux, icebergFreshWaterFlux, riverRunoffFlux, iceRunoffFlux - real (kind=RKIND), dimension(:), pointer :: subglacialRunoffFlux real (kind=RKIND), dimension(:), pointer :: rainFlux err = 0 @@ -327,7 +325,6 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur call mpas_pool_get_array(forcingPool, 'seaIceFreshWaterFlux', seaIceFreshWaterFlux) call mpas_pool_get_array(forcingPool, 'icebergFreshWaterFlux', icebergFreshWaterFlux) call mpas_pool_get_array(forcingPool, 'riverRunoffFlux', riverRunoffFlux) - call mpas_pool_get_array(forcingPool, 'subglacialRunoffFlux', subglacialRunoffFlux) call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) call mpas_pool_get_array(forcingPool, 'rainFlux', rainFlux) @@ -357,7 +354,68 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur !$omp end parallel #endif - if (trim(config_subglacial_runoff_mode) == 'data') then +#ifdef MPAS_OPENACC + !$acc exit data delete(evaporationFlux, snowFlux, seaIceFreshWaterFlux, icebergFreshWaterFlux, & + !$acc riverRunoffFlux, iceRunoffFlux, rainFlux) +#endif + + call mpas_timer_stop("bulk_thick") + + end subroutine ocn_surface_bulk_forcing_thick!}}} + +!*********************************************************************** +! +! routine ocn_surface_bulk_forcing_thick_subglacial_runoff +! +!> \brief Determines the thickness forcing array used for the bulk forcing. +!> \author Doug Jacobsen +!> \date 04/25/12 +!> \details +!> This routine computes the thickness forcing arrays used later in MPAS. +! +!----------------------------------------------------------------------- + + subroutine ocn_surface_bulk_forcing_thick_subglacial_runoff(forcingPool, surfaceThicknessFluxSubglacialRunoff, err)!{{{ + + !----------------------------------------------------------------- + ! + ! input/output variables + ! + !----------------------------------------------------------------- + type (mpas_pool_type), intent(inout) :: forcingPool !< Input: Forcing information + real (kind=RKIND), dimension(:), intent(inout) :: & + surfaceThicknessFluxSubglacialRunoff !< Input/Output: Array for surface thickness flux due to subglacial runoff + + !----------------------------------------------------------------- + ! + ! output variables + ! + !----------------------------------------------------------------- + + integer, intent(out) :: err !< Output: Error flag + + !----------------------------------------------------------------- + ! + ! local variables + ! + !----------------------------------------------------------------- + + integer :: iCell, nCells + + real (kind=RKIND), dimension(:), pointer :: subglacialRunoffFlux + + err = 0 + + if (bulkThicknessFluxOff) return + + call mpas_timer_start("bulk_thick", .false.) + + call mpas_pool_get_array(forcingPool, 'subglacialRunoffFlux', subglacialRunoffFlux) + + nCells = nCellsHalo( 2 ) + + ! Build surface fluxes at cell centers + #ifdef MPAS_OPENACC !$acc enter data copyin(subglacialRunoffFlux) @@ -374,16 +432,15 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur !$omp end do !$omp end parallel #endif - end if #ifdef MPAS_OPENACC - !$acc exit data delete(evaporationFlux, snowFlux, seaIceFreshWaterFlux, icebergFreshWaterFlux, & - !$acc riverRunoffFlux, subglacialRunoffFlux, iceRunoffFlux, rainFlux) + !$acc exit data delete(subglacialRunoffFlux) #endif call mpas_timer_stop("bulk_thick") - end subroutine ocn_surface_bulk_forcing_thick!}}} + end subroutine ocn_surface_bulk_forcing_thick_subglacial_runoff!}}} + !*********************************************************************** ! diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index c6d9e72f83a9..e3f47d1316b0 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -220,8 +220,12 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ ! Compute surface mass flux array from bulk forcing call ocn_surface_bulk_forcing_thick(forcingPool, & surfaceThicknessFlux, & - surfaceThicknessFluxRunoff, & + surfaceThicknessFluxRunoff, err) + + if (trim(config_subglacial_runoff_mode) == 'data') then + call ocn_surface_bulk_forcing_thick_subglacial_runoff(forcingPool, & surfaceThicknessFluxSubglacialRunoff, err) + end if ! Compute surface thickness flux from land ice call ocn_surface_land_ice_fluxes_thick(forcingPool, & @@ -242,11 +246,14 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ ! Compute surface flux tendency call ocn_thick_surface_flux_tend(fractionAbsorbed, & fractionAbsorbedRunoff, & - fractionAbsorbedSubglacialRunoff, & surfaceThicknessFlux, & surfaceThicknessFluxRunoff, & + tendThick, err) + if (trim(config_subglacial_runoff_mode) == 'data') then + call ocn_thick_surface_flux_tend(fractionAbsorbedSubglacialRunoff, & surfaceThicknessFluxSubglacialRunoff, & tendThick, err) + end if ! Compute contribution from frazil ice formation call ocn_frazil_forcing_layer_thickness(forcingPool, & @@ -1244,10 +1251,15 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & endif ! compute budgets call ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, & - fractionAbsorbedRunoff, fractionAbsorbedSubglacialRunoff, & + fractionAbsorbedRunoff, & layerThickness, tracerGroupSurfaceFlux, & - tracerGroupSurfaceFluxRunoff, tracerGroupSurfaceFluxSubglacialRunoff, & + tracerGroupSurfaceFluxRunoff, & + tracerGroupTend, err) + if (trim(config_subglacial_runoff_mode) == 'data') then + call ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbedSubglacialRunoff, & + layerThickness, tracerGroupSurfaceFluxSubglacialRunoff, & tracerGroupTend, err) + end if ! ! Compute shortwave absorption diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F index eb531a999582..8e389a74ba37 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F @@ -46,7 +46,8 @@ module ocn_thick_surface_flux !-------------------------------------------------------------------- public :: ocn_thick_surface_flux_tend, & - ocn_thick_surface_flux_init + ocn_thick_surface_flux_init, & + ocn_thick_surface_flux_tend_subglacial_runoff !-------------------------------------------------------------------- ! @@ -74,8 +75,7 @@ module ocn_thick_surface_flux !----------------------------------------------------------------------- subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoefficientsRunoff, & - transmissionCoefficientsSubglacialRunoff, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & - surfaceThicknessFluxSubglacialRunoff, tend, err)!{{{ + surfaceThicknessFlux, surfaceThicknessFluxRunoff, tend, err)!{{{ !----------------------------------------------------------------- ! ! input variables @@ -84,13 +84,11 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe real (kind=RKIND), dimension(:,:), intent(in) :: & transmissionCoefficients, &!< Input: Coefficients for the transmission of surface fluxes - transmissionCoefficientsRunoff, &!< Input: Coefficients for the transmission of surface fluxes due to river runoff - transmissionCoefficientsSubglacialRunoff !< Input: Coefficients for the transmission of surface fluxes due to subglacial runoff + transmissionCoefficientsRunoff !< Input: Coefficients for the transmission of surface fluxes due to river runoff real (kind=RKIND), dimension(:), intent(in) :: & - surfaceThicknessFlux, &!< Input: surface flux of thickness - surfaceThicknessFluxRunoff, &!< Input: surface flux of thickness due to river runoff - surfaceThicknessFluxSubglacialRunoff !< Input: surface flux of thickness due to subglacial runoff + surfaceThicknessFlux, &!< Input: surface flux of thickness + surfaceThicknessFluxRunoff !< Input: surface flux of thickness due to river runoff !----------------------------------------------------------------- @@ -118,7 +116,7 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe integer :: iCell, k - real (kind=RKIND) :: remainingFlux, remainingFluxRunoff, remainingFluxSubglacialRunoff + real (kind=RKIND) :: remainingFlux, remainingFluxRunoff err = 0 @@ -127,32 +125,24 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe call mpas_timer_start("thick surface flux") #ifdef MPAS_OPENACC - !$acc enter data copyin(transmissionCoefficients, transmissionCoefficientsRunoff, transmissionCoefficientsSubglacialRunoff) + !$acc enter data copyin(transmissionCoefficients, transmissionCoefficientsRunoff) !$acc parallel loop & !$acc present(tend, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & - !$acc surfaceThicknessFluxSubglacialRunoff, transmissionCoefficientsSubglacialRunoff, & !$acc transmissionCoefficients, transmissionCoefficientsRunoff, minLevelCell, maxLevelCell) & - !$acc private(k, remainingFlux, remainingFluxRunoff, remainingFluxSubglacialRunoff) + !$acc private(k, remainingFlux, remainingFluxRunoff) #else !$omp parallel - !$omp do schedule(runtime) private(remainingFlux, remainingFluxRunoff, remainingFluxSubglacialRunoff, k) + !$omp do schedule(runtime) private(remainingFlux, remainingFluxRunoff, k) #endif do iCell = 1, nCellsOwned remainingFlux = 1.0_RKIND remainingFluxRunoff = 1.0_RKIND - if (trim(config_subglacial_runoff_mode) == 'data') then - remainingFluxSubglacialRunoff = 1.0_RKIND - end if do k = minLevelCell(iCell), maxLevelCell(iCell) remainingFlux = remainingFlux - transmissionCoefficients(k, iCell) remainingFluxRunoff = remainingFluxRunoff - transmissionCoefficientsRunoff(k, iCell) tend(k, iCell) = tend(k, iCell) + surfaceThicknessFlux(iCell) * transmissionCoefficients(k, iCell) & + surfaceThicknessFluxRunoff(iCell) * transmissionCoefficientsRunoff(k, iCell) - if (trim(config_subglacial_runoff_mode) == 'data') then - remainingFluxSubglacialRunoff = remainingFluxSubglacialRunoff - transmissionCoefficientsSubglacialRunoff(k, iCell) - tend(k, iCell) = tend(k, iCell) + surfaceThicknessFluxSubglacialRunoff(iCell) * transmissionCoefficientsSubglacialRunoff(k, iCell) - end if end do if(maxLevelCell(iCell) > 0 .and. remainingFlux > 0.0_RKIND) then @@ -164,12 +154,6 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe + remainingFluxRunoff * surfaceThicknessFluxRunoff(iCell) end if - if (trim(config_subglacial_runoff_mode) == 'data') then - if(maxLevelCell(iCell) > 0 .and. remainingFluxSubglacialRunoff > 0.0_RKIND) then - tend(maxLevelCell(iCell), iCell) = tend(maxLevelCell(iCell), iCell) & - + remainingFluxSubglacialRunoff * surfaceThicknessFluxSubglacialRunoff(iCell) - end if - end if end do #ifndef MPAS_OPENACC !$omp end do @@ -177,7 +161,7 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe #endif #ifdef MPAS_OPENACC - !$acc exit data delete(transmissionCoefficients, transmissionCoefficientsRunoff, transmissionCoefficientsSubglacialRunoff) + !$acc exit data delete(transmissionCoefficients, transmissionCoefficientsRunoff) #endif call mpas_timer_stop("thick surface flux") @@ -186,6 +170,107 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe end subroutine ocn_thick_surface_flux_tend!}}} +!*********************************************************************** +! +! routine ocn_thick_surface_flux_tend_subglacial_runoff +! +!> \brief Computes tendency term from horizontal advection of thickness +!> \author Doug Jacobsen +!> \date 15 September 2011 +!> \details +!> This routine computes the horizontal advection tendency for +!> thickness based on current state and user choices of forcings. +! +!----------------------------------------------------------------------- + + subroutine ocn_thick_surface_flux_tend_subglacial_runoff(transmissionCoefficientsSubglacialRunoff, & + surfaceThicknessFluxSubglacialRunoff, tend, err)!{{{ + !----------------------------------------------------------------- + ! + ! input variables + ! + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:,:), intent(in) :: & + transmissionCoefficientsSubglacialRunoff !< Input: Coefficients for the transmission of surface fluxes due to subglacial runoff + + real (kind=RKIND), dimension(:), intent(in) :: & + surfaceThicknessFluxSubglacialRunoff !< Input: surface flux of thickness due to subglacial runoff + + + !----------------------------------------------------------------- + ! + ! input/output variables + ! + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:,:), intent(inout) :: & + tend !< Input/Output: thickness tendency + + !----------------------------------------------------------------- + ! + ! output variables + ! + !----------------------------------------------------------------- + + integer, intent(out) :: err !< Output: error flag + + !----------------------------------------------------------------- + ! + ! local variables + ! + !----------------------------------------------------------------- + + integer :: iCell, k + + real (kind=RKIND) :: remainingFlux, remainingFluxRunoff, remainingFluxSubglacialRunoff + + err = 0 + + if (.not. surfaceThicknessFluxOn) return + + call mpas_timer_start("thick surface flux") + +#ifdef MPAS_OPENACC + !$acc enter data copyin(transmissionCoefficientsSubglacialRunoff) + + !$acc parallel loop & + !$acc present(tend, & + !$acc surfaceThicknessFluxSubglacialRunoff, transmissionCoefficientsSubglacialRunoff, & + !$acc minLevelCell, maxLevelCell) & + !$acc private(k, remainingFluxSubglacialRunoff) +#else + !$omp parallel + !$omp do schedule(runtime) private(remainingFluxSubglacialRunoff, k) +#endif + do iCell = 1, nCellsOwned + remainingFluxSubglacialRunoff = 1.0_RKIND + do k = minLevelCell(iCell), maxLevelCell(iCell) + remainingFluxSubglacialRunoff = remainingFluxSubglacialRunoff - transmissionCoefficientsSubglacialRunoff(k, iCell) + tend(k, iCell) = tend(k, iCell) + surfaceThicknessFluxSubglacialRunoff(iCell) * transmissionCoefficientsSubglacialRunoff(k, iCell) + end do + + if(maxLevelCell(iCell) > 0 .and. remainingFluxSubglacialRunoff > 0.0_RKIND) then + tend(maxLevelCell(iCell), iCell) = tend(maxLevelCell(iCell), iCell) & + + remainingFluxSubglacialRunoff * surfaceThicknessFluxSubglacialRunoff(iCell) + end if + + end do +#ifndef MPAS_OPENACC + !$omp end do + !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc exit data delete(transmissionCoefficientsSubglacialRunoff) +#endif + + call mpas_timer_stop("thick surface flux") + + !-------------------------------------------------------------------- + + end subroutine ocn_thick_surface_flux_tend_subglacial_runoff!}}} + !*********************************************************************** ! ! routine ocn_thick_surface_flux_init diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F index 4a1dc5fce098..fd754562a579 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F @@ -45,7 +45,8 @@ module ocn_tracer_surface_flux_to_tend !-------------------------------------------------------------------- public :: ocn_tracer_surface_flux_tend, & - ocn_tracer_surface_flux_init + ocn_tracer_surface_flux_init, & + ocn_tracer_surface_flux_tend_subglacial_runoff !-------------------------------------------------------------------- ! @@ -71,8 +72,8 @@ module ocn_tracer_surface_flux_to_tend ! !----------------------------------------------------------------------- - subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbsorbedRunoff, fractionAbsorbedSubglacialRunoff, & - layerThickness, surfaceTracerFlux, surfaceTracerFluxRunoff, surfaceTracerFluxSubglacialRunoff, tend, err)!{{{ + subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbsorbedRunoff, & + layerThickness, surfaceTracerFlux, surfaceTracerFluxRunoff, tend, err)!{{{ !----------------------------------------------------------------- ! ! input variables @@ -97,13 +98,6 @@ subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbso real (kind=RKIND), dimension(:,:), intent(in) :: & fractionAbsorbedRunoff !< Input: Coefficients for the application of surface fluxes due to river runoff - real (kind=RKIND), dimension(:,:), intent(in), pointer :: & - surfaceTracerFluxSubglacialRunoff !< Input: surface tracer fluxes from subglacial runoff - - real (kind=RKIND), dimension(:,:), intent(in) :: & - fractionAbsorbedSubglacialRunoff !< Input: Coefficients for the application of surface fluxes due to subglacial runoff - - !----------------------------------------------------------------- ! ! input/output variables @@ -204,14 +198,96 @@ subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbso call mpas_timer_stop("surface_tracer_runoff_flux") end if + !-------------------------------------------------------------------- + + end subroutine ocn_tracer_surface_flux_tend!}}} + + +!*********************************************************************** +! +! routine ocn_tracer_surface_flux_tend_subglacial_runoff +! +!> \brief Computes tendency term for surface fluxes +!> \author Doug Jacobsen +!> \date 12/17/12 +!> \details +!> This routine computes the tendency for tracers based on surface fluxes. +! +!----------------------------------------------------------------------- + + subroutine ocn_tracer_surface_flux_tend_subglacial_runoff(meshPool, fractionAbsorbedSubglacialRunoff, & + layerThickness, surfaceTracerFluxSubglacialRunoff, tend, err)!{{{ + !----------------------------------------------------------------- + ! + ! input variables + ! + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(in) :: & + meshPool !< Input: mesh information + + real (kind=RKIND), dimension(:,:), intent(in) :: & + layerThickness !< Input: Layer thickness + + real (kind=RKIND), dimension(:,:), intent(in), pointer :: & + surfaceTracerFluxSubglacialRunoff !< Input: surface tracer fluxes from subglacial runoff + + real (kind=RKIND), dimension(:,:), intent(in) :: & + fractionAbsorbedSubglacialRunoff !< Input: Coefficients for the application of surface fluxes due to subglacial runoff + + + !----------------------------------------------------------------- + ! + ! input/output variables + ! + !----------------------------------------------------------------- + + real (kind=RKIND), dimension(:,:,:), intent(inout) :: & + tend !< Input/Output: velocity tendency + + !----------------------------------------------------------------- + ! + ! output variables + ! + !----------------------------------------------------------------- + + integer, intent(out) :: err !< Output: error flag + + !----------------------------------------------------------------- + ! + ! local variables + ! + !----------------------------------------------------------------- + + integer :: iCell, k, iTracer, nTracers, nCells + integer, pointer :: nVertLevels + integer, dimension(:), pointer :: nCellsArray + integer, dimension(:), pointer :: minLevelCell, maxLevelCell + + real (kind=RKIND) :: remainingFlux + + err = 0 + + if (.not. surfaceTracerFluxOn) return + + call mpas_timer_start("surface_tracer_flux") + + call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) + call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) + nTracers = size(tend, dim=1) + + call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) + call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) + + nCells = nCellsArray( 1 ) + ! now do subglacial runoff component - if (trim(config_subglacial_runoff_mode) == 'data') then - call mpas_timer_start("surface_tracer_subglacial_runoff_flux") + call mpas_timer_start("surface_tracer_subglacial_runoff_flux") - !$omp parallel - !$omp do schedule(runtime) private(remainingFlux, k, iTracer) - do iCell = 1, nCells + !$omp parallel + !$omp do schedule(runtime) private(remainingFlux, k, iTracer) + do iCell = 1, nCells remainingFlux = 1.0_RKIND do k = minLevelCell(iCell), maxLevelCell(iCell) remainingFlux = remainingFlux - fractionAbsorbedSubglacialRunoff(k, iCell) @@ -228,15 +304,13 @@ subroutine ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbed, fractionAbso + surfaceTracerFluxSubglacialRunoff(iTracer, iCell) * remainingFlux end do end if - end do - !$omp end do - !$omp end parallel + end do + !$omp end do + !$omp end parallel - call mpas_timer_stop("surface_tracer_subglacial_runoff_flux") - end if !-------------------------------------------------------------------- - end subroutine ocn_tracer_surface_flux_tend!}}} + end subroutine ocn_tracer_surface_flux_tend_subglacial_runoff!}}} !*********************************************************************** ! From 9ff4253ce41be1da822001d6cb7f3208e0a92260 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Sun, 21 Jul 2024 18:53:48 -0600 Subject: [PATCH 331/904] fix function call typos --- components/mpas-ocean/src/shared/mpas_ocn_tendency.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index e3f47d1316b0..b855512fb294 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -250,7 +250,7 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ surfaceThicknessFluxRunoff, & tendThick, err) if (trim(config_subglacial_runoff_mode) == 'data') then - call ocn_thick_surface_flux_tend(fractionAbsorbedSubglacialRunoff, & + call ocn_thick_surface_flux_tend_subglacial_runoff(fractionAbsorbedSubglacialRunoff, & surfaceThicknessFluxSubglacialRunoff, & tendThick, err) end if @@ -1256,7 +1256,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & tracerGroupSurfaceFluxRunoff, & tracerGroupTend, err) if (trim(config_subglacial_runoff_mode) == 'data') then - call ocn_tracer_surface_flux_tend(meshPool, fractionAbsorbedSubglacialRunoff, & + call ocn_tracer_surface_flux_tend_subglacial_runoff(meshPool, fractionAbsorbedSubglacialRunoff, & layerThickness, tracerGroupSurfaceFluxSubglacialRunoff, & tracerGroupTend, err) end if From 476908cbf811143a1066833171c2b1c9f250f925 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Sun, 21 Jul 2024 21:15:47 -0600 Subject: [PATCH 332/904] more new subrutines with _subglacial_runoff --- .../shared/mpas_ocn_surface_bulk_forcing.F | 218 +++++++++++++++--- .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 6 +- 2 files changed, 187 insertions(+), 37 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 168f8c31e279..81732b637a97 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -50,8 +50,10 @@ module ocn_surface_bulk_forcing ocn_surface_bulk_forcing_vel, & ocn_surface_bulk_forcing_thick, & ocn_surface_bulk_forcing_init, & + ocn_surface_bulk_forcing_tracers_subglacial_runoff, & ocn_surface_bulk_forcing_thick_subglacial_runoff + !-------------------------------------------------------------------- ! ! Private module variables @@ -79,7 +81,7 @@ module ocn_surface_bulk_forcing !----------------------------------------------------------------------- subroutine ocn_surface_bulk_forcing_tracers(meshPool, groupName, forcingPool, tracerGroup, & - tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxSubglacialRunoff, tracersSurfaceFluxRemoved, dt, layerThickness, err)!{{{ + tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxRemoved, dt, layerThickness, err)!{{{ !----------------------------------------------------------------- ! @@ -99,8 +101,6 @@ subroutine ocn_surface_bulk_forcing_tracers(meshPool, groupName, forcingPool, tr real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFlux !< Input/Output: Surface flux for tracer group real (kind=RKIND), dimension(:,:), intent(inout) :: & tracersSurfaceFluxRunoff !< Input/Output: Surface flux for tracer group due to river runoff - real (kind=RKIND), dimension(:,:), intent(inout) :: & - tracersSurfaceFluxSubglacialRunoff !< Input/Output: Surface flux for tracer group due to subglacial runoff real (kind=RKIND), dimension(:,:), intent(inout) :: & tracersSurfaceFluxRemoved !< Input/Output: Accumulator for ignored Surface flux for tracer group real (kind=RKIND), dimension(:,:), intent(in) :: layerThickness @@ -125,12 +125,69 @@ subroutine ocn_surface_bulk_forcing_tracers(meshPool, groupName, forcingPool, tr call mpas_timer_start("bulk_" // trim(groupName)) if ( trim(groupName) == 'activeTracers' ) then call ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracerGroup, & - tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxSubglacialRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err) + tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err) end if call mpas_timer_stop("bulk_" // trim(groupName)) end subroutine ocn_surface_bulk_forcing_tracers!}}} +!*********************************************************************** +! +! routine ocn_surface_bulk_forcing_tracers_subglacial_runoff +! +!> \brief Determines the tracers forcing array used for the bulk forcing. +!> \author Doug Jacobsen +!> \date 04/25/12 +!> \details +!> This routine computes the tracers forcing arrays used later in MPAS. +! +!----------------------------------------------------------------------- + + subroutine ocn_surface_bulk_forcing_tracers_subglacial_runoff(meshPool, groupName, forcingPool, & + tracersSurfaceFluxSubglacialRunoff, err)!{{{ + + !----------------------------------------------------------------- + ! + ! input variables + ! + !----------------------------------------------------------------- + type (mpas_pool_type), intent(in) :: meshPool !< Input: mesh information + character (len=*) :: groupName !< Input: Name of tracer group + + !----------------------------------------------------------------- + ! + ! input/output variables + ! + !----------------------------------------------------------------- + type (mpas_pool_type), intent(inout) :: forcingPool !< Input: Forcing information + real (kind=RKIND), dimension(:,:), intent(inout) :: & + tracersSurfaceFluxSubglacialRunoff !< Input/Output: Surface flux for tracer group due to subglacial runoff + + !----------------------------------------------------------------- + ! + ! output variables + ! + !----------------------------------------------------------------- + + integer, intent(out) :: err !< Output: Error flag + + !----------------------------------------------------------------- + ! + ! local variables + ! + !----------------------------------------------------------------- + + err = 0 + + call mpas_timer_start("bulk_" // trim(groupName)) + if ( trim(groupName) == 'activeTracers' ) then + call ocn_surface_bulk_forcing_active_tracers_subglacial_runoff(meshPool, forcingPool, & + tracersSurfaceFluxSubglacialRunoff, err) + end if + call mpas_timer_stop("bulk_" // trim(groupName)) + + end subroutine ocn_surface_bulk_forcing_tracers_subglacial_runoff!}}} + !*********************************************************************** ! ! routine ocn_surface_bulk_forcing_vel @@ -503,7 +560,7 @@ end subroutine ocn_surface_bulk_forcing_init!}}} !----------------------------------------------------------------------- subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracerGroup, & - tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxSubglacialRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err)!{{{ + tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err)!{{{ !----------------------------------------------------------------- ! @@ -520,7 +577,6 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer type (mpas_pool_type), intent(inout) :: forcingPool !< Input: Forcing information real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFlux real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFluxRunoff - real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFluxSubglacialRunoff real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFluxRemoved real (kind=RKIND), dimension(:,:,:), intent(inout) :: tracerGroup real (kind=RKIND), dimension(:,:), intent(in) :: layerThickness @@ -548,7 +604,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer type(mpas_pool_type),pointer :: tracersSurfaceFluxPool real (kind=RKIND), dimension(:), pointer :: latentHeatFlux, sensibleHeatFlux, longWaveHeatFluxUp, longWaveHeatFluxDown, & - seaIceHeatFlux, icebergHeatFlux, evaporationFlux, riverRunoffFlux, subglacialRunoffFlux + seaIceHeatFlux, icebergHeatFlux, evaporationFlux, riverRunoffFlux real (kind=RKIND), dimension(:), pointer :: seaIceFreshWaterFlux, icebergFreshWaterFlux, seaIceSalinityFlux, iceRunoffFlux real (kind=RKIND), dimension(:), pointer :: shortWaveHeatFlux, penetrativeTemperatureFlux real (kind=RKIND), dimension(:), pointer :: snowFlux, rainFlux @@ -589,7 +645,6 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer call mpas_pool_get_array(forcingPool, 'seaIceSalinityFlux', seaIceSalinityFlux) call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) call mpas_pool_get_array(forcingPool, 'riverRunoffFlux', riverRunoffFlux) - call mpas_pool_get_array(forcingPool, 'subglacialRunoffFlux', subglacialRunoffFlux) call mpas_pool_get_array(forcingPool, 'penetrativeTemperatureFlux', penetrativeTemperatureFlux) call mpas_pool_get_array(forcingPool, 'landIcePressure', landIcePressure) @@ -653,28 +708,6 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer tracersSurfaceFluxRunoff(index_temperature_flux,iCell) = riverRunoffFlux(iCell) & * max(tracerGroup(index_temperature_flux,minLevelCell(iCell),iCell), 0.0_RKIND) / rho_sw - if (trim(config_subglacial_runoff_mode) == 'data') then - if ( config_use_sgr_opt_temp_prescribed ) then - !sgr with fixed prescribed temperature - tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & - config_sgr_temperature_prescribed / rho_sw - else - !sgr with temperature equal to the local freezing point of freshwater - tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & - ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & - inLandIceCavity=.true.) / rho_sw - end if - - if ( config_use_sgr_opt_salt_prescribed ) then - !sgr with fixed prescribed temperature - tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = subglacialRunoffFlux(iCell) * & - config_sgr_salinity_prescribed / rho_sw - else - !sgr with temperature equal to the local freezing point of freshwater - tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = 0.0_RKIND - end if - end if - ! Accumulate fluxes that use the freezing point ! mrp performance note: should call ocn_freezing_temperature just once here seaIceTemperatureFlux(iCell) = seaIceFreshWaterFlux(iCell) * & @@ -694,12 +727,6 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer totalFreshWaterTemperatureFlux(iCell) = surfaceTemperatureFluxWithoutRunoff & + tracersSurfaceFluxRunoff(index_temperature_flux,iCell) - ! add subglacial runoff contribution for sending through coupler - if (trim(config_subglacial_runoff_mode) == 'data') then - totalFreshWaterTemperatureFlux(iCell) = totalFreshWaterTemperatureFlux(iCell) & - + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) - end if - ! Fields with zero temperature are not accumulated. These include: ! snowFlux ! iceRunoffFlux @@ -720,6 +747,125 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer end subroutine ocn_surface_bulk_forcing_active_tracers!}}} +!*********************************************************************** +! +! routine ocn_surface_bulk_forcing_active_tracers_subglacial_runoff +! +!> \brief Determines the active tracers forcing array used for the bulk forcing. +!> \author Doug Jacobsen +!> \date 04/25/12 +!> \details +!> This routine computes the active tracers forcing arrays used later in MPAS. +! +!----------------------------------------------------------------------- + + subroutine ocn_surface_bulk_forcing_active_tracers_subglacial_runoff(meshPool, forcingPool, & + tracersSurfaceFluxSubglacialRunoff, err)!{{{ + + !----------------------------------------------------------------- + ! + ! input variables + ! + !----------------------------------------------------------------- + type (mpas_pool_type), intent(in) :: meshPool !< Input: mesh information + + !----------------------------------------------------------------- + ! + ! input/output variables + ! + !----------------------------------------------------------------- + type (mpas_pool_type), intent(inout) :: forcingPool !< Input: Forcing information + real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFluxSubglacialRunoff + + !----------------------------------------------------------------- + ! + ! output variables + ! + !----------------------------------------------------------------- + + integer, intent(out) :: err !< Output: Error flag + + !----------------------------------------------------------------- + ! + ! local variables + ! + !----------------------------------------------------------------- + + integer :: iCell, nCells + integer, pointer :: index_temperature_fluxPtr, index_salinity_fluxPtr + integer :: index_temperature_flux, index_salinity_flux + integer, dimension(:), pointer :: nCellsArray + + type(mpas_pool_type),pointer :: tracersSurfaceFluxPool + + real (kind=RKIND), dimension(:), pointer :: subglacialRunoffFlux + real (kind=RKIND), dimension(:), pointer :: totalFreshWaterTemperatureFlux + real (kind=RKIND), dimension(:), pointer :: landIcePressure + + err = 0 + + call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) + + call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) + + call mpas_pool_get_dimension(tracersSurfaceFluxPool, & + 'index_temperatureSurfaceFlux', & + index_temperature_fluxPtr) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, & + 'index_salinitySurfaceFlux', & + index_salinity_fluxPtr) + index_temperature_flux = index_temperature_fluxPtr + index_salinity_flux = index_salinity_fluxPtr + + call mpas_pool_get_array(forcingPool, 'subglacialRunoffFlux', subglacialRunoffFlux) + + call mpas_pool_get_array(forcingPool, 'landIcePressure', landIcePressure) + + call mpas_pool_get_array(forcingPool, 'totalFreshWaterTemperatureFlux', totalFreshWaterTemperatureFlux) + + nCells = nCellsArray( 3 ) + + ! Surface fluxes of water have an associated heat content, but the coupled system does not account for this + ! Assume surface fluxes of water have a temperature dependent on the incoming mass flux. + ! Assume surface fluxes of water have zero salinity. So the RHS forcing is zero for salinity. + ! Only include this heat forcing when bulk thickness is turned on + ! indices on tracerGroup are (iTracer, iLevel, iCell) + if (.not. bulkThicknessFluxOff) then + !$omp parallel + !$omp do schedule(runtime) + do iCell = 1, nCells + + if ( config_use_sgr_opt_temp_prescribed ) then + !sgr with fixed prescribed temperature + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & + config_sgr_temperature_prescribed / rho_sw + else + !sgr with temperature equal to the local freezing point of freshwater + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) = subglacialRunoffFlux(iCell) * & + ocn_freezing_temperature(salinity=0.0_RKIND, pressure=landIcePressure(iCell), & + inLandIceCavity=.true.) / rho_sw + end if + + if ( config_use_sgr_opt_salt_prescribed ) then + !sgr with fixed prescribed temperature + tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = subglacialRunoffFlux(iCell) * & + config_sgr_salinity_prescribed / rho_sw + else + !sgr with temperature equal to the local freezing point of freshwater + tracersSurfaceFluxSubglacialRunoff(index_salinity_flux,iCell) = 0.0_RKIND + end if + + ! add subglacial runoff contribution for sending through coupler + totalFreshWaterTemperatureFlux(iCell) = totalFreshWaterTemperatureFlux(iCell) & + + tracersSurfaceFluxSubglacialRunoff(index_temperature_flux,iCell) + + end do + !$omp end do + !$omp end parallel + endif ! bulkThicknessFluxOn + + end subroutine ocn_surface_bulk_forcing_active_tracers_subglacial_runoff!}}} + end module ocn_surface_bulk_forcing diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index b855512fb294..f42df59a629c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -891,9 +891,13 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & forcingPool, tracerGroup, & tracerGroupSurfaceFlux, & tracerGroupSurfaceFluxRunoff, & - tracerGroupSurfaceFluxSubglacialRunoff, & tracerGroupSurfaceFluxRemoved, & dt, layerThickness, err) + if (trim(config_subglacial_runoff_mode) == 'data') then + call ocn_surface_bulk_forcing_tracers(meshPool, groupName, & + forcingPool, & + tracerGroupSurfaceFluxSubglacialRunoff, err) + end if end if ! sfc bulk forcing From 5a54e97cf2577dbc9bed3bff0c25379ad4fc72e1 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Sun, 21 Jul 2024 21:43:49 -0600 Subject: [PATCH 333/904] fix function call typo _subglacial_runoff --- components/mpas-ocean/src/shared/mpas_ocn_tendency.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index f42df59a629c..a0f4ae74e5ec 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -894,7 +894,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & tracerGroupSurfaceFluxRemoved, & dt, layerThickness, err) if (trim(config_subglacial_runoff_mode) == 'data') then - call ocn_surface_bulk_forcing_tracers(meshPool, groupName, & + call ocn_surface_bulk_forcing_tracers_subglacial_runoff(meshPool, groupName, & forcingPool, & tracerGroupSurfaceFluxSubglacialRunoff, err) end if From bdd8761563b4e46fb575c11830289c0fd4facd13 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Sun, 21 Jul 2024 23:12:28 -0600 Subject: [PATCH 334/904] fix timer for sgr --- .../src/shared/mpas_ocn_tracer_surface_flux_to_tend.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F index fd754562a579..280ebc72844b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F @@ -270,8 +270,6 @@ subroutine ocn_tracer_surface_flux_tend_subglacial_runoff(meshPool, fractionAbso if (.not. surfaceTracerFluxOn) return - call mpas_timer_start("surface_tracer_flux") - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) nTracers = size(tend, dim=1) @@ -308,6 +306,8 @@ subroutine ocn_tracer_surface_flux_tend_subglacial_runoff(meshPool, fractionAbso !$omp end do !$omp end parallel + call mpas_timer_stop("surface_tracer_subglacial_runoff_flux") + !-------------------------------------------------------------------- end subroutine ocn_tracer_surface_flux_tend_subglacial_runoff!}}} From cc0c4ce0d2f7171fdbc229780194e8905af66795 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Jul 2024 07:27:07 -0700 Subject: [PATCH 335/904] Adds missing links for obklen and ustar in shoc --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp | 1 - components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 80485bf67130..657f9487611c 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -395,7 +395,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); - add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 1e9b4746701f..38758bf04a12 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -685,7 +685,9 @@ Int Functions::shoc_main( wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables - shoc_output.pblh(i) = pblh_s; + shoc_output.obklen(i) = obklen_s; + shoc_output.ustar(i) = ustar_s; + shoc_output.pblh(i) s= pblh_s; }); Kokkos::fence(); #else From d71781f58999349579e6a88d7378ffc05a5741d2 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 08:43:37 -0600 Subject: [PATCH 336/904] Remove obklen postcondition check --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 80485bf67130..657f9487611c 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -395,7 +395,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); - add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. From 6e5097d6a8b2922d44b845fdff5dff0eeebbc78b Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 08:44:17 -0600 Subject: [PATCH 337/904] Set output correctly for new variables --- components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 1e9b4746701f..11383e772454 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -686,6 +686,8 @@ Int Functions::shoc_main( w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables shoc_output.pblh(i) = pblh_s; + shoc_output.ustar(i) = ustar_s; + shoc_output.obklen(i) = obklen_s; }); Kokkos::fence(); #else From f25ec79b6b7c0c46de11d4b9795fcdaf9550a94a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Jul 2024 07:46:10 -0700 Subject: [PATCH 338/904] Fixes a typo --- components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 38758bf04a12..807eccdfe559 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -687,7 +687,7 @@ Int Functions::shoc_main( shoc_output.obklen(i) = obklen_s; shoc_output.ustar(i) = ustar_s; - shoc_output.pblh(i) s= pblh_s; + shoc_output.pblh(i) = pblh_s; }); Kokkos::fence(); #else From 3bea55f109ccdca502d26909fbb0a66054a16c4b Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 22 Jul 2024 12:59:58 -0500 Subject: [PATCH 339/904] new argument for apply weights --- driver-moab/main/seq_map_mod.F90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/driver-moab/main/seq_map_mod.F90 b/driver-moab/main/seq_map_mod.F90 index be47473f444c..82e67b8b195e 100644 --- a/driver-moab/main/seq_map_mod.F90 +++ b/driver-moab/main/seq_map_mod.F90 @@ -355,6 +355,7 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, real(r8), dimension(:), allocatable :: wghts real(kind=r8) , allocatable :: targtags(:,:), targtags_ini(:,:) real(kind=r8) :: factor + integer :: filter_type ! used for caas projection #endif ! ! Local Variables @@ -649,7 +650,8 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, call shr_sys_flush(logunit) endif #endif - ierr = iMOAB_ApplyScalarProjectionWeights ( mapper%intx_mbid, mapper%weight_identifier, fldlist_moab, fldlist_moab) + filter_type = 0 ! no + ierr = iMOAB_ApplyScalarProjectionWeights ( mapper%intx_mbid, filter_type, mapper%weight_identifier, fldlist_moab, fldlist_moab) if (ierr .ne. 0) then write(logunit,*) subname,' error in applying weights ' call shr_sys_abort(subname//' ERROR in applying weights') From 45af3b53e5aee4d9a54a79e78f3066ec5ecf1fcc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Jul 2024 11:56:15 -0700 Subject: [PATCH 340/904] Adds a multi-process test with dynamics --- .../dynamics_physics/CMakeLists.txt | 1 + .../CMakeLists.txt | 109 +++++++++++++++ .../input.yaml | 109 +++++++++++++++ .../output.yaml | 124 ++++++++++++++++++ 4 files changed, 343 insertions(+) create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml diff --git a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 64ac2187c280..51d0bd456479 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -13,6 +13,7 @@ if (SCREAM_DOUBLE_PRECISION) # initial conditions. #add_subdirectory(homme_mam4xx_pg2) add_subdirectory(mam/homme_shoc_cld_p3_mam_optics_rrtmgp) + add_subdirectory(mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep) endif() endif() endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt new file mode 100644 index 000000000000..e3c562cfcd13 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt @@ -0,0 +1,109 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 41) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction ${dynLibName} shoc p3 scream_rrtmgp mam + LABELS dynamics shoc cld p3 rrtmgp physics mam4_optics mam4_aci mam4_drydep + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 2) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 1) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp mam4_optics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml new file mode 100644 index 000000000000..1857bc534dd2 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml @@ -0,0 +1,109 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + mass_column_conservation_error_tolerance: 1e-3 + energy_column_conservation_error_tolerance: 1e-4 + column_conservation_checks_fail_handling_type: Warning + property_check_data_fields: [phis] + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + phis : 1.0 + + #variables required for shoc + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variables required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + + #variables required for mam4_aci + dgnum: 1e-3 + + #mam4 drydep + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 + ram1: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mac_mic,mam4_optics,rrtmgp,mam4_drydep] + schedule_type: Sequential + Type: Group + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + schedule_type: Sequential + Type: Group + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 + p3: + do_prescribed_ccn: false + enable_column_conservation_checks: true + max_total_ni: 740.0e3 + shoc: + enable_column_conservation_checks: true + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + enable_column_conservation_checks: true + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml new file mode 100644 index 000000000000..f2408ab189a5 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml @@ -0,0 +1,124 @@ +%YAML 1.1 +--- +filename_prefix: homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + # SHOC + - cldfrac_liq + - eddy_diff_mom + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + HOMME + - horiz_winds + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + HOMME + - T_mid + #mam_optics + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + # Diagnostics + - T_mid_at_lev_2 + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_500mb + - T_mid_at_500hPa + - T_mid_at_50000Pa + - PotentialTemperature + - AtmosphereDensity + - Exner + - VirtualTemperature + - z_int + - geopotential_int_at_lev_2 + - z_mid_at_500mb + - geopotential_mid + - dz + - DryStaticEnergy + - SeaLevelPressure + - LiqWaterPath + - IceWaterPath + - VapWaterPath + - RainWaterPath + - RimeWaterPath + - ShortwaveCloudForcing + - LongwaveCloudForcing + - RelativeHumidity + - ZonalVapFlux + - MeridionalVapFlux + - PotentialTemperature_at_model_top + - PotentialTemperature_at_500mb + #drydep + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols + # GLL output for homme states. These + # represent all current possible homme + # states available. + Dynamics: + Field Names: + - v_dyn + - vtheta_dp_dyn + - dp3d_dyn + - phi_int_dyn + - ps_dyn + - phis_dyn + - omega_dyn + - Qdp_dyn + IO Grid Name: Physics GLL + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: true +... From 0b72d7532ee6d28b926ed2365b3d68c86a022f54 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 22 Jul 2024 13:20:13 -0600 Subject: [PATCH 341/904] corrected subglacialRunoffFlux add up in conservation check so that it is as mass, not volume flux --- .../src/analysis_members/mpas_ocn_conservation_check.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index 756328a3d5dd..beed937d6767 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -983,7 +983,7 @@ subroutine mass_conservation(domain, err) sumArray( 8) = sumArray( 8) + areaCell(iCell) * removedIceRunoffFlux(iCell) sumArray( 9) = sumArray( 9) + areaCell(iCell) * icebergFreshwaterFlux(iCell) if (trim(config_subglacial_runoff_mode) == 'data') then - sumArray(13) = sumArray(13) + areaCell(iCell) * subglacialRunoffFlux(iCell) / areaCell(iCell) * rho_sw + sumArray(13) = sumArray(13) + areaCell(iCell) * subglacialRunoffFlux(iCell) end if enddo From 00fc2618b0ad0d064870659e92ac51d8f4246c13 Mon Sep 17 00:00:00 2001 From: irenavankova Date: Wed, 17 Jul 2024 12:18:10 -0600 Subject: [PATCH 342/904] Change if statement for config_sgr_flux_vertical_location init check Co-authored-by: Carolyn Begeman --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 5046e19d5d59..a2d6586a7d63 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -4742,10 +4742,9 @@ subroutine ocn_diagnostics_init(domain, err)!{{{ call ocn_diagnostics_variables_init(domain, jenkinsOn, & hollandJenkinsOn, err) - if ( trim(config_sgr_flux_vertical_location) == 'top' ) then - else if ( trim(config_sgr_flux_vertical_location) == 'uniform' ) then - else if ( trim(config_sgr_flux_vertical_location) == 'bottom' ) then - else + if ( trim(config_sgr_flux_vertical_location) /= 'top' .and. & + trim(config_sgr_flux_vertical_location) /= 'uniform' .and. & + trim(config_sgr_flux_vertical_location) /= 'bottom' ) then call mpas_log_write("config_sgr_flux_vertical_location not one of 'bottom', 'uniform', 'top'.", MPAS_LOG_CRIT) end if From 71af25a190706654cb5ae9e52ec253d9f54d03e8 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 22 Jul 2024 15:12:35 -0500 Subject: [PATCH 343/904] extra endif in mpas ocean mct module --- components/mpas-ocean/driver/ocn_comp_mct.F | 1 - 1 file changed, 1 deletion(-) diff --git a/components/mpas-ocean/driver/ocn_comp_mct.F b/components/mpas-ocean/driver/ocn_comp_mct.F index bd425e7e806a..5bedda574003 100644 --- a/components/mpas-ocean/driver/ocn_comp_mct.F +++ b/components/mpas-ocean/driver/ocn_comp_mct.F @@ -3936,7 +3936,6 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_dmpar_exch_halo_field(iceFluxDMSPField) endif endif - endif !----------------------------------------------------------------------- !EOC From f130c27bf6375d16a470e43af9cbebbb5c6af0dc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Jul 2024 13:14:23 -0700 Subject: [PATCH 344/904] Adds variables for drydep only CIME simulation --- components/eamxx/cime_config/namelist_defaults_scream.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index e586c6ce2151..e3da0b0c9eee 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -493,8 +493,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0,0.0 - 2.6e-08 - 0.0 + 2.6e-08 + 0.41417721820867320E-007 + 0.15100083211582764E+004 + 0.0 0.0 0.0 0.0 From 77feb929dcad7ab939f7108c6b4f292d716e5530 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 23 Jul 2024 13:45:38 -0600 Subject: [PATCH 345/904] change author on new subroutines --- .../src/shared/mpas_ocn_surface_bulk_forcing.F | 12 ++++++------ .../src/shared/mpas_ocn_thick_surface_flux.F | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 81732b637a97..11d70ae9c40d 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -136,8 +136,8 @@ end subroutine ocn_surface_bulk_forcing_tracers!}}} ! routine ocn_surface_bulk_forcing_tracers_subglacial_runoff ! !> \brief Determines the tracers forcing array used for the bulk forcing. -!> \author Doug Jacobsen -!> \date 04/25/12 +!> \author Irena Vankova +!> \date July 2024 !> \details !> This routine computes the tracers forcing arrays used later in MPAS. ! @@ -425,8 +425,8 @@ end subroutine ocn_surface_bulk_forcing_thick!}}} ! routine ocn_surface_bulk_forcing_thick_subglacial_runoff ! !> \brief Determines the thickness forcing array used for the bulk forcing. -!> \author Doug Jacobsen -!> \date 04/25/12 +!> \author Irena Vankova +!> \date July 2024 !> \details !> This routine computes the thickness forcing arrays used later in MPAS. ! @@ -752,8 +752,8 @@ end subroutine ocn_surface_bulk_forcing_active_tracers!}}} ! routine ocn_surface_bulk_forcing_active_tracers_subglacial_runoff ! !> \brief Determines the active tracers forcing array used for the bulk forcing. -!> \author Doug Jacobsen -!> \date 04/25/12 +!> \author Irena Vankova +!> \date July 2024 !> \details !> This routine computes the active tracers forcing arrays used later in MPAS. ! diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F index 8e389a74ba37..b959a89bff1d 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F @@ -175,8 +175,8 @@ end subroutine ocn_thick_surface_flux_tend!}}} ! routine ocn_thick_surface_flux_tend_subglacial_runoff ! !> \brief Computes tendency term from horizontal advection of thickness -!> \author Doug Jacobsen -!> \date 15 September 2011 +!> \author Irena Vankova +!> \date July 2024 !> \details !> This routine computes the horizontal advection tendency for !> thickness based on current state and user choices of forcings. @@ -276,8 +276,8 @@ end subroutine ocn_thick_surface_flux_tend_subglacial_runoff!}}} ! routine ocn_thick_surface_flux_init ! !> \brief Initializes ocean horizontal thickness surface fluxes -!> \author Doug Jacobsen -!> \date 12/17/12 +!> \author Irena Vankova +!> \date July 2024 !> \details !> This routine initializes quantities related to thickness !> surface fluxes in the ocean. From 181b07ccc550fd079e11d2ce4ff3b9ae723fefac Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 24 Jul 2024 11:44:33 -0500 Subject: [PATCH 346/904] local compile changes --- components/mpas-ocean/driver/ocn_comp_mct.F | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/driver/ocn_comp_mct.F b/components/mpas-ocean/driver/ocn_comp_mct.F index bd425e7e806a..d1b140563bb7 100644 --- a/components/mpas-ocean/driver/ocn_comp_mct.F +++ b/components/mpas-ocean/driver/ocn_comp_mct.F @@ -2952,6 +2952,8 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ block_ptr => block_ptr % next end do + errorCode = 0 + !----------------------------------------------------------------------- !EOC @@ -3936,7 +3938,6 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_dmpar_exch_halo_field(iceFluxDMSPField) endif endif - endif !----------------------------------------------------------------------- !EOC From 076bf9195e3105453150a5844876543dfcfa1310 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 18:38:38 -0700 Subject: [PATCH 347/904] Fixes dimensions of dry dep flux variables and fixes input.yaml for a test --- .../eamxx_mam_dry_deposition_process_interface.cpp | 11 ++++++++--- .../eamxx/tests/single-process/mam/drydep/input.yaml | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 047878796d9a..433c9d4f6063 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -48,6 +48,11 @@ void MAMDryDep::set_grids( const FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; const FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // layout for 2D (ncol, pcnst) + constexpr int pcnst = mam4::aero_model::pcnst; + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constants"); + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points auto make_layout = [](const std::vector &extents, @@ -198,11 +203,11 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------- // FIXME: These are diagnostics, remove them from FM after initial evaluation // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, + add_field("deposition_flux_of_cloud_borne_aerosols", scalar2d_pcnct, 1 / m2 / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, - 1 / m2 / s, grid_name); + add_field("deposition_flux_of_interstitial_aerosols", + scalar2d_pcnct, 1 / m2 / s, grid_name); } // set_grids // ================================================================ diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 685a68fcea04..86c6dd8766a2 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -28,12 +28,12 @@ initial_conditions: fraction_landuse: 0.2 w_updraft: 1e-5 pbl_height: 0.0 - Obukhov_length: -0.48167567011091336E+002 + obklen: -0.48167567011091336E+002 fv: 0.47569674433601039E+000 landfrac: 0.14675684171817760E+000 icefrac: 0.00000000000000000E+000 ocnfrac: 0.85324315828182240E+000 - surface_friction_velocty: 0.27482164092221828E+000 + ustar: 0.27482164092221828E+000 ram1: 0.45506166067091662E+002 dgnumwet: 0.41417721820867320E-007 wetdens: 0.15100083211582764E+004 From dbaeea26beec89d0a7a72187fc813766cabd2b5d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 25 Jul 2024 08:40:52 -0600 Subject: [PATCH 348/904] EAMxx: add runtime and buildnml-time checks on output frequency Ensures dt_output is not smaller than dt_atm --- components/eamxx/cime_config/eamxx_buildnml.py | 17 +++++++++++++++++ .../src/share/io/scream_output_manager.cpp | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 0c7ed8e52aac..121506f7eff3 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -1017,6 +1017,23 @@ def do_cime_vars_on_yaml_output_files(case, caseroot): print (" - setting skip_t0_output=true\n") print (" - setting freq and freq_units to HIST_N and HIST_OPTION respectively\n") + # If frequency_units is not nsteps, verify that we don't request + # a frequency faster than the model timestep + if content['output_control']['frequency_units'] in ['nsecs','nmins','nhours']: + freq = content['output_control']['Frequency'] + units = content['output_control']['frequency_units'] + dt_out = 1 if units=="nsecs" else 60 if units=="nmins" else 3600 + dt_out = dt_out*int(freq) + + dt_atm = 86400 / case.get_value("ATM_NCPL") + expect (dt_atm<=dt_out, + "Cannot have output frequency faster than atm timestep.\n" + f" yaml file: {fn.strip()}\n" + f" Frequency: {freq}\n" + f" frequency_units: {units}\n" + f" ATM_NCPL: {case.get_value('ATM_NCPL')}\n" + f" This yields dt_atm={dt_atm} > dt_output={dt_out}. Please, adjust 'Frequency' and/or 'frequency_units'\n") + ordered_dump(content, open(dst_yaml, "w")) output_yaml_files.append(dst_yaml) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 35b1c753b03e..539204c66a1b 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -328,6 +328,13 @@ void OutputManager::run(const util::TimeStamp& timestamp) return; } + EKAT_REQUIRE_MSG (timestamp<=m_output_control.next_write_ts, + "Error! The input timestamp is past the next scheduled write timestamp.\n" + " - current time stamp : " + timestamp.to_string() + "\n" + " - next write time stamp: " + m_output_control.next_write_ts.to_stirng() + "\n" + "The most likely cause is an output frequency that is faster than the atm timestep.\n" + "Try to increase 'Frequency' and/or 'frequency_units' in your output yaml file.\n"); + // Update counters ++m_output_control.nsamples_since_last_write; ++m_checkpoint_control.nsamples_since_last_write; From 0288938beae15cf92f0a86d077e0cd12b3ffaf3c Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 25 Jul 2024 11:06:20 -0500 Subject: [PATCH 349/904] Make bld files consistent with Registry --- components/mpas-ocean/bld/build-namelist | 10 ++++---- .../mpas-ocean/bld/build-namelist-section | 2 +- .../namelist_defaults_mpaso.xml | 2 +- .../namelist_definition_mpaso.xml | 23 ++++++++++--------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 490100040c00..37c8b88f67e6 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -708,6 +708,11 @@ add_default($nl, 'config_use_bulk_wind_stress'); add_default($nl, 'config_use_bulk_thickness_flux'); add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); +if ($OCN_SGR eq 'data') { + add_default($nl, 'config_subglacial_runoff_mode', 'val'=>"data"); +} else { + add_default($nl, 'config_subglacial_runoff_mode'); +} add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); add_default($nl, 'config_sgr_flux_vertical_location'); add_default($nl, 'config_use_sgr_opt_kpp'); @@ -715,11 +720,6 @@ add_default($nl, 'config_use_sgr_opt_temp_prescribed'); add_default($nl, 'config_use_sgr_opt_salt_prescribed'); add_default($nl, 'config_sgr_temperature_prescribed'); add_default($nl, 'config_sgr_salinity_prescribed'); -if ($OCN_SGR eq 'data') { - add_default($nl, 'config_subglacial_runoff_mode', 'val'=>"data"); -} else { - add_default($nl, 'config_subglacial_runoff_mode'); -} ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index 189d32fc7b52..9e646beb6b5e 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -224,6 +224,7 @@ add_default($nl, 'config_use_bulk_wind_stress'); add_default($nl, 'config_use_bulk_thickness_flux'); add_default($nl, 'config_flux_attenuation_coefficient'); add_default($nl, 'config_flux_attenuation_coefficient_runoff'); +add_default($nl, 'config_subglacial_runoff_mode'); add_default($nl, 'config_flux_attenuation_coefficient_subglacial_runoff'); add_default($nl, 'config_sgr_flux_vertical_location'); add_default($nl, 'config_use_sgr_opt_kpp'); @@ -231,7 +232,6 @@ add_default($nl, 'config_use_sgr_opt_temp_prescribed'); add_default($nl, 'config_use_sgr_opt_salt_prescribed'); add_default($nl, 'config_sgr_temperature_prescribed'); add_default($nl, 'config_sgr_salinity_prescribed'); -add_default($nl, 'config_subglacial_runoff_mode'); ############################ # Namelist group: coupling # diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index c57920e5c388..b70f7942686e 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -342,6 +342,7 @@ .true. 0.001 10.0 +'off' 0.001 'bottom' .true. @@ -349,7 +350,6 @@ .false. 0.0 0.0 -'off' .false. diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index fed7828e6a8b..601b121e5768 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1188,9 +1188,17 @@ Valid values: Any positive real number. Default: Defined in namelist_defaults.xml + +Selects the mode in which subglacial runoff fluxes are implemented. + +Valid values: 'off', 'data' +Default: Defined in namelist_defaults.xml + + -The length scale of exponential decay of subglacial runoff. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$. +The length scale of exponential decay of subglacial runoff, used when config_sgr_flux_vertical_location is 'top' or 'bottom'. Fluxes are multiplied by $e^{z/\gamma}$, where this coefficient is $\gamma$. Valid values: Any positive real number. Default: Defined in namelist_defaults.xml @@ -1198,9 +1206,9 @@ Default: Defined in namelist_defaults.xml -Selects the vertical location where subglacial runoff is fluxed +Selects the vertical location where subglacial runoff is fluxed. -Valid values: 'top','uniform', 'bottom'. +Valid values: 'top','uniform', 'bottom' Default: Defined in namelist_defaults.xml @@ -1240,17 +1248,10 @@ Default: Defined in namelist_defaults.xml category="forcing" group="forcing"> Prescribed subglacial runoff salinity value, applied when config_use_sgr_opt_salt_prescribed = .true. -Valid values: Any real number. +Valid values: Any positive real number Default: Defined in namelist_defaults.xml - -Selects the mode in which subglacial runoff fluxes are computed. - -Valid values: 'off', 'data' -Default: Defined in namelist_defaults.xml - From 48cee50966dfefddfad8b90e8502c9296bf62573 Mon Sep 17 00:00:00 2001 From: irenavankova Date: Thu, 25 Jul 2024 10:12:31 -0600 Subject: [PATCH 350/904] Correct DSGR file name Co-authored-by: Xylar Asay-Davis --- components/mpas-ocean/cime_config/buildnml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index f3eb22881683..8f5193c334be 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -380,7 +380,7 @@ def buildnml(case, caseroot, compname): if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.IcoswISC30E3r5.20231120.nc' if ocn_sgr == 'data': - data_sgr_file = 'DSGR.massFlux.massFlux.MALI.out2055.IcoswISC30E3r5.20240328.nc' + data_sgr_file = 'DSGR.massFlux.MALI.out2055.IcoswISC30E3r5.20240328.nc' elif ocn_grid == 'IcosXISC30E3r7': decomp_date = '20240314' From 5a0f2842c03cb058d51dd8c09a133efece7cd2ca Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 25 Jul 2024 10:44:04 -0600 Subject: [PATCH 351/904] Corrected authorship --- .../mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F | 4 ++-- .../src/shared/mpas_ocn_tracer_surface_flux_to_tend.F | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F index b959a89bff1d..1c535532cea2 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F @@ -276,8 +276,8 @@ end subroutine ocn_thick_surface_flux_tend_subglacial_runoff!}}} ! routine ocn_thick_surface_flux_init ! !> \brief Initializes ocean horizontal thickness surface fluxes -!> \author Irena Vankova -!> \date July 2024 +!> \author Doug Jacobsen +!> \date 12/17/12 !> \details !> This routine initializes quantities related to thickness !> surface fluxes in the ocean. diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F index 280ebc72844b..ba97203c4ec3 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_surface_flux_to_tend.F @@ -208,8 +208,8 @@ end subroutine ocn_tracer_surface_flux_tend!}}} ! routine ocn_tracer_surface_flux_tend_subglacial_runoff ! !> \brief Computes tendency term for surface fluxes -!> \author Doug Jacobsen -!> \date 12/17/12 +!> \author Irena Vankova +!> \date July 2024 !> \details !> This routine computes the tendency for tracers based on surface fluxes. ! From 9ddd011645f60d3c59e7783476f75d5359ed06ae Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Thu, 25 Jul 2024 10:49:40 -0600 Subject: [PATCH 352/904] Added DSGR to stealth feature tests --- cime_config/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/tests.py b/cime_config/tests.py index c1f6cfaa1b70..12ab829c51dd 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -274,6 +274,7 @@ "ERS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "PEM_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-TMIX.mpaso-jra_1958", + "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", ) }, From d0369c9a21a6d10a39bea0a674b68dba4a14a842 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Thu, 25 Jul 2024 12:17:34 -0700 Subject: [PATCH 353/904] Add support for ne512pg2 bi-grid --- cime_config/config_grids.xml | 46 +++++++++++++++++++ .../bld/namelist_files/namelist_defaults.xml | 2 + .../namelist_files/namelist_definition.xml | 2 +- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 7c8eb0ff75bd..7c3ebe83b0b0 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -1683,6 +1683,26 @@ oRRS18to6v3 + + ne512np4.pg2 + ne512np4.pg2 + ICOS10 + r0125 + null + null + ICOS10 + + + + ne512np4.pg2 + ne512np4.pg2 + ne512np4.pg2 + r0125 + null + null + ICOS10 + + ne512np4 360x720cru @@ -2894,6 +2914,8 @@ 1 $DIN_LOC_ROOT/share/domains/domain.lnd.ne512pg2_oRRS18to6v3.200212.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne512pg2_oRRS18to6v3.200212.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne512pg2_ICOS10.240602.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne512pg2_ICOS10.240602.nc ne512np4.pg2 is Spectral Elem 6km grid w/ 2x2 FV physics grid per element: @@ -3934,6 +3956,11 @@ cpl/gridmaps/ne256pg2/map_ne256pg2_to_r0125_bilin.200212.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_oRRS18to6v3_nco.200212.nc @@ -3943,6 +3970,15 @@ cpl/gridmaps/ne512pg2/map_oRRS18to6v3_to_ne512pg2_nco.200212.nc + + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ICOS10_to_ne512pg2_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ICOS10_to_ne512pg2_nco_c240531.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.200212.nc @@ -4717,6 +4753,11 @@ cpl/gridmaps/ne256pg2/map_r0125_to_ne256pg2_mono.200212.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_r0125_to_ne512pg2_mono.c20240625.nc + + cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc @@ -4797,6 +4838,11 @@ lnd/clm2/mappingdata/maps/ne240np4/map_0.5x0.5_nomask_to_ne240np4_nomask_aave_da_c121019.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_r0125_to_ne512pg2_mono.c20240625.nc + + cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 44cf85c73f3f..7ae61a1d4893 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -306,6 +306,8 @@ lnd/clm2/surfdata_map/surfdata_ne120pg2_simyr2010_c230119.nc lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr1850_c240131.nc lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr2010_c230207.nc + +lnd/clm2/surfdata_map/surfdata_ne512pg2_simyr2010_c240522.nc lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index ec3a74cbd965..72c7b1c3d654 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -1409,7 +1409,7 @@ CLM run type. +"512x1024,360x720cru,128x256,64x128,48x96,32x64,8x16,94x192,0.23x0.31,0.9x1.25,1.9x2.5,2.5x3.33,4x5,10x15,5x5_amazon,1x1_tropicAtl,1x1_camdenNJ,1x1_vancouverCAN,1x1_mexicocityMEX,1x1_asphaltjungleNJ,1x1_brazil,1x1_urbanc_alpha,1x1_numaIA,1x1_smallvilleIA,0.1x0.1,0.5x0.5,3x3min,5x5min,10x10min,0.33x0.33,ne4np4,ne4np4.pg2,ne11np4,ne16np4,ne30np4,ne30np4.pg2,ne60np4,ne120np4,ne120np4.pg2,ne240np4,ne256np4,ne256np4.pg2,ne512np4.pg2,ne1024np4,ne1024np4.pg2,1km-merge-10min,ne0np4_arm_x8v3_lowcon,ne0np4_conus_x4v1_lowcon,ne0np4_enax4v1,ne0np4_twpx4v1,r2,r05,r0125,NLDAS,ne0np4_northamericax4v1.pg2,ne0np4_arcticx4v1.pg2,r025"> Horizontal resolutions Note: 0.1x0.1, 0.5x0.5, 5x5min, 10x10min, 3x3min and 0.33x0.33 are only used for CLM tools From e5845eaaf8d2bb1cd07c0223c0848c76cc21617f Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 10:20:20 -0600 Subject: [PATCH 354/904] Fix how IOP reads time --- .../src/share/iop/intensive_observation_period.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 6f8474f8d85e..32b120c57002 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -248,15 +248,17 @@ initialize_iop_file(const util::TimeStamp& run_t0, else if (scorpio::has_dim(iop_file, "tsec")) time_dimname = "tsec"; else EKAT_ERROR_MSG("Error! No valid dimension for tsec in "+iop_file+".\n"); - const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); - m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); - scorpio::read_var(iop_file,"tsec",m_time_info.iop_file_times_in_sec.data()); - - // From now on, when we read vars, "time" must be treated as unlimited, to avoid issues + // When we read vars, "time" must be treated as unlimited, to avoid issues if (not scorpio::is_dim_unlimited(iop_file,time_dimname)) { scorpio::pretend_dim_is_unlimited(iop_file,time_dimname); } + const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); + m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); + for (int t=0; t Date: Thu, 25 Jul 2024 17:21:20 -0700 Subject: [PATCH 355/904] reduce the number of compsets down to one and make use of testmods to drive the tests of various cases --- cime_config/tests.py | 4 ++-- components/eamxx/cime_config/config_compsets.xml | 12 ------------ .../cime_config/namelist_defaults_scream.xml | 14 -------------- .../scream/dpxx/arm97/shell_commands | 15 +++++++++++++++ .../scream/dpxx/dycomsrf01/shell_commands | 15 +++++++++++++++ 5 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands diff --git a/cime_config/tests.py b/cime_config/tests.py index 275754e26a40..9c17744d0e64 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -657,8 +657,8 @@ "time" : "01:00:00", # each test 225 phys cols, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General", - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-ARM97", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-arm97", "ERS_P16_Ln22.ne30_ne30.F2000-SCREAMv1-DP-RCE", ) }, diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index e8c94534b919..c41bdb870ec7 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -80,12 +80,6 @@ Experimental, under development - - F2010-SCREAMv1-DP-ARM97 - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%ARM97 - Experimental, under development - - F2000-SCREAMv1-DP-RCE 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx @@ -98,8 +92,6 @@ 2016-08-01 2020-01-20 - 1999-07-10 - 1997-06-23 2000-01-01 @@ -150,16 +142,12 @@ - 31.5 - 36.605 0.0 - 238.5 - 262.515 0.0 diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index f1264b8b4e08..6b89e1fe7a3d 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -594,25 +594,11 @@ be lost if SCREAM_HACK_XML is not enabled. false - - ${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc - 36.605 - 262.515 - true - true - ${DIN_LOC_ROOT}/atm/cam/scam/iop/RCE_iopfile_4scam.nc 0.0 0.0 - - ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc - 31.5 - 238.5 - true - true - diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands new file mode 100644 index 000000000000..bda1791d66d7 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands @@ -0,0 +1,15 @@ +# Sets up DPxx case to test ARM97 (continental deep convection) +# case over land using prescribed surface fluxes and nudged winds + +./xmlchange RUN_STARTDATE="1997-06-23" +./xmlchange PTS_LAT=36.605 +./xmlchange PTS_LON=262.515 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file=${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=36.605 -b +$ATMCHANGE target_longitude=262.515 -b +$ATMCHANGE iop_nudge_uv=true -b +$ATMCHANGE iop_srf_prop=true -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands new file mode 100644 index 000000000000..58c235807872 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands @@ -0,0 +1,15 @@ +# Sets up DPxx case to test DYCOMSrf01 (marine stratocumulus) +# case over ocean using prescribed surface fluxes + +./xmlchange RUN_STARTDATE="1999-07-10" +./xmlchange PTS_LAT=31.5 +./xmlchange PTS_LON=238.5 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file=${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=31.5 -b +$ATMCHANGE target_longitude=238.5 -b +$ATMCHANGE iop_dosubsidence=true -b +$ATMCHANGE iop_srf_prop=true -b From e74604f2bed779b7a2439d9fa0fc0d4196605f90 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 25 Jul 2024 22:24:07 -0400 Subject: [PATCH 356/904] remove sunlit sync to host --- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 15de175da67d..8090a9300fd9 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -109,7 +109,6 @@ TEST_CASE("aodvis") { // Check result tau.sync_to_host(); - sunlit.sync_to_host(); diag->get_diagnostic().sync_to_host(); const auto tau_h = tau.get_view(); From 2ed55ace931870c0e12adfb7c50c94cfdcffd28c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Jul 2024 12:11:20 -0600 Subject: [PATCH 357/904] EAMxx: fix typo --- components/eamxx/src/share/io/scream_output_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 539204c66a1b..ccec5f516fd5 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -331,7 +331,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) EKAT_REQUIRE_MSG (timestamp<=m_output_control.next_write_ts, "Error! The input timestamp is past the next scheduled write timestamp.\n" " - current time stamp : " + timestamp.to_string() + "\n" - " - next write time stamp: " + m_output_control.next_write_ts.to_stirng() + "\n" + " - next write time stamp: " + m_output_control.next_write_ts.to_string() + "\n" "The most likely cause is an output frequency that is faster than the atm timestep.\n" "Try to increase 'Frequency' and/or 'frequency_units' in your output yaml file.\n"); From 59f9a9be68570796fc8e9c8dd9a0d0025bed728e Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 26 Jul 2024 13:04:18 -0600 Subject: [PATCH 358/904] Remove overuse of templates --- .../rrtmgp/scream_rrtmgp_interface.hpp | 234 ++++++++---------- 1 file changed, 109 insertions(+), 125 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index c82d9e38a3b6..0bceb1282e5f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -175,21 +175,40 @@ using hview_t = Kokkos::View; using pool_t = conv::MemPoolSingleton; +using real1dk = view_t; +using real2dk = view_t; +using real3dk = view_t; +using creal1dk = view_t; +using creal2dk = view_t; +using creal3dk = view_t; +using int1dk = view_t; +using int3dk = view_t; + +using gas_optics_t = GasOpticsRRTMGPK; +using cloud_optics_t = CloudOpticsK; +using gas_concs_t = GasConcsK; +using fluxes_t = FluxesBybandK; +using fluxes_broadband_t = FluxesBroadbandK; +using optical_props_t = OpticalPropsK; +using optical_props1_t = OpticalProps1sclK; +using optical_props2_t = OpticalProps2strK; +using source_func_t = SourceFuncLWK; + /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -static inline GasOpticsRRTMGPK k_dist_sw_k; -static inline GasOpticsRRTMGPK k_dist_lw_k; +static inline gas_optics_t k_dist_sw_k; +static inline gas_optics_t k_dist_lw_k; /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -static inline CloudOpticsK cloud_optics_sw_k; -static inline CloudOpticsK cloud_optics_lw_k; +static inline cloud_optics_t cloud_optics_sw_k; +static inline cloud_optics_t cloud_optics_lw_k; /* * Flag to indicate whether or not we have initialized RRTMGP @@ -200,7 +219,7 @@ static inline bool initialized_k = false; * Initialize data for RRTMGP driver */ static void rrtmgp_initialize( - GasConcsK &gas_concs, + const gas_concs_t &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger) @@ -238,13 +257,11 @@ static void rrtmgp_initialize( /* * Compute band-by-band surface albedos from broadband albedos. */ -template static void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, - SfcAlbDirVisT &sfc_alb_dir_vis, SfcAlbDirNirT &sfc_alb_dir_nir, - SfcAlbDifVisT &sfc_alb_dif_vis, SfcAlbDifNirT &sfc_alb_dif_nir, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif) + const creal1dk &sfc_alb_dir_vis, const creal1dk &sfc_alb_dir_nir, + const creal1dk &sfc_alb_dif_vis, const creal1dk &sfc_alb_dif_nir, + const real2dk &sfc_alb_dir, const real2dk &sfc_alb_dif) { EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); @@ -289,13 +306,11 @@ static void compute_band_by_band_surface_albedos( /* * Compute broadband visible/UV and near-infrared surface fluxes. */ -template static void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, - SwBndFluxDirT &sw_bnd_flux_dir , SwBndFluxDifT &sw_bnd_flux_dif , - SfcFluxDirVisT &sfc_flux_dir_vis, SfcFluxDirNirT &sfc_flux_dir_nir, - SfcFluxDifVisT &sfc_flux_dif_vis, SfcFluxDifNirT &sfc_flux_dif_nir) + const real3dk &sw_bnd_flux_dir , const real3dk &sw_bnd_flux_dif , + const real1dk &sfc_flux_dir_vis, const real1dk &sfc_flux_dir_nir, + const real1dk &sfc_flux_dif_vis, const real1dk &sfc_flux_dif_nir) { // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when @@ -349,42 +364,26 @@ static void compute_broadband_surface_fluxes( * The input logger is in charge of outputing info to * screen and/or to file (or neither), depending on how it was set up. */ -template static void rrtmgp_main( const int ncol, const int nlay, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei, CldfracT &cldfrac, - AerTauSwT &aer_tau_sw, AerSsaSwT &aer_ssa_sw, AerAsmSwT &aer_asm_sw, AerTauLwT &aer_tau_lw, - CldTauSwBndT &cld_tau_sw_bnd, CldTauLwBndT &cld_tau_lw_bnd, - CldTauSwGptT &cld_tau_sw_gpt, CldTauLwGptT &cld_tau_lw_gpt, - SwFluxUpT &sw_flux_up, SwFluxDnT &sw_flux_dn, SwFluxDnDirT &sw_flux_dn_dir, - LwFluxUpT &lw_flux_up, LwFluxDnT &lw_flux_dn, - SwClnclrskyFluxUpT &sw_clnclrsky_flux_up, SwClnclrskyFluxDnT &sw_clnclrsky_flux_dn, SwClnclrskyFluxDnDirT &sw_clnclrsky_flux_dn_dir, - SwClrskyFluxUpT &sw_clrsky_flux_up, SwClrskyFluxDnT &sw_clrsky_flux_dn, SwClrskyFluxDnDirT &sw_clrsky_flux_dn_dir, - SwClnskyFluxUpT &sw_clnsky_flux_up, SwClnskyFluxDnT &sw_clnsky_flux_dn, SwClnskyFluxDnDirT &sw_clnsky_flux_dn_dir, - LwClnclrskyFluxUpT &lw_clnclrsky_flux_up, LwClnclrskyFluxDnT &lw_clnclrsky_flux_dn, - LwClrskyFluxUpT &lw_clrsky_flux_up, LwClrskyFluxDnT &lw_clrsky_flux_dn, - LwClnskyFluxUpT &lw_clnsky_flux_up, LwClnskyFluxDnT &lw_clnsky_flux_dn, - SwBndFluxUpT &sw_bnd_flux_up, SwBndFluxDnT &sw_bnd_flux_dn, SwBndFluxDnDirT &sw_bnd_flux_dn_dir, - LwBndFluxUpT &lw_bnd_flux_up, LwBndFluxDnT &lw_bnd_flux_dn, - const RealT tsi_scaling, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei, const real2dk &cldfrac, + const real3dk &aer_tau_sw, const real3dk &aer_ssa_sw, const real3dk &aer_asm_sw, const real3dk &aer_tau_lw, + const real3dk &cld_tau_sw_bnd, const real3dk &cld_tau_lw_bnd, + const real3dk &cld_tau_sw_gpt, const real3dk &cld_tau_lw_gpt, + const real2dk &sw_flux_up, const real2dk &sw_flux_dn, const real2dk &sw_flux_dn_dir, + const real2dk &lw_flux_up, const real2dk &lw_flux_dn, + const real2dk &sw_clnclrsky_flux_up, const real2dk &sw_clnclrsky_flux_dn, const real2dk &sw_clnclrsky_flux_dn_dir, + const real2dk &sw_clrsky_flux_up, const real2dk &sw_clrsky_flux_dn, const real2dk &sw_clrsky_flux_dn_dir, + const real2dk &sw_clnsky_flux_up, const real2dk &sw_clnsky_flux_dn, const real2dk &sw_clnsky_flux_dn_dir, + const real2dk &lw_clnclrsky_flux_up, const real2dk &lw_clnclrsky_flux_dn, + const real2dk &lw_clrsky_flux_up, const real2dk &lw_clrsky_flux_dn, + const real2dk &lw_clnsky_flux_up, const real2dk &lw_clnsky_flux_dn, + const real3dk &sw_bnd_flux_up, const real3dk &sw_bnd_flux_dn, const real3dk &sw_bnd_flux_dn_dir, + const real3dk &lw_bnd_flux_up, const real3dk &lw_bnd_flux_dn, + const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) { @@ -404,7 +403,7 @@ static void rrtmgp_main( #endif // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; + fluxes_t fluxes_sw; fluxes_sw.flux_up = sw_flux_up; fluxes_sw.flux_dn = sw_flux_dn; fluxes_sw.flux_dn_dir = sw_flux_dn_dir; @@ -412,37 +411,37 @@ static void rrtmgp_main( fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; + fluxes_broadband_t clnclrsky_fluxes_sw; clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; + fluxes_broadband_t clrsky_fluxes_sw; clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; + fluxes_broadband_t clnsky_fluxes_sw; clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; + fluxes_t fluxes_lw; fluxes_lw.flux_up = lw_flux_up; fluxes_lw.flux_dn = lw_flux_dn; fluxes_lw.bnd_flux_up = lw_bnd_flux_up; fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; + fluxes_broadband_t clnclrsky_fluxes_lw; clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; + fluxes_broadband_t clrsky_fluxes_lw; clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; + fluxes_broadband_t clnsky_fluxes_lw; clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; @@ -450,8 +449,8 @@ static void rrtmgp_main( auto nlwbands = k_dist_lw_k.get_nband(); // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; + optical_props2_t aerosol_sw; + optical_props1_t aerosol_lw; aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); aerosol_sw.alloc_2str(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { @@ -476,8 +475,8 @@ static void rrtmgp_main( #endif // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + optical_props2_t clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + optical_props1_t clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); @@ -549,17 +548,15 @@ static void rrtmgp_finalize() /* * Shortwave driver (called by rrtmgp_main) */ -template static void rrtmgp_sw( const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const RealT tsi_scaling, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + optical_props2_t &aerosol, optical_props2_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, + const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { @@ -647,7 +644,7 @@ static void rrtmgp_sw( // Subset gases auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; + gas_concs_t gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { auto vmr_day = view_t("vmr_day", nday, nlay); @@ -660,7 +657,7 @@ static void rrtmgp_sw( } // Subset aerosol optics - OpticalProps2strK aerosol_day; + optical_props2_t aerosol_day; aerosol_day.init(k_dist.get_band_lims_wavenumber()); aerosol_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { @@ -671,7 +668,7 @@ static void rrtmgp_sw( // Subset cloud optics // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; + optical_props2_t clouds_day; clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); clouds_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { @@ -697,7 +694,7 @@ static void rrtmgp_sw( auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; + fluxes_t fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; fluxes_day.flux_dn_dir = flux_dn_dir_day; @@ -706,10 +703,10 @@ static void rrtmgp_sw( fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; // Allocate space for optical properties - OpticalProps2strK optics; + optical_props2_t optics; optics.alloc_2str(nday, nlay, k_dist); - OpticalProps2strK optics_no_aerosols; + optical_props2_t optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_2str(nday, nlay, k_dist); @@ -811,14 +808,13 @@ static void rrtmgp_sw( /* * Longwave driver (called by rrtmgp_main) */ -template static void rrtmgp_lw( const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + optical_props1_t &aerosol, optical_props1_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Problem size @@ -856,16 +852,16 @@ static void rrtmgp_lw( }); // Allocate space for optical properties - OpticalProps1sclK optics; + optical_props1_t optics; optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; + optical_props1_t optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); } // Boundary conditions - SourceFuncLWK lw_sources; + source_func_t lw_sources; lw_sources.alloc(ncol, nlay, k_dist); view_t t_sfc ("t_sfc" ,ncol); view_t emis_sfc("emis_sfc",nbnd,ncol); @@ -950,12 +946,10 @@ static void rrtmgp_lw( /* * Return a subcolumn mask consistent with a specified overlap assumption */ -template -static view_t get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, - CldfT &cldf, const int overlap_option, SeedsT &seeds) +static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds) { // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = view_t("subcolumn_mask", ncol, nlay, ngpt); + int3dk subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); // Subcolumn generators are a means for producing a variable x(i,j,k), where // @@ -1029,10 +1023,9 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i /* * Compute cloud area from 3d subcol cloud property */ -template static void compute_cloud_area( - int ncol, int nlay, int ngpt, RealT pmin, RealT pmax, - const PmidT& pmid, const CldTauGptT& cld_tau_gpt, CldAreaT& cld_area) + int ncol, int nlay, int ngpt, Real pmin, Real pmax, + const creal2dk& pmid, const real3dk& cld_tau_gpt, const real1dk& cld_area) { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 @@ -1058,20 +1051,15 @@ static void compute_cloud_area( /* * Return select cloud-top diagnostics following AeroCom recommendation */ -template static void compute_aerocom_cloudtop( - int ncol, int nlay, const TmidT &tmid, const PmidT &pmid, - const PdelT &p_del, const ZdelT &z_del, const QcT &qc, - const QiT &qi, const RelT &rel, const ReiT &rei, - const CldfracTotT &cldfrac_tot, const NcT &nc, - TmidAtCldtopT &T_mid_at_cldtop, PmidAtCldtopT &p_mid_at_cldtop, - CldfracIceAtCldtopT &cldfrac_ice_at_cldtop, CldfracLiqAtCldtopT &cldfrac_liq_at_cldtop, - CldfracTotAtCldtopT &cldfrac_tot_at_cldtop, CdncAtCldtopT &cdnc_at_cldtop, - EffRadiusQcAtCldtopT &eff_radius_qc_at_cldtop, EffRadiusQiAtCldTopT &eff_radius_qi_at_cldtop) + int ncol, int nlay, const creal2dk &tmid, const creal2dk &pmid, + const creal2dk &p_del, const real2dk &z_del, const creal2dk &qc, + const creal2dk &qi, const creal2dk &rel, const creal2dk &rei, + const real2dk &cldfrac_tot, const creal2dk &nc, + const real1dk &T_mid_at_cldtop, const real1dk &p_mid_at_cldtop, + const real1dk &cldfrac_ice_at_cldtop, const real1dk &cldfrac_liq_at_cldtop, + const real1dk &cldfrac_tot_at_cldtop, const real1dk &cdnc_at_cldtop, + const real1dk &eff_radius_qc_at_cldtop, const real1dk &eff_radius_qi_at_cldtop) { /* The goal of this routine is to calculate properties at cloud top * based on the AeroCom recommendation. See reference for routine @@ -1217,7 +1205,7 @@ static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, O } -static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) +static int get_wavelength_index(optical_props_t &kdist, RealT wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -1251,14 +1239,13 @@ static inline int get_wavelength_index_lw_k(RealT wavelength) return get_wavelength_index(k_dist_lw_k, wavelength); } -template -static OpticalProps2strK get_cloud_optics_sw( +static optical_props2_t get_cloud_optics_sw( const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { // Initialize optics - OpticalProps2strK clouds; + optical_props2_t clouds; clouds.init(kdist.get_band_lims_wavenumber()); clouds.alloc_2str(ncol, nlay); @@ -1278,14 +1265,13 @@ static OpticalProps2strK get_cloud_optics_sw( return clouds; } -template -static OpticalProps1sclK get_cloud_optics_lw( +static optical_props1_t get_cloud_optics_lw( const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { // Initialize optics - OpticalProps1sclK clouds; + optical_props1_t clouds; clouds.init(kdist.get_band_lims_wavenumber()); clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! @@ -1305,12 +1291,11 @@ static OpticalProps1sclK get_cloud_optics_lw( return clouds; } -template -static OpticalProps2strK get_subsampled_clouds( +static optical_props2_t get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { + optical_props2_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { // Initialized subsampled optics - OpticalProps2strK subsampled_optics; + optical_props2_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_2str(ncol, nlay); // Check that we do not have clouds with no optical properties; this would get corrected @@ -1356,13 +1341,12 @@ static OpticalProps2strK get_subsampled_clouds( return subsampled_optics; } -template -static OpticalProps1sclK get_subsampled_clouds( +static optical_props1_t get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, - CldT &cld, PlayT &p_lay) { + optical_props1_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { + // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; + optical_props1_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_1scl(ncol, nlay); // Check that we do not have clouds with no optical properties; this would get corrected From de1862c61a0824480c3ff21d18f8f5f31a1023e9 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 26 Jul 2024 15:35:47 -0500 Subject: [PATCH 359/904] Downgrade libunwind for machine frontier-scream-gpu Downgrade libunwind from 1.6.2 to 1.5.0 to prevent ums002/default, which is required to load libunwind/1.6.2, from resetting rocm/5.4.0 back to rocm/5.3.0. --- cime_config/machines/config_machines.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3617bb88f9ec..7cd34fdb0247 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1437,9 +1437,7 @@ PrgEnv-cray craype-accel-amd-gfx90a rocm/5.4.0 - ums/default - ums002/default - libunwind/1.6.2 + libunwind/1.5.0 cce/15.0.1 From d4d7e0e15f457223ccea74d9e661d7d555f29451 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 26 Jul 2024 15:56:28 -0500 Subject: [PATCH 360/904] Run hipInit prior to MPI_Init in subroutine cime_pre_init1() This workaround is used to avoid occasional segfaults during MPI_Init. Not required for ROCm 5.5.1 or newer releases. --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 9 +++++++++ .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 6 ++++++ components/eamxx/src/mct_coupling/scream_f2c_mod.F90 | 3 +++ driver-mct/main/cime_comp_mod.F90 | 2 ++ 4 files changed, 20 insertions(+) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 34bbbedcc5c4..bc79f6c11a7a 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -25,6 +25,7 @@ module atm_comp_mct ! Public interfaces !-------------------------------------------------------------------------- + public :: atm_init_hip_mct public :: atm_init_mct public :: atm_run_mct public :: atm_final_mct @@ -46,6 +47,14 @@ module atm_comp_mct CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + !=============================================================================== + subroutine atm_init_hip_mct() + use scream_f2c_mod, only: scream_init_hip_atm + + call scream_init_hip_atm() + + end subroutine atm_init_hip_mct + !=============================================================================== subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use iso_c_binding, only: c_ptr, c_loc, c_int, c_char, c_bool diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 0bdf90eeb71c..99a7ff0d8240 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -21,6 +21,8 @@ #include "ekat/ekat_pack.hpp" #include "ekat/ekat_assert.hpp" +#include + // Anonymous namespace, for some utility functions namespace { @@ -202,6 +204,10 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor }); } +void scream_init_hip_atm () { + hipInit(0); +} + void scream_init_atm (const char* caseid, const char* hostname, const char* username) diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 51c3eca91230..25abb1495ba5 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -69,6 +69,9 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices integer(kind=c_int), intent(in) :: import_field_size, export_field_size end subroutine scream_setup_surface_coupling + subroutine scream_init_hip_atm () bind(c) + end subroutine scream_init_hip_atm + ! This subroutine performs completes the initialization of the atm instance. ! In particular, this routine must be called *after* scream_create_atm_instance, ! and *after* scream_setup_surface_coupling. diff --git a/driver-mct/main/cime_comp_mod.F90 b/driver-mct/main/cime_comp_mod.F90 index 2131d0c86844..8282a76109c8 100644 --- a/driver-mct/main/cime_comp_mod.F90 +++ b/driver-mct/main/cime_comp_mod.F90 @@ -714,6 +714,7 @@ module cime_comp_mod subroutine cime_pre_init1(esmf_log_option) use shr_pio_mod, only : shr_pio_init1, shr_pio_init2 use seq_comm_mct, only: num_inst_driver + use atm_comp_mct, only: atm_init_hip_mct !---------------------------------------------------------- !| Initialize MCT and MPI communicators and IO !---------------------------------------------------------- @@ -736,6 +737,7 @@ subroutine cime_pre_init1(esmf_log_option) beg_count = shr_sys_irtc(irtc_rate) + call atm_init_hip_mct() call mpi_init(ierr) call shr_mpi_chkerr(ierr,subname//' mpi_init') From 4411d44834873c7d988f4af4b47e2c54a7483133 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 26 Jul 2024 18:06:26 -0500 Subject: [PATCH 361/904] Add SCREAM_SYSTEM_WORKAROUND macro This macro is used to optionally run hipInit prior to MPI_Init to avoid occasional segfaults. It can be turned off for ROCm 5.5.1 or newer releases. --- .../cmake_macros/crayclang-scream_frontier-scream-gpu.cmake | 2 +- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 4 ++++ .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 4 ++++ components/eamxx/src/mct_coupling/scream_f2c_mod.F90 | 2 ++ driver-mct/main/cime_comp_mod.F90 | 4 ++++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index cc90b369244a..a5c89c3318ed 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -5,7 +5,7 @@ set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") -string(APPEND CPPDEFS " -DLINUX") +string(APPEND CPPDEFS " -DLINUX -DSCREAM_SYSTEM_WORKAROUND=1") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index bc79f6c11a7a..dc57d828bbdb 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -25,7 +25,9 @@ module atm_comp_mct ! Public interfaces !-------------------------------------------------------------------------- +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) public :: atm_init_hip_mct +#endif public :: atm_init_mct public :: atm_run_mct public :: atm_final_mct @@ -47,6 +49,7 @@ module atm_comp_mct CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) !=============================================================================== subroutine atm_init_hip_mct() use scream_f2c_mod, only: scream_init_hip_atm @@ -54,6 +57,7 @@ subroutine atm_init_hip_mct() call scream_init_hip_atm() end subroutine atm_init_hip_mct +#endif !=============================================================================== subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 99a7ff0d8240..e9129e6fcea3 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -21,7 +21,9 @@ #include "ekat/ekat_pack.hpp" #include "ekat/ekat_assert.hpp" +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) #include +#endif // Anonymous namespace, for some utility functions namespace { @@ -204,9 +206,11 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor }); } +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) void scream_init_hip_atm () { hipInit(0); } +#endif void scream_init_atm (const char* caseid, const char* hostname, diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 25abb1495ba5..f7a0bc9c3794 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -69,8 +69,10 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices integer(kind=c_int), intent(in) :: import_field_size, export_field_size end subroutine scream_setup_surface_coupling +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) subroutine scream_init_hip_atm () bind(c) end subroutine scream_init_hip_atm +#endif ! This subroutine performs completes the initialization of the atm instance. ! In particular, this routine must be called *after* scream_create_atm_instance, diff --git a/driver-mct/main/cime_comp_mod.F90 b/driver-mct/main/cime_comp_mod.F90 index 8282a76109c8..7cf442b9413d 100644 --- a/driver-mct/main/cime_comp_mod.F90 +++ b/driver-mct/main/cime_comp_mod.F90 @@ -714,7 +714,9 @@ module cime_comp_mod subroutine cime_pre_init1(esmf_log_option) use shr_pio_mod, only : shr_pio_init1, shr_pio_init2 use seq_comm_mct, only: num_inst_driver +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) use atm_comp_mct, only: atm_init_hip_mct +#endif !---------------------------------------------------------- !| Initialize MCT and MPI communicators and IO !---------------------------------------------------------- @@ -737,7 +739,9 @@ subroutine cime_pre_init1(esmf_log_option) beg_count = shr_sys_irtc(irtc_rate) +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) call atm_init_hip_mct() +#endif call mpi_init(ierr) call shr_mpi_chkerr(ierr,subname//' mpi_init') From 24f24b88665c4e671f91bde4aacb0f3007c20d3b Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Mon, 29 Jul 2024 09:28:12 -0600 Subject: [PATCH 362/904] added package condition in tendency tracers --- .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index a0f4ae74e5ec..601d134ceec8 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -830,13 +830,15 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & modifiedGroupName, & tracerGroupSurfaceFluxRunoff) - ! Get surface flux due to subglacial runoff array - ! only active tracers have subglacial runoff flux for now, - ! but we still need to associate for ALL tracers - modifiedGroupName = groupName // "SurfaceFluxSubglacialRunoff" - call mpas_pool_get_array(tracersSurfaceFluxPool, & - modifiedGroupName, & - tracerGroupSurfaceFluxSubglacialRunoff) + if (trim(config_subglacial_runoff_mode) == 'data') then + ! Get surface flux due to subglacial runoff array + ! only active tracers have subglacial runoff flux for now, + ! but we still need to associate for ALL tracers + modifiedGroupName = groupName // "SurfaceFluxSubglacialRunoff" + call mpas_pool_get_array(tracersSurfaceFluxPool, & + modifiedGroupName, & + tracerGroupSurfaceFluxSubglacialRunoff) + end if ! Get surface flux removed array to keep track of how much ! flux is ignored From 1f77bf0e48729dc1f06fad26933fa885a9e1e0d2 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 29 Jul 2024 10:55:32 -0600 Subject: [PATCH 363/904] Changes due to code review feedback. --- .../eamxx_mam_wetscav_process_interface.cpp | 68 +++++++++++++------ .../eamxx_mam_wetscav_process_interface.hpp | 36 ++++++++-- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index b598fa6d7483..2a81c1a9c427 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -29,7 +29,6 @@ void MAMWetscav::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. auto q_unit = kg / kg; - auto dqdt_unit = kg / kg / s; auto n_unit = 1 / kg; // units of number mixing ratios of tracers m_grid = grids_manager->get_grid("Physics"); @@ -206,7 +205,7 @@ void MAMWetscav::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- - static constexpr auto m3 = m2 * m; + static constexpr auto m3 = m * m * m; // Aerosol dry particle diameter [m] add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); @@ -357,6 +356,44 @@ void MAMWetscav::initialize_impl(const RunType run_type) { const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now + + sh_frac_ = view_2d("sh_frac", ncol_, nlev_); + Kokkos::deep_copy(sh_frac_, 0); + + // Deep convective cloud fraction [fraction] + dp_frac_ = view_2d("dp_frac", ncol_, nlev_); + Kokkos::deep_copy(dp_frac_, 0); + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + evapcsh_ = view_2d("evapcsh", ncol_, nlev_); + Kokkos::deep_copy(evapcsh_, 0); + + // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] + evapcdp_ = view_2d("evapcdp", ncol_, nlev_); + Kokkos::deep_copy(evapcdp_, 0); + + // Rain production, shallow convection [kg/kg/s] + rprdsh_ = view_2d("rprdsh", ncol_, nlev_); + Kokkos::deep_copy(rprdsh_, 0); + + // Rain production, deep convection [kg/kg/s] + rprddp_ = view_2d("rprddp", ncol_, nlev_); + Kokkos::deep_copy(rprddp_, 0); + + // In cloud water mixing ratio, deep convection + icwmrdp_ = view_2d("icwmrdp", ncol_, nlev_); + Kokkos::deep_copy(icwmrdp_, 0); + + // In cloud water mixing ratio, shallow convection + icwmrsh_ = view_2d("icwmrsh", ncol_, nlev_); + Kokkos::deep_copy(icwmrsh_, 0); + + // Detraining cld H20 from deep convection [kg/kg/s] + dlf_ = view_2d("dlf", ncol_, nlev_); + Kokkos::deep_copy(dlf_, 0); + //--------------------------------------------------------------------------------- // Setup preprocessing and post processing //--------------------------------------------------------------------------------- @@ -394,40 +431,31 @@ void MAMWetscav::run_impl(const double dt) { // TODO: Following variables are from convective parameterization (not // implemented yet in EAMxx), so should be zero for now - auto sh_frac = view_2d("sh_frac", ncol_, nlev_); - Kokkos::deep_copy(sh_frac, 0); + auto sh_frac = sh_frac_; // Deep convective cloud fraction [fraction] - auto dp_frac = view_2d("dp_frac", ncol_, nlev_); - Kokkos::deep_copy(dp_frac, 0); + auto dp_frac = dp_frac_; // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] - auto evapcsh = view_2d("evapcsh", ncol_, nlev_); - Kokkos::deep_copy(evapcsh, 0); + auto evapcsh = evapcsh_; // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] - auto evapcdp = view_2d("evapcdp", ncol_, nlev_); - Kokkos::deep_copy(evapcdp, 0); + auto evapcdp = evapcdp_; // Rain production, shallow convection [kg/kg/s] - auto rprdsh = view_2d("rprdsh", ncol_, nlev_); - Kokkos::deep_copy(rprdsh, 0); + auto rprdsh = rprdsh_; // Rain production, deep convection [kg/kg/s] - auto rprddp = view_2d("rprddp", ncol_, nlev_); - Kokkos::deep_copy(rprddp, 0); + auto rprddp = rprddp_; // In cloud water mixing ratio, deep convection - auto icwmrdp = view_2d("icwmrdp", ncol_, nlev_); - Kokkos::deep_copy(icwmrdp, 0); + auto icwmrdp = icwmrdp_; // In cloud water mixing ratio, shallow convection - auto icwmrsh = view_2d("icwmrsh", ncol_, nlev_); - Kokkos::deep_copy(icwmrsh, 0); + auto icwmrsh = icwmrsh_; // Detraining cld H20 from deep convection [kg/kg/s] - auto dlf = view_2d("dlf", ncol_, nlev_); - Kokkos::deep_copy(dlf, 0); + auto dlf = dlf_; //----------- Variables from macrophysics scheme ------------- // Total cloud fraction diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index 8bd81115d8a3..ac811cab34d1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -40,10 +40,10 @@ class MAMWetscav : public scream::AtmosphereProcess { MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The type of subcomponent - AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type() const override { return AtmosphereProcessType::Physics; } // The name of the subcomponent - std::string name() const { return "mam4_wetscav"; } + std::string name() const override { return "mam4_wetscav"; } // Set the grid and input output variables void set_grids( @@ -52,7 +52,7 @@ class MAMWetscav : public scream::AtmosphereProcess { // management of common atm process memory // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels - size_t requested_buffer_size_in_bytes() const { + size_t requested_buffer_size_in_bytes() const override { return mam_coupling::buffer_size(ncol_, nlev_); } void init_buffers(const ATMBufferManager &buffer_manager) override; @@ -64,7 +64,7 @@ class MAMWetscav : public scream::AtmosphereProcess { void run_impl(const double dt) override; // Finalize - void finalize_impl(){/*Do nothing*/}; + void finalize_impl() override {/*Do nothing*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. @@ -168,6 +168,34 @@ class MAMWetscav : public scream::AtmosphereProcess { // Work arrays view_2d work_; + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now + + view_2d sh_frac_; + + // Deep convective cloud fraction [fraction] + view_2d dp_frac_; + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + view_2d evapcsh_; + + view_2d evapcdp_; + + // Rain production, shallow convection [kg/kg/s] + view_2d rprdsh_; + + // Rain production, deep convection [kg/kg/s] + view_2d rprddp_; + + // In cloud water mixing ratio, deep convection + view_2d icwmrdp_; + + // In cloud water mixing ratio, shallow convection + view_2d icwmrsh_; + + // Detraining cld H20 from deep convection [kg/kg/s] + view_2d dlf_; + // Aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; From 2890ec1f87bdf12f910d28667e8a35a822e7df89 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 29 Jul 2024 11:30:38 -0600 Subject: [PATCH 364/904] Changes due to code review feedback. --- .../physics/mam/eamxx_mam_wetscav_process_interface.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index ac811cab34d1..9d1dc20bc88f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -10,14 +10,6 @@ // For component name #include -#ifndef KOKKOS_ENABLE_CUDA -#define protected_except_cuda public -#define private_except_cuda public -#else -#define protected_except_cuda protected -#define private_except_cuda private -#endif - namespace scream { /* From 846e5044429caf921a74c9b49e194e004b76115a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 29 Jul 2024 15:00:28 -0600 Subject: [PATCH 365/904] EAMxx: fix check for io --- components/eamxx/src/share/io/scream_output_manager.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index ccec5f516fd5..13e2e97101b1 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -328,7 +328,11 @@ void OutputManager::run(const util::TimeStamp& timestamp) return; } - EKAT_REQUIRE_MSG (timestamp<=m_output_control.next_write_ts, + // Ensure we did not go past the scheduled write time without hitting it + EKAT_REQUIRE_MSG ( + (m_output_control.frequency_units=="nsteps" + ? timestamp.get_num_steps()<=m_output_control.next_write_ts.get_num_steps() + : timestamp<=m_output_control.next_write_ts), "Error! The input timestamp is past the next scheduled write timestamp.\n" " - current time stamp : " + timestamp.to_string() + "\n" " - next write time stamp: " + m_output_control.next_write_ts.to_string() + "\n" From eabd0fed3fb21bce4e634d76b673e65cb78f7cf4 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 29 Jul 2024 16:27:32 -0600 Subject: [PATCH 366/904] Changes due to code review feedback. --- ...x_mam_dry_deposition_process_interface.cpp | 24 +++++++------------ .../eamxx/src/physics/mam/mam_coupling.hpp | 1 + 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 433c9d4f6063..4a9dcbf1eca6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -41,28 +41,22 @@ void MAMDryDep::set_grids( using namespace ShortFieldTagsNames; // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d{{COL}, {ncol_}}; + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - const FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + const FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + const FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); // layout for 2D (ncol, pcnst) constexpr int pcnst = mam4::aero_model::pcnst; - const FieldLayout scalar2d_pcnct = + const FieldLayout vector2d_pcnst = grid_->get_2d_vector_layout(pcnst, "num_phys_constants"); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points - auto make_layout = [](const std::vector &extents, - const std::vector &names) { - std::vector tags(extents.size(), CMP); - return FieldLayout(tags, extents, names); - }; const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "num_modes", "lev"}); + const FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true, num_aero_modes, "num_modes"); using namespace ekat::units; @@ -150,10 +144,10 @@ void MAMDryDep::set_grids( //----------- Variables from other mam4xx processes ------------ // Geometric mean wet diameter for number distribution [m] - add_field("dgnumwet", scalar4d_mid, m, grid_name); + add_field("dgnumwet", vector3d_mid, m, grid_name); // Wet density of interstitial aerosol [kg/m3] - add_field("wetdens", scalar4d_mid, kg / m3, grid_name); + add_field("wetdens", vector3d_mid, kg / m3, grid_name); // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process @@ -203,11 +197,11 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------- // FIXME: These are diagnostics, remove them from FM after initial evaluation // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_cloud_borne_aerosols", scalar2d_pcnct, + add_field("deposition_flux_of_cloud_borne_aerosols", vector2d_pcnst, 1 / m2 / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", - scalar2d_pcnct, 1 / m2 / s, grid_name); + vector2d_pcnst, 1 / m2 / s, grid_name); } // set_grids // ================================================================ diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 1e13cece98d9..961ad675fbd1 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -606,6 +606,7 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); + // NOTE: we are not currently allowing surface topography: EKAT_KERNEL_ASSERT_MSG(dry_atm.z_surf == 0, "dry_atm.z_surf must be zero"); PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output From 405486e3e5420b72de8120ec0792b31ce7a5c128 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 29 Jul 2024 18:43:26 -0700 Subject: [PATCH 367/904] add pyatmproc list field methods --- .../src/python/libpyscream/pyatmproc.hpp | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 000a7d64c9a9..108edc2551b3 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -131,12 +131,32 @@ struct PyAtmProc { return pybind11::cast(missing); } - pybind11::list dump_fields() { - std::vector all_fields; - for (auto it : fields) { - all_fields.push_back(it.first); + pybind11::list list_fields(std::string ftype) { + std::vector fields_list; + for (const auto& field_pair : fields) { + const auto& field_identifier = field_pair.second.f.get_header().get_identifier(); + + if (ftype == "required" && ap->has_required_field(field_identifier)) { + fields_list.push_back(field_pair.first); + } else if (ftype == "computed" && ap->has_computed_field(field_identifier)) { + fields_list.push_back(field_pair.first); + } else if (ftype == "all") { + fields_list.push_back(field_pair.first); + } } - return pybind11::cast(all_fields); + return pybind11::cast(fields_list); + } + + pybind11::list list_all_fields() { + return list_fields("all"); + } + + pybind11::list list_required_fields() { + return list_fields("required"); + } + + pybind11::list list_computed_fields() { + return list_fields("computed"); } void setup_output (const std::string& yaml_file) { @@ -184,7 +204,9 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) .def("read_ic",&PyAtmProc::read_ic) - .def("dump_fields",&PyAtmProc::dump_fields); + .def("list_all_fields",&PyAtmProc::list_all_fields) + .def("list_required_fields",&PyAtmProc::list_required_fields) + .def("list_computed_fields",&PyAtmProc::list_computed_fields); } } // namespace scream From 8d49236cf761aae4e6d20ae768f1c8c87070f60c Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 29 Jul 2024 19:02:25 -0700 Subject: [PATCH 368/904] replace std::numeric_limits with kokkos norm_min --- .../shoc/impl/shoc_assumed_pdf_impl.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp index a407e83c2ee7..84e76c5941a1 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp @@ -9,6 +9,20 @@ namespace scream { namespace shoc { +/* +Add some functions to avoid cuda compilation warnings +TODO: move this to ekat or something +*/ +#ifdef KOKKOS_ENABLE_CUDA +KOKKOS_INLINE_FUNCTION constexpr Real safe_min() { + return Kokkos::Experimental::norm_min_v; +} +#else +KOKKOS_INLINE_FUNCTION constexpr Real safe_min() { + return std::numeric_limits::min(); +} +#endif + /* * Implementation of shoc shoc_assumed_pdf. Clients should NOT * #include this file, but include shoc_functions.hpp instead. @@ -317,7 +331,7 @@ void Functions::shoc_assumed_pdf( std_s1 = ekat::sqrt(ekat::max(0, ekat::square(cthl1)*thl2_1 + ekat::square(cqt1)*qw2_1 - 2*cthl1*sqrtthl2_1*cqt1*sqrtqw2_1*r_qwthl_1)); - const auto std_s1_not_small = std_s1 > std::sqrt(std::numeric_limits::min()) * 100; + const auto std_s1_not_small = std_s1 > std::sqrt(safe_min()) * 100; s1 = qw1_1-qs1*((1 + beta1*qw1_1)/(1 + beta1*qs1)); if (std_s1_not_small.any()) { C1.set(std_s1_not_small, sp(0.5)*(1 + ekat::erf(s1/(sqrt2*std_s1)))); @@ -357,7 +371,7 @@ void Functions::shoc_assumed_pdf( ekat::square(cthl2)*thl2_2 + ekat::square(cqt2)*qw2_2 - 2*cthl2*sqrtthl2_2*cqt2*sqrtqw2_2*r_qwthl_1))); s2.set(nequal, qw1_2-qs2*((1 + beta2*qw1_2)/(1 + beta2*qs2))); - const auto std_s2_not_small = std_s2 > std::sqrt(std::numeric_limits::min()) * 100; + const auto std_s2_not_small = std_s2 > std::sqrt(safe_min()) * 100; const auto nequal_std_s2_not_small = nequal && std_s2_not_small; if (nequal_std_s2_not_small.any()) { C2.set(nequal_std_s2_not_small, sp(0.5)*(1 + ekat::erf(s2/(sqrt2*std_s2)))); From dd4b09a565540e6bd20e908fbdf55644df845902 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 29 Jul 2024 19:23:53 -0700 Subject: [PATCH 369/904] update haero module --- externals/haero | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/haero b/externals/haero index 8e4c96ded375..0dff12afaf72 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit 8e4c96ded375bf03bb5e83b0779e45a3aec9659e +Subproject commit 0dff12afaf72f3dff7831676260812d41dd68683 From 8fd0eae1ad3d3ae389b10e1e685d4061714e256d Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 30 Jul 2024 08:47:05 -0600 Subject: [PATCH 370/904] Changes due to code review feedback. --- .../eamxx_mam_wetscav_process_interface.cpp | 38 ++++++------------- .../eamxx_mam_wetscav_process_interface.hpp | 12 +++++- .../mam/p3_mam4_wetscav/output.yaml | 4 -- .../mam/shoc_cldfrac_p3_wetscav/output.yaml | 4 -- .../single-process/mam/wet_scav/output.yaml | 4 -- 5 files changed, 23 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 2a81c1a9c427..81f13fd6ae9a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -47,10 +47,6 @@ void MAMWetscav::set_grids( // layout for 2D (1d horiz X 1d vertical) variables FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); - // layout for 3D (ncol, nmodes, nlevs) - FieldLayout scalar3d_mid_nmodes = - m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); - // layout for 2D (ncol, pcnst) FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "num_phys_constants"); @@ -205,22 +201,6 @@ void MAMWetscav::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- - static constexpr auto m3 = m * m * m; - - // Aerosol dry particle diameter [m] - add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); - - // Wet aerosol density [kg/m3] - add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); - - // Aerosol water [kg/kg] - add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); - - // Wet aerosol diameter [m] - add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); - - // Fraction of transported species that are insoluble [fraction] - add_field("fracis", scalar3d_mid, nondim, grid_name); // Aerosol wet deposition (interstitial) [kg/m2/s] add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); @@ -352,6 +332,14 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } + const int nmodes = mam4::AeroConfig::num_modes(); + + // Aerosol dry particle diameter [m] + dgncur_a_ = view_3d("dgncur_a", ncol_, nmodes, nlev_); + wetdens_ = view_3d("wetdens", ncol_, nmodes, nlev_); + qaerwat_ = view_3d("qaerwat", ncol_, nmodes, nlev_); + dgnumwet_ = view_3d("dgnumwet", ncol_, nmodes, nlev_); + // Allocate work array const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); @@ -475,12 +463,10 @@ void MAMWetscav::run_impl(const double dt) { const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); - const auto wet_geometric_mean_diameter_i = - get_field_out("dgnumwet").get_view(); - const auto dry_geometric_mean_diameter_i = - get_field_out("dgncur_a").get_view(); - const auto qaerwat = get_field_out("qaerwat").get_view(); - const auto wetdens = get_field_out("wetdens").get_view(); + const auto wet_geometric_mean_diameter_i = dgnumwet_; + const auto dry_geometric_mean_diameter_i = dgncur_a_; + const auto qaerwat = qaerwat_; + const auto wetdens = wetdens_; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index 9d1dc20bc88f..3c3dec6cc623 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -23,6 +23,7 @@ namespace scream { class MAMWetscav : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; using view_2d = typename KT::template view_2d; + using view_3d = typename KT::template view_3d; // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; @@ -160,9 +161,18 @@ class MAMWetscav : public scream::AtmosphereProcess { // Work arrays view_2d work_; + // Aerosol dry particle diameter [m] + // Dimensions: [cols, modes, levels] + view_3d dgncur_a_; + // Wet aerosol density [kg/m3] + view_3d wetdens_; + // Aerosol water [kg/kg] + view_3d qaerwat_; + // Wet aerosol diameter [m] + view_3d dgnumwet_; + // TODO: Following variables are from convective parameterization (not // implemented yet in EAMxx), so should be zero for now - view_2d sh_frac_; // Deep convective cloud fraction [fraction] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml index c26db8eb6814..049dcdad35a5 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -56,10 +56,6 @@ Field Names: - num_c4 - aerdepwetis - aerdepwetcw - - dgnumwet - - dgncur_a - - wetdens - - qaerwat output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml index 1ae53e6c1c30..c38bafaa65e3 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -90,10 +90,6 @@ Field Names: - num_c4 - aerdepwetis - aerdepwetcw - - dgnumwet - - dgncur_a - - wetdens - - qaerwat output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 9cd0a16bc414..75172823870e 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -7,10 +7,6 @@ Fields: Field Names: - aerdepwetis - aerdepwetcw - - dgnumwet - - dgncur_a - - wetdens - - qaerwat - bc_c1 - bc_c3 - bc_c4 From 99e46d6b768ea1422246f2e5a6998b49ce5d3737 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 30 Jul 2024 09:14:31 -0700 Subject: [PATCH 371/904] add test for COMBLE and various fixes --- cime_config/tests.py | 1 + .../scream/dpxx/arm97/shell_commands | 2 +- .../scream/dpxx/comble/shell_commands | 20 +++++++++++++++++++ .../scream/dpxx/dycomsrf01/shell_commands | 2 +- 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands diff --git a/cime_config/tests.py b/cime_config/tests.py index 9c17744d0e64..dd08de8c2540 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -659,6 +659,7 @@ "tests" : ( "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-dycomsrf01", "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-arm97", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-comble", "ERS_P16_Ln22.ne30_ne30.F2000-SCREAMv1-DP-RCE", ) }, diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands index bda1791d66d7..4cba77f45f81 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands @@ -8,7 +8,7 @@ # Scripts location ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange -$ATMCHANGE iop_file=${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc -b +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/ARM97_iopfile_4scam.nc -b $ATMCHANGE target_latitude=36.605 -b $ATMCHANGE target_longitude=262.515 -b $ATMCHANGE iop_nudge_uv=true -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands new file mode 100644 index 000000000000..166e77b1f4bb --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -0,0 +1,20 @@ +# Sets up DPxx case to test COMBLE (cold air outbreak) +# lagrangian case over ocean with interactive surface fluxes and coriolis forcing applied to winds + +./xmlchange RUN_STARTDATE="2020-03-12" +./xmlchange PTS_LAT=74.5 +./xmlchange PTS_LON=10.0 + +./xmlchange SSTICE_DATA_FILENAME="${DIN_LOC_ROOT}/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc" +./xmlchange SSTICE_YEAR_ALIGN=2020 +./xmlchange SSTICE_YEAR_START=2020 +./xmlchange SSTICE_YEAR_END=2021 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/COMBLE_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=74.5 -b +$ATMCHANGE target_longitude=10.0 -b +$ATMCHANGE iop_coriolis=true + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands index 58c235807872..8c442acee75b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands @@ -8,7 +8,7 @@ # Scripts location ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange -$ATMCHANGE iop_file=${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -b +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -b $ATMCHANGE target_latitude=31.5 -b $ATMCHANGE target_longitude=238.5 -b $ATMCHANGE iop_dosubsidence=true -b From 29d83eba554da13a41dc590ea4354c2585d1686d Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 30 Jul 2024 11:15:57 -0600 Subject: [PATCH 372/904] Revert "Changes due to code review feedback." This reverts commit 8fd0eae1ad3d3ae389b10e1e685d4061714e256d. There are fields that need to be in the FieldManager since this is still in the evaluation stage. --- .../eamxx_mam_wetscav_process_interface.cpp | 38 +++++++++++++------ .../eamxx_mam_wetscav_process_interface.hpp | 12 +----- .../mam/p3_mam4_wetscav/output.yaml | 4 ++ .../mam/shoc_cldfrac_p3_wetscav/output.yaml | 4 ++ .../single-process/mam/wet_scav/output.yaml | 4 ++ 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 81f13fd6ae9a..2a81c1a9c427 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -47,6 +47,10 @@ void MAMWetscav::set_grids( // layout for 2D (1d horiz X 1d vertical) variables FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + // layout for 3D (ncol, nmodes, nlevs) + FieldLayout scalar3d_mid_nmodes = + m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); + // layout for 2D (ncol, pcnst) FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "num_phys_constants"); @@ -201,6 +205,22 @@ void MAMWetscav::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- + static constexpr auto m3 = m * m * m; + + // Aerosol dry particle diameter [m] + add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); + + // Wet aerosol density [kg/m3] + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); + + // Aerosol water [kg/kg] + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); + + // Wet aerosol diameter [m] + add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); + + // Fraction of transported species that are insoluble [fraction] + add_field("fracis", scalar3d_mid, nondim, grid_name); // Aerosol wet deposition (interstitial) [kg/m2/s] add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); @@ -332,14 +352,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } - const int nmodes = mam4::AeroConfig::num_modes(); - - // Aerosol dry particle diameter [m] - dgncur_a_ = view_3d("dgncur_a", ncol_, nmodes, nlev_); - wetdens_ = view_3d("wetdens", ncol_, nmodes, nlev_); - qaerwat_ = view_3d("qaerwat", ncol_, nmodes, nlev_); - dgnumwet_ = view_3d("dgnumwet", ncol_, nmodes, nlev_); - // Allocate work array const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); @@ -463,10 +475,12 @@ void MAMWetscav::run_impl(const double dt) { const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); - const auto wet_geometric_mean_diameter_i = dgnumwet_; - const auto dry_geometric_mean_diameter_i = dgncur_a_; - const auto qaerwat = qaerwat_; - const auto wetdens = wetdens_; + const auto wet_geometric_mean_diameter_i = + get_field_out("dgnumwet").get_view(); + const auto dry_geometric_mean_diameter_i = + get_field_out("dgncur_a").get_view(); + const auto qaerwat = get_field_out("qaerwat").get_view(); + const auto wetdens = get_field_out("wetdens").get_view(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index 3c3dec6cc623..9d1dc20bc88f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -23,7 +23,6 @@ namespace scream { class MAMWetscav : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; using view_2d = typename KT::template view_2d; - using view_3d = typename KT::template view_3d; // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; @@ -161,18 +160,9 @@ class MAMWetscav : public scream::AtmosphereProcess { // Work arrays view_2d work_; - // Aerosol dry particle diameter [m] - // Dimensions: [cols, modes, levels] - view_3d dgncur_a_; - // Wet aerosol density [kg/m3] - view_3d wetdens_; - // Aerosol water [kg/kg] - view_3d qaerwat_; - // Wet aerosol diameter [m] - view_3d dgnumwet_; - // TODO: Following variables are from convective parameterization (not // implemented yet in EAMxx), so should be zero for now + view_2d sh_frac_; // Deep convective cloud fraction [fraction] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml index 049dcdad35a5..c26db8eb6814 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -56,6 +56,10 @@ Field Names: - num_c4 - aerdepwetis - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml index c38bafaa65e3..1ae53e6c1c30 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -90,6 +90,10 @@ Field Names: - num_c4 - aerdepwetis - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 75172823870e..9cd0a16bc414 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -7,6 +7,10 @@ Fields: Field Names: - aerdepwetis - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat - bc_c1 - bc_c3 - bc_c4 From fc80af369da2416e71d5b5f68196c7cb3c1ef023 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 30 Jul 2024 11:03:40 -0700 Subject: [PATCH 373/904] update comment --- components/eamxx/cime_config/config_compsets.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index c41bdb870ec7..c684e1f6c6b6 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -75,7 +75,7 @@ - F2010-SCREAMv1-DP-General + F2010-SCREAMv1-DP-General 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%General Experimental, under development From ffe22ad4c463e6fce1c8f9fbec2daab1089fd6d3 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 11 Jul 2024 12:45:16 -0700 Subject: [PATCH 374/904] read in IOP lev data before surface pressure adjustment is applied --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 657f7ad742a0..70f1c41b603a 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -541,6 +541,15 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); + // Read in IOP lev data + auto data = iop_file_pressure.get_view().data(); + scorpio::read_var(iop_file,"lev",data); + + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilev(); From 3a6fcc9c5500030580d6eac5ec6716748e978128 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 11 Jul 2024 16:21:55 -0700 Subject: [PATCH 375/904] add minimum threshold when checking lat and lon relative error for IOP file to prevent NaN --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 70f1c41b603a..42ea7d0ec03f 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - m_params.get("target_latitude"); + ekat::impl::max(m_params.get("target_latitude"),0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - m_params.get("target_longitude"); + ekat::impl::max(m_params.get("target_longitude"),0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), From 613beffa39b50655cfb28430d68d91bff5ed52b1 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 16 Jul 2024 10:28:54 -0700 Subject: [PATCH 376/904] change ekat::impl::max instances to std::max and remove unneeded m_helper_fields line --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 42ea7d0ec03f..cf20435fd2ce 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - ekat::impl::max(m_params.get("target_latitude"),0.1); + std::max(m_params.get("target_latitude"),0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - ekat::impl::max(m_params.get("target_longitude"),0.1); + std::max(m_params.get("target_longitude"),0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -548,7 +548,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) // Convert to pressure to millibar (file gives pressure in Pa) for (int ilev=0; ilev Date: Wed, 17 Jul 2024 09:42:48 -0700 Subject: [PATCH 377/904] cast minimum lat/lon threshold as a real --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index cf20435fd2ce..6f8474f8d85e 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),0.1); + std::max(m_params.get("target_latitude"),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - std::max(m_params.get("target_longitude"),0.1); + std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), From 861565c882d829a70e9375a3c800b34ea67af165 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 19 Jul 2024 15:04:44 -0400 Subject: [PATCH 378/904] fix capture this warning in mam micro --- .../physics/mam/eamxx_mam_microphysics_process_interface.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7ea2adfddb63..8b93936148ce 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,6 +317,8 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index @@ -448,7 +450,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, some_step, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, From 44be05960805fc3539a2736ecef3072c3dfeaeab Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:10:21 -0500 Subject: [PATCH 379/904] Add ums and ums002 to load libunwind 1.6.2 After recent Frontier maintenance, ums/default and ums002/default are required to load libunwind/1.6.2. --- cime_config/machines/config_machines.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index c04dff5976cd..3617bb88f9ec 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1437,6 +1437,8 @@ PrgEnv-cray craype-accel-amd-gfx90a rocm/5.4.0 + ums/default + ums002/default libunwind/1.6.2 From 08fb2d1bd4f14ba8a1e279b049129f61b86546f3 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:12:27 -0500 Subject: [PATCH 380/904] Add to scream_scorpio_types.hpp to use std::int64_t This fixes a known build error after recent Frontier maintenance: no type named 'int64_t' in namespace 'std'; did you mean simply 'int64_t'? --- components/eamxx/src/share/io/scream_scorpio_types.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/share/io/scream_scorpio_types.hpp b/components/eamxx/src/share/io/scream_scorpio_types.hpp index 645b1b26937b..3b16cebc06c8 100644 --- a/components/eamxx/src/share/io/scream_scorpio_types.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_types.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace scream { namespace scorpio { From ac7280cdaea785027609f1306798a44ae806294e Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:16:28 -0500 Subject: [PATCH 381/904] Explicitly add gcc/12.2.0 lib path to CMAKE_EXE_LINKER_FLAGS This is a simple workaround to fix scream linking errors after recent Frontier maintenance before we find better solutions. --- .../cmake_macros/crayclang-scream_frontier-scream-gpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index afcca8f479e5..cc90b369244a 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -19,7 +19,7 @@ endif() string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64 -L/opt/gcc/12.2.0/snos/lib64") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{ROCM_PATH}/include") # Crusher: this resolves a crash in mct in docn init From bcf0a02e6e053b67eea98d8eb0dbec03b3f2dbbd Mon Sep 17 00:00:00 2001 From: Hassan Beydoun Date: Fri, 19 Jul 2024 13:05:12 -0700 Subject: [PATCH 382/904] add output variable for rain effective diameter --- components/eam/src/physics/p3/scream/micro_p3_interface.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 index edea27959d25..11d13c71a7a7 100644 --- a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 @@ -834,6 +834,8 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) real(rtype) :: icwmrst(pcols,pver) ! stratus water mixing ratio - on grid real(rtype) :: rho(pcols,pver) real(rtype) :: reff_rain(pcols,pver) + real(rtype) :: deff_rain(pcols,pver) + real(rtype) :: col_location(pcols,3),tmp_loc(pcols) ! Array of column lon (index 1) and lat (index 2) integer :: tmpi_loc(pcols) ! Global column index temp array @@ -1358,6 +1360,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) aqrain = 0._rtype anrain = 0._rtype freqr = 0._rtype + deff_rain = reff_rain*2._rtype*1.e6_rtype !rain effective diameter in microns ! Prognostic precipitation where (rain(:ncol,top_lev:) >= 1.e-7_rtype) aqrain(:ncol,top_lev:) = rain(:ncol,top_lev:) * cld_frac_r(:ncol,top_lev:) @@ -1412,6 +1415,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) call outfld('FREQI', freqi, pcols, lchnk) call outfld('FREQR', freqr, pcols, lchnk) call outfld('CDNUMC', cdnumc, pcols, lchnk) + call outfld('ADRAIN', deff_rain, pcols, lchnk) call outfld('CLOUDFRAC_LIQ_MICRO', cld_frac_l, pcols, lchnk) call outfld('CLOUDFRAC_ICE_MICRO', cld_frac_i, pcols, lchnk) From e55a1217ed8723f0623b8397a07dfb00522c1b27 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 18 Jul 2024 15:46:35 -0600 Subject: [PATCH 383/904] Add ustar and obklen to FM --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 657f9487611c..80485bf67130 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -395,6 +395,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); + add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. From 4d4fc7492c359f1094a3bed5b5a027c986dc1333 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 08:43:37 -0600 Subject: [PATCH 384/904] Remove obklen postcondition check --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 80485bf67130..657f9487611c 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -395,7 +395,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); - add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. From 237c7f524f01291055a4b3dff72c891c8072e328 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 10:20:20 -0600 Subject: [PATCH 385/904] Fix how IOP reads time --- .../src/share/iop/intensive_observation_period.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 6f8474f8d85e..32b120c57002 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -248,15 +248,17 @@ initialize_iop_file(const util::TimeStamp& run_t0, else if (scorpio::has_dim(iop_file, "tsec")) time_dimname = "tsec"; else EKAT_ERROR_MSG("Error! No valid dimension for tsec in "+iop_file+".\n"); - const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); - m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); - scorpio::read_var(iop_file,"tsec",m_time_info.iop_file_times_in_sec.data()); - - // From now on, when we read vars, "time" must be treated as unlimited, to avoid issues + // When we read vars, "time" must be treated as unlimited, to avoid issues if (not scorpio::is_dim_unlimited(iop_file,time_dimname)) { scorpio::pretend_dim_is_unlimited(iop_file,time_dimname); } + const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); + m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); + for (int t=0; t Date: Fri, 19 Jul 2024 15:48:20 -0700 Subject: [PATCH 386/904] filter aodvis by day indices --- components/eamxx/src/diagnostics/aodvis.cpp | 19 ++++++-- .../src/diagnostics/tests/aodvis_test.cpp | 46 ++++++++++++++++--- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index efb9b5ba9510..ac16c0033730 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -2,6 +2,8 @@ #include +#include "share/util/scream_universal_constants.hpp" + namespace scream { AODVis::AODVis(const ekat::Comm &comm, const ekat::ParameterList ¶ms) @@ -23,11 +25,13 @@ void AODVis::set_grids( m_nlevs = grid->get_num_vertical_levels(); // Define layouts we need (both inputs and outputs) - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,m_swbands,"swband"); - FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, m_swbands, "swband"); + FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); // The fields required for this diagnostic to be computed add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + add_field("sunlit", scalar1d_layout, nondim, grid_name); // Construct and allocate the aodvis field FieldIdentifier fid("AerosolOpticalDepth550nm", scalar1d_layout, nondim, @@ -41,18 +45,25 @@ void AODVis::compute_diagnostic_impl() { using MT = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; + Real var_fill_value = constants::DefaultFillValue().value; + const auto aod = m_diagnostic_output.get_view(); const auto tau_vis = get_field_in("aero_tau_sw") .subfield(1, m_vis_bnd) .get_view(); + const auto sunlit = get_field_in("sunlit").get_view(); const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); - auto tau_icol = ekat::subview(tau_vis, icol); - aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + if(sunlit(icol) == 0.0) { + aod(icol) = var_fill_value; + } else { + auto tau_icol = ekat::subview(tau_vis, icol); + aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + } }); } diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 140114812215..15de175da67d 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -1,10 +1,11 @@ +#include + #include "catch2/catch.hpp" #include "diagnostics/register_diagnostics.hpp" #include "share/field/field_utils.hpp" #include "share/grid/mesh_free_grids_manager.hpp" #include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_utils.hpp" - +#include "share/util/scream_universal_constants.hpp" namespace scream { std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, @@ -30,6 +31,10 @@ TEST_CASE("aodvis") { using namespace ShortFieldTagsNames; using namespace ekat::units; + Real var_fill_value = constants::DefaultFillValue().value; + + Real some_limit = 0.0025; + // A world comm ekat::Comm comm(MPI_COMM_WORLD); @@ -40,7 +45,7 @@ TEST_CASE("aodvis") { // Create a grids manager - single column for these tests constexpr int nlevs = 33; - const int ngcols = 2 * comm.size(); + const int ngcols = 10 * comm.size(); int nbnds = eamxx_swbands(); int swvis = eamxx_vis_swband_idx(); @@ -49,16 +54,23 @@ TEST_CASE("aodvis") { auto grid = gm->get_grid("Physics"); // Input (randomized) tau - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,nbnds,"swband"); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, nbnds, "swband"); FieldIdentifier tau_fid("aero_tau_sw", scalar3d_swband_layout, nondim, grid->name()); Field tau(tau_fid); tau.allocate_view(); tau.get_header().get_tracking().update_time_stamp(t0); + // Input (randomized) sunlit + FieldLayout scalar2d_layout = grid->get_2d_scalar_layout(); + FieldIdentifier sunlit_fid("sunlit", scalar2d_layout, nondim, grid->name()); + Field sunlit(sunlit_fid); + sunlit.allocate_view(); + sunlit.get_header().get_tracking().update_time_stamp(t0); // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(0, 0.05); + RPDF pdf(0, 0.005); auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -71,18 +83,33 @@ TEST_CASE("aodvis") { // Randomize tau randomize(tau, engine, pdf); + // Randomize sunlit + randomize(sunlit, engine, pdf); + // Create and set up the diagnostic ekat::ParameterList params; auto diag = diag_factory.create("AerosolOpticalDepth550nm", comm, params); diag->set_grids(gm); diag->set_required_field(tau); + diag->set_required_field(sunlit); diag->initialize(t0, RunType::Initial); + auto sun_h = sunlit.get_view(); + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + // zero out all sun_h if below 0.05 + if(sun_h(icol) < some_limit) { + sun_h(icol) = 0.0; + } + } + // sync to device + sunlit.sync_to_dev(); + // Run diag diag->compute_diagnostic(); // Check result tau.sync_to_host(); + sunlit.sync_to_host(); diag->get_diagnostic().sync_to_host(); const auto tau_h = tau.get_view(); @@ -93,12 +120,17 @@ TEST_CASE("aodvis") { auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - for(int ilev = 0; ilev < nlevs; ++ilev) { - aod_t(icol) += tau_h(icol, swvis, ilev); + if(sun_h(icol) < some_limit) { + aod_t(icol) = var_fill_value; + } else { + for(int ilev = 0; ilev < nlevs; ++ilev) { + aod_t(icol) += tau_h(icol, swvis, ilev); + } } } aod_hf.sync_to_dev(); aod_tf.sync_to_dev(); + // Workaround for non-bfb behavior of view_reduction() in release builds if(SCREAM_BFB_TESTING) { REQUIRE(views_are_equal(aod_hf, aod_tf)); From 7d8d166ec3d48d8465c982b85f277fa6c6ccf957 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 25 Jul 2024 22:24:07 -0400 Subject: [PATCH 387/904] remove sunlit sync to host --- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 15de175da67d..8090a9300fd9 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -109,7 +109,6 @@ TEST_CASE("aodvis") { // Check result tau.sync_to_host(); - sunlit.sync_to_host(); diag->get_diagnostic().sync_to_host(); const auto tau_h = tau.get_view(); From 5230b6081fb88817921b49460ee5b51d92e65e2b Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 12 Jul 2024 16:42:47 -0500 Subject: [PATCH 388/904] rename pyeamxx to pyscream --- components/eamxx/src/python/CMakeLists.txt | 4 ++-- components/eamxx/src/python/pyatmproc.hpp | 2 +- components/eamxx/src/python/pyeamxx.cpp | 2 +- components/eamxx/src/python/pyeamxx.hpp | 6 +++--- components/eamxx/src/python/pyeamxx/__init__.py | 16 ++++++++-------- components/eamxx/src/python/pygrid.hpp | 4 ++-- components/eamxx/src/python/pyproject.toml | 6 +++--- components/eamxx/src/python/readme | 6 +++--- .../eamxx/tests/python/pyp3/CMakeLists.txt | 2 +- .../eamxx/tests/python/pyp3/p3_standalone_py | 12 ++++++------ 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 43e2db73e533..41eae6d65463 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,8 +1,8 @@ find_package(pybind11 REQUIRED) find_package(mpi4py REQUIRED) -pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC +pybind11_add_module(pyscream pyscream.cpp) +target_link_libraries(pyscream PUBLIC mpi4py scream_share scream_io diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index 98d7b9a525ed..9e45bc513f1b 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -8,7 +8,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" -#include "pyeamxx.hpp" +#include "pyscream.hpp" #include diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index c4ec360d80a1..2efb8b6f8df9 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -45,7 +45,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyeamxx,m) { +PYBIND11_MODULE (pyscream,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/pyeamxx.hpp index 59371844a763..781c195919dd 100644 --- a/components/eamxx/src/python/pyeamxx.hpp +++ b/components/eamxx/src/python/pyeamxx.hpp @@ -1,5 +1,5 @@ -#ifndef PYEAMXX_HPP -#define PYEAMXX_HPP +#ifndef PYSCREAM_HPP +#define PYSCREAM_HPP #include "physics/register_physics.hpp" #include "diagnostics/register_diagnostics.hpp" @@ -27,4 +27,4 @@ struct PySession { } // namespace scream -#endif // PYEAMXX_HPP +#endif // PYSCREAM_HPP diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py index 75a36e148291..98c31744eb9f 100644 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ b/components/eamxx/src/python/pyeamxx/__init__.py @@ -1,15 +1,15 @@ """ This file will serve as a way to organize and expose - libpyeamxx internals to the rest of pyeamxx + libpyscream internals to the rest of pyscream """ -from libpyeamxx.libpyeamxx_ext import AtmProc -from libpyeamxx.libpyeamxx_ext import Grid -from libpyeamxx.libpyeamxx_ext import ParameterList -from libpyeamxx.libpyeamxx_ext import init -from libpyeamxx.libpyeamxx_ext import Field -from libpyeamxx.libpyeamxx_ext import P3 -from libpyeamxx.libpyeamxx_ext import finalize +from libpyscream.libpyscream_ext import AtmProc +from libpyscream.libpyscream_ext import Grid +from libpyscream.libpyscream_ext import ParameterList +from libpyscream.libpyscream_ext import init +from libpyscream.libpyscream_ext import Field +from libpyscream.libpyscream_ext import P3 +from libpyscream.libpyscream_ext import finalize __all__ = [ diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp index 102cf6037c62..e49a0443a345 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/pygrid.hpp @@ -3,7 +3,7 @@ #include "share/grid/mesh_free_grids_manager.hpp" -#include "pyeamxx.hpp" +#include "pyscream.hpp" #include @@ -14,7 +14,7 @@ namespace scream { inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) { EKAT_REQUIRE_MSG (PySession::get().inited, - "Error! You did not initialize pyeamxx, or you already finalized it!\n"); + "Error! You did not initialize pyscream, or you already finalized it!\n"); auto& comm = PySession::get().comm; ekat::ParameterList gm_params; std::vector grids_names = {"Physics"}; diff --git a/components/eamxx/src/python/pyproject.toml b/components/eamxx/src/python/pyproject.toml index fa5d6fc44817..490ea99c143e 100644 --- a/components/eamxx/src/python/pyproject.toml +++ b/components/eamxx/src/python/pyproject.toml @@ -3,15 +3,15 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "pyeamxx" +name = "pyscream" version = "0.0.2" dependencies = ["numpy", "mpi4py"] [tool.setuptools.packages.find] where = ["."] -include = ["pyeamxx", "libpyeamxx"] +include = ["pyscream", "libpyscream"] exclude = ["build_src", "tests"] namespaces = true [tool.setuptools.package-data] -"libpyeamxx" = ["*.so*"] +"libpyscream" = ["*.so*"] diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme index f5ed48f058ee..c16c32a5d217 100644 --- a/components/eamxx/src/python/readme +++ b/components/eamxx/src/python/readme @@ -1,7 +1,7 @@ INFO: - EAMxx python bindings -- pyeamxx is where we will house the python code -- libpyeamxx is where we will house the extensions +- pyscream is where we will house the python code +- libpyscream is where we will house the extensions - packaging moved to https://github.com/mahf708/experimental-scream-feedstock TODO: @@ -11,5 +11,5 @@ TODO: - decide archs/pythons/mpis to tgt USER: -- conda install pyeamxx -c mahf708/label/$mac (mac is chrysalis or pm-cpu) +- conda install pyscream -c mahf708/label/$mac (mac is chrysalis or pm-cpu) - see example in components/eamxx/tests/python/pyp3 diff --git a/components/eamxx/tests/python/pyp3/CMakeLists.txt b/components/eamxx/tests/python/pyp3/CMakeLists.txt index 7142be8dd8f8..945f15faaf65 100644 --- a/components/eamxx/tests/python/pyp3/CMakeLists.txt +++ b/components/eamxx/tests/python/pyp3/CMakeLists.txt @@ -53,6 +53,6 @@ foreach (rank IN LISTS MpiRanks) SRC_FILE p3_standalone_cxx.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc TGT_FILE p3_standalone_py.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_cxx${suffix} ${FIXTURES_BASE_NAME}_py${suffix} - LABELS pyeamxx + LABELS pyscream ) endforeach() diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index 0db6bcfcef61..34747943d081 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -5,7 +5,7 @@ import sys # Add path to scream libs sys.path.append('@SCREAM_BASE_DIR@/scripts') -# Add path to pyeamxx libs +# Add path to pyscream libs sys.path.append('@CMAKE_BINARY_DIR@/src/python') # Without these, and manual init/finalize, on my laptop I get @@ -15,7 +15,7 @@ mpi4py.rc.initialize = False # do not initialize MPI automatically mpi4py.rc.finalize = False # do not finalize MPI automatically from mpi4py import MPI -import pyeamxx +import pyscream from pathlib import Path from utils import ensure_yaml @@ -38,9 +38,9 @@ def main (): # Create the grid ncols = 218 nlevs = 72 - pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) + pyscream.create_grids_manager(ncols,nlevs,str(ic_file)) - p3 = pyeamxx.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') + p3 = pyscream.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') params = p3.get_params() old = params.get_dbl('max_total_ni') print (f"max_total_ni: {params.get_dbl('max_total_ni')}") @@ -64,7 +64,7 @@ if __name__ == "__main__": # This level of indirection ensures all pybind structs are destroyed # before we finalize eamxx (and hence kokkos) MPI.Init() - pyeamxx.init() + pyscream.init() main () - pyeamxx.finalize() + pyscream.finalize() MPI.Finalize() From a6f14a9d4482df92de08ac5eb8e56639ddd02a38 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 12 Jul 2024 16:54:10 -0500 Subject: [PATCH 389/904] more renaming --- components/eamxx/src/python/CMakeLists.txt | 13 +---------- .../src/python/libpyscream/CMakeLists.txt | 12 ++++++++++ .../python/{ => libpyscream}/pyatmproc.hpp | 2 +- .../src/python/{ => libpyscream}/pyfield.hpp | 0 .../src/python/{ => libpyscream}/pygrid.hpp | 2 +- .../python/{ => libpyscream}/pyparamlist.hpp | 0 .../pyscream_ext.cpp} | 2 +- .../pyscream_ext.hpp} | 0 .../src/python/{ => libpyscream}/pyutils.hpp | 0 .../eamxx/src/python/pyeamxx/__init__.py | 23 ------------------- .../eamxx/src/python/pyscream/__init__.py | 10 ++++++++ 11 files changed, 26 insertions(+), 38 deletions(-) create mode 100644 components/eamxx/src/python/libpyscream/CMakeLists.txt rename components/eamxx/src/python/{ => libpyscream}/pyatmproc.hpp (99%) rename components/eamxx/src/python/{ => libpyscream}/pyfield.hpp (100%) rename components/eamxx/src/python/{ => libpyscream}/pygrid.hpp (97%) rename components/eamxx/src/python/{ => libpyscream}/pyparamlist.hpp (100%) rename components/eamxx/src/python/{pyeamxx.cpp => libpyscream/pyscream_ext.cpp} (97%) rename components/eamxx/src/python/{pyeamxx.hpp => libpyscream/pyscream_ext.hpp} (100%) rename components/eamxx/src/python/{ => libpyscream}/pyutils.hpp (100%) delete mode 100644 components/eamxx/src/python/pyeamxx/__init__.py create mode 100644 components/eamxx/src/python/pyscream/__init__.py diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 41eae6d65463..4c8adf40fca7 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,12 +1 @@ -find_package(pybind11 REQUIRED) -find_package(mpi4py REQUIRED) - -pybind11_add_module(pyscream pyscream.cpp) -target_link_libraries(pyscream PUBLIC - mpi4py - scream_share - scream_io - diagnostics - eamxx_physics - scream_test_support -) +add_subdirectory(libpyscream) \ No newline at end of file diff --git a/components/eamxx/src/python/libpyscream/CMakeLists.txt b/components/eamxx/src/python/libpyscream/CMakeLists.txt new file mode 100644 index 000000000000..103c6a80b888 --- /dev/null +++ b/components/eamxx/src/python/libpyscream/CMakeLists.txt @@ -0,0 +1,12 @@ +find_package(pybind11 REQUIRED) +find_package(mpi4py REQUIRED) + +pybind11_add_module(pyscream_ext pyscream_ext.cpp) +target_link_libraries(pyscream_ext PUBLIC + mpi4py + scream_share + scream_io + diagnostics + eamxx_physics + scream_test_support +) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp similarity index 99% rename from components/eamxx/src/python/pyatmproc.hpp rename to components/eamxx/src/python/libpyscream/pyatmproc.hpp index 9e45bc513f1b..0acc371c1f71 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -8,7 +8,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" -#include "pyscream.hpp" +#include "pyscream_ext.hpp" #include diff --git a/components/eamxx/src/python/pyfield.hpp b/components/eamxx/src/python/libpyscream/pyfield.hpp similarity index 100% rename from components/eamxx/src/python/pyfield.hpp rename to components/eamxx/src/python/libpyscream/pyfield.hpp diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp similarity index 97% rename from components/eamxx/src/python/pygrid.hpp rename to components/eamxx/src/python/libpyscream/pygrid.hpp index e49a0443a345..e8adfa3bbc49 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -3,7 +3,7 @@ #include "share/grid/mesh_free_grids_manager.hpp" -#include "pyscream.hpp" +#include "pyscream_ext.hpp" #include diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/libpyscream/pyparamlist.hpp similarity index 100% rename from components/eamxx/src/python/pyparamlist.hpp rename to components/eamxx/src/python/libpyscream/pyparamlist.hpp diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp similarity index 97% rename from components/eamxx/src/python/pyeamxx.cpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.cpp index 2efb8b6f8df9..e0ad9ceec260 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp @@ -45,7 +45,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyscream,m) { +PYBIND11_MODULE (pyscream_ext,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/libpyscream/pyscream_ext.hpp similarity index 100% rename from components/eamxx/src/python/pyeamxx.hpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.hpp diff --git a/components/eamxx/src/python/pyutils.hpp b/components/eamxx/src/python/libpyscream/pyutils.hpp similarity index 100% rename from components/eamxx/src/python/pyutils.hpp rename to components/eamxx/src/python/libpyscream/pyutils.hpp diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py deleted file mode 100644 index 98c31744eb9f..000000000000 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" - This file will serve as a way to organize and expose - libpyscream internals to the rest of pyscream -""" - -from libpyscream.libpyscream_ext import AtmProc -from libpyscream.libpyscream_ext import Grid -from libpyscream.libpyscream_ext import ParameterList -from libpyscream.libpyscream_ext import init -from libpyscream.libpyscream_ext import Field -from libpyscream.libpyscream_ext import P3 -from libpyscream.libpyscream_ext import finalize - - -__all__ = [ - 'init', - 'finalize', - 'AtmProc', - 'Grid', - 'ParameterList', - 'Field', - 'P3', -] diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py new file mode 100644 index 000000000000..8159f5da47c7 --- /dev/null +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -0,0 +1,10 @@ +""" + This file will serve as a way to organize and expose + libpyscream internals to the rest of pyscream +""" + +from libpyscream import pyscream_ext as pyscream + +__all__ = [ + "pyscream" +] From 8c81c698d7a1a6b5c61292884f2f64acd185b464 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 12:06:01 -0500 Subject: [PATCH 390/904] skip cosp when building python --- components/eamxx/src/physics/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e0e89e60f80d..e5bce027bcc4 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -7,7 +7,11 @@ add_subdirectory(share) add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) - add_subdirectory(cosp) + if (EAMXX_ENABLE_PYBIND) + message(STATUS "WARNING: skipping COSP when buidling Python") + else() + add_subdirectory(cosp) + endif() else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() From ae7dfedd0778f743e1ab0df24d15d12b4211ffeb Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 14:02:39 -0500 Subject: [PATCH 391/904] Add geo_data_source --- components/eamxx/src/python/libpyscream/pygrid.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index e8adfa3bbc49..93a8a5395dba 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -25,6 +25,7 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo gm_params.set("grids_names",grids_names); if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } From a38380b9cf2928404ba618a1d91e29f2978a57a0 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 23:04:05 -0500 Subject: [PATCH 392/904] dump fields and reorg things --- components/eamxx/src/physics/CMakeLists.txt | 6 +----- components/eamxx/src/python/CMakeLists.txt | 2 +- .../eamxx/src/python/libpyscream/pyatmproc.hpp | 9 +++++++++ components/eamxx/src/python/libpyscream/pygrid.hpp | 1 - components/eamxx/src/python/pyscream/__init__.py | 14 ++++++++++++-- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e5bce027bcc4..e0e89e60f80d 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -7,11 +7,7 @@ add_subdirectory(share) add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) - if (EAMXX_ENABLE_PYBIND) - message(STATUS "WARNING: skipping COSP when buidling Python") - else() - add_subdirectory(cosp) - endif() + add_subdirectory(cosp) else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 4c8adf40fca7..d19bc05c240e 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1 +1 @@ -add_subdirectory(libpyscream) \ No newline at end of file +add_subdirectory(libpyscream) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 0acc371c1f71..5248ccf1f965 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -131,6 +131,14 @@ struct PyAtmProc { return pybind11::cast(missing); } + pybind11::list dump_fields() { + std::vector all_fields; + for (auto it : fields) { + all_fields.push_back(it.first); + } + return pybind11::cast(all_fields); + } + void setup_output (const std::string& yaml_file) { auto comm = PySession::get().comm; @@ -176,6 +184,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) .def("read_ic",&PyAtmProc::read_ic); + .def("dump_fields",&PyAtmProc::dump_fields); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index 93a8a5395dba..e8adfa3bbc49 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -25,7 +25,6 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo gm_params.set("grids_names",grids_names); if (latlon_nc_file!="") { - gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py index 8159f5da47c7..e76f52170fb3 100644 --- a/components/eamxx/src/python/pyscream/__init__.py +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -3,8 +3,18 @@ libpyscream internals to the rest of pyscream """ -from libpyscream import pyscream_ext as pyscream +from libpyscream.pyscream_ext import init +from libpyscream.pyscream_ext import finalize +from libpyscream.pyscream_ext import Field +from libpyscream.pyscream_ext import AtmProc +from libpyscream.pyscream_ext import ParameterList +from libpyscream.pyscream_ext import create_grids_manager __all__ = [ - "pyscream" + "init", + "finalize", + "Field", + "AtmProc", + "ParameterList", + "create_grids_manager", ] From 2c8d6a7a3757742d56bc85248f9add846b019ca4 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 14 Jul 2024 09:56:44 -0500 Subject: [PATCH 393/904] fix minor typo in def dumpfields --- components/eamxx/src/python/libpyscream/pyatmproc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 5248ccf1f965..000a7d64c9a9 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -183,7 +183,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("get_params",&PyAtmProc::get_params) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) - .def("read_ic",&PyAtmProc::read_ic); + .def("read_ic",&PyAtmProc::read_ic) .def("dump_fields",&PyAtmProc::dump_fields); } } // namespace scream From fc7b9e6a2961da698f2d54059feadfdfc568fedb Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 14 Jul 2024 11:48:48 -0500 Subject: [PATCH 394/904] add geo data source --- components/eamxx/src/python/libpyscream/pygrid.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index e8adfa3bbc49..1c0d56eb08b3 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -23,8 +23,10 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo pl.set("number_of_global_columns",ncols); pl.set("number_of_vertical_levels",nlevs); gm_params.set("grids_names",grids_names); + gm_params.set("geo_data_source",std::string("CREATE_EMPTY_DATA")); if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } From c1344ad585738915bbb9a2cf076d3bdb0f1776a3 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 5 Jun 2024 17:08:33 -0600 Subject: [PATCH 395/904] Port scream_rrtmgp_interface to new pattern --- .../eamxx/src/physics/rrtmgp/CMakeLists.txt | 1 - .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 6 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 1167 +------------- .../rrtmgp/scream_rrtmgp_interface.hpp | 1432 ++++++++++++++--- 4 files changed, 1243 insertions(+), 1363 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index fcb819f599a8..0d42d61fa662 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -109,7 +109,6 @@ set(EAM_RRTMGP_DIR ${SCREAM_BASE_DIR}/../eam/src/physics/rrtmgp) # NOTE: The external RRTMGP build needs some fixes to work with CUDA in a library build, so for now we will build these ourselves set(EXTERNAL_SRC ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels/mo_gas_optics_kernels.cpp - ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_constants.cpp ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_util_reorder.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/expand_and_transpose.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_fluxes_broadband_kernels.cpp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 0dfe4945e0c8..119093d5f9d3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -109,14 +109,14 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, RealT xmin, RealT xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - bool1dk bad_mask("bad_mask", x.size()); + Kokkos::View bad_mask("bad_mask", x.size()); Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { if (x.data()[i] < xmin or x.data()[i] > xmax) { bad_mask.data()[i] = true; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 450d7ad216be..1696e7933685 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,5 +1,4 @@ #include "scream_rrtmgp_interface.hpp" -#include "rrtmgp_utils.hpp" #include "cpp/examples/mo_load_coefficients.h" #include "examples/all-sky/mo_load_cloud_coefficients.h" #include "cpp/rrtmgp/mo_gas_concentrations.h" @@ -8,10 +7,6 @@ #include "cpp/rte/mo_rte_sw.h" #include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" -#include "ekat/util/ekat_math_utils.hpp" -#ifdef RRTMGP_ENABLE_KOKKOS -#include "Kokkos_Random.hpp" -#endif namespace scream { @@ -38,41 +33,27 @@ void finalize_kls() #endif } +#ifdef RRTMGP_ENABLE_YAKL namespace rrtmgp { -#ifdef RRTMGP_ENABLE_YAKL using yakl::fortran::parallel_for; using yakl::fortran::SimpleBounds; using yakl::intrinsics::merge; -#endif - /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -#ifdef RRTMGP_ENABLE_YAKL GasOpticsRRTMGP k_dist_sw; GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -GasOpticsRRTMGPK k_dist_sw_k; -GasOpticsRRTMGPK k_dist_lw_k; -#endif /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -#ifdef RRTMGP_ENABLE_YAKL CloudOptics cloud_optics_sw; CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -CloudOpticsK cloud_optics_sw_k; -CloudOpticsK cloud_optics_lw_k; -#endif bool initialized = false; bool initialized_k = false; @@ -80,7 +61,6 @@ bool initialized_k = false; // local functions namespace { -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_cloud_optics_sw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -106,36 +86,7 @@ OpticalProps2str get_cloud_optics_sw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_cloud_optics_sw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps2strK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_2str(ncol, nlay); - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_cloud_optics_lw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -161,36 +112,7 @@ OpticalProps1scl get_cloud_optics_lw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_cloud_optics_lw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps1sclK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2str &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -242,60 +164,7 @@ OpticalProps2str get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps2strK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_2str(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, - // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), - // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should - // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very - // high resolution. - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); - subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - subsampled_optics.ssa(icol,ilay,igpt) = 0; - subsampled_optics.g (icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1scl &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -339,51 +208,6 @@ OpticalProps1scl get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_1scl(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM; use different - // seed values for longwave and shortwave - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif } @@ -392,7 +216,6 @@ OpticalProps1sclK get_subsampled_clouds( * can be used as-is, but are intended to be wrapped by the SCREAM AD * interface to radiation. */ -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_initialize(GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, @@ -419,65 +242,15 @@ void rrtmgp_initialize(GasConcs &gas_concs, // We are now initialized! initialized = true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger) -{ - // If we've already initialized, just exit - if (initialized_k) { - if (logger) - logger->info("RRTMGP is already initialized; skipping\n"); - return; - } - - // Initialize Kokkos - if (!Kokkos::is_initialized()) { Kokkos::initialize(); } - - // Load and initialize absorption coefficient data - load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); - load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); - - // Load and initialize cloud optical property look-up table information - load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); - load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); - - // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 18000; - const size_t ncol = gas_concs.ncol; - const size_t nlay = gas_concs.nlay; - const size_t nlev = SCREAM_NUM_VERTICAL_LEV; - const size_t my_size_ref = ncol * nlay * nlev; - conv::MemPoolSingleton::init(2e6 * (float(my_size_ref) / base_ref)); - - // We are now initialized! - initialized_k = true; -} -#endif - void rrtmgp_finalize() { -#ifdef RRTMGP_ENABLE_YAKL initialized = false; k_dist_sw.finalize(); k_dist_lw.finalize(); cloud_optics_sw.finalize(); //~CloudOptics(); cloud_optics_lw.finalize(); //~CloudOptics(); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - initialized_k = false; - k_dist_sw_k.finalize(); - k_dist_lw_k.finalize(); - cloud_optics_sw_k.finalize(); //~CloudOptics(); - cloud_optics_lw_k.finalize(); //~CloudOptics(); - conv::MemPoolSingleton::finalize(); -#endif } -#ifdef RRTMGP_ENABLE_YAKL void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, @@ -527,56 +300,7 @@ void compute_band_by_band_surface_albedos( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif) { - - EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - - EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, - "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); - EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), - "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); - - // Loop over bands, and determine for each band whether it is broadly in the - // visible or infrared part of the spectrum (visible or "not visible") - Kokkos::parallel_for(conv::get_mdrp<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); - } - else { - // Band straddles the visible to near-infrared transition, so we take - // the albedo to be the average of the visible and near-infrared - // broadband albedos - sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); - sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , @@ -631,62 +355,7 @@ void compute_broadband_surface_fluxes( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir) { - // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums - // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when - // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This - // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. - //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - - // Initialize sums over bands - Kokkos::deep_copy(sfc_flux_dir_nir, 0); - Kokkos::deep_copy(sfc_flux_dir_vis, 0); - Kokkos::deep_copy(sfc_flux_dif_nir, 0); - Kokkos::deep_copy(sfc_flux_dif_vis, 0); - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { - for (int ibnd = 0; ibnd < nswbands; ++ibnd) { - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else { - // Band straddles the visible to near-infrared transition, so put half - // the flux in visible and half in near-infrared fluxes - sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - } - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -855,191 +524,19 @@ void rrtmgp_main( ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, - real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - -#ifdef SCREAM_RRTMGP_DEBUG - // Sanity check inputs, and possibly repair - check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); - check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); - check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); - check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); - check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); - check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); - check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); - check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); - check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); - check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); - check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); -#endif - - // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; - fluxes_sw.flux_up = sw_flux_up; - fluxes_sw.flux_dn = sw_flux_dn; - fluxes_sw.flux_dn_dir = sw_flux_dn_dir; - fluxes_sw.bnd_flux_up = sw_bnd_flux_up; - fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; - fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; - clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; - clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; - clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; - clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; - clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; - clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; - clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; - clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; - clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; - // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; - fluxes_lw.flux_up = lw_flux_up; - fluxes_lw.flux_dn = lw_flux_dn; - fluxes_lw.bnd_flux_up = lw_bnd_flux_up; - fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; - clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; - clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; - clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; - clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; - clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; - clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; +int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - auto nswbands = k_dist_sw_k.get_nband(); - auto nlwbands = k_dist_lw_k.get_nband(); + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; - aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); - aerosol_sw.alloc_2str(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); - aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); - aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); - }); - aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); - aerosol_lw.alloc_1scl(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Check aerosol optical properties - // NOTE: these should already have been checked by precondition checks, but someday we might have - // non-trivial aerosol optics, so this is still good to do here. - check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); - check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); - check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); - check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); -#endif - - // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); - Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); - Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); - - // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; - // This implements the Monte Carlo Independing Column Approximation by mapping only a single - // subcolumn (cloud state) to each gpoint. - auto nswgpts = k_dist_sw_k.get_ngpt(); - auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); - // Longwave - auto nlwgpts = k_dist_lw_k.get_ngpt(); - auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); - - // Copy cloud properties to outputs (is this needed, or can we just use pointers?) - // Alternatively, just compute and output a subcolumn cloud mask - Kokkos::parallel_for(conv::get_mdrp<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, - // but we might want to provide additional debug info here. NOTE: we may actually want to move this - // up higher in the code, I think optical props should go up higher since optical props are kind of - // a parameterization of their own, and we might want to swap different choices. These checks go here - // only because we need to run them on computed optical props, so if the optical props themselves get - // computed up higher, then perform these checks higher as well - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); - check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); - check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); -#endif - - // Do shortwave - rrtmgp_sw( - ncol, nlay, - k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, - fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, - tsi_scaling, logger, - extra_clnclrsky_diag, extra_clnsky_diag - ); - - // Do longwave - rrtmgp_lw( - ncol, nlay, - k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - aerosol_lw, clouds_lw_gpt, - fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, - extra_clnclrsky_diag, extra_clnsky_diag - ); - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL -int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3d("cldx", ncol, nlay, ngpt); + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = real3d("cldx", ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1090,84 +587,7 @@ int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d }); return subcolumn_mask; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3dk("cldx", ncol, nlay, ngpt); - - // Apply overlap assumption to set cldx - if (overlap_option == 0) { // Dummy mask, always cloudy - Kokkos::deep_copy(cldx, 1); - } else { // Default case, maximum-random overlap - // Maximum-random overlap: - // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, - // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same - // algorithm used in RRTMG implementation of maximum-random overlap (see - // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) - // - // First, fill cldx with random numbers. Need to use a unique seed for each column! - // auto seeds_host = Kokkos::create_mirror_view(seeds); - // Kokkos::deep_copy(seeds_host, seeds); - // for (int icol = 0; icol < ncol; ++icol) { - // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { - // auto generator = random_pool.get_state(); - // cldx(icol,ilay,igpt) = generator.drand(0., 1.); - // random_pool.free_state(generator); - // }); - // } - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - conv::Random rand(seeds(icol)); - for (int igpt = 0; igpt < ngpt; igpt++) { - for (int ilay = 0; ilay < nlay; ilay++) { - cldx(icol,ilay,igpt) = rand.genFP(); - } - } - }); - - // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { - for (int ilay = 1; ilay < nlay; ilay++) { - // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn - if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { - // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent - // layers are maximimally overlapped - cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); - } else { - // Cloud-less above, use new random number so that clouds are distributed - // randomly in this layer. Need to scale new random number to range - // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution - // of random numbers in this layer with the above branch of the conditional, - // which would otherwise inflate cloud fraction in this layer. - cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); - } - } - }); - } - - // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { - subcolumn_mask(icol,ilay,igpt) = 1; - } else { - subcolumn_mask(icol,ilay,igpt) = 0; - } - }); - return subcolumn_mask; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1423,267 +843,7 @@ void rrtmgp_sw( }); } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &flux_dn_dir = fluxes.flux_dn_dir; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; - - // Reset fluxes to zero - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up (icol,ilev) = 0; - flux_dn (icol,ilev) = 0; - flux_dn_dir(icol,ilev) = 0; - clnclrsky_flux_up (icol,ilev) = 0; - clnclrsky_flux_dn (icol,ilev) = 0; - clnclrsky_flux_dn_dir(icol,ilev) = 0; - clrsky_flux_up (icol,ilev) = 0; - clrsky_flux_dn (icol,ilev) = 0; - clrsky_flux_dn_dir(icol,ilev) = 0; - clnsky_flux_up (icol,ilev) = 0; - clnsky_flux_dn (icol,ilev) = 0; - clnsky_flux_dn_dir(icol,ilev) = 0; - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up (icol,ilev,ibnd) = 0; - bnd_flux_dn (icol,ilev,ibnd) = 0; - bnd_flux_dn_dir(icol,ilev,ibnd) = 0; - }); - - // Get daytime indices - auto dayIndices = int1dk("dayIndices", ncol); - Kokkos::deep_copy(dayIndices, -1); - - int nday = 0; - // Serialized for now. - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { - for (int icol = 0; icol < ncol; ++icol) { - if (mu0(icol) > 0) { - dayIndices(nday_inner++) = icol; - } - } - }, Kokkos::Sum(nday)); - - if (nday == 0) { - // No daytime columns in this chunk, skip the rest of this routine - return; - } - - // Subset mu0 - auto mu0_day = real1dk("mu0_day", nday); - Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { - mu0_day(iday) = mu0(dayIndices(iday)); - }); - - // subset state variables - auto p_lay_day = real2dk("p_lay_day", nday, nlay); - auto t_lay_day = real2dk("t_lay_day", nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); - t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); - }); - auto p_lev_day = real2dk("p_lev_day", nday, nlay+1); - auto t_lev_day = real2dk("t_lev_day", nday, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); - t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); - }); - - // Subset gases - auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; - gas_concs_day.init(gas_names, nday, nlay); - for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = real2dk("vmr_day", nday, nlay); - auto vmr = real2dk("vmr" , ncol, nlay); - gas_concs.get_vmr(gas_names[igas], vmr); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); - }); - gas_concs_day.set_vmr(gas_names[igas], vmr_day); - } - - // Subset aerosol optics - OpticalProps2strK aerosol_day; - aerosol_day.init(k_dist.get_band_lims_wavenumber()); - aerosol_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { - aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); - aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); - aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); - }); - - // Subset cloud optics - // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; - clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); - clouds_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { - clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); - clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); - clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); - }); - - // RRTMGP assumes surface albedos have a screwy dimension ordering - // for some strange reason, so we need to transpose these; also do - // daytime subsetting in the same kernel - real2dk sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - real2dk sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); - Kokkos::parallel_for(conv::get_mdrp<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { - sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); - sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); - }); - - // Temporaries we need for daytime-only fluxes - auto flux_up_day = real2dk("flux_up_day", nday, nlay+1); - auto flux_dn_day = real2dk("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = real2dk("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = real3dk("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = real3dk("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = real3dk("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; - fluxes_day.flux_up = flux_up_day; - fluxes_day.flux_dn = flux_dn_day; - fluxes_day.flux_dn_dir = flux_dn_dir_day; - fluxes_day.bnd_flux_up = bnd_flux_up_day; - fluxes_day.bnd_flux_dn = bnd_flux_dn_day; - fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; - - // Allocate space for optical properties - OpticalProps2strK optics; - optics.alloc_2str(nday, nlay, k_dist); - - OpticalProps2strK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_2str(nday, nlay, k_dist); - } - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", nday, nlay); - limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); - - // Do gas optics - real2dk toa_flux("toa_flux", nday, ngpt); - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); - if (extra_clnsky_diag) { - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); - check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); - check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); -#endif - - // Apply tsi_scaling - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { - toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); - }); - - if (extra_clnclrsky_diag) { - // Compute clear-clean-sky (just gas) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } - - // Combine gas and aerosol optics - aerosol_day.delta_scale(); - aerosol_day.increment(optics); - - // Compute clearsky (gas + aerosol) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - - // Now merge in cloud optics and do allsky calculations - - // Combine gas and cloud optics - clouds_day.delta_scale(); - clouds_day.increment(optics); - // Compute fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - flux_up (icol,ilev) = flux_up_day (iday,ilev); - flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { - const int icol = dayIndices(iday); - bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); - bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); - bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); - }); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds_day.increment(optics_no_aerosols); - // Compute cleansky (gas + clouds) fluxes on daytime columns - rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1816,147 +976,7 @@ void rrtmgp_lw( } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, - OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Problem size - int nbnd = k_dist.get_nband(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - - // Reset fluxes to zero - Kokkos::parallel_for( - conv::get_mdrp<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up(icol, ilev) = 0; - flux_dn(icol, ilev) = 0; - clnclrsky_flux_up(icol, ilev) = 0; - clnclrsky_flux_dn(icol, ilev) = 0; - clrsky_flux_up(icol, ilev) = 0; - clrsky_flux_dn(icol, ilev) = 0; - clnsky_flux_up(icol, ilev) = 0; - clnsky_flux_dn(icol, ilev) = 0; - }); - Kokkos::parallel_for( - conv::get_mdrp<3>({nbnd, nlay + 1, ncol}), - KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up(icol, ilev, ibnd) = 0; - bnd_flux_dn(icol, ilev, ibnd) = 0; - }); - // Allocate space for optical properties - OpticalProps1sclK optics; - optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); - } - - // Boundary conditions - SourceFuncLWK lw_sources; - lw_sources.alloc(ncol, nlay, k_dist); - real1dk t_sfc ("t_sfc" ,ncol); - real2dk emis_sfc("emis_sfc",nbnd,ncol); - - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - // Surface temperature - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); - }); - Kokkos::deep_copy(emis_sfc , 0.98); - - // Get Gaussian quadrature weights - // TODO: move this crap out of userland! - // Weights and angle secants for first order (k=1) Gaussian quadrature. - // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 - // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - realHost2dk gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - realHost2dk gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - real2dk gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - real2dk gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); - Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); - Kokkos::deep_copy(gauss_wts, gauss_wts_host); - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", ncol, nlay); - auto t_lev_limited = real2dk("t_lev_limited", ncol, nlay+1); - limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); - limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); - - // Do gas optics - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, real2dk(), t_lev_limited); - if (extra_clnsky_diag) { - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, real2dk(), t_lev_limited); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); -#endif - - if (extra_clnclrsky_diag) { - // Compute clean-clear-sky fluxes before we add in aerosols and clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); - } - - // Combine gas and aerosol optics - aerosol.increment(optics); - - // Compute clear-sky fluxes before we add in clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); - - // Combine gas and cloud optics - clouds.increment(optics); - - // Compute allsky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds.increment(optics_no_aerosols); - // Compute clean-sky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); - } - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area) { @@ -1981,46 +1001,11 @@ void compute_cloud_area( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area) { - // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy - // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = real2dk("subcol_mask", ncol, ngpt); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but - // using play/pmid does not - if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { - subcol_mask(icol,igpt) = 1; - } - }); - // Compute average over subcols to get cloud area - auto ngpt_inv = 1.0 / ngpt; - Kokkos::deep_copy(cld_area, 0); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // This loop needs to be serial because of the atomic reduction - for (int igpt = 0; igpt < ngpt; ++igpt) { - cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index_sw_k(double wavelength) { return get_wavelength_index(k_dist_sw_k, wavelength); } - -int get_wavelength_index_lw_k(double wavelength) { return get_wavelength_index(k_dist_lw_k, wavelength); } -#endif - -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index(OpticalProps &kdist, double wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -2043,33 +1028,7 @@ int get_wavelength_index(OpticalProps &kdist, double wavelength) { }); return band_index.hostRead(); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength) { - // Get wavelength bounds for all wavelength bands - auto wavelength_bounds = kdist.get_band_lims_wavelength(); - - // Find the band index for the specified wavelength - // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength - // in units of meters, we need a conversion factor of 10^2 - const int nbnds = kdist.get_nband(); - int band_index = -1; - Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { - if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { - if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } else { - if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } - }, Kokkos::Max(band_index)); - return band_index; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -2166,109 +1125,7 @@ void compute_aerocom_cloudtop( cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop) { - /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCom recommendation. See reference for routine - * get_subcolumn_mask above, where equation 14 is used for the - * maximum-random overlap assumption for subcolumn generation. We use - * equation 13, the column counterpart. - */ - // Set outputs to zero - Kokkos::deep_copy(T_mid_at_cldtop, 0.0); - Kokkos::deep_copy(p_mid_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); - Kokkos::deep_copy(cdnc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); - - // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = real1dk("aerocom_clr", ncol); - Kokkos::deep_copy(aerocom_clr, 1.0); - - // Get gravity acceleration constant from constants - using physconst = scream::physics::Constants; - - // TODO: move tunable constant to namelist - constexpr real q_threshold = 0.0; // BAD_CONSTANT! - - // TODO: move tunable constant to namelist - constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! - - // Loop over all columns in parallel - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // Loop over all layers in serial (due to accumulative - // product), starting at 2 (second highest) layer because the - // highest is assumed to hav no clouds - for(int ilay = 1; ilay < nlay; ++ilay) { - // Only do the calculation if certain conditions are met - if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && - (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { - /* PART I: Probabilistically determining cloud top */ - // Populate aerocom_tmp as the clear-sky fraction - // probability of this level, where aerocom_clr is that of - // the previous level - auto aerocom_tmp = - aerocom_clr(icol) * - (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), - cldfrac_tot(icol, ilay))) / - (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), - 1.0 - cldfrac_tot_threshold)); - // Temporary variable for probability "weights" - auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; - // Temporary variable for liquid "phase" - auto aerocom_phi = - qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); - /* PART II: The inferred properties */ - /* In general, converting a 3D property X to a 2D cloud-top - * counterpart x follows: x(i) += X(i,k) * weights * Phase - * but X and Phase are not always needed */ - // T_mid_at_cldtop - T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; - // p_mid_at_cldtop - p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; - // cldfrac_ice_at_cldtop - cldfrac_ice_at_cldtop(icol) += - (1.0 - aerocom_phi) * aerocom_wts; - // cldfrac_liq_at_cldtop - cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; - // cdnc_at_cldtop - /* We need to convert nc from 1/mass to 1/volume first, and - * from grid-mean to in-cloud, but after that, the - * calculation follows the general logic */ - auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / - z_del(icol, ilay) / physconst::gravit / - cldfrac_tot(icol, ilay); - cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; - // eff_radius_qc_at_cldtop - eff_radius_qc_at_cldtop(icol) += - rel(icol, ilay) * aerocom_phi * aerocom_wts; - // eff_radius_qi_at_cldtop - eff_radius_qi_at_cldtop(icol) += - rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset aerocom_clr to aerocom_tmp to accumulate - aerocom_clr(icol) = aerocom_tmp; - } - } - // After the serial loop over levels, the cloudy fraction is - // defined as (1 - aerocom_clr). This is true because - // aerocom_clr is the result of accumulative probabilities - // (their products) - cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); - }); -} -#endif } // namespace rrtmgp +#endif } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 49fbfeb4486b..c40be4904302 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -6,10 +6,17 @@ #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" #include "cpp/rrtmgp_const.h" +#include "rrtmgp_utils.hpp" + #include "physics/share/physics_constants.hpp" #include "ekat/mpi/ekat_comm.hpp" #include "ekat/logging/ekat_logger.hpp" +#include "ekat/util/ekat_math_utils.hpp" + +#ifdef RRTMGP_ENABLE_KOKKOS +#include "Kokkos_Random.hpp" +#endif namespace scream { @@ -18,105 +25,34 @@ void finalize_kls(); namespace rrtmgp { -/* - * Objects containing k-distribution information need to be initialized - * once and then persist throughout the life of the program, so we - * declare them here within the rrtmgp namespace. - */ #ifdef RRTMGP_ENABLE_YAKL extern GasOpticsRRTMGP k_dist_sw; extern GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern GasOpticsRRTMGPK k_dist_sw_k; -extern GasOpticsRRTMGPK k_dist_lw_k; -#endif -/* - * Objects containing cloud optical property look-up table information. - * We want to initialize these once and use throughout the life of the - * program, so declare here and read data in during rrtmgp_initialize(). - */ -#ifdef RRTMGP_ENABLE_YAKL extern CloudOptics cloud_optics_sw; extern CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern CloudOpticsK cloud_optics_sw_k; -extern CloudOpticsK cloud_optics_lw_k; -#endif -/* - * Flag to indicate whether or not we have initialized RRTMGP - */ -#ifdef RRTMGP_ENABLE_YAKL extern bool initialized; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern bool initialized_k; -#endif -/* - * Initialize data for RRTMGP driver - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_initialize( +void rrtmgp_initialize( GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger); -#endif -/* - * Compute band-by-band surface albedos from broadband albedos. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_band_by_band_surface_albedos( +void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, real2d &sfc_alb_dir, real2d &sfc_alb_dif); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif); -#endif -/* - * Compute broadband visible/UV and near-infrared surface fluxes. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_broadband_surface_fluxes( +void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir); -#endif -/* - * Main driver code to run RRTMGP. - * The input logger is in charge of outputing info to - * screen and/or to file (or neither), depending on how it was set up. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_main( +void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, GasConcs &gas_concs, @@ -138,42 +74,10 @@ extern void rrtmgp_main( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -/* - * Perform any clean-up tasks - */ -extern void rrtmgp_finalize(); +void rrtmgp_finalize(); -/* - * Shortwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_sw( +void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -184,26 +88,8 @@ extern void rrtmgp_sw( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Longwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_lw( +void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -211,46 +97,13 @@ extern void rrtmgp_lw( OpticalProps1scl &aerosol, OpticalProps1scl &clouds, FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Return a subcolumn mask consistent with a specified overlap assumption - */ -#ifdef RRTMGP_ENABLE_YAKL int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds); -#endif -/* - * Compute cloud area from 3d subcol cloud property - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, Real pmin, Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, Real pmin, Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area); -#endif -/* - * Return select cloud-top diagnostics following AeroCom recommendation - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -260,26 +113,7 @@ void compute_aerocom_cloudtop( real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop); -#endif -/* - * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area - * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion - * with the standard practice of using "water path" to refer to the total column-integrated - * quantities). - */ -#ifdef RRTMGP_ENABLE_YAKL template void mixing_ratio_to_cloud_mass( yakl::Array const &mixing_ratio, @@ -303,10 +137,1035 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif + +template +void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { + yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { + arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + }); +} + +int get_wavelength_index(OpticalProps &kdist, double wavelength); +int get_wavelength_index_sw(double wavelength); +int get_wavelength_index_lw(double wavelength); +#endif // RRTMGP_ENABLE_YAKL + +// New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS +template +struct rrtmgp_interface { + +using MDRP = typename conv::MDRP; + +template +using view_t = Kokkos::View; + +template +using oview_t = Kokkos::Experimental::OffsetView; + +template +using hview_t = Kokkos::View; + +using pool_t = conv::MemPoolSingleton; + +/* + * Objects containing k-distribution information need to be initialized + * once and then persist throughout the life of the program, so we + * declare them here within the rrtmgp namespace. + */ +static inline GasOpticsRRTMGPK k_dist_sw_k; +static inline GasOpticsRRTMGPK k_dist_lw_k; + +/* + * Objects containing cloud optical property look-up table information. + * We want to initialize these once and use throughout the life of the + * program, so declare here and read data in during rrtmgp_initialize(). + */ +static inline CloudOpticsK cloud_optics_sw_k; +static inline CloudOpticsK cloud_optics_lw_k; + +/* + * Flag to indicate whether or not we have initialized RRTMGP + */ +static inline bool initialized_k = false; + +/* + * Initialize data for RRTMGP driver + */ +static void rrtmgp_initialize( + GasConcsK &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger) +{ + // If we've already initialized, just exit + if (initialized_k) { + if (logger) + logger->info("RRTMGP is already initialized; skipping\n"); + return; + } + + // Initialize Kokkos + if (!Kokkos::is_initialized()) { Kokkos::initialize(); } + + // Load and initialize absorption coefficient data + load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); + load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); + + // Load and initialize cloud optical property look-up table information + load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); + load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); + + // initialize kokkos rrtmgp pool allocator + const size_t base_ref = 18000; + const size_t ncol = gas_concs.ncol; + const size_t nlay = gas_concs.nlay; + const size_t nlev = SCREAM_NUM_VERTICAL_LEV; + const size_t my_size_ref = ncol * nlay * nlev; + pool_t::init(2e6 * (float(my_size_ref) / base_ref)); + + // We are now initialized! + initialized_k = true; +} + +/* + * Compute band-by-band surface albedos from broadband albedos. + */ +template +static void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + SfcAlbDirVisT &sfc_alb_dir_vis, SfcAlbDirNirT &sfc_alb_dir_nir, + SfcAlbDifVisT &sfc_alb_dif_vis, SfcAlbDifNirT &sfc_alb_dif_nir, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif) +{ + EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + + EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, + "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); + EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), + "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); + + // Loop over bands, and determine for each band whether it is broadly in the + // visible or infrared part of the spectrum (visible or "not visible") + Kokkos::parallel_for(MDRP::template get<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); + } + else { + // Band straddles the visible to near-infrared transition, so we take + // the albedo to be the average of the visible and near-infrared + // broadband albedos + sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); + sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); + } + }); +} + +/* + * Compute broadband visible/UV and near-infrared surface fluxes. + */ +template +static void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + SwBndFluxDirT &sw_bnd_flux_dir , SwBndFluxDifT &sw_bnd_flux_dif , + SfcFluxDirVisT &sfc_flux_dir_vis, SfcFluxDirNirT &sfc_flux_dir_nir, + SfcFluxDifVisT &sfc_flux_dif_vis, SfcFluxDifNirT &sfc_flux_dif_nir) +{ + // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums + // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when + // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This + // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. + //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + + // Initialize sums over bands + Kokkos::deep_copy(sfc_flux_dir_nir, 0); + Kokkos::deep_copy(sfc_flux_dir_vis, 0); + Kokkos::deep_copy(sfc_flux_dif_nir, 0); + Kokkos::deep_copy(sfc_flux_dif_vis, 0); + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { + for (int ibnd = 0; ibnd < nswbands; ++ibnd) { + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else { + // Band straddles the visible to near-infrared transition, so put half + // the flux in visible and half in near-infrared fluxes + sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + } + } + }); +} + +/* + * Main driver code to run RRTMGP. + * The input logger is in charge of outputing info to + * screen and/or to file (or neither), depending on how it was set up. + */ +template +static void rrtmgp_main( + const int ncol, const int nlay, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei, CldfracT &cldfrac, + AerTauSwT &aer_tau_sw, AerSsaSwT &aer_ssa_sw, AerAsmSwT &aer_asm_sw, AerTauLwT &aer_tau_lw, + CldTauSwBndT &cld_tau_sw_bnd, CldTauLwBndT &cld_tau_lw_bnd, + CldTauSwGptT &cld_tau_sw_gpt, CldTauLwGptT &cld_tau_lw_gpt, + SwFluxUpT &sw_flux_up, SwFluxDnT &sw_flux_dn, SwFluxDnDirT &sw_flux_dn_dir, + LwFluxUpT &lw_flux_up, LwFluxDnT &lw_flux_dn, + SwClnclrskyFluxUpT &sw_clnclrsky_flux_up, SwClnclrskyFluxDnT &sw_clnclrsky_flux_dn, SwClnclrskyFluxDnDirT &sw_clnclrsky_flux_dn_dir, + SwClrskyFluxUpT &sw_clrsky_flux_up, SwClrskyFluxDnT &sw_clrsky_flux_dn, SwClrskyFluxDnDirT &sw_clrsky_flux_dn_dir, + SwClnskyFluxUpT &sw_clnsky_flux_up, SwClnskyFluxDnT &sw_clnsky_flux_dn, SwClnskyFluxDnDirT &sw_clnsky_flux_dn_dir, + LwClnclrskyFluxUpT &lw_clnclrsky_flux_up, LwClnclrskyFluxDnT &lw_clnclrsky_flux_dn, + LwClrskyFluxUpT &lw_clrsky_flux_up, LwClrskyFluxDnT &lw_clrsky_flux_dn, + LwClnskyFluxUpT &lw_clnsky_flux_up, LwClnskyFluxDnT &lw_clnsky_flux_dn, + SwBndFluxUpT &sw_bnd_flux_up, SwBndFluxDnT &sw_bnd_flux_dn, SwBndFluxDnDirT &sw_bnd_flux_dn_dir, + LwBndFluxUpT &lw_bnd_flux_up, LwBndFluxDnT &lw_bnd_flux_dn, + const RealT tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) +{ +#ifdef SCREAM_RRTMGP_DEBUG + // Sanity check inputs, and possibly repair + check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); + check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); + check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); + check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); + check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); + check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); + check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); + check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); + check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); + check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); + check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); +#endif + + // Setup pointers to RRTMGP SW fluxes + FluxesBybandK fluxes_sw; + fluxes_sw.flux_up = sw_flux_up; + fluxes_sw.flux_dn = sw_flux_dn; + fluxes_sw.flux_dn_dir = sw_flux_dn_dir; + fluxes_sw.bnd_flux_up = sw_bnd_flux_up; + fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; + fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_sw; + clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; + clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; + clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_sw; + clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; + clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; + clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_sw; + clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; + clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; + clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; + + // Setup pointers to RRTMGP LW fluxes + FluxesBybandK fluxes_lw; + fluxes_lw.flux_up = lw_flux_up; + fluxes_lw.flux_dn = lw_flux_dn; + fluxes_lw.bnd_flux_up = lw_bnd_flux_up; + fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_lw; + clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; + clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_lw; + clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; + clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_lw; + clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; + clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; + + auto nswbands = k_dist_sw_k.get_nband(); + auto nlwbands = k_dist_lw_k.get_nband(); + + // Setup aerosol optical properties + OpticalProps2strK aerosol_sw; + OpticalProps1sclK aerosol_lw; + aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); + aerosol_sw.alloc_2str(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); + aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); + aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); + }); + aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); + aerosol_lw.alloc_1scl(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Check aerosol optical properties + // NOTE: these should already have been checked by precondition checks, but someday we might have + // non-trivial aerosol optics, so this is still good to do here. + check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); + check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); + check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); + check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); +#endif + + // Convert cloud physical properties to optical properties for input to RRTMGP + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); + Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); + + // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; + // This implements the Monte Carlo Independing Column Approximation by mapping only a single + // subcolumn (cloud state) to each gpoint. + auto nswgpts = k_dist_sw_k.get_ngpt(); + auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); + // Longwave + auto nlwgpts = k_dist_lw_k.get_ngpt(); + auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); + + // Copy cloud properties to outputs (is this needed, or can we just use pointers?) + // Alternatively, just compute and output a subcolumn cloud mask + Kokkos::parallel_for(MDRP::template get<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); + }); + Kokkos::parallel_for(MDRP::template get<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, + // but we might want to provide additional debug info here. NOTE: we may actually want to move this + // up higher in the code, I think optical props should go up higher since optical props are kind of + // a parameterization of their own, and we might want to swap different choices. These checks go here + // only because we need to run them on computed optical props, so if the optical props themselves get + // computed up higher, then perform these checks higher as well + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); + check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); +#endif + + // Do shortwave + rrtmgp_sw( + ncol, nlay, + k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, + fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, + tsi_scaling, logger, + extra_clnclrsky_diag, extra_clnsky_diag + ); + + // Do longwave + rrtmgp_lw( + ncol, nlay, + k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + aerosol_lw, clouds_lw_gpt, + fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, + extra_clnclrsky_diag, extra_clnsky_diag + ); + +} + +/* + * Perform any clean-up tasks + */ +static void rrtmgp_finalize() +{ + initialized_k = false; + k_dist_sw_k.finalize(); + k_dist_lw_k.finalize(); + cloud_optics_sw_k.finalize(); //~CloudOptics(); + cloud_optics_lw_k.finalize(); //~CloudOptics(); + pool_t::finalize(); +} + +/* + * Shortwave driver (called by rrtmgp_main) + */ +template +static void rrtmgp_sw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, + OpticalProps2strK &aerosol, OpticalProps2strK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const RealT tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Get problem sizes + int nbnd = k_dist.get_nband(); + int ngpt = k_dist.get_ngpt(); + int ngas = gas_concs.get_num_gases(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &flux_dn_dir = fluxes.flux_dn_dir; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; + + // Reset fluxes to zero + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up (icol,ilev) = 0; + flux_dn (icol,ilev) = 0; + flux_dn_dir(icol,ilev) = 0; + clnclrsky_flux_up (icol,ilev) = 0; + clnclrsky_flux_dn (icol,ilev) = 0; + clnclrsky_flux_dn_dir(icol,ilev) = 0; + clrsky_flux_up (icol,ilev) = 0; + clrsky_flux_dn (icol,ilev) = 0; + clrsky_flux_dn_dir(icol,ilev) = 0; + clnsky_flux_up (icol,ilev) = 0; + clnsky_flux_dn (icol,ilev) = 0; + clnsky_flux_dn_dir(icol,ilev) = 0; + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up (icol,ilev,ibnd) = 0; + bnd_flux_dn (icol,ilev,ibnd) = 0; + bnd_flux_dn_dir(icol,ilev,ibnd) = 0; + }); + + // Get daytime indices + auto dayIndices = view_t("dayIndices", ncol); + Kokkos::deep_copy(dayIndices, -1); + + int nday = 0; + // Serialized for now. + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { + for (int icol = 0; icol < ncol; ++icol) { + if (mu0(icol) > 0) { + dayIndices(nday_inner++) = icol; + } + } + }, Kokkos::Sum(nday)); + + if (nday == 0) { + // No daytime columns in this chunk, skip the rest of this routine + return; + } + + // Subset mu0 + auto mu0_day = view_t("mu0_day", nday); + Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { + mu0_day(iday) = mu0(dayIndices(iday)); + }); + + // subset state variables + auto p_lay_day = view_t("p_lay_day", nday, nlay); + auto t_lay_day = view_t("t_lay_day", nday, nlay); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); + t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); + }); + auto p_lev_day = view_t("p_lev_day", nday, nlay+1); + auto t_lev_day = view_t("t_lev_day", nday, nlay+1); + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); + t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); + }); + + // Subset gases + auto gas_names = gas_concs.get_gas_names(); + GasConcsK gas_concs_day; + gas_concs_day.init(gas_names, nday, nlay); + for (int igas = 0; igas < ngas; igas++) { + auto vmr_day = view_t("vmr_day", nday, nlay); + auto vmr = view_t("vmr" , ncol, nlay); + gas_concs.get_vmr(gas_names[igas], vmr); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); + }); + gas_concs_day.set_vmr(gas_names[igas], vmr_day); + } + + // Subset aerosol optics + OpticalProps2strK aerosol_day; + aerosol_day.init(k_dist.get_band_lims_wavenumber()); + aerosol_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { + aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); + aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); + aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); + }); + + // Subset cloud optics + // TODO: nbnd -> ngpt once we pass sub-sampled cloud state + OpticalProps2strK clouds_day; + clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); + clouds_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { + clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); + clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); + clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); + }); + + // RRTMGP assumes surface albedos have a screwy dimension ordering + // for some strange reason, so we need to transpose these; also do + // daytime subsetting in the same kernel + view_t sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); + view_t sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); + Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { + sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); + sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); + }); + + // Temporaries we need for daytime-only fluxes + auto flux_up_day = view_t("flux_up_day", nday, nlay+1); + auto flux_dn_day = view_t("flux_dn_day", nday, nlay+1); + auto flux_dn_dir_day = view_t("flux_dn_dir_day", nday, nlay+1); + auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); + FluxesBybandK fluxes_day; + fluxes_day.flux_up = flux_up_day; + fluxes_day.flux_dn = flux_dn_day; + fluxes_day.flux_dn_dir = flux_dn_dir_day; + fluxes_day.bnd_flux_up = bnd_flux_up_day; + fluxes_day.bnd_flux_dn = bnd_flux_dn_day; + fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; + + // Allocate space for optical properties + OpticalProps2strK optics; + optics.alloc_2str(nday, nlay, k_dist); + + OpticalProps2strK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_2str(nday, nlay, k_dist); + } + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = view_t("t_lay_limited", nday, nlay); + limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); + + // Do gas optics + view_t toa_flux("toa_flux", nday, ngpt); + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); + if (extra_clnsky_diag) { + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); + check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); + check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); +#endif + + // Apply tsi_scaling + Kokkos::parallel_for(MDRP::template get<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { + toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); + }); + + if (extra_clnclrsky_diag) { + // Compute clear-clean-sky (just gas) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + + // Combine gas and aerosol optics + aerosol_day.delta_scale(); + aerosol_day.increment(optics); + + // Compute clearsky (gas + aerosol) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + + // Now merge in cloud optics and do allsky calculations + + // Combine gas and cloud optics + clouds_day.delta_scale(); + clouds_day.increment(optics); + // Compute fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + flux_up (icol,ilev) = flux_up_day (iday,ilev); + flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { + const int icol = dayIndices(iday); + bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); + bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); + bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); + }); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds_day.increment(optics_no_aerosols); + // Compute cleansky (gas + clouds) fluxes on daytime columns + rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } +} + +/* + * Longwave driver (called by rrtmgp_main) + */ +template +static void rrtmgp_lw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Problem size + int nbnd = k_dist.get_nband(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + + // Reset fluxes to zero + Kokkos::parallel_for( + MDRP::template get<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up(icol, ilev) = 0; + flux_dn(icol, ilev) = 0; + clnclrsky_flux_up(icol, ilev) = 0; + clnclrsky_flux_dn(icol, ilev) = 0; + clrsky_flux_up(icol, ilev) = 0; + clrsky_flux_dn(icol, ilev) = 0; + clnsky_flux_up(icol, ilev) = 0; + clnsky_flux_dn(icol, ilev) = 0; + }); + Kokkos::parallel_for( + MDRP::template get<3>({nbnd, nlay + 1, ncol}), + KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up(icol, ilev, ibnd) = 0; + bnd_flux_dn(icol, ilev, ibnd) = 0; + }); + + // Allocate space for optical properties + OpticalProps1sclK optics; + optics.alloc_1scl(ncol, nlay, k_dist); + OpticalProps1sclK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); + } + + // Boundary conditions + SourceFuncLWK lw_sources; + lw_sources.alloc(ncol, nlay, k_dist); + view_t t_sfc ("t_sfc" ,ncol); + view_t emis_sfc("emis_sfc",nbnd,ncol); + + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + // Surface temperature + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); + }); + Kokkos::deep_copy(emis_sfc , 0.98); + + // Get Gaussian quadrature weights + // TODO: move this crap out of userland! + // Weights and angle secants for first order (k=1) Gaussian quadrature. + // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 + // after Abramowitz & Stegun 1972, page 921 + int constexpr max_gauss_pts = 4; + hview_t gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; + gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; + gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; + gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; + + hview_t gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); + gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; + gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; + gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; + gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; + + view_t gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + view_t gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); + Kokkos::deep_copy(gauss_wts, gauss_wts_host); + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = view_t("t_lay_limited", ncol, nlay); + auto t_lev_limited = view_t("t_lev_limited", ncol, nlay+1); + limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); + limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); + + // Do gas optics + oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); + if (extra_clnsky_diag) { + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); +#endif + + if (extra_clnclrsky_diag) { + // Compute clean-clear-sky fluxes before we add in aerosols and clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); + } + + // Combine gas and aerosol optics + aerosol.increment(optics); + + // Compute clear-sky fluxes before we add in clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); + + // Combine gas and cloud optics + clouds.increment(optics); + + // Compute allsky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds.increment(optics_no_aerosols); + // Compute clean-sky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); + } +} + +/* + * Return a subcolumn mask consistent with a specified overlap assumption + */ +template +static view_t get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, + CldfT &cldf, const int overlap_option, SeedsT &seeds) +{ + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = view_t("subcolumn_mask", ncol, nlay, ngpt); + + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = view_t("cldx", ncol, nlay, ngpt); + + // Apply overlap assumption to set cldx + if (overlap_option == 0) { // Dummy mask, always cloudy + Kokkos::deep_copy(cldx, 1); + } else { // Default case, maximum-random overlap + // Maximum-random overlap: + // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, + // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same + // algorithm used in RRTMG implementation of maximum-random overlap (see + // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) + // + // First, fill cldx with random numbers. Need to use a unique seed for each column! + // auto seeds_host = Kokkos::create_mirror_view(seeds); + // Kokkos::deep_copy(seeds_host, seeds); + // for (int icol = 0; icol < ncol; ++icol) { + // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); + // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // auto generator = random_pool.get_state(); + // cldx(icol,ilay,igpt) = generator.drand(0., 1.); + // random_pool.free_state(generator); + // }); + // } + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + conv::Random rand(seeds(icol)); + for (int igpt = 0; igpt < ngpt; igpt++) { + for (int ilay = 0; ilay < nlay; ilay++) { + cldx(icol,ilay,igpt) = rand.genFP(); + } + } + }); + + // Step down columns and apply algorithm from eq (14) + Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + for (int ilay = 1; ilay < nlay; ilay++) { + // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn + if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { + // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent + // layers are maximimally overlapped + cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); + } else { + // Cloud-less above, use new random number so that clouds are distributed + // randomly in this layer. Need to scale new random number to range + // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution + // of random numbers in this layer with the above branch of the conditional, + // which would otherwise inflate cloud fraction in this layer. + cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); + } + } + }); + } + + // Use cldx array to create subcolumn mask + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { + subcolumn_mask(icol,ilay,igpt) = 1; + } else { + subcolumn_mask(icol,ilay,igpt) = 0; + } + }); + return subcolumn_mask; +} + +/* + * Compute cloud area from 3d subcol cloud property + */ +template +static void compute_cloud_area( + int ncol, int nlay, int ngpt, RealT pmin, RealT pmax, + const PmidT& pmid, const CldTauGptT& cld_tau_gpt, CldAreaT& cld_area) +{ + // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy + // then 2d subcol mask is 1, otherwise it is 0 + auto subcol_mask = view_t("subcol_mask", ncol, ngpt); + Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but + // using play/pmid does not + if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { + subcol_mask(icol,igpt) = 1; + } + }); + // Compute average over subcols to get cloud area + auto ngpt_inv = 1.0 / ngpt; + Kokkos::deep_copy(cld_area, 0); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // This loop needs to be serial because of the atomic reduction + for (int igpt = 0; igpt < ngpt; ++igpt) { + cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; + } + }); +} + +/* + * Return select cloud-top diagnostics following AeroCom recommendation + */ +template +static void compute_aerocom_cloudtop( + int ncol, int nlay, const TmidT &tmid, const PmidT &pmid, + const PdelT &p_del, const ZdelT &z_del, const QcT &qc, + const QiT &qi, const RelT &rel, const ReiT &rei, + const CldfracTotT &cldfrac_tot, const NcT &nc, + TmidAtCldtopT &T_mid_at_cldtop, PmidAtCldtopT &p_mid_at_cldtop, + CldfracIceAtCldtopT &cldfrac_ice_at_cldtop, CldfracLiqAtCldtopT &cldfrac_liq_at_cldtop, + CldfracTotAtCldtopT &cldfrac_tot_at_cldtop, CdncAtCldtopT &cdnc_at_cldtop, + EffRadiusQcAtCldtopT &eff_radius_qc_at_cldtop, EffRadiusQiAtCldTopT &eff_radius_qi_at_cldtop) +{ + /* The goal of this routine is to calculate properties at cloud top + * based on the AeroCom recommendation. See reference for routine + * get_subcolumn_mask above, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. + */ + // Set outputs to zero + Kokkos::deep_copy(T_mid_at_cldtop, 0.0); + Kokkos::deep_copy(p_mid_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); + Kokkos::deep_copy(cdnc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); + + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto aerocom_clr = view_t("aerocom_clr", ncol); + Kokkos::deep_copy(aerocom_clr, 1.0); + + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + + // TODO: move tunable constant to namelist + constexpr RealT q_threshold = 0.0; // BAD_CONSTANT! + + // TODO: move tunable constant to namelist + constexpr RealT cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! + + // Loop over all columns in parallel + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds + for(int ilay = 1; ilay < nlay; ++ilay) { + // Only do the calculation if certain conditions are met + if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && + (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + aerocom_clr(icol) * + (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), + cldfrac_tot(icol, ilay))) / + (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = + qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; + // p_mid_at_cldtop + p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + cldfrac_ice_at_cldtop(icol) += + (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / + z_del(icol, ilay) / physconst::gravit / + cldfrac_tot(icol, ilay); + cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtop + eff_radius_qc_at_cldtop(icol) += + rel(icol, ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtop + eff_radius_qi_at_cldtop(icol) += + rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + aerocom_clr(icol) = aerocom_tmp; + } + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); + }); +} + +/* + * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area + * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion + * with the standard practice of using "water path" to refer to the total column-integrated + * quantities). + */ template -void mixing_ratio_to_cloud_mass( +static void mixing_ratio_to_cloud_mass( View1 const& mixing_ratio, View2 const& cloud_fraction, View3 const& dp, @@ -315,7 +1174,7 @@ void mixing_ratio_to_cloud_mass( int ncol = mixing_ratio.extent(0); int nlay = mixing_ratio.extent(1); using physconst = scream::physics::Constants; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { // Compute in-cloud mixing ratio (mixing ratio of the cloudy part of the layer) // NOTE: these thresholds (from E3SM) seem arbitrary, but included here for consistency // This limits in-cloud mixing ratio to 0.005 kg/kg. According to note in cloud_diagnostics @@ -329,7 +1188,6 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif /* * Routine to limit a quantity to set bounds. Used to make sure @@ -337,33 +1195,199 @@ void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -#ifdef RRTMGP_ENABLE_YAKL template -void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { - yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { +static void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { + Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -template -void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + +static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) +{ + // Get wavelength bounds for all wavelength bands + auto wavelength_bounds = kdist.get_band_lims_wavelength(); + + // Find the band index for the specified wavelength + // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength + // in units of meters, we need a conversion factor of 10^2 + const int nbnds = kdist.get_nband(); + int band_index = -1; + Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { + if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { + if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } else { + if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } + }, Kokkos::Max(band_index)); + return band_index; +} + +static inline int get_wavelength_index_sw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_sw_k, wavelength); +} + +static inline int get_wavelength_index_lw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_lw_k, wavelength); +} + +template +static OpticalProps2strK get_cloud_optics_sw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + + // Initialize optics + OpticalProps2strK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_2str(ncol, nlay); + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = view_t("rel_limited", ncol, nlay); + auto rei_limited = view_t("rei_limited", ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} + +template +static OpticalProps1sclK get_cloud_optics_lw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + + // Initialize optics + OpticalProps1sclK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = view_t("rel_limited", ncol, nlay); + auto rei_limited = view_t("rei_limited", ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} + +template +OpticalProps2strK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { + // Initialized subsampled optics + OpticalProps2strK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_2str(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, + // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), + // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should + // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very + // high resolution. + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM + auto seeds = view_t("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); + subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + subsampled_optics.ssa(icol,ilay,igpt) = 0; + subsampled_optics.g (icol,ilay,igpt) = 0; + } }); + return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_YAKL -int get_wavelength_index(OpticalProps &kdist, double wavelength); -int get_wavelength_index_sw(double wavelength); -int get_wavelength_index_lw(double wavelength); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength); -int get_wavelength_index_sw_k(double wavelength); -int get_wavelength_index_lw_k(double wavelength); -#endif +template +OpticalProps1sclK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, + CldT &cld, PlayT &p_lay) { + // Initialized subsampled optics + OpticalProps1sclK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_1scl(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM; use different + // seed values for longwave and shortwave + auto seeds = view_t("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + } + }); + return subsampled_optics; +} + +}; // struct rrtmgp_interface +#endif // RRTMGP_ENABLE_KOKKOS } // namespace rrtmgp } // namespace scream From 8677a1f45b071573f6142c46cdbc941048ab0fc9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 6 Jun 2024 16:55:23 -0600 Subject: [PATCH 396/904] Builds and runs --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 109 +++++----- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 130 ++++++------ .../src/physics/rrtmgp/rrtmgp_test_utils.cpp | 200 +----------------- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 128 ++++++++++- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 5 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 7 - .../rrtmgp/scream_rrtmgp_interface.hpp | 33 +-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 68 +++--- 9 files changed, 306 insertions(+), 376 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 59e6f72cc922..3d7fba5ac72d 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 59e6f72cc92268c69d1bd6ede570a959c1ebc28a +Subproject commit 3d7fba5ac72dab7c8448411e7452e2b409f22d93 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1ff3f27b2fce..a24f4d5ddc32 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -624,17 +624,17 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.init(gas_names_yakl_offset,m_col_chunk_size,m_nlay); - rrtmgp::rrtmgp_initialize( + interface_t::rrtmgp_initialize( m_gas_concs_k, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, m_atm_logger ); VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_sw, rrtmgp::k_dist_sw_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_lw, rrtmgp::k_dist_lw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, rrtmgp::cloud_optics_sw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, rrtmgp::cloud_optics_lw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_sw, interface_t::k_dist_sw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_lw, interface_t::k_dist_lw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, interface_t::cloud_optics_sw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, interface_t::cloud_optics_lw_k); #endif // Set property checks for fields in this process @@ -924,30 +924,30 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS // If YAKL is on, we don't want aliased memory in both the yakl and kokos // subviews. - auto subview_1dk = [&](const real1dk v) -> real1dk { - real1dk subv(v, std::make_pair(0, ncol)); + auto subview_1dk = [&](const view_1d_real v) -> view_1d_real { + view_1d_real subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - real1dk rv(v.label(), ncol); + view_1d_real rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_2dk = [&](const real2dk v) -> real2dk { - real2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_2dk = [&](const view_2d_real v) -> view_2d_real { + view_2d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); + view_2d_real rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_3dk = [&](const real3dk v) -> real3dk { - real3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + auto subview_3dk = [&](const view_3d_real v) -> view_3d_real { + view_3d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); + view_3d_real rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); return rv; #else @@ -1194,7 +1194,7 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::fence(); #ifdef RRTMGP_ENABLE_KOKKOS COMPARE_ALL_WRAP(std::vector({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1206,7 +1206,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real2d tmp2d = subview_2d(m_buffer.tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); + view_2d_real tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1308,14 +1308,14 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + interface_t::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + interface_t::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); + interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); + interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), - std::vector({lwp_k, iwp_k})); + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1350,13 +1350,13 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dir, sfc_alb_dif); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis_k, sfc_alb_dir_nir_k, sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), - std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1384,7 +1384,7 @@ void RRTMGPRadiation::run_impl (const double dt) { ); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, m_nlay, p_lay_k, t_lay_k, p_lev_k, t_lev_k, m_gas_concs_k, @@ -1412,7 +1412,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, lw_clnsky_flux_up, lw_clnsky_flux_dn}), - std::vector({ + std::vector({ sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, @@ -1422,7 +1422,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), - std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1472,7 +1472,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), - std::vector({sw_heating_k, lw_heating_k})); + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1509,14 +1509,14 @@ void RRTMGPRadiation::run_impl (const double dt) { sw_bnd_flux_dif_k(icol,ilev,ibnd) = sw_bnd_flux_dn_k(icol,ilev,ibnd) - sw_bnd_flux_dir_k(icol,ilev,ibnd); }); // Compute surface fluxes - rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot_k, nswbands, sw_bnd_flux_dir_k, sw_bnd_flux_dif_k, sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), - std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1537,22 +1537,22 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); - real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); - real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); - real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably // does not matter in practice, as clouds probably should not be produced above 50 hPa and we // should not be encountering surface pressure above 1200 hPa, but in the event that things go off // the rails we might want to look at these still. - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), - std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Compute cloud-top diagnostics following AeroCOM recommendation @@ -1580,20 +1580,20 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // Get visible 0.67 micron band for COSP - auto idx_067_k = rrtmgp::get_wavelength_index_sw_k(0.67e-6); + auto idx_067_k = interface_t::get_wavelength_index_sw_k(0.67e-6); // Get IR 10.5 micron band for COSP - auto idx_105_k = rrtmgp::get_wavelength_index_lw_k(10.5e-6); - - real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - - rrtmgp::compute_aerocom_cloudtop( + auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); + + view_1d_real T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + + interface_t::compute_aerocom_cloudtop( ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, @@ -1602,7 +1602,7 @@ void RRTMGPRadiation::run_impl (const double dt) { T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), - std::vector({ + std::vector({ T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); @@ -1699,7 +1699,7 @@ void RRTMGPRadiation::run_impl (const double dt) { }); #ifdef RRTMGP_ENABLE_YAKL // Sync back to gas_concs_k - real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + view_3d_real temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); Kokkos::deep_copy(temp, m_gas_concs_k.concs); #endif #endif @@ -1769,11 +1769,12 @@ void RRTMGPRadiation::run_impl (const double dt) { void RRTMGPRadiation::finalize_impl () { #ifdef RRTMGP_ENABLE_YAKL m_gas_concs.reset(); + rrtmgp::rrtmgp_finalize(); #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.reset(); + interface_t::rrtmgp_finalize(); #endif - rrtmgp::rrtmgp_finalize(); finalize_kls(); } diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 66c89476028b..28c5033c46d4 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -28,6 +28,10 @@ class RRTMGPRadiation : public AtmosphereProcess { template using uview_2d = Unmanaged>; + using layout_t = typename ekat::KokkosTypes::Layout; + + using interface_t = rrtmgp::rrtmgp_interface; + // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -92,7 +96,7 @@ class RRTMGPRadiation : public AtmosphereProcess { GasConcs m_gas_concs; #endif #ifdef RRTMGP_ENABLE_KOKKOS - GasConcsK m_gas_concs_k; + GasConcsK m_gas_concs_k; #endif // Prescribed greenhouse gas surface concentrations in moles / moles air @@ -137,15 +141,15 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + view_1d_real mu0_k; + view_1d_real sfc_alb_dir_vis_k; + view_1d_real sfc_alb_dir_nir_k; + view_1d_real sfc_alb_dif_vis_k; + view_1d_real sfc_alb_dif_nir_k; + view_1d_real sfc_flux_dir_vis_k; + view_1d_real sfc_flux_dir_nir_k; + view_1d_real sfc_flux_dif_vis_k; + view_1d_real sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) @@ -168,21 +172,21 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + view_2d_real p_lay_k; + view_2d_real t_lay_k; + view_2d_real z_del_k; + view_2d_real p_del_k; + view_2d_real qc_k; + view_2d_real nc_k; + view_2d_real qi_k; + view_2d_real cldfrac_tot_k; + view_2d_real eff_radius_qc_k; + view_2d_real eff_radius_qi_k; + view_2d_real tmp2d_k; + view_2d_real lwp_k; + view_2d_real iwp_k; + view_2d_real sw_heating_k; + view_2d_real lw_heating_k; #endif // 2d size (ncol, nlay+1) @@ -212,28 +216,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + view_2d_real p_lev_k; + view_2d_real t_lev_k; + view_2d_real sw_flux_up_k; + view_2d_real sw_flux_dn_k; + view_2d_real sw_flux_dn_dir_k; + view_2d_real lw_flux_up_k; + view_2d_real lw_flux_dn_k; + view_2d_real sw_clnclrsky_flux_up_k; + view_2d_real sw_clnclrsky_flux_dn_k; + view_2d_real sw_clnclrsky_flux_dn_dir_k; + view_2d_real sw_clrsky_flux_up_k; + view_2d_real sw_clrsky_flux_dn_k; + view_2d_real sw_clrsky_flux_dn_dir_k; + view_2d_real sw_clnsky_flux_up_k; + view_2d_real sw_clnsky_flux_dn_k; + view_2d_real sw_clnsky_flux_dn_dir_k; + view_2d_real lw_clnclrsky_flux_up_k; + view_2d_real lw_clnclrsky_flux_dn_k; + view_2d_real lw_clrsky_flux_up_k; + view_2d_real lw_clrsky_flux_dn_k; + view_2d_real lw_clnsky_flux_up_k; + view_2d_real lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -244,10 +248,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + view_3d_real sw_bnd_flux_up_k; + view_3d_real sw_bnd_flux_dn_k; + view_3d_real sw_bnd_flux_dir_k; + view_3d_real sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -256,8 +260,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + view_3d_real lw_bnd_flux_up_k; + view_3d_real lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -266,8 +270,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + view_2d_real sfc_alb_dir_k; + view_2d_real sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -278,10 +282,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + view_3d_real aero_tau_sw_k; + view_3d_real aero_ssa_sw_k; + view_3d_real aero_g_sw_k; + view_3d_real aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -290,8 +294,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + view_3d_real cld_tau_sw_bnd_k; + view_3d_real cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -300,8 +304,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + view_3d_real cld_tau_sw_gpt_k; + view_3d_real cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp index 54a32133da92..65cc99e09cc3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp @@ -1,25 +1,13 @@ -#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" #include "physics/rrtmgp/rrtmgp_test_utils.hpp" #ifdef RRTMGP_ENABLE_YAKL #include "YAKL_netcdf.h" #endif -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rte/mo_fluxes.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" - #include #include namespace rrtmgpTest { -#ifdef RRTMGP_ENABLE_YAKL -using yakl::fortran::parallel_for; -using yakl::fortran::SimpleBounds; -using yakl::intrinsics::mod; -using yakl::intrinsics::merge; -#endif - bool file_exists(const char *filename) { if (auto file = fopen(filename, "r")) { fclose(file); @@ -29,8 +17,12 @@ bool file_exists(const char *filename) { } } -// TODO: use YAKL intrinsics for this to avoid needing to make host copies #ifdef RRTMGP_ENABLE_YAKL +using yakl::fortran::parallel_for; +using yakl::fortran::SimpleBounds; +using yakl::intrinsics::mod; +using yakl::intrinsics::merge; + bool all_close(real2d &arr1, real2d &arr2, double tolerance) { int nx = arr1.dimension[0]; int ny = arr2.dimension[1]; @@ -46,28 +38,7 @@ bool all_close(real2d &arr1, real2d &arr2, double tolerance) { } return true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { - int nx = arr1.extent(0); - int ny = arr2.extent(1); - auto arr1_h = Kokkos::create_mirror_view(arr1); - auto arr2_h = Kokkos::create_mirror_view(arr2); - Kokkos::deep_copy(arr1_h, arr1); - Kokkos::deep_copy(arr2_h, arr2); - for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { - printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); - return false; - } - } - } - return true; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void dummy_atmos( std::string inputfile, int ncol, real2d &p_lay, real2d &t_lay, @@ -94,37 +65,7 @@ void dummy_atmos( // create a dummy atmosphere. dummy_clouds(scream::rrtmgp::cloud_optics_sw, p_lay, t_lay, lwp, iwp, rel, rei, cld); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_atmos( - std::string inputfile, - int ncol, real2dk &p_lay, real2dk &t_lay, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld) { - - // Setup boundary conditions, solar zenith angle, etc - // NOTE: this stuff would come from the model in a real run - // Ocean-ish values for surface albedos, just for example - Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); - - // Pick a solar zenith angle; this should come from the model - Kokkos::deep_copy(mu0, 0.86 ); - - // Get dummy cloud PHYSICAL properties. Note that this function call - // needs the CloudOptics object only because it uses the min and max - // valid values from the lookup tables for liquid and ice water path to - // create a dummy atmosphere. - dummy_clouds(scream::rrtmgp::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void dummy_clouds( CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cloud_mask) { @@ -151,38 +92,7 @@ void dummy_clouds( rei(icol,ilay) = merge(rei_val, 0._wp, iwp(icol,ilay) > 0._wp); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask) { - - // Problem sizes - int ncol = t_lay.extent(0); - int nlay = t_lay.extent(1); - - // Generate some fake liquid and ice water data. We pick values to be midway between - // the min and max of the valid lookup table values for effective radii - real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); - real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); - // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and - // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. - // Set sane values for liquid and ice water path. - // NOTE: these "sane" values are in g/m2! - Kokkos::parallel_for( conv::get_mdrp<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { - cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; - // Ice and liquid will overlap in a few layers - lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); - iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); - rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); - rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); - }); -} -#endif - -// Function to read fluxes from input file so we can compare our answers against the reference -#ifdef RRTMGP_ENABLE_YAKL void read_fluxes( std::string inputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -208,36 +118,7 @@ void read_fluxes( io.read(lw_flux_up, "lw_flux_up" ); io.read(lw_flux_dn, "lw_flux_dn" ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void read_fluxes( - std::string inputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - // Initialize netcdf reader - conv::SimpleNetCDF io; - io.open(inputfile, NC_NOWRITE); - - // Initialize arrays to hold fluxes - int nlev = io.getDimSize("lev"); - int ncol = io.getDimSize("col_flx"); - sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); - sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); - sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); - lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); - lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); - - // Read data - io.read(sw_flux_up, "sw_flux_up" ); - io.read(sw_flux_dn, "sw_flux_dn" ); - io.read(sw_flux_dir, "sw_flux_dir"); - io.read(lw_flux_up, "lw_flux_up" ); - io.read(lw_flux_dn, "lw_flux_dn" ); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void write_fluxes( std::string outputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -253,76 +134,5 @@ void write_fluxes( io.close(); } #endif -#ifdef RRTMGP_ENABLE_KOKKOS -void write_fluxes( - std::string outputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - conv::SimpleNetCDF io; - io.create(outputfile); - io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); - io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); - io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); - io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); - io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); - io.close(); -} -#endif - -// TODO: This function should instead take values, not file names, -// because for the test we do not want to write to file -#ifdef RRTMGP_ENABLE_YAKL -int compare_y(std::string file1, std::string file2) { - // Read data from baseline and test file - real2d sw_flux_up_1; - real2d sw_flux_dn_1; - real2d sw_flux_dir_1; - real2d lw_flux_up_1; - real2d lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2d sw_flux_up_2; - real2d sw_flux_dn_2; - real2d sw_flux_dir_2; - real2d lw_flux_up_2; - real2d lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int compare_k(std::string file1, std::string file2) { - // Read data from baseline and test file - real2dk sw_flux_up_1; - real2dk sw_flux_dn_1; - real2dk sw_flux_dir_1; - real2dk lw_flux_up_1; - real2dk lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2dk sw_flux_up_2; - real2dk sw_flux_dn_2; - real2dk sw_flux_dir_2; - real2dk lw_flux_up_2; - real2dk lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index b9498a9a34f9..79f6e570950d 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -1,5 +1,10 @@ #ifndef RRTMGP_TEST_UTILS_HPP #define RRTMGP_TEST_UTILS_HPP + +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rte/mo_fluxes.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" namespace rrtmgpTest { @@ -37,33 +42,134 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance); +template +struct rrtmgp_test_utils { -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +using real1dk = typename ekat::KokkosTypes::template view_1d; +using real2dk = typename ekat::KokkosTypes::template view_2d; +using real3dk = typename ekat::KokkosTypes::template view_3d; +using MDRP = typename conv::MDRP; +using interface_t = scream::rrtmgp::rrtmgp_interface; -void dummy_atmos( +static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) +{ + int nx = arr1.extent(0); + int ny = arr2.extent(1); + auto arr1_h = Kokkos::create_mirror_view(arr1); + auto arr2_h = Kokkos::create_mirror_view(arr2); + Kokkos::deep_copy(arr1_h, arr1); + Kokkos::deep_copy(arr2_h, arr2); + for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { + printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); + return false; + } + } + } + return true; +} + +static void dummy_clouds( + CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask +) +{ + // Problem sizes + int ncol = t_lay.extent(0); + int nlay = t_lay.extent(1); + + // Generate some fake liquid and ice water data. We pick values to be midway between + // the min and max of the valid lookup table values for effective radii + real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); + real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); + + // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and + // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. + // Set sane values for liquid and ice water path. + // NOTE: these "sane" values are in g/m2! + Kokkos::parallel_for( MDRP::template get<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { + cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; + // Ice and liquid will overlap in a few layers + lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); + iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); + rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); + rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); + }); +} + +static void dummy_atmos( std::string inputfile, int ncol, real2dk &p_lay, real2dk &t_lay, real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, real1dk &mu0, real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +) +{ + // Setup boundary conditions, solar zenith angle, etc + // NOTE: this stuff would come from the model in a real run -void read_fluxes( + // Ocean-ish values for surface albedos, just for example + Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); + + // Pick a solar zenith angle; this should come from the model + Kokkos::deep_copy(mu0, 0.86 ); + + // Get dummy cloud PHYSICAL properties. Note that this function call + // needs the CloudOptics object only because it uses the min and max + // valid values from the lookup tables for liquid and ice water path to + // create a dummy atmosphere. + dummy_clouds(interface_t::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); +} + +static void read_fluxes( std::string inputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + // Initialize netcdf reader + conv::SimpleNetCDF io; + io.open(inputfile, NC_NOWRITE); -void write_fluxes( + // Initialize arrays to hold fluxes + int nlev = io.getDimSize("lev"); + int ncol = io.getDimSize("col_flx"); + sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); + sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); + sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); + lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); + lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); + + // Read data + io.read(sw_flux_up, "sw_flux_up" ); + io.read(sw_flux_dn, "sw_flux_dn" ); + io.read(sw_flux_dir, "sw_flux_dir"); + io.read(lw_flux_up, "lw_flux_up" ); + io.read(lw_flux_dn, "lw_flux_dn" ); +} + +static void write_fluxes( std::string outputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + conv::SimpleNetCDF io; + io.create(outputfile); + io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); + io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); + io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); + io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); + io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); + io.close(); +} + +}; #endif } diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 119093d5f9d3..6e924103229f 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -109,8 +109,9 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, RealT xmin, RealT xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 1696e7933685..10281df8078c 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,11 +1,4 @@ #include "scream_rrtmgp_interface.hpp" -#include "cpp/examples/mo_load_coefficients.h" -#include "examples/all-sky/mo_load_cloud_coefficients.h" -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" -#include "cpp/rte/mo_rte_sw.h" -#include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" namespace scream { diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index c40be4904302..2038cb62ff79 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -4,7 +4,14 @@ #include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" +#include "cpp/examples/mo_load_coefficients.h" #include "cpp/rrtmgp_const.h" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "cpp/rte/mo_rte_sw.h" +#include "cpp/rte/mo_rte_lw.h" +#include "examples/all-sky/mo_load_cloud_coefficients.h" #include "rrtmgp_utils.hpp" @@ -152,7 +159,7 @@ int get_wavelength_index_lw(double wavelength); // New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_interface { using MDRP = typename conv::MDRP; @@ -469,8 +476,8 @@ static void rrtmgp_main( #endif // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); @@ -640,7 +647,7 @@ static void rrtmgp_sw( // Subset gases auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; + GasConcsK gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { auto vmr_day = view_t("vmr_day", nday, nlay); @@ -653,7 +660,7 @@ static void rrtmgp_sw( } // Subset aerosol optics - OpticalProps2strK aerosol_day; + OpticalProps2strK aerosol_day; aerosol_day.init(k_dist.get_band_lims_wavenumber()); aerosol_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { @@ -664,7 +671,7 @@ static void rrtmgp_sw( // Subset cloud optics // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; + OpticalProps2strK clouds_day; clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); clouds_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { @@ -690,7 +697,7 @@ static void rrtmgp_sw( auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; + FluxesBybandK fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; fluxes_day.flux_dn_dir = flux_dn_dir_day; @@ -699,10 +706,10 @@ static void rrtmgp_sw( fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; // Allocate space for optical properties - OpticalProps2strK optics; + OpticalProps2strK optics; optics.alloc_2str(nday, nlay, k_dist); - OpticalProps2strK optics_no_aerosols; + OpticalProps2strK optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_2str(nday, nlay, k_dist); @@ -1195,8 +1202,8 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template -static void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { +template +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); @@ -1291,7 +1298,7 @@ static OpticalProps1sclK get_cloud_optics_lw( } template -OpticalProps2strK get_subsampled_clouds( +static OpticalProps2strK get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { // Initialized subsampled optics @@ -1342,7 +1349,7 @@ OpticalProps2strK get_subsampled_clouds( } template -OpticalProps1sclK get_subsampled_clouds( +static OpticalProps1sclK get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index ddef61808fe0..36e2a59e02ff 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -849,6 +849,14 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { #endif #ifdef RRTMGP_ENABLE_KOKKOS +using interface_t = scream::rrtmgp::rrtmgp_interface<>; +using real1dk = interface_t::view_t; +using real2dk = interface_t::view_t; +using real3dk = interface_t::view_t; +using int1dk = interface_t::view_t; +using int2dk = interface_t::view_t; +using int3dk = interface_t::view_t; + TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos scream::init_kls(); @@ -921,7 +929,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 1.0; }); auto cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with no cloud @@ -931,7 +939,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.0; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with empty clouds (cloud fraction but with no associated mixing ratio) @@ -943,7 +951,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.1; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with cell half filled with cloud @@ -953,7 +961,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.5; }); cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up @@ -977,14 +985,14 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { }); // Limit to bounds that contain the data; should be no change in values - scream::rrtmgp::limit_to_bounds(arr, 0.0, 5.0, arr_limited); + interface_t::limit_to_bounds_k(arr, 0.0, 5.0, arr_limited); REQUIRE(chc(arr)(0,0) == chc(arr_limited)(0,0)); REQUIRE(chc(arr)(0,1) == chc(arr_limited)(0,1)); REQUIRE(chc(arr)(1,0) == chc(arr_limited)(1,0)); REQUIRE(chc(arr)(1,1) == chc(arr_limited)(1,1)); // Limit to bounds that do not completely contain the data; should be a change in values! - scream::rrtmgp::limit_to_bounds(arr, 1.5, 3.5, arr_limited); + interface_t::limit_to_bounds_k(arr, 1.5, 3.5, arr_limited); REQUIRE(chc(arr_limited)(0,0) == 1.5); REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); @@ -1078,7 +1086,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Create simple test cases; We expect, given the input data, that band 10 // will straddle the NIR and VIS, bands 1-9 will be purely NIR, and bands 11-14 @@ -1106,7 +1114,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1138,7 +1146,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1169,7 +1177,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1200,7 +1208,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1216,7 +1224,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Finalize YAKL logger->info("Free memory...\n"); - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); gas_concs.reset(); scream::finalize_kls(); } @@ -1282,7 +1290,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { @@ -1316,7 +1324,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); auto cldmask_h = chc(cldmask); for (int igpt = 0; igpt < ngpt; igpt++) { if (cldmask_h(0,0,igpt) == 1) { @@ -1359,7 +1367,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); // Case: @@ -1376,7 +1384,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 1; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); // Case: @@ -1393,11 +1401,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1.0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0 / 3.0); // Case: @@ -1414,11 +1422,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); scream::finalize_kls(); } @@ -1463,7 +1471,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { Kokkos::deep_copy(rel, 10.0); Kokkos::deep_copy(rei, 10.0); // Call the function - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1481,7 +1489,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { // Case 2: if all clouds, everything goes to 1 * its value Kokkos::deep_copy(cldfrac_tot, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1504,7 +1512,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 3) = 0.3; cldfrac_tot(0, 4) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1520,7 +1528,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 5) = 0.4; cldfrac_tot(0, 6) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1534,7 +1542,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 4) = 0.0; cldfrac_tot(0, 5) = 0.1; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1550,7 +1558,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 1.0); Kokkos::deep_copy(qi, 0.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1568,7 +1576,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 0.0); Kokkos::deep_copy(qi, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1604,7 +1612,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { qc(0, 6) = 50; qc(0, 7) = 10; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, From 5c8349144ea47bbc4e27bc2bdd2b96227689173f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 14:15:42 -0600 Subject: [PATCH 397/904] Everything working --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 327 +++++++++--------- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 158 ++++----- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 10 +- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 89 ++++- .../rrtmgp/scream_rrtmgp_interface.hpp | 24 +- .../src/physics/rrtmgp/tests/rrtmgp_tests.cpp | 64 ++-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 15 +- .../rrtmgp/rrtmgp_standalone_unit.cpp | 37 +- 9 files changed, 405 insertions(+), 321 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 3d7fba5ac72d..4c542032b3ac 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 3d7fba5ac72dab7c8448411e7452e2b409f22d93 +Subproject commit 4c542032b3ac3f842a9619c9eadc005c507a0366 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index a24f4d5ddc32..1556280eab4b 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -19,6 +19,28 @@ namespace scream { +template +void check_views2(const View1& view1, const View2& view2) +{ + constexpr auto krank1 = View1::rank; + constexpr auto krank2 = View2::rank; + + RRT_REQUIRE(krank1 == krank2, "Rank mismatch for: " << view1.label()); + RRT_REQUIRE(krank1 == 2, "Rank mismatch for: " << view1.label()); + + for (auto r = 0; r < krank1; ++r) { + RRT_REQUIRE(view1.extent(r) == view2.extent(r), "Dim mismatch for: " << view1.label() << ", rank: " << r << ", " << view1.extent(r) << " != " << view2.extent(r)); + } + + for (auto i = 0; i < view1.extent(0); ++i) { + for (auto j = 0; j < view1.extent(1); ++j) { + const auto data1 = view1(i, j); + const auto data2 = view2(i, j); + RRT_REQUIRE(data1 == data2, "Data mismatch for: " << view1.label() << ", i: " << i << ", j: " << j << ", " << data1 << " != " << data2); + } + } +} + using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; @@ -596,7 +618,7 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // Names of active gases auto gas_names_yakl_offset = string1dv(m_ngas); - m_gas_mol_weights = view_1d_real("gas_mol_weights",m_ngas); + m_gas_mol_weights = real1dk("gas_mol_weights",m_ngas); // the lookup function for getting the gas mol weights doesn't work on device auto gas_mol_w_host = Kokkos::create_mirror_view(m_gas_mol_weights); for (int igas = 0; igas < m_ngas; igas++) { @@ -848,11 +870,9 @@ void RRTMGPRadiation::run_impl (const double dt) { "[RRTMGP::run_impl] Col chunk beg,end: " + std::to_string(beg) + ", " + std::to_string(beg+ncol) + "\n"); +#ifdef RRTMGP_ENABLE_YAKL // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. - // JGF: this doesn't appear to be copying the data, just returning a new array - // pointing to the same memory. -#ifdef RRTMGP_ENABLE_YAKL auto subview_1d = [&](const real1d v) -> real1d { return real1d(v.label(),v.myData,ncol); }; @@ -923,31 +943,64 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // If YAKL is on, we don't want aliased memory in both the yakl and kokos - // subviews. - auto subview_1dk = [&](const view_1d_real v) -> view_1d_real { - view_1d_real subv(v, std::make_pair(0, ncol)); + // subviews, so make new views and deep_copy. Also, be sure to trim the excess + // items from the field manager views due to simd packs. If we don't trim, then + // check_range_k will fail due to looking at unused values. Once rrtmgp can handle + // packs, this won't be necessary. + auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { + ureal1dk subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - view_1d_real rv(v.label(), ncol); + real1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_2dk = [&](const view_2d_real v) -> view_2d_real { - view_2d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { + cureal1dk subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - view_2d_real rv(v.label(), ncol, v.extent(1)); + creal1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_3dk = [&](const view_3d_real v) -> view_3d_real { - view_3d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + auto subview_2dk = [&](const ureal2dk& v) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - view_3d_real rv(v.label(), ncol, v.extent(1), v.extent(2)); + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_2dkc = [&](const cureal2dk& v, const int inner_dim) -> cureal2dk { + cureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); +#ifdef RRTMGP_ENABLE_YAKL + creal2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_2dk_new = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); +#ifdef RRTMGP_ENABLE_YAKL + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { + ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); +#ifdef RRTMGP_ENABLE_YAKL + real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); return rv; #else @@ -955,59 +1008,59 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; - auto p_lay_k = subview_2dk(m_buffer.p_lay_k); - auto t_lay_k = subview_2dk(m_buffer.t_lay_k); - auto p_lev_k = subview_2dk(m_buffer.p_lev_k); - auto z_del_k = subview_2dk(m_buffer.z_del_k); - auto p_del_k = subview_2dk(m_buffer.p_del_k); - auto t_lev_k = subview_2dk(m_buffer.t_lev_k); - auto mu0_k = subview_1dk(m_buffer.mu0_k); - auto sfc_alb_dir_k = subview_2dk(m_buffer.sfc_alb_dir_k); - auto sfc_alb_dif_k = subview_2dk(m_buffer.sfc_alb_dif_k); - auto sfc_alb_dir_vis_k = subview_1dk(m_buffer.sfc_alb_dir_vis_k); - auto sfc_alb_dir_nir_k = subview_1dk(m_buffer.sfc_alb_dir_nir_k); - auto sfc_alb_dif_vis_k = subview_1dk(m_buffer.sfc_alb_dif_vis_k); - auto sfc_alb_dif_nir_k = subview_1dk(m_buffer.sfc_alb_dif_nir_k); - auto qc_k = subview_2dk(m_buffer.qc_k); - auto nc_k = subview_2dk(m_buffer.nc_k); - auto qi_k = subview_2dk(m_buffer.qi_k); - auto cldfrac_tot_k = subview_2dk(m_buffer.cldfrac_tot_k); - auto rel_k = subview_2dk(m_buffer.eff_radius_qc_k); - auto rei_k = subview_2dk(m_buffer.eff_radius_qi_k); - auto sw_flux_up_k = subview_2dk(m_buffer.sw_flux_up_k); - auto sw_flux_dn_k = subview_2dk(m_buffer.sw_flux_dn_k); - auto sw_flux_dn_dir_k = subview_2dk(m_buffer.sw_flux_dn_dir_k); - auto lw_flux_up_k = subview_2dk(m_buffer.lw_flux_up_k); - auto lw_flux_dn_k = subview_2dk(m_buffer.lw_flux_dn_k); - auto sw_clnclrsky_flux_up_k = subview_2dk(m_buffer.sw_clnclrsky_flux_up_k); - auto sw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_k); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_dir_k); - auto sw_clrsky_flux_up_k = subview_2dk(m_buffer.sw_clrsky_flux_up_k); - auto sw_clrsky_flux_dn_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_k); - auto sw_clrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_dir_k); - auto sw_clnsky_flux_up_k = subview_2dk(m_buffer.sw_clnsky_flux_up_k); - auto sw_clnsky_flux_dn_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_k); - auto sw_clnsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_dir_k); - auto lw_clnclrsky_flux_up_k = subview_2dk(m_buffer.lw_clnclrsky_flux_up_k); - auto lw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.lw_clnclrsky_flux_dn_k); - auto lw_clrsky_flux_up_k = subview_2dk(m_buffer.lw_clrsky_flux_up_k); - auto lw_clrsky_flux_dn_k = subview_2dk(m_buffer.lw_clrsky_flux_dn_k); - auto lw_clnsky_flux_up_k = subview_2dk(m_buffer.lw_clnsky_flux_up_k); - auto lw_clnsky_flux_dn_k = subview_2dk(m_buffer.lw_clnsky_flux_dn_k); - auto sw_bnd_flux_up_k = subview_3dk(m_buffer.sw_bnd_flux_up_k); - auto sw_bnd_flux_dn_k = subview_3dk(m_buffer.sw_bnd_flux_dn_k); - auto sw_bnd_flux_dir_k = subview_3dk(m_buffer.sw_bnd_flux_dir_k); - auto sw_bnd_flux_dif_k = subview_3dk(m_buffer.sw_bnd_flux_dif_k); - auto lw_bnd_flux_up_k = subview_3dk(m_buffer.lw_bnd_flux_up_k); - auto lw_bnd_flux_dn_k = subview_3dk(m_buffer.lw_bnd_flux_dn_k); - auto sfc_flux_dir_vis_k = subview_1dk(m_buffer.sfc_flux_dir_vis_k); - auto sfc_flux_dir_nir_k = subview_1dk(m_buffer.sfc_flux_dir_nir_k); - auto sfc_flux_dif_vis_k = subview_1dk(m_buffer.sfc_flux_dif_vis_k); - auto sfc_flux_dif_nir_k = subview_1dk(m_buffer.sfc_flux_dif_nir_k); - auto aero_tau_sw_k = subview_3dk(m_buffer.aero_tau_sw_k); - auto aero_ssa_sw_k = subview_3dk(m_buffer.aero_ssa_sw_k); - auto aero_g_sw_k = subview_3dk(m_buffer.aero_g_sw_k); - auto aero_tau_lw_k = subview_3dk(m_buffer.aero_tau_lw_k); + // Note, ncol will not necessary be m_col_chunk_size because the number of cols + // will not always be evenly divided by m_col_chunk_size. In most cases, the + // extra space will not cause any problems, but it does sometimes. + auto p_lay_k = subview_2dkc(d_pmid, m_nlay); + auto t_lay_k = subview_2dkc(d_tmid, m_nlay); + auto p_lev_k = subview_2dkc(d_pint, m_nlay+1); + auto p_del_k = subview_2dkc(d_pdel, m_nlay); + auto sfc_alb_dir_k = m_buffer.sfc_alb_dir_k; + auto sfc_alb_dif_k = m_buffer.sfc_alb_dif_k; + auto sfc_alb_dir_vis_k = subview_1dkc(d_sfc_alb_dir_vis); + auto sfc_alb_dir_nir_k = subview_1dkc(d_sfc_alb_dir_nir); + auto sfc_alb_dif_vis_k = subview_1dkc(d_sfc_alb_dif_vis); + auto sfc_alb_dif_nir_k = subview_1dkc(d_sfc_alb_dif_nir); + auto qc_k = subview_2dkc(d_qc, m_nlay); + auto nc_k = subview_2dkc(d_nc, m_nlay); + auto qi_k = subview_2dkc(d_qi, m_nlay); + auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; + auto rel_k = subview_2dkc(d_rel, m_nlay); + auto rei_k = subview_2dkc(d_rei, m_nlay); + auto sw_flux_up_k = subview_2dk_new(d_sw_flux_up, m_nlay+1); + auto sw_flux_dn_k = subview_2dk_new(d_sw_flux_dn, m_nlay+1); + auto sw_flux_dn_dir_k = subview_2dk_new(d_sw_flux_dn_dir, m_nlay+1); + auto lw_flux_up_k = subview_2dk_new(d_lw_flux_up, m_nlay+1); + auto lw_flux_dn_k = subview_2dk_new(d_lw_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_up_k = subview_2dk_new(d_sw_clnclrsky_flux_up, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = subview_2dk_new(d_sw_clnclrsky_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); + auto sw_clrsky_flux_up_k = subview_2dk_new(d_sw_clrsky_flux_up, m_nlay+1); + auto sw_clrsky_flux_dn_k = subview_2dk_new(d_sw_clrsky_flux_dn, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clrsky_flux_dn_dir, m_nlay+1); + auto sw_clnsky_flux_up_k = subview_2dk_new(d_sw_clnsky_flux_up, m_nlay+1); + auto sw_clnsky_flux_dn_k = subview_2dk_new(d_sw_clnsky_flux_dn, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnsky_flux_dn_dir, m_nlay+1); + auto lw_clnclrsky_flux_up_k = subview_2dk_new(d_lw_clnclrsky_flux_up, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = subview_2dk_new(d_lw_clnclrsky_flux_dn, m_nlay+1); + auto lw_clrsky_flux_up_k = subview_2dk_new(d_lw_clrsky_flux_up, m_nlay+1); + auto lw_clrsky_flux_dn_k = subview_2dk_new(d_lw_clrsky_flux_dn, m_nlay+1); + auto lw_clnsky_flux_up_k = subview_2dk_new(d_lw_clnsky_flux_up, m_nlay+1); + auto lw_clnsky_flux_dn_k = subview_2dk_new(d_lw_clnsky_flux_dn, m_nlay+1); + auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; + auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; + auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; + auto sw_bnd_flux_dif_k = m_buffer.sw_bnd_flux_dif_k; + auto lw_bnd_flux_up_k = m_buffer.lw_bnd_flux_up_k; + auto lw_bnd_flux_dn_k = m_buffer.lw_bnd_flux_dn_k; + auto sfc_flux_dir_vis_k = subview_1dk(d_sfc_flux_dir_vis); + auto sfc_flux_dir_nir_k = subview_1dk(d_sfc_flux_dir_nir); + auto sfc_flux_dif_vis_k = subview_1dk(d_sfc_flux_dif_vis); + auto sfc_flux_dif_nir_k = subview_1dk(d_sfc_flux_dif_nir); + auto aero_tau_sw_k = m_buffer.aero_tau_sw_k; + auto aero_ssa_sw_k = m_buffer.aero_ssa_sw_k; + auto aero_g_sw_k = m_buffer.aero_g_sw_k; + auto aero_tau_lw_k = m_buffer.aero_tau_lw_k; auto cld_tau_sw_bnd_k = subview_3dk(m_buffer.cld_tau_sw_bnd_k); auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); @@ -1015,7 +1068,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif auto d_tint = m_buffer.d_tint; auto d_dz = m_buffer.d_dz; - + auto d_mu0 = m_buffer.cosine_zenith; // Set gas concs to "view" only the first ncol columns #ifdef RRTMGP_ENABLE_YAKL @@ -1032,7 +1085,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // Determine the cosine zenith angle // NOTE: Since we are bridging to F90 arrays this must be done on HOST and then // deep copied to a device view. - auto d_mu0 = m_buffer.cosine_zenith; auto h_mu0 = Kokkos::create_mirror_view(d_mu0); if (m_fixed_solar_zenith_angle > 0) { for (int i=0; i({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1206,7 +1226,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real2d tmp2d = subview_2d(m_buffer.tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real tmp2d_k = subview_2dk(m_buffer.tmp2d_k); + real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1315,7 +1335,7 @@ void RRTMGPRadiation::run_impl (const double dt) { interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), - std::vector({lwp_k, iwp_k})); + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1356,7 +1376,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), - std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1386,9 +1406,9 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS interface_t::rrtmgp_main( ncol, m_nlay, - p_lay_k, t_lay_k, p_lev_k, t_lev_k, + p_lay_k, t_lay_k, p_lev_k, d_tint, m_gas_concs_k, - sfc_alb_dir_k, sfc_alb_dif_k, mu0_k, + sfc_alb_dir_k, sfc_alb_dif_k, d_mu0, lwp_k, iwp_k, rel_k, rei_k, cldfrac_tot_k, aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k, cld_tau_sw_bnd_k, cld_tau_lw_bnd_k, @@ -1412,7 +1432,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, lw_clnsky_flux_up, lw_clnsky_flux_dn}), - std::vector({ + std::vector({ sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, @@ -1420,9 +1440,8 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up_k, lw_clnclrsky_flux_dn_k, lw_clrsky_flux_up_k, lw_clrsky_flux_dn_k, lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); - COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), - std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1472,7 +1491,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), - std::vector({sw_heating_k, lw_heating_k})); + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1516,7 +1535,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), - std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1537,10 +1556,10 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_1d_real cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably @@ -1552,7 +1571,7 @@ void RRTMGPRadiation::run_impl (const double dt) { interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), - std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Compute cloud-top diagnostics following AeroCOM recommendation @@ -1584,17 +1603,17 @@ void RRTMGPRadiation::run_impl (const double dt) { // Get IR 10.5 micron band for COSP auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); - view_1d_real T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); interface_t::compute_aerocom_cloudtop( - ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, + ncol, nlay, t_lay_k, p_lay_k, p_del_k, d_dz, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k); @@ -1602,7 +1621,7 @@ void RRTMGPRadiation::run_impl (const double dt) { T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), - std::vector({ + std::vector({ T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); @@ -1658,34 +1677,8 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; - d_sfc_flux_dir_nir(icol) = sfc_flux_dir_nir_k(i); - d_sfc_flux_dir_vis(icol) = sfc_flux_dir_vis_k(i); - d_sfc_flux_dif_nir(icol) = sfc_flux_dif_nir_k(i); - d_sfc_flux_dif_vis(icol) = sfc_flux_dif_vis_k(i); d_sfc_flux_sw_net(icol) = sw_flux_dn_k(i,kbot_k) - sw_flux_up_k(i,kbot_k); d_sfc_flux_lw_dn(icol) = lw_flux_dn_k(i,kbot_k); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - d_sw_flux_up(icol,k) = sw_flux_up_k(i,k); - d_sw_flux_dn(icol,k) = sw_flux_dn_k(i,k); - d_sw_flux_dn_dir(icol,k) = sw_flux_dn_dir_k(i,k); - d_lw_flux_up(icol,k) = lw_flux_up_k(i,k); - d_lw_flux_dn(icol,k) = lw_flux_dn_k(i,k); - d_sw_clnclrsky_flux_up(icol,k) = sw_clnclrsky_flux_up_k(i,k); - d_sw_clnclrsky_flux_dn(icol,k) = sw_clnclrsky_flux_dn_k(i,k); - d_sw_clnclrsky_flux_dn_dir(icol,k) = sw_clnclrsky_flux_dn_dir_k(i,k); - d_sw_clrsky_flux_up(icol,k) = sw_clrsky_flux_up_k(i,k); - d_sw_clrsky_flux_dn(icol,k) = sw_clrsky_flux_dn_k(i,k); - d_sw_clrsky_flux_dn_dir(icol,k) = sw_clrsky_flux_dn_dir_k(i,k); - d_sw_clnsky_flux_up(icol,k) = sw_clnsky_flux_up_k(i,k); - d_sw_clnsky_flux_dn(icol,k) = sw_clnsky_flux_dn_k(i,k); - d_sw_clnsky_flux_dn_dir(icol,k) = sw_clnsky_flux_dn_dir_k(i,k); - d_lw_clnclrsky_flux_up(icol,k) = lw_clnclrsky_flux_up_k(i,k); - d_lw_clnclrsky_flux_dn(icol,k) = lw_clnclrsky_flux_dn_k(i,k); - d_lw_clrsky_flux_up(icol,k) = lw_clrsky_flux_up_k(i,k); - d_lw_clrsky_flux_dn(icol,k) = lw_clrsky_flux_dn_k(i,k); - d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up_k(i,k); - d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn_k(i,k); - }); // Extract optical properties for COSP Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067_k); @@ -1699,7 +1692,7 @@ void RRTMGPRadiation::run_impl (const double dt) { }); #ifdef RRTMGP_ENABLE_YAKL // Sync back to gas_concs_k - view_3d_real temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); Kokkos::deep_copy(temp, m_gas_concs_k.concs); #endif #endif diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 28c5033c46d4..c60a1cb3181a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -16,17 +16,21 @@ namespace scream { class RRTMGPRadiation : public AtmosphereProcess { public: - using view_1d_real = typename ekat::KokkosTypes::template view_1d; - using view_2d_real = typename ekat::KokkosTypes::template view_2d; - using view_3d_real = typename ekat::KokkosTypes::template view_3d; - using view_2d_real_const = typename ekat::KokkosTypes::template view_2d; - using ci_string = ekat::CaseInsensitiveString; - - using KT = ekat::KokkosTypes; - template - using uview_1d = Unmanaged>; - template - using uview_2d = Unmanaged>; + using KT = ekat::KokkosTypes; + using real1dk = typename KT::template view_1d; + using real2dk = typename KT::template view_2d; + using real3dk = typename KT::template view_3d; + using creal1dk = typename KT::template view_1d; + using creal2dk = typename KT::template view_2d; + using creal3dk = typename KT::template view_3d; + using ureal1dk = Unmanaged; + using ureal2dk = Unmanaged; + using ureal3dk = Unmanaged; + using cureal1dk = Unmanaged; + using cureal2dk = Unmanaged; + using cureal3dk = Unmanaged; + + using ci_string = ekat::CaseInsensitiveString; using layout_t = typename ekat::KokkosTypes::Layout; @@ -91,7 +95,7 @@ class RRTMGPRadiation : public AtmosphereProcess { // These are the gases that we keep track of int m_ngas; std::vector m_gas_names; - view_1d_real m_gas_mol_weights; + real1dk m_gas_mol_weights; #ifdef RRTMGP_ENABLE_YAKL GasConcs m_gas_concs; #endif @@ -128,7 +132,7 @@ class RRTMGPRadiation : public AtmosphereProcess { static constexpr int num_3d_nlay_nlwgpts = 1; // 1d size (ncol) - uview_1d cosine_zenith; + ureal1dk cosine_zenith; #ifdef RRTMGP_ENABLE_YAKL real1d mu0; real1d sfc_alb_dir_vis; @@ -141,19 +145,19 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_1d_real mu0_k; - view_1d_real sfc_alb_dir_vis_k; - view_1d_real sfc_alb_dir_nir_k; - view_1d_real sfc_alb_dif_vis_k; - view_1d_real sfc_alb_dif_nir_k; - view_1d_real sfc_flux_dir_vis_k; - view_1d_real sfc_flux_dir_nir_k; - view_1d_real sfc_flux_dif_vis_k; - view_1d_real sfc_flux_dif_nir_k; + real1dk mu0_k; + real1dk sfc_alb_dir_vis_k; + real1dk sfc_alb_dir_nir_k; + real1dk sfc_alb_dif_vis_k; + real1dk sfc_alb_dif_nir_k; + real1dk sfc_flux_dir_vis_k; + real1dk sfc_flux_dir_nir_k; + real1dk sfc_flux_dif_vis_k; + real1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) - uview_2d d_dz; + ureal2dk d_dz; #ifdef RRTMGP_ENABLE_YAKL real2d p_lay; real2d t_lay; @@ -172,25 +176,25 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real p_lay_k; - view_2d_real t_lay_k; - view_2d_real z_del_k; - view_2d_real p_del_k; - view_2d_real qc_k; - view_2d_real nc_k; - view_2d_real qi_k; - view_2d_real cldfrac_tot_k; - view_2d_real eff_radius_qc_k; - view_2d_real eff_radius_qi_k; - view_2d_real tmp2d_k; - view_2d_real lwp_k; - view_2d_real iwp_k; - view_2d_real sw_heating_k; - view_2d_real lw_heating_k; + real2dk p_lay_k; + real2dk t_lay_k; + real2dk z_del_k; + real2dk p_del_k; + real2dk qc_k; + real2dk nc_k; + real2dk qi_k; + real2dk cldfrac_tot_k; + real2dk eff_radius_qc_k; + real2dk eff_radius_qi_k; + real2dk tmp2d_k; + real2dk lwp_k; + real2dk iwp_k; + real2dk sw_heating_k; + real2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) - uview_2d d_tint; + ureal2dk d_tint; #ifdef RRTMGP_ENABLE_YAKL real2d p_lev; real2d t_lev; @@ -216,28 +220,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real p_lev_k; - view_2d_real t_lev_k; - view_2d_real sw_flux_up_k; - view_2d_real sw_flux_dn_k; - view_2d_real sw_flux_dn_dir_k; - view_2d_real lw_flux_up_k; - view_2d_real lw_flux_dn_k; - view_2d_real sw_clnclrsky_flux_up_k; - view_2d_real sw_clnclrsky_flux_dn_k; - view_2d_real sw_clnclrsky_flux_dn_dir_k; - view_2d_real sw_clrsky_flux_up_k; - view_2d_real sw_clrsky_flux_dn_k; - view_2d_real sw_clrsky_flux_dn_dir_k; - view_2d_real sw_clnsky_flux_up_k; - view_2d_real sw_clnsky_flux_dn_k; - view_2d_real sw_clnsky_flux_dn_dir_k; - view_2d_real lw_clnclrsky_flux_up_k; - view_2d_real lw_clnclrsky_flux_dn_k; - view_2d_real lw_clrsky_flux_up_k; - view_2d_real lw_clrsky_flux_dn_k; - view_2d_real lw_clnsky_flux_up_k; - view_2d_real lw_clnsky_flux_dn_k; + real2dk p_lev_k; + real2dk t_lev_k; + real2dk sw_flux_up_k; + real2dk sw_flux_dn_k; + real2dk sw_flux_dn_dir_k; + real2dk lw_flux_up_k; + real2dk lw_flux_dn_k; + real2dk sw_clnclrsky_flux_up_k; + real2dk sw_clnclrsky_flux_dn_k; + real2dk sw_clnclrsky_flux_dn_dir_k; + real2dk sw_clrsky_flux_up_k; + real2dk sw_clrsky_flux_dn_k; + real2dk sw_clrsky_flux_dn_dir_k; + real2dk sw_clnsky_flux_up_k; + real2dk sw_clnsky_flux_dn_k; + real2dk sw_clnsky_flux_dn_dir_k; + real2dk lw_clnclrsky_flux_up_k; + real2dk lw_clnclrsky_flux_dn_k; + real2dk lw_clrsky_flux_up_k; + real2dk lw_clrsky_flux_dn_k; + real2dk lw_clnsky_flux_up_k; + real2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -248,10 +252,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real sw_bnd_flux_up_k; - view_3d_real sw_bnd_flux_dn_k; - view_3d_real sw_bnd_flux_dir_k; - view_3d_real sw_bnd_flux_dif_k; + real3dk sw_bnd_flux_up_k; + real3dk sw_bnd_flux_dn_k; + real3dk sw_bnd_flux_dir_k; + real3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -260,8 +264,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real lw_bnd_flux_up_k; - view_3d_real lw_bnd_flux_dn_k; + real3dk lw_bnd_flux_up_k; + real3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -270,8 +274,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real sfc_alb_dir_k; - view_2d_real sfc_alb_dif_k; + real2dk sfc_alb_dir_k; + real2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -282,10 +286,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real aero_tau_sw_k; - view_3d_real aero_ssa_sw_k; - view_3d_real aero_g_sw_k; - view_3d_real aero_tau_lw_k; + real3dk aero_tau_sw_k; + real3dk aero_ssa_sw_k; + real3dk aero_g_sw_k; + real3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -294,8 +298,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real cld_tau_sw_bnd_k; - view_3d_real cld_tau_lw_bnd_k; + real3dk cld_tau_sw_bnd_k; + real3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -304,8 +308,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real cld_tau_sw_gpt_k; - view_3d_real cld_tau_lw_gpt_k; + real3dk cld_tau_sw_gpt_k; + real3dk cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index 79f6e570950d..466467919fcf 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -42,14 +42,14 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_test_utils { -using real1dk = typename ekat::KokkosTypes::template view_1d; -using real2dk = typename ekat::KokkosTypes::template view_2d; -using real3dk = typename ekat::KokkosTypes::template view_3d; -using MDRP = typename conv::MDRP; using interface_t = scream::rrtmgp::rrtmgp_interface; +using real1dk = typename interface_t::view_t; +using real2dk = typename interface_t::view_t; +using real3dk = typename interface_t::view_t; +using MDRP = typename conv::MDRP; static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 6e924103229f..8c236e64e811 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -57,9 +57,10 @@ void compute_heating_rate ( View4 const &heating_rate) { using physconst = scream::physics::Constants; + using MDRP = typename conv::MDRP; auto ncol = flux_up.extent(0); auto nlay = flux_up.extent(1)-1; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { heating_rate(icol,ilay) = ( flux_up(icol,ilay+1) - flux_up(icol,ilay) - flux_dn(icol,ilay+1) + flux_dn(icol,ilay) @@ -98,18 +99,20 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } }); auto num_bad = sum(bad_mask); - pass = false; - out << msg << ": " - << num_bad << " values outside range " - << "[" << xmin << "," << xmax << "]" - << "; minval = " << _xmin - << "; maxval = " << _xmax << "\n"; + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } } return pass; } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template ::type* = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; @@ -117,10 +120,10 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - Kokkos::View bad_mask("bad_mask", x.size()); - Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { - if (x.data()[i] < xmin or x.data()[i] > xmax) { - bad_mask.data()[i] = true; + Kokkos::View bad_mask("bad_mask", x.extent(0)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + if (x(i) < xmin or x(i) > xmax) { + bad_mask(i) = true; } }); auto num_bad = conv::sum(bad_mask); @@ -133,6 +136,68 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val } return pass; } + +template ::type* = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + if (x(i, j) < xmin or x(i, j) > xmax) { + bad_mask(i, j) = true; + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + +template ::type* = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1), x.extent(2)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + for (size_t k = 0; k < x.extent(2); ++k) { + if (x(i, j, k) < xmin or x(i, j, k) > xmax) { + bad_mask(i, j, k) = true; + } + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + + #endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 2038cb62ff79..c82d9e38a3b6 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -159,7 +159,7 @@ int get_wavelength_index_lw(double wavelength); // New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_interface { using MDRP = typename conv::MDRP; @@ -980,7 +980,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i // Kokkos::deep_copy(seeds_host, seeds); // for (int icol = 0; icol < ncol; ++icol) { // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // Kokkos::parallel_for(MDRP::template get<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { // auto generator = random_pool.get_state(); // cldx(icol,ilay,igpt) = generator.drand(0., 1.); // random_pool.free_state(generator); @@ -996,7 +996,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i }); // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { for (int ilay = 1; ilay < nlay; ilay++) { // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { @@ -1016,7 +1016,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i } // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { subcolumn_mask(icol,ilay,igpt) = 1; } else { @@ -1202,13 +1202,21 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template +template::type* = nullptr> static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + Kokkos::parallel_for(arr_out.size(), KOKKOS_LAMBDA(int i) { + arr_out(i) = std::min(std::max(arr_in(i), lower), upper); + }); +} + +template::type* = nullptr> +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { + Kokkos::parallel_for(MDRP::template get<2>({arr_out.extent(0), arr_out.extent(1)}), KOKKOS_LAMBDA(int i, int j) { + arr_out(i, j) = std::min(std::max(arr_in(i, j), lower), upper); }); } + static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) { // Get wavelength bounds for all wavelength bands @@ -1382,7 +1390,7 @@ static OpticalProps1sclK get_subsampled_clouds( auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { auto ibnd = gpoint_bands(igpt); if (cldmask(icol,ilay,igpt) == 1) { subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 65d1da9f1d68..0d3f18e7d841 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -320,6 +320,12 @@ int run_yakl(int argc, char** argv) { int run_kokkos(int argc, char** argv) { using namespace ekat::logger; using logger_t = Logger; + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using MDRP = utils_t::MDRP; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; ekat::Comm comm(MPI_COMM_WORLD); auto logger = std::make_shared("",LogLevel::info,comm); @@ -374,7 +380,7 @@ int run_kokkos(int argc, char** argv) { real2dk sw_flux_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Get dimension sizes int ncol = sw_flux_up_ref.extent(0); @@ -394,12 +400,12 @@ int run_kokkos(int argc, char** argv) { real2dk p_lev("p_lev", ncol, nlay+1); real2dk t_lev("t_lev", ncol, nlay+1); real2dk col_dry; - GasConcsK gas_concs; + GasConcsK gas_concs; read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry, ncol); // Initialize absorption coefficients logger->info("Initialize RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Setup our dummy atmosphere based on the input data we read in logger->info("Setup dummy atmos...\n"); @@ -413,7 +419,7 @@ int run_kokkos(int argc, char** argv) { real2dk rel("rel", ncol, nlay); real2dk rei("rei", ncol, nlay); real2dk cld("cld", ncol, nlay); - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol, p_lay, t_lay, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -426,8 +432,8 @@ int run_kokkos(int argc, char** argv) { // we would just have to setup the pointers to them in the // FluxesBroadband object logger->info("Setup fluxes...\n"); - const auto nswbands = scream::rrtmgp::k_dist_sw_k.get_nband(); - const auto nlwbands = scream::rrtmgp::k_dist_lw_k.get_nband(); + const auto nswbands = interface_t::k_dist_sw_k.get_nband(); + const auto nlwbands = interface_t::k_dist_lw_k.get_nband(); real2dk sw_flux_up ("sw_flux_up" , ncol, nlay+1); real2dk sw_flux_dn ("sw_flux_dn" , ncol, nlay+1); real2dk sw_flux_dir("sw_flux_dir", ncol, nlay+1); @@ -457,7 +463,7 @@ int run_kokkos(int argc, char** argv) { // Compute band-by-band surface_albedos. real2dk sfc_alb_dir("sfc_alb_dir", ncol, nswbands); real2dk sfc_alb_dif("sfc_alb_dif", ncol, nswbands); - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -468,19 +474,19 @@ int run_kokkos(int argc, char** argv) { auto aer_ssa_sw = real3dk("aer_ssa_sw", ncol, nlay, nswbands); auto aer_asm_sw = real3dk("aer_asm_sw", ncol, nlay, nswbands); auto aer_tau_lw = real3dk("aer_tau_lw", ncol, nlay, nlwbands); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_sw(icol,ilay,ibnd) = 0; aer_ssa_sw(icol,ilay,ibnd) = 0; aer_asm_sw(icol,ilay,ibnd) = 0; }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_lw(icol,ilay,ibnd) = 0; }); // These are returned as outputs now from rrtmgp_main // TODO: provide as inputs consistent with how aerosol is treated? - const auto nswgpts = scream::rrtmgp::k_dist_sw_k.get_ngpt(); - const auto nlwgpts = scream::rrtmgp::k_dist_lw_k.get_ngpt(); + const auto nswgpts = interface_t::k_dist_sw_k.get_ngpt(); + const auto nlwgpts = interface_t::k_dist_lw_k.get_ngpt(); auto cld_tau_sw_bnd = real3dk("cld_tau_sw_bnd", ncol, nlay, nswbands); auto cld_tau_lw_bnd = real3dk("cld_tau_lw_bnd", ncol, nlay, nlwbands); auto cld_tau_sw = real3dk("cld_tau_sw", ncol, nlay, nswgpts); @@ -489,7 +495,7 @@ int run_kokkos(int argc, char** argv) { // Run RRTMGP code on dummy atmosphere logger->info("Run RRTMGP...\n"); const Real tsi_scaling = 1; - scream::rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, nlay, p_lay, t_lay, p_lev, t_lev, gas_concs, sfc_alb_dir, sfc_alb_dif, mu0, @@ -513,33 +519,33 @@ int run_kokkos(int argc, char** argv) { // Check values against baseline logger->info("Check values...\n"); - rrtmgpTest::read_fluxes( + utils_t::read_fluxes( baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; // Because the aerosol optical properties are all set to zero, these fluxes must be equal - if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; logger->info("Cleaning up...\n"); // Clean up or else YAKL will throw errors - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); scream::finalize_kls(); return nerr != 0 ? 1 : 0; diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 36e2a59e02ff..99a582445616 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -856,6 +856,7 @@ using real3dk = interface_t::view_t; using int1dk = interface_t::view_t; using int2dk = interface_t::view_t; using int3dk = interface_t::view_t; +using MDRP = interface_t::MDRP; TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos @@ -1082,7 +1083,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Need to initialize RRTMGP with dummy gases logger->info("Init gases...\n"); - GasConcsK gas_concs; + GasConcsK gas_concs; string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); @@ -1100,7 +1101,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { auto sw_bnd_flux_dir = real3dk("sw_bnd_flux_dir", ncol, nlay+1, nbnd); auto sw_bnd_flux_dif = real3dk("sw_bnd_flux_dif", ncol, nlay+1, nbnd); logger->info("Populate band-resolved 3d fluxes for test case with only transition band flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1132,7 +1133,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in NIR bands logger->info("Populate band-resolved 3d fluxes for test case with only NIR flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1; sw_bnd_flux_dif(icol,ilay,ibnd) = 1; @@ -1163,7 +1164,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in VIS bands logger->info("Populate band-resolved 3d fluxes for test case with only VIS/UV flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1194,7 +1195,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in all bands logger->info("Populate band-resolved 3d fluxes for test with non-zero flux in all bands...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1.0; sw_bnd_flux_dif(icol,ilay,ibnd) = 2.0; @@ -1293,13 +1294,13 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { for (int igpt = 0; igpt < ngpt; ++igpt) { real cldmask_real = cldmask(icol,ilay,igpt); cldfrac_from_mask(icol,ilay) += cldmask_real; } }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { cldfrac_from_mask(icol,ilay) = cldfrac_from_mask(icol,ilay) / ngpt; }); // For cldfrac 1 we should get 1, for cldfrac 0 we should get 0, but in between we cannot be sure diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 9e6e94a2164f..75feeb1b8dd4 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -422,6 +422,13 @@ TEST_CASE("rrtmgp_scream_standalone", "") { #endif #ifdef RRTMGP_ENABLE_KOKKOS TEST_CASE("rrtmgp_scream_standalone_k", "") { + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using MDRP = interface_t::MDRP; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; + // Get baseline name (needs to be passed as an arg) std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); @@ -439,7 +446,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { real2dk sw_flux_dn_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Load ad parameter list std::string fname = "input_unit.yaml"; @@ -526,11 +533,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { // this will copy the first column of the input data (the first profile) ncol // times. We will then fill some fraction of these columns with clouds for // the test problem. - GasConcsK gas_concs; + GasConcsK gas_concs; real2dk col_dry; read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, col_dry, ncol_all); // Setup dummy problem; need to use tmp arrays with ncol_all size - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol_all, p_lay_all, t_lay_all, sfc_alb_dir_vis_all, sfc_alb_dir_nir_all, sfc_alb_dif_vis_all, sfc_alb_dif_nir_all, @@ -547,7 +554,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { sfc_alb_dif_nir(icol) = sfc_alb_dif_nir_all(icol_all); mu0(icol) = mu0_all(icol_all); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lay(icol,ilay) = p_lay_all(icol_all,ilay); t_lay(icol,ilay) = t_lay_all(icol_all,ilay); @@ -557,14 +564,14 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { rei(icol,ilay) = rei_all(icol_all,ilay); cld(icol,ilay) = cld_all(icol_all,ilay); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lev(icol,ilay) = p_lev_all(icol_all,ilay); t_lev(icol,ilay) = t_lev_all(icol_all,ilay); }); // Need to calculate a dummy pseudo_density for our test problem - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { p_del(icol,ilay) = abs(p_lev(icol,ilay+1) - p_lev(icol,ilay)); }); @@ -573,13 +580,13 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto qc = real2dk("qc", ncol, nlay); auto nc = real2dk("nc", ncol, nlay); auto qi = real2dk("qi", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); qi(icol,ilay) = 1e-3 * iwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); }); // Copy gases from gas_concs to gas_vmr array - Kokkos::parallel_for(conv::get_mdrp<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { + Kokkos::parallel_for(MDRP::template get<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { auto icol_all = ncol * irank + icol; gas_vmr(icol,ilay,igas) = gas_concs.concs(icol_all,ilay,igas); }); @@ -691,7 +698,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { const int i = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - if (k < nlay) t_lay(i+1,k+1) = d_tmid(i,k); + if (k < nlay) t_lay(i,k) = d_tmid(i,k); sw_flux_up_test(i,k) = d_sw_flux_up(i,k); sw_flux_dn_test(i,k) = d_sw_flux_dn(i,k); @@ -722,7 +729,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto sw_flux_dn_dir_loc = real2dk("sw_flux_dn_dir_loc", ncol, nlay+1); auto lw_flux_up_loc = real2dk("lw_flux_up_loc" , ncol, nlay+1); auto lw_flux_dn_loc = real2dk("lw_flux_dn_loc" , ncol, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; sw_flux_up_loc(icol,ilay) = sw_flux_up_ref(icol_all,ilay); sw_flux_dn_loc(icol,ilay) = sw_flux_dn_ref(icol_all,ilay); @@ -730,11 +737,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { lw_flux_up_loc(icol,ilay) = lw_flux_up_ref(icol_all,ilay); lw_flux_dn_loc(icol,ilay) = lw_flux_dn_ref(icol_all,ilay); }); - REQUIRE(rrtmgpTest::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); + REQUIRE(utils_t::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); // Finalize the driver. YAKL will be finalized inside // RRTMGPRadiation::finalize_impl after RRTMGP has had the From b7e006d8070926c19c218b940398adf225052578 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 14:50:19 -0600 Subject: [PATCH 398/904] cleanup --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 89 ++++++------------- 1 file changed, 27 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1556280eab4b..cde298cc8cd9 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -19,28 +19,6 @@ namespace scream { -template -void check_views2(const View1& view1, const View2& view2) -{ - constexpr auto krank1 = View1::rank; - constexpr auto krank2 = View2::rank; - - RRT_REQUIRE(krank1 == krank2, "Rank mismatch for: " << view1.label()); - RRT_REQUIRE(krank1 == 2, "Rank mismatch for: " << view1.label()); - - for (auto r = 0; r < krank1; ++r) { - RRT_REQUIRE(view1.extent(r) == view2.extent(r), "Dim mismatch for: " << view1.label() << ", rank: " << r << ", " << view1.extent(r) << " != " << view2.extent(r)); - } - - for (auto i = 0; i < view1.extent(0); ++i) { - for (auto j = 0; j < view1.extent(1); ++j) { - const auto data1 = view1(i, j); - const auto data2 = view2(i, j); - RRT_REQUIRE(data1 == data2, "Data mismatch for: " << view1.label() << ", i: " << i << ", j: " << j << ", " << data1 << " != " << data2); - } - } -} - using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; @@ -967,8 +945,8 @@ void RRTMGPRadiation::run_impl (const double dt) { return subv; #endif }; - auto subview_2dk = [&](const ureal2dk& v) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); #ifdef RRTMGP_ENABLE_YAKL real2dk rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); @@ -985,16 +963,6 @@ void RRTMGPRadiation::run_impl (const double dt) { return rv; #else return subv; -#endif - }; - auto subview_2dk_new = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); -#ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; #endif }; auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { @@ -1027,26 +995,26 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; auto rel_k = subview_2dkc(d_rel, m_nlay); auto rei_k = subview_2dkc(d_rei, m_nlay); - auto sw_flux_up_k = subview_2dk_new(d_sw_flux_up, m_nlay+1); - auto sw_flux_dn_k = subview_2dk_new(d_sw_flux_dn, m_nlay+1); - auto sw_flux_dn_dir_k = subview_2dk_new(d_sw_flux_dn_dir, m_nlay+1); - auto lw_flux_up_k = subview_2dk_new(d_lw_flux_up, m_nlay+1); - auto lw_flux_dn_k = subview_2dk_new(d_lw_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_up_k = subview_2dk_new(d_sw_clnclrsky_flux_up, m_nlay+1); - auto sw_clnclrsky_flux_dn_k = subview_2dk_new(d_sw_clnclrsky_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); - auto sw_clrsky_flux_up_k = subview_2dk_new(d_sw_clrsky_flux_up, m_nlay+1); - auto sw_clrsky_flux_dn_k = subview_2dk_new(d_sw_clrsky_flux_dn, m_nlay+1); - auto sw_clrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clrsky_flux_dn_dir, m_nlay+1); - auto sw_clnsky_flux_up_k = subview_2dk_new(d_sw_clnsky_flux_up, m_nlay+1); - auto sw_clnsky_flux_dn_k = subview_2dk_new(d_sw_clnsky_flux_dn, m_nlay+1); - auto sw_clnsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnsky_flux_dn_dir, m_nlay+1); - auto lw_clnclrsky_flux_up_k = subview_2dk_new(d_lw_clnclrsky_flux_up, m_nlay+1); - auto lw_clnclrsky_flux_dn_k = subview_2dk_new(d_lw_clnclrsky_flux_dn, m_nlay+1); - auto lw_clrsky_flux_up_k = subview_2dk_new(d_lw_clrsky_flux_up, m_nlay+1); - auto lw_clrsky_flux_dn_k = subview_2dk_new(d_lw_clrsky_flux_dn, m_nlay+1); - auto lw_clnsky_flux_up_k = subview_2dk_new(d_lw_clnsky_flux_up, m_nlay+1); - auto lw_clnsky_flux_dn_k = subview_2dk_new(d_lw_clnsky_flux_dn, m_nlay+1); + auto sw_flux_up_k = subview_2dk(d_sw_flux_up, m_nlay+1); + auto sw_flux_dn_k = subview_2dk(d_sw_flux_dn, m_nlay+1); + auto sw_flux_dn_dir_k = subview_2dk(d_sw_flux_dn_dir, m_nlay+1); + auto lw_flux_up_k = subview_2dk(d_lw_flux_up, m_nlay+1); + auto lw_flux_dn_k = subview_2dk(d_lw_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_up_k = subview_2dk(d_sw_clnclrsky_flux_up, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = subview_2dk(d_sw_clnclrsky_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); + auto sw_clrsky_flux_up_k = subview_2dk(d_sw_clrsky_flux_up, m_nlay+1); + auto sw_clrsky_flux_dn_k = subview_2dk(d_sw_clrsky_flux_dn, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = subview_2dk(d_sw_clrsky_flux_dn_dir, m_nlay+1); + auto sw_clnsky_flux_up_k = subview_2dk(d_sw_clnsky_flux_up, m_nlay+1); + auto sw_clnsky_flux_dn_k = subview_2dk(d_sw_clnsky_flux_dn, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = subview_2dk(d_sw_clnsky_flux_dn_dir, m_nlay+1); + auto lw_clnclrsky_flux_up_k = subview_2dk(d_lw_clnclrsky_flux_up, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = subview_2dk(d_lw_clnclrsky_flux_dn, m_nlay+1); + auto lw_clrsky_flux_up_k = subview_2dk(d_lw_clrsky_flux_up, m_nlay+1); + auto lw_clrsky_flux_dn_k = subview_2dk(d_lw_clrsky_flux_dn, m_nlay+1); + auto lw_clnsky_flux_up_k = subview_2dk(d_lw_clnsky_flux_up, m_nlay+1); + auto lw_clnsky_flux_dn_k = subview_2dk(d_lw_clnsky_flux_dn, m_nlay+1); auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; @@ -1224,9 +1192,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // correct size that uses m_buffer.tmp2d's pointer #ifdef RRTMGP_ENABLE_YAKL real2d tmp2d = subview_2d(m_buffer.tmp2d); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1235,22 +1200,22 @@ void RRTMGPRadiation::run_impl (const double dt) { // 'o3' is marked as 'Required' rather than 'Computed', so we need to get the proper field auto f = name=="o3" ? get_field_in(full_name) : get_field_out(full_name); auto d_vmr = f.get_view(); +#ifdef RRTMGP_ENABLE_KOKKOS + auto tmp2d_k = subview_2dkc(d_vmr, m_nlay); +#endif +#ifdef RRTMGP_ENABLE_YAKL // Copy to YAKL const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { -#ifdef RRTMGP_ENABLE_YAKL tmp2d(i+1,k+1) = d_vmr(icol,k); // Note that for YAKL arrays i and k start with index 1 -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - tmp2d_k(i,k) = d_vmr(icol,k); -#endif }); }); Kokkos::fence(); +#endif // Populate GasConcs object #ifdef RRTMGP_ENABLE_YAKL From f1e2d98e68a43e98cf212e1ae5c4d70254963a75 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 15:48:32 -0600 Subject: [PATCH 399/904] subview_2dk bug fix --- .../eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index cde298cc8cd9..9fab85444fcc 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -946,7 +946,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); + ureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); #ifdef RRTMGP_ENABLE_YAKL real2dk rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); From 23f89ce9d5a34e15d01857492a369e550af77185 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 17:44:50 -0600 Subject: [PATCH 400/904] More fixes --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 9fab85444fcc..7a5af5843d19 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -926,7 +926,7 @@ void RRTMGPRadiation::run_impl (const double dt) { // check_range_k will fail due to looking at unused values. Once rrtmgp can handle // packs, this won't be necessary. auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { - ureal1dk subv(v, std::make_pair(0, ncol)); + ureal1dk subv(v, std::make_pair(beg, beg+ncol)); #ifdef RRTMGP_ENABLE_YAKL real1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); @@ -936,7 +936,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { - cureal1dk subv(v, std::make_pair(0, ncol)); + cureal1dk subv(v, std::make_pair(beg, beg+ncol)); #ifdef RRTMGP_ENABLE_YAKL creal1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); @@ -966,7 +966,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { - ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); // The range assumes these are buffer views, not fields #ifdef RRTMGP_ENABLE_YAKL real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); From 8cae9b12fc801bb9649c6182ac85fd4194ccb546 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 13:04:44 -0600 Subject: [PATCH 401/904] Set to correct rrtmgp branch --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 4c542032b3ac..02c2e03df33e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 4c542032b3ac3f842a9619c9eadc005c507a0366 +Subproject commit 02c2e03df33edca6101670168ca187cb94c0768e From 0eddf307ad51587d4bc1a98bed8077e614e0c174 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 14:01:18 -0600 Subject: [PATCH 402/904] Add buffer views should be unmanaged --- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index c60a1cb3181a..5dece435fb0a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -145,15 +145,15 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + ureal1dk mu0_k; + ureal1dk sfc_alb_dir_vis_k; + ureal1dk sfc_alb_dir_nir_k; + ureal1dk sfc_alb_dif_vis_k; + ureal1dk sfc_alb_dif_nir_k; + ureal1dk sfc_flux_dir_vis_k; + ureal1dk sfc_flux_dir_nir_k; + ureal1dk sfc_flux_dif_vis_k; + ureal1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) @@ -176,21 +176,21 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + ureal2dk p_lay_k; + ureal2dk t_lay_k; + ureal2dk z_del_k; + ureal2dk p_del_k; + ureal2dk qc_k; + ureal2dk nc_k; + ureal2dk qi_k; + ureal2dk cldfrac_tot_k; + ureal2dk eff_radius_qc_k; + ureal2dk eff_radius_qi_k; + ureal2dk tmp2d_k; + ureal2dk lwp_k; + ureal2dk iwp_k; + ureal2dk sw_heating_k; + ureal2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) @@ -220,28 +220,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + ureal2dk p_lev_k; + ureal2dk t_lev_k; + ureal2dk sw_flux_up_k; + ureal2dk sw_flux_dn_k; + ureal2dk sw_flux_dn_dir_k; + ureal2dk lw_flux_up_k; + ureal2dk lw_flux_dn_k; + ureal2dk sw_clnclrsky_flux_up_k; + ureal2dk sw_clnclrsky_flux_dn_k; + ureal2dk sw_clnclrsky_flux_dn_dir_k; + ureal2dk sw_clrsky_flux_up_k; + ureal2dk sw_clrsky_flux_dn_k; + ureal2dk sw_clrsky_flux_dn_dir_k; + ureal2dk sw_clnsky_flux_up_k; + ureal2dk sw_clnsky_flux_dn_k; + ureal2dk sw_clnsky_flux_dn_dir_k; + ureal2dk lw_clnclrsky_flux_up_k; + ureal2dk lw_clnclrsky_flux_dn_k; + ureal2dk lw_clrsky_flux_up_k; + ureal2dk lw_clrsky_flux_dn_k; + ureal2dk lw_clnsky_flux_up_k; + ureal2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -252,10 +252,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + ureal3dk sw_bnd_flux_up_k; + ureal3dk sw_bnd_flux_dn_k; + ureal3dk sw_bnd_flux_dir_k; + ureal3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -264,8 +264,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + ureal3dk lw_bnd_flux_up_k; + ureal3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -274,8 +274,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + ureal2dk sfc_alb_dir_k; + ureal2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -286,10 +286,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + ureal3dk aero_tau_sw_k; + ureal3dk aero_ssa_sw_k; + ureal3dk aero_g_sw_k; + ureal3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -298,8 +298,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + ureal3dk cld_tau_sw_bnd_k; + ureal3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -308,8 +308,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + ureal3dk cld_tau_sw_gpt_k; + ureal3dk cld_tau_lw_gpt_k; #endif }; From 5efd49e03c3f097723de4db33f72483dd14769ec Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 15:55:26 -0600 Subject: [PATCH 403/904] Build fixes for yakl mode --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 4 ++-- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 7a5af5843d19..62de7f8416cd 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -1293,8 +1293,8 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - interface_t::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - interface_t::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 5dece435fb0a..e036e7ac08c0 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -34,7 +34,9 @@ class RRTMGPRadiation : public AtmosphereProcess { using layout_t = typename ekat::KokkosTypes::Layout; +#ifdef RRTMGP_ENABLE_KOKKOS using interface_t = rrtmgp::rrtmgp_interface; +#endif // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); From dd83e12a9e977da10104e0b4338cffb3e8241885 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 26 Jul 2024 13:04:18 -0600 Subject: [PATCH 404/904] Remove overuse of templates --- .../rrtmgp/scream_rrtmgp_interface.hpp | 234 ++++++++---------- 1 file changed, 109 insertions(+), 125 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index c82d9e38a3b6..0bceb1282e5f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -175,21 +175,40 @@ using hview_t = Kokkos::View; using pool_t = conv::MemPoolSingleton; +using real1dk = view_t; +using real2dk = view_t; +using real3dk = view_t; +using creal1dk = view_t; +using creal2dk = view_t; +using creal3dk = view_t; +using int1dk = view_t; +using int3dk = view_t; + +using gas_optics_t = GasOpticsRRTMGPK; +using cloud_optics_t = CloudOpticsK; +using gas_concs_t = GasConcsK; +using fluxes_t = FluxesBybandK; +using fluxes_broadband_t = FluxesBroadbandK; +using optical_props_t = OpticalPropsK; +using optical_props1_t = OpticalProps1sclK; +using optical_props2_t = OpticalProps2strK; +using source_func_t = SourceFuncLWK; + /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -static inline GasOpticsRRTMGPK k_dist_sw_k; -static inline GasOpticsRRTMGPK k_dist_lw_k; +static inline gas_optics_t k_dist_sw_k; +static inline gas_optics_t k_dist_lw_k; /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -static inline CloudOpticsK cloud_optics_sw_k; -static inline CloudOpticsK cloud_optics_lw_k; +static inline cloud_optics_t cloud_optics_sw_k; +static inline cloud_optics_t cloud_optics_lw_k; /* * Flag to indicate whether or not we have initialized RRTMGP @@ -200,7 +219,7 @@ static inline bool initialized_k = false; * Initialize data for RRTMGP driver */ static void rrtmgp_initialize( - GasConcsK &gas_concs, + const gas_concs_t &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger) @@ -238,13 +257,11 @@ static void rrtmgp_initialize( /* * Compute band-by-band surface albedos from broadband albedos. */ -template static void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, - SfcAlbDirVisT &sfc_alb_dir_vis, SfcAlbDirNirT &sfc_alb_dir_nir, - SfcAlbDifVisT &sfc_alb_dif_vis, SfcAlbDifNirT &sfc_alb_dif_nir, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif) + const creal1dk &sfc_alb_dir_vis, const creal1dk &sfc_alb_dir_nir, + const creal1dk &sfc_alb_dif_vis, const creal1dk &sfc_alb_dif_nir, + const real2dk &sfc_alb_dir, const real2dk &sfc_alb_dif) { EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); @@ -289,13 +306,11 @@ static void compute_band_by_band_surface_albedos( /* * Compute broadband visible/UV and near-infrared surface fluxes. */ -template static void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, - SwBndFluxDirT &sw_bnd_flux_dir , SwBndFluxDifT &sw_bnd_flux_dif , - SfcFluxDirVisT &sfc_flux_dir_vis, SfcFluxDirNirT &sfc_flux_dir_nir, - SfcFluxDifVisT &sfc_flux_dif_vis, SfcFluxDifNirT &sfc_flux_dif_nir) + const real3dk &sw_bnd_flux_dir , const real3dk &sw_bnd_flux_dif , + const real1dk &sfc_flux_dir_vis, const real1dk &sfc_flux_dir_nir, + const real1dk &sfc_flux_dif_vis, const real1dk &sfc_flux_dif_nir) { // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when @@ -349,42 +364,26 @@ static void compute_broadband_surface_fluxes( * The input logger is in charge of outputing info to * screen and/or to file (or neither), depending on how it was set up. */ -template static void rrtmgp_main( const int ncol, const int nlay, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei, CldfracT &cldfrac, - AerTauSwT &aer_tau_sw, AerSsaSwT &aer_ssa_sw, AerAsmSwT &aer_asm_sw, AerTauLwT &aer_tau_lw, - CldTauSwBndT &cld_tau_sw_bnd, CldTauLwBndT &cld_tau_lw_bnd, - CldTauSwGptT &cld_tau_sw_gpt, CldTauLwGptT &cld_tau_lw_gpt, - SwFluxUpT &sw_flux_up, SwFluxDnT &sw_flux_dn, SwFluxDnDirT &sw_flux_dn_dir, - LwFluxUpT &lw_flux_up, LwFluxDnT &lw_flux_dn, - SwClnclrskyFluxUpT &sw_clnclrsky_flux_up, SwClnclrskyFluxDnT &sw_clnclrsky_flux_dn, SwClnclrskyFluxDnDirT &sw_clnclrsky_flux_dn_dir, - SwClrskyFluxUpT &sw_clrsky_flux_up, SwClrskyFluxDnT &sw_clrsky_flux_dn, SwClrskyFluxDnDirT &sw_clrsky_flux_dn_dir, - SwClnskyFluxUpT &sw_clnsky_flux_up, SwClnskyFluxDnT &sw_clnsky_flux_dn, SwClnskyFluxDnDirT &sw_clnsky_flux_dn_dir, - LwClnclrskyFluxUpT &lw_clnclrsky_flux_up, LwClnclrskyFluxDnT &lw_clnclrsky_flux_dn, - LwClrskyFluxUpT &lw_clrsky_flux_up, LwClrskyFluxDnT &lw_clrsky_flux_dn, - LwClnskyFluxUpT &lw_clnsky_flux_up, LwClnskyFluxDnT &lw_clnsky_flux_dn, - SwBndFluxUpT &sw_bnd_flux_up, SwBndFluxDnT &sw_bnd_flux_dn, SwBndFluxDnDirT &sw_bnd_flux_dn_dir, - LwBndFluxUpT &lw_bnd_flux_up, LwBndFluxDnT &lw_bnd_flux_dn, - const RealT tsi_scaling, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei, const real2dk &cldfrac, + const real3dk &aer_tau_sw, const real3dk &aer_ssa_sw, const real3dk &aer_asm_sw, const real3dk &aer_tau_lw, + const real3dk &cld_tau_sw_bnd, const real3dk &cld_tau_lw_bnd, + const real3dk &cld_tau_sw_gpt, const real3dk &cld_tau_lw_gpt, + const real2dk &sw_flux_up, const real2dk &sw_flux_dn, const real2dk &sw_flux_dn_dir, + const real2dk &lw_flux_up, const real2dk &lw_flux_dn, + const real2dk &sw_clnclrsky_flux_up, const real2dk &sw_clnclrsky_flux_dn, const real2dk &sw_clnclrsky_flux_dn_dir, + const real2dk &sw_clrsky_flux_up, const real2dk &sw_clrsky_flux_dn, const real2dk &sw_clrsky_flux_dn_dir, + const real2dk &sw_clnsky_flux_up, const real2dk &sw_clnsky_flux_dn, const real2dk &sw_clnsky_flux_dn_dir, + const real2dk &lw_clnclrsky_flux_up, const real2dk &lw_clnclrsky_flux_dn, + const real2dk &lw_clrsky_flux_up, const real2dk &lw_clrsky_flux_dn, + const real2dk &lw_clnsky_flux_up, const real2dk &lw_clnsky_flux_dn, + const real3dk &sw_bnd_flux_up, const real3dk &sw_bnd_flux_dn, const real3dk &sw_bnd_flux_dn_dir, + const real3dk &lw_bnd_flux_up, const real3dk &lw_bnd_flux_dn, + const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) { @@ -404,7 +403,7 @@ static void rrtmgp_main( #endif // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; + fluxes_t fluxes_sw; fluxes_sw.flux_up = sw_flux_up; fluxes_sw.flux_dn = sw_flux_dn; fluxes_sw.flux_dn_dir = sw_flux_dn_dir; @@ -412,37 +411,37 @@ static void rrtmgp_main( fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; + fluxes_broadband_t clnclrsky_fluxes_sw; clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; + fluxes_broadband_t clrsky_fluxes_sw; clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; + fluxes_broadband_t clnsky_fluxes_sw; clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; + fluxes_t fluxes_lw; fluxes_lw.flux_up = lw_flux_up; fluxes_lw.flux_dn = lw_flux_dn; fluxes_lw.bnd_flux_up = lw_bnd_flux_up; fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; + fluxes_broadband_t clnclrsky_fluxes_lw; clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; + fluxes_broadband_t clrsky_fluxes_lw; clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; + fluxes_broadband_t clnsky_fluxes_lw; clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; @@ -450,8 +449,8 @@ static void rrtmgp_main( auto nlwbands = k_dist_lw_k.get_nband(); // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; + optical_props2_t aerosol_sw; + optical_props1_t aerosol_lw; aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); aerosol_sw.alloc_2str(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { @@ -476,8 +475,8 @@ static void rrtmgp_main( #endif // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + optical_props2_t clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + optical_props1_t clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); @@ -549,17 +548,15 @@ static void rrtmgp_finalize() /* * Shortwave driver (called by rrtmgp_main) */ -template static void rrtmgp_sw( const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const RealT tsi_scaling, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + optical_props2_t &aerosol, optical_props2_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, + const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { @@ -647,7 +644,7 @@ static void rrtmgp_sw( // Subset gases auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; + gas_concs_t gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { auto vmr_day = view_t("vmr_day", nday, nlay); @@ -660,7 +657,7 @@ static void rrtmgp_sw( } // Subset aerosol optics - OpticalProps2strK aerosol_day; + optical_props2_t aerosol_day; aerosol_day.init(k_dist.get_band_lims_wavenumber()); aerosol_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { @@ -671,7 +668,7 @@ static void rrtmgp_sw( // Subset cloud optics // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; + optical_props2_t clouds_day; clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); clouds_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { @@ -697,7 +694,7 @@ static void rrtmgp_sw( auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; + fluxes_t fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; fluxes_day.flux_dn_dir = flux_dn_dir_day; @@ -706,10 +703,10 @@ static void rrtmgp_sw( fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; // Allocate space for optical properties - OpticalProps2strK optics; + optical_props2_t optics; optics.alloc_2str(nday, nlay, k_dist); - OpticalProps2strK optics_no_aerosols; + optical_props2_t optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_2str(nday, nlay, k_dist); @@ -811,14 +808,13 @@ static void rrtmgp_sw( /* * Longwave driver (called by rrtmgp_main) */ -template static void rrtmgp_lw( const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + optical_props1_t &aerosol, optical_props1_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Problem size @@ -856,16 +852,16 @@ static void rrtmgp_lw( }); // Allocate space for optical properties - OpticalProps1sclK optics; + optical_props1_t optics; optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; + optical_props1_t optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); } // Boundary conditions - SourceFuncLWK lw_sources; + source_func_t lw_sources; lw_sources.alloc(ncol, nlay, k_dist); view_t t_sfc ("t_sfc" ,ncol); view_t emis_sfc("emis_sfc",nbnd,ncol); @@ -950,12 +946,10 @@ static void rrtmgp_lw( /* * Return a subcolumn mask consistent with a specified overlap assumption */ -template -static view_t get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, - CldfT &cldf, const int overlap_option, SeedsT &seeds) +static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds) { // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = view_t("subcolumn_mask", ncol, nlay, ngpt); + int3dk subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); // Subcolumn generators are a means for producing a variable x(i,j,k), where // @@ -1029,10 +1023,9 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i /* * Compute cloud area from 3d subcol cloud property */ -template static void compute_cloud_area( - int ncol, int nlay, int ngpt, RealT pmin, RealT pmax, - const PmidT& pmid, const CldTauGptT& cld_tau_gpt, CldAreaT& cld_area) + int ncol, int nlay, int ngpt, Real pmin, Real pmax, + const creal2dk& pmid, const real3dk& cld_tau_gpt, const real1dk& cld_area) { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 @@ -1058,20 +1051,15 @@ static void compute_cloud_area( /* * Return select cloud-top diagnostics following AeroCom recommendation */ -template static void compute_aerocom_cloudtop( - int ncol, int nlay, const TmidT &tmid, const PmidT &pmid, - const PdelT &p_del, const ZdelT &z_del, const QcT &qc, - const QiT &qi, const RelT &rel, const ReiT &rei, - const CldfracTotT &cldfrac_tot, const NcT &nc, - TmidAtCldtopT &T_mid_at_cldtop, PmidAtCldtopT &p_mid_at_cldtop, - CldfracIceAtCldtopT &cldfrac_ice_at_cldtop, CldfracLiqAtCldtopT &cldfrac_liq_at_cldtop, - CldfracTotAtCldtopT &cldfrac_tot_at_cldtop, CdncAtCldtopT &cdnc_at_cldtop, - EffRadiusQcAtCldtopT &eff_radius_qc_at_cldtop, EffRadiusQiAtCldTopT &eff_radius_qi_at_cldtop) + int ncol, int nlay, const creal2dk &tmid, const creal2dk &pmid, + const creal2dk &p_del, const real2dk &z_del, const creal2dk &qc, + const creal2dk &qi, const creal2dk &rel, const creal2dk &rei, + const real2dk &cldfrac_tot, const creal2dk &nc, + const real1dk &T_mid_at_cldtop, const real1dk &p_mid_at_cldtop, + const real1dk &cldfrac_ice_at_cldtop, const real1dk &cldfrac_liq_at_cldtop, + const real1dk &cldfrac_tot_at_cldtop, const real1dk &cdnc_at_cldtop, + const real1dk &eff_radius_qc_at_cldtop, const real1dk &eff_radius_qi_at_cldtop) { /* The goal of this routine is to calculate properties at cloud top * based on the AeroCom recommendation. See reference for routine @@ -1217,7 +1205,7 @@ static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, O } -static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) +static int get_wavelength_index(optical_props_t &kdist, RealT wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -1251,14 +1239,13 @@ static inline int get_wavelength_index_lw_k(RealT wavelength) return get_wavelength_index(k_dist_lw_k, wavelength); } -template -static OpticalProps2strK get_cloud_optics_sw( +static optical_props2_t get_cloud_optics_sw( const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { // Initialize optics - OpticalProps2strK clouds; + optical_props2_t clouds; clouds.init(kdist.get_band_lims_wavenumber()); clouds.alloc_2str(ncol, nlay); @@ -1278,14 +1265,13 @@ static OpticalProps2strK get_cloud_optics_sw( return clouds; } -template -static OpticalProps1sclK get_cloud_optics_lw( +static optical_props1_t get_cloud_optics_lw( const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { // Initialize optics - OpticalProps1sclK clouds; + optical_props1_t clouds; clouds.init(kdist.get_band_lims_wavenumber()); clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! @@ -1305,12 +1291,11 @@ static OpticalProps1sclK get_cloud_optics_lw( return clouds; } -template -static OpticalProps2strK get_subsampled_clouds( +static optical_props2_t get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { + optical_props2_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { // Initialized subsampled optics - OpticalProps2strK subsampled_optics; + optical_props2_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_2str(ncol, nlay); // Check that we do not have clouds with no optical properties; this would get corrected @@ -1356,13 +1341,12 @@ static OpticalProps2strK get_subsampled_clouds( return subsampled_optics; } -template -static OpticalProps1sclK get_subsampled_clouds( +static optical_props1_t get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, - CldT &cld, PlayT &p_lay) { + optical_props1_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { + // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; + optical_props1_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_1scl(ncol, nlay); // Check that we do not have clouds with no optical properties; this would get corrected From eb6bcca697ccadb0fee647091d9f44520d970b74 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 30 Jul 2024 17:19:29 -0600 Subject: [PATCH 405/904] EAMxx: add routine to reset unlimited dim length This allows future snapshots to effectively overwrite what was already stored --- .../src/share/io/scream_scorpio_interface.cpp | 24 +++++++++++++++++++ .../src/share/io/scream_scorpio_interface.hpp | 1 + 2 files changed, 25 insertions(+) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index cb2b501f2b5c..8d2f64994ddd 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -717,6 +717,30 @@ std::string get_time_name (const std::string& filename) return pf.file->time_dim->name; } +void reset_unlimited_dim_len(const std::string& filename, const int new_length) +{ + auto& f = impl::get_file(filename,"scorpio::reset_unlimited_dim_len"); + + // Reset dim length + EKAT_REQUIRE_MSG (f.time_dim!=nullptr, + "Error! Cannot reset unlimited dim length. No unlimited dim stored.\n" + " - file name: " + filename + "\n"); + EKAT_REQUIRE_MSG (new_lengthlength, + "Error! New time dimension length must be shorter than the current one.\n" + " - file name: " + filename + "\n" + " - curr len : " + std::to_string(f.time_dim->length) + "\n" + " - new len : " + std::to_string(new_length) + "\n"); + f.time_dim->length = new_length; + + // Reset number of records counter for each time dep var + for (auto it : f.vars) { + auto& v = *it.second; + if (v.time_dep) { + v.num_records = new_length; + } + } +} + // =================== Decompositions operations ==================== // // NOTES: diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 549de680fe45..54e3d9027651 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -108,6 +108,7 @@ bool is_dim_unlimited (const std::string& filename, // NOTE: these throw if time dim is not present. Use has_dim to check first. int get_time_len (const std::string& filename); std::string get_time_name (const std::string& filename); +void reset_unlimited_dim_len(const std::string& filename, const int new_length); // =================== Decompositions operations ==================== // From 0bdead86c0f3a3b240d92efd3076a62cdc67dd65 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 30 Jul 2024 17:21:19 -0600 Subject: [PATCH 406/904] EAMxx: two fixes to hist restart logic * Only open last output file if the next write timestamp will fit * When resuming a file, reset time dim length to what was stored in the rhist file (will overwrite any timestamp that was written to file after rhist was written) --- .../src/share/io/scream_output_manager.cpp | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 13e2e97101b1..d6be54d09181 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -242,17 +242,18 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const auto& last_output_filename = get_attribute(rhist_file,"GLOBAL","last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { - scorpio::register_file(last_output_filename,scorpio::Read,m_output_file_specs.iotype); - int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); - scorpio::release_file(last_output_filename); + int num_snaps = scorpio::get_attribute(rhist_file,"GLOBAL","last_output_file_num_snaps"); m_output_file_specs.filename = last_output_filename; m_output_file_specs.is_open = true; m_output_file_specs.storage.num_snapshots_in_file = num_snaps; - // The setup_file call will not register any new variable (the file is in Append mode, - // so all dims/vars must already be in the file). However, it will register decompositions, - // since those are a property of the run, not of the file. - setup_file(m_output_file_specs,m_output_control); + + if (m_output_file_specs.storage.snapshot_fits(m_output_control.next_write_ts)) { + // The setup_file call will not register any new variable (the file is in Append mode, + // so all dims/vars must already be in the file). However, it will register decompositions, + // since those are a property of the run, not of the file. + setup_file(m_output_file_specs,m_output_control); + } } scorpio::release_file(rhist_file); } @@ -494,6 +495,9 @@ void OutputManager::run(const util::TimeStamp& timestamp) write_timestamp (filespecs.filename,"last_write",m_output_control.last_write_ts,true); scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_filename",m_output_file_specs.filename); scorpio::set_attribute (filespecs.filename,"GLOBAL","num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); + + int nsnaps = scorpio::get_dimlen(m_output_file_specs.filename,"time"); + scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_file_num_snaps",nsnaps); } // Write these in both output and rhist file. The former, b/c we need these info when we postprocess // output, and the latter b/c we want to make sure these params don't change across restarts @@ -789,6 +793,24 @@ setup_file ( IOFileSpecs& filespecs, auto mode = m_resume_output_file ? scorpio::Append : scorpio::Write; scorpio::register_file(filename,mode,filespecs.iotype); if (m_resume_output_file) { + // We may have resumed an output file that contains extra snapshots *after* the restart time. + // E.g., if we output every step and the run crashed a few steps after writing the restart. + // In that case, we need to reset the time dimension in the output file, so that the extra + // snapshots will be overwritten. + const auto all_times = scorpio::get_all_times(filename); + int ntimes = all_times.size(); + int ngood = 0; + for (const auto& t : all_times) { + auto keep = t<=m_output_control.last_write_ts.days_from(m_case_t0); + if (keep) { + ++ngood; + } else { + break; + } + } + if (ngood Date: Tue, 30 Jul 2024 17:01:29 -0700 Subject: [PATCH 407/904] fixes to comble test case and update comments for dycomsrf01 --- .../testdefs/testmods_dirs/scream/dpxx/comble/shell_commands | 4 ++-- .../testmods_dirs/scream/dpxx/dycomsrf01/shell_commands | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands index 166e77b1f4bb..5b3b6989aa41 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -5,7 +5,7 @@ ./xmlchange PTS_LAT=74.5 ./xmlchange PTS_LON=10.0 -./xmlchange SSTICE_DATA_FILENAME="${DIN_LOC_ROOT}/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc" +./xmlchange SSTICE_DATA_FILENAME='$DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc' ./xmlchange SSTICE_YEAR_ALIGN=2020 ./xmlchange SSTICE_YEAR_START=2020 ./xmlchange SSTICE_YEAR_END=2021 @@ -16,5 +16,5 @@ ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange $ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/COMBLE_iopfile_4scam.nc -b $ATMCHANGE target_latitude=74.5 -b $ATMCHANGE target_longitude=10.0 -b -$ATMCHANGE iop_coriolis=true +$ATMCHANGE iop_coriolis=true -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands index 8c442acee75b..675448efedc6 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands @@ -1,5 +1,5 @@ # Sets up DPxx case to test DYCOMSrf01 (marine stratocumulus) -# case over ocean using prescribed surface fluxes +# case over ocean using prescribed surface fluxes and compute tendencies of large scale subsidence from prescribed omega ./xmlchange RUN_STARTDATE="1999-07-10" ./xmlchange PTS_LAT=31.5 From b4352409b890093a34ab600915543a02bbd811be Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 30 Jul 2024 20:39:22 -0400 Subject: [PATCH 408/904] small mods to v3 script --- run_e3sm.template.sh | 59 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 623867c01904..1a55dc933d87 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -5,9 +5,6 @@ # Bash coding style inspired by: # http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming -# TO DO: -# - custom pelayout - main() { # For debugging, uncomment line below @@ -46,8 +43,8 @@ readonly GET_REFCASE=TRUE #readonly RUN_REFDATE="" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' # Set paths -readonly CODE_ROOT="${HOME}/E3SMv3/code/${CHECKOUT}" -readonly CASE_ROOT="/pscratch/sd/r/${USER}/e3sm-scratch/${CASE_NAME}" +readonly CODE_ROOT="/pscratch/sd/${USER:0:1}/${USER}/e3sm-scratch/${MACHINE}/your_casename/code/${CHECKOUT}" +readonly CASE_ROOT="/pscratch/sd/${USER:0:1}/${USER}/e3sm-scratch/${MACHINE}/${CASE_NAME}" # Sub-directories readonly CASE_BUILD_DIR=${CASE_ROOT}/build @@ -56,6 +53,7 @@ readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive # Define type of run # short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', # 'M_1x10_ndays', 'M2_1x10_ndays', 'M80_1x10_ndays', 'L_1x10_ndays' +# * can replace XS, M, etc with custom-XY with XY being the node count # or 'production' for full simulation readonly run='XS_2x5_ndays' if [ "${run}" != "production" ]; then @@ -118,6 +116,9 @@ fetch_code # Create case create_newcase +# Custom PE layout +custom_pelayout + # Setup case_setup @@ -322,6 +323,14 @@ create_newcase() { echo $'\n----- Starting create_newcase -----\n' + + if [[ ${PELAYOUT} == custom-* ]]; + then + layout="M" # temporary placeholder for create_newcase + else + layout=${PELAYOUT} + fi + if [[ -z "$CASE_GROUP" ]]; then ${CODE_ROOT}/cime/scripts/create_newcase \ --case ${CASE_NAME} \ @@ -333,7 +342,7 @@ create_newcase() { --machine ${MACHINE} \ --project ${PROJECT} \ --walltime ${WALLTIME} \ - --pecount ${PELAYOUT} + --pecount ${layout} else ${CODE_ROOT}/cime/scripts/create_newcase \ --case ${CASE_NAME} \ @@ -346,7 +355,7 @@ create_newcase() { --machine ${MACHINE} \ --project ${PROJECT} \ --walltime ${WALLTIME} \ - --pecount ${PELAYOUT} + --pecount ${layout} fi @@ -398,6 +407,37 @@ case_setup() { popd } +#----------------------------------------------------- +custom_pelayout() { + +if [[ ${PELAYOUT} == custom-* ]]; +then + echo $'\n CUSTOMIZE PROCESSOR CONFIGURATION:' + + # Number of cores per node (machine specific) + if [ "${MACHINE}" == "pm-cpu" ]; then + ncore=128 + else + echo 'ERROR: MACHINE = '${MACHINE}' is not supported for custom PE layout.' + exit 400 + fi + + # Extract number of nodes + tmp=($(echo ${PELAYOUT} | tr "-" " ")) + nnodes=${tmp[1]} + + # Customize + pushd ${CASE_SCRIPTS_DIR} + ./xmlchange NTASKS=$(( $nnodes * $ncore )) + ./xmlchange NTHRDS=1 + ./xmlchange MAX_MPITASKS_PER_NODE=$ncore + ./xmlchange MAX_TASKS_PER_NODE=$ncore + popd + +fi + +} + #----------------------------------------------------- case_build() { @@ -534,9 +574,10 @@ copy_script() { local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance mkdir -p ${script_provenance_dir} - local this_script_name=`basename $0` + local this_script_name=$( basename -- "$0"; ) + local this_script_dir=$( dirname -- "$0"; ) local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` - cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + cp -vp "${this_script_dir}/${this_script_name}" ${script_provenance_dir}/${script_provenance_name} } From 33b4eaa5c04b7f1a8e866c5ef4b174e1c8016b1d Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 30 Jul 2024 20:44:30 -0400 Subject: [PATCH 409/904] Use $PSCRATCH instead --- run_e3sm.template.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 1a55dc933d87..e2a5b603ba79 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -43,8 +43,8 @@ readonly GET_REFCASE=TRUE #readonly RUN_REFDATE="" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' # Set paths -readonly CODE_ROOT="/pscratch/sd/${USER:0:1}/${USER}/e3sm-scratch/${MACHINE}/your_casename/code/${CHECKOUT}" -readonly CASE_ROOT="/pscratch/sd/${USER:0:1}/${USER}/e3sm-scratch/${MACHINE}/${CASE_NAME}" +readonly CASE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}" +readonly CODE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}/code/${CHECKOUT}" # Sub-directories readonly CASE_BUILD_DIR=${CASE_ROOT}/build From 31f5b2934f871bf348c9c3e2ee2cc223ef102596 Mon Sep 17 00:00:00 2001 From: Peter Andrew Bogenschutz Date: Wed, 31 Jul 2024 08:29:08 -0700 Subject: [PATCH 410/904] fix to COMBLE longitude --- .../testdefs/testmods_dirs/scream/dpxx/comble/shell_commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands index 5b3b6989aa41..0cc2f5575bf3 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -3,7 +3,7 @@ ./xmlchange RUN_STARTDATE="2020-03-12" ./xmlchange PTS_LAT=74.5 -./xmlchange PTS_LON=10.0 +./xmlchange PTS_LON=9.9 ./xmlchange SSTICE_DATA_FILENAME='$DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc' ./xmlchange SSTICE_YEAR_ALIGN=2020 @@ -15,6 +15,6 @@ ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange $ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/COMBLE_iopfile_4scam.nc -b $ATMCHANGE target_latitude=74.5 -b -$ATMCHANGE target_longitude=10.0 -b +$ATMCHANGE target_longitude=9.9 -b $ATMCHANGE iop_coriolis=true -b From 408e876005c21fec791b21fe5a8989611b6a50cc Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 31 Jul 2024 10:34:53 -0600 Subject: [PATCH 411/904] Fix a couple of FIXME comments from code review. Added comments containing descriptions and units for locally allocated arrays. Comments only, no chnage in logic. --- ...x_mam_dry_deposition_process_interface.cpp | 31 ++++++--- ...x_mam_dry_deposition_process_interface.hpp | 65 +++++++++++++++++-- .../src/physics/shoc/impl/shoc_main_impl.hpp | 4 +- 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 4a9dcbf1eca6..034972f85258 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -318,24 +318,39 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Allocate memory //----------------------------------------------------------------- const int pcnst = mam4::aero_model::pcnst; - // FIXME: comment what they are and units..... - qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - rho_ = view_2d("rho", ncol_, nlev_); + // Output of the the mixing ratio tendencies [kg/kg/s or 1/kg/s] ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); - vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), - aerosol_categories_, ncol_, nlev_); - vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), - aerosol_categories_, ncol_, nlev_); + // Deposition velocity of turbulent dry deposition [m/s] vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_); + // Deposition velocity of gravitational settling [m/s] + vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + // Deposition velocity, [m/s] + // Fraction landuse weighted sum of vlc_grv and vlc_trb + vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + + // Work array to hold the mixing ratios [kg/kg or 1/kg] + // Packs AerosolState::int_aero_nmr and AerosolState::int_aero_nmr + // into one array. + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + + // Work array to hold the air density [kg/m3] + rho_ = view_2d("rho", ncol_, nlev_); + + // Work array to hold cloud borne aerosols mixing ratios [kg/kg or 1/kg] + // Filled with Prognostics::n_mode_c and Prognostics::q_aero_c + qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); - qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); + // Work array to hold tendency for 1 species [kg/kg/s] or [1/kg/s] dqdt_tmp_ = view_3d("dqdt_tmp_", pcnst, ncol_, nlev_); static constexpr int n_land_type = mam4::DryDeposition::n_land_type; // FIXME: This should come from a file reading + // The fraction of land use for the column. [non-dimentional] fraction_landuse_ = view_2d("fraction_landuse_", n_land_type, ncol_); //----------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 8feb71e04693..e39090cecdea 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -45,19 +45,70 @@ class MAMDryDep final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - view_3d qtracers_; - view_3d ptend_q_; - // inputs - // FIXME: collect all inputs and outputs together - view_2d fraction_landuse_; + /* Note on mam4::DryDeposition::aerosol_categories = 4 + used in deposition velocity dimension defined below. These + correspond to the two attachment states and two moments: + 0 - interstitial aerosol, 0th moment (i.e., number) + 1 - interstitial aerosol, 3rd moment (i.e., volume/mass) + 2 - cloud-borne aerosol, 0th moment (i.e., number) + 3 - cloud-borne aerosol, 3rd moment (i.e., volume/mass) + see comments in the DryDeposition class in mam4xx. + */ + // Output deposition velocity of turbulent dry deposition [m/s] + // Dimensions + // [numer of modes, aerosol_categories_, num columns] view_3d vlc_trb_; - view_2d rho_; - view_4d vlc_dry_; + // Output deposition velocity of gravitational settling [m/s] + // Dimensions + // [num_modes, aerosol_categories_, num columns, num levels] view_4d vlc_grv_; + + // Output deposition velocity, [m/s] + // fraction landuse weighted sum of vlc_grv and vlc_trb + // Dimensions + // [num_modes, aerosol_categories_, num columns, num levels] + view_4d vlc_dry_; + + + // Output of the the mixing ratio tendencies [kg/kg/s or 1/kg/s] + // Dimensions + // [num columns, num levels, mam4::aero_model::pcnst] + // Packed the same way qtracers_ is layed out. + view_3d ptend_q_; + + // Work array to hold the mixing ratios [kg/kg or 1/kg] + // Dimensions + // [num columns, num levels, mam4::aero_model::pcnst] + // Packs AerosolState::int_aero_nmr + // and AerosolState::int_aero_nmr + // into one array, hence is mixed kg/kg and 1/kg. + view_3d qtracers_; + + // Work array to hold the fraction [non-dimentional] + // of land use for column. + // Dimensions + // [MAMDryDep::n_land_type, num columns] + // Values should sum to 1. + view_2d fraction_landuse_; + + // Work array to hold the air density [kg/m3] + // Dimensions + // [num columns, num levels] + // Calculated from air pressure at layer midpoint, + // Constants::r_gas_dry_air and air temperture. + view_2d rho_; + + // Work array to hold tendency for 1 species [kg/kg/s] or [1/kg/s] + // Dimensions + // [mam4::aero_model::pcnst, num column, num level] view_3d dqdt_tmp_; + // Work array to hold cloud borne aerosols mixing ratios [kg/kg or 1/kg] + // Dimensions + // [mam4::aero_model::pcnst, num column, num level] + // Filled with Prognostics::n_mode_c and Prognostics::q_aero_c view_3d qqcw_; public: diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 807eccdfe559..11383e772454 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -685,9 +685,9 @@ Int Functions::shoc_main( wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables + shoc_output.pblh(i) = pblh_s; + shoc_output.ustar(i) = ustar_s; shoc_output.obklen(i) = obklen_s; - shoc_output.ustar(i) = ustar_s; - shoc_output.pblh(i) = pblh_s; }); Kokkos::fence(); #else From ec1c09b1084fdbc29f935780ed20776724010f5e Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 31 Jul 2024 11:32:41 -0600 Subject: [PATCH 412/904] Update EKAT --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index b39fd9fbd814..9ffec9a4fb4c 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit b39fd9fbd81439470e146d7915ed9a22c688ac6a +Subproject commit 9ffec9a4fb4c03596b7e1729241b6348aad70ec1 From 48fb3c0faee1391a078c76dcd1d80b2cf2a5de38 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 31 Jul 2024 17:17:03 -0500 Subject: [PATCH 413/904] update PAM submodule --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 5931666c95a3..3ea20ad38f28 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 5931666c95a3b75ae63fbe5bff6e743e74cb2c0a +Subproject commit 3ea20ad38f286730973429e0d491420a6f599f11 From 7d69007b0ec4a15872584ebd6faa38642cfe4729 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 31 Jul 2024 17:17:35 -0500 Subject: [PATCH 414/904] add negative mass checks for pam_debug.h --- components/eam/src/physics/crm/pam/pam_debug.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/pam_debug.h b/components/eam/src/physics/crm/pam/pam_debug.h index 7e54f437f691..a168710afc6d 100644 --- a/components/eam/src/physics/crm/pam/pam_debug.h +++ b/components/eam/src/physics/crm/pam/pam_debug.h @@ -134,7 +134,9 @@ void pam_debug_check_state( pam::PamCoupler &coupler, int id, int nstep ) { const auto is_neg_t_atm = temp(k,j,i,iens)<0; const auto is_neg_d_atm = rhod(k,j,i,iens)<0; const auto is_neg_q_atm = rhov(k,j,i,iens)<0; - if ( is_neg_t_atm || is_neg_q_atm || is_neg_d_atm ) { + const auto is_neg_c_atm = rhoc(k,j,i,iens)<0; + const auto is_neg_i_atm = rhoi(k,j,i,iens)<0; + if ( is_neg_t_atm || is_neg_q_atm || is_neg_d_atm || is_neg_c_atm || is_neg_i_atm ) { auto phis = input_phis(iens)/grav; printf("PAM-DEBUG neg-found - st:%3.3d id:%2.2d k:%3.3d i:%3.3d n:%3.3d y:%5.1f x:%5.1f ph:%6.1f -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g -- t:%8.2g rd:%8.2g rv:%8.2g rc:%8.2g ri:%8.2g u:%8.2g w:%8.2g \n", nstep,id,k,i,iens,lat(iens),lon(iens),phis, From 1d9dae3ad3016776b8aad0a03be1fd7b5d14fb4d Mon Sep 17 00:00:00 2001 From: Yunpeng Shan Date: Wed, 31 Jul 2024 21:34:19 -0500 Subject: [PATCH 415/904] Add outfld lines for in-cloud properties output. --- components/eam/src/physics/p3/eam/micro_p3_interface.F90 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/eam/src/physics/p3/eam/micro_p3_interface.F90 b/components/eam/src/physics/p3/eam/micro_p3_interface.F90 index bb6e164b4a14..c6c533329a3f 100644 --- a/components/eam/src/physics/p3/eam/micro_p3_interface.F90 +++ b/components/eam/src/physics/p3/eam/micro_p3_interface.F90 @@ -1630,6 +1630,12 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) call outfld('FREQI', freqi, pcols, lchnk) call outfld('FREQR', freqr, psetcols, lchnk) call outfld('CDNUMC', cdnumc, pcols, lchnk) +! call outfld('CLOUDFRAC_LIQ_MICRO', cld_frac_l, pcols, lchnk) call outfld('CLOUDFRAC_ICE_MICRO', cld_frac_i, pcols, lchnk) From b393388533598f6842992d9a64f7e21ef6000829 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 1 Aug 2024 10:34:38 -0700 Subject: [PATCH 416/904] add correct starting time for COMBLE case --- .../testdefs/testmods_dirs/scream/dpxx/comble/shell_commands | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands index 0cc2f5575bf3..5b5df60cf822 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -2,6 +2,7 @@ # lagrangian case over ocean with interactive surface fluxes and coriolis forcing applied to winds ./xmlchange RUN_STARTDATE="2020-03-12" +./xmlchange START_TOD=79200 ./xmlchange PTS_LAT=74.5 ./xmlchange PTS_LON=9.9 From 89392e4a91afae5dc3cc9370fdc661f637bbe0fb Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 1 Aug 2024 15:24:20 -0600 Subject: [PATCH 417/904] remove multiprocess tests for compute sanitizer --- components/eamxx/tests/CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 5fd23a41d9d8..7602f654356d 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -68,9 +68,13 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) # Testing individual atm processes add_subdirectory(single-process) - # Testing multiple atm processes coupled together - add_subdirectory(multi-process) - + # Testing multiple atm processes coupled together. + # Some compute-sanitizer tests time out with these + # larger multiprocess tests, so disable in that case. + if (NOT EKAT_ENABLE_COMPUTE_SANITIZER) + add_subdirectory(multi-process) + endif() + if (EAMXX_ENABLE_PYBIND) add_subdirectory(python) endif() From b3c5aee66a2d69cc6daf3668011c276d0edd0130 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Thu, 1 Aug 2024 21:08:41 -0700 Subject: [PATCH 418/904] Set default cloud condensate in IC and increase tolerance for difference in atm/lnd lat/lon in domain checking --- components/eamxx/cime_config/namelist_defaults_scream.xml | 8 ++++---- driver-mct/cime_config/namelist_definition_drv.xml | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50188236e4e0..2714c34d9c20 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -517,10 +517,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 - 0.0 - 0.0 - 0.0 - 0.0 + 0.0 + 0.0 + 0.0 + 0.0 0.0 diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index 7fbf83688c8a..982acabe64d8 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -1527,6 +1527,7 @@ 1.e-10 1.e-10 1.e-10 + 1.e-10 From fc26235f0a0ac961111d83e0db1239cec55a554c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 06:43:52 -0700 Subject: [PATCH 419/904] Adds a CIME drydep test --- cime_config/tests.py | 1 + .../scream/mam4xx/drydep/shell_commands | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index b4fb91cc5ee1..4750de448765 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -725,6 +725,7 @@ "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-drydep", ) }, diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands index d219b2a489e0..c5acf055a882 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands @@ -1,12 +1,16 @@ -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ -#modify initial condition file to get aerosol species ICs -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b -# Add drydep process +#------------------------------------------------------ +#Update IC file and add drydep process +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_drydep" -b From f0fc790936de3d22f0a412ddc7a5cbd73797eff6 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Fri, 2 Aug 2024 14:04:44 -0700 Subject: [PATCH 420/904] Update Livermore Computing machine config files Replaces gdata with workspace for Livermore Machines Removes quartz machine Adds dane machine [BFB] --- .../{intel_quartz.cmake => intel_dane.cmake} | 0 cime_config/machines/config_machines.xml | 34 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) rename cime_config/machines/cmake_macros/{intel_quartz.cmake => intel_dane.cmake} (100%) diff --git a/cime_config/machines/cmake_macros/intel_quartz.cmake b/cime_config/machines/cmake_macros/intel_dane.cmake similarity index 100% rename from cime_config/machines/cmake_macros/intel_quartz.cmake rename to cime_config/machines/cmake_macros/intel_dane.cmake diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7cd34fdb0247..f821ff66ba4f 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2978,14 +2978,14 @@ mpich cbronze /p/lustre2/$USER/e3sm_scratch/ruby - /usr/gdata/e3sm/ccsm3data/inputdata - /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 + /usr/workspace/e3sm/ccsm3data/inputdata + /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/e3sm/tools/cprnc + /usr/workspace/e3sm/tools/cprnc 8 lc_slurm - donahue5 -at- llnl.gov + boutte3 -at- llnl.gov 56 56 @@ -3010,7 +3010,7 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/e3sm/install/quartz/modulefiles + /usr/workspace/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 @@ -3022,28 +3022,28 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/e3sm/install/quartz/netcdf-fortran/ + /usr/workspace/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 - - LLNL Linux Cluster, Linux (pgi), 36 pes/node, batch system is Slurm + + LLNL Linux Cluster, 112 pes/node, batch system is Slurm LINUX intel mpich cbronze - /p/lustre2/$USER/e3sm_scratch/quartz - /usr/gdata/e3sm/ccsm3data/inputdata - /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 + /p/lustre2/$USER/e3sm_scratch/dane + /usr/workspace/e3sm/ccsm3data/inputdata + /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/e3sm/tools/cprnc + /usr/workspace/e3sm/tools/cprnc 8 lc_slurm - donahue5 -at- llnl.gov - 72 - 36 + boutte3 -at- llnl.gov + 224 + 112 @@ -3066,7 +3066,7 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/e3sm/install/quartz/modulefiles + /usr/workspace/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 @@ -3078,7 +3078,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/e3sm/install/quartz/netcdf-fortran/ + /usr/workspace/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 From faa559bf8ca778bc6c0738d8994bc56236fcf599 Mon Sep 17 00:00:00 2001 From: dqwu Date: Sun, 4 Aug 2024 11:35:49 -0500 Subject: [PATCH 421/904] Use cpe/22.12 instead of PrgEnv-cray to undo unintended changes On Frontier, the "one module to rule them all" is typically cpe. Changing cpe versions alters defaults for most other modules. The change in the default cpe version (22.12 -> 23.12) may be one of the main breaking changes for post-maintenance SCREAM builds. Trey White recommended using cpe/22.12 to get as close as possible to the pre-maintenance configuration. This also allows us to use libunwind/1.6.2 without loading the ums modules. --- cime_config/machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index f821ff66ba4f..eeb80756bb8c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1434,10 +1434,10 @@ /usr/share/lmod/lmod/libexec/lmod python - PrgEnv-cray + cpe/22.12 craype-accel-amd-gfx90a rocm/5.4.0 - libunwind/1.5.0 + libunwind/1.6.2 cce/15.0.1 From 7581ae045f308b6544e61f9c30db20536441a27a Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 5 Aug 2024 09:19:07 -0400 Subject: [PATCH 422/904] fix link to best practices, save code in home --- run_e3sm.template.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index e2a5b603ba79..413f2bf75286 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -3,7 +3,7 @@ # E3SM Coupled Model Group run_e3sm script template. # # Bash coding style inspired by: -# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming +# https://web.archive.org/web/20200620202413/http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming main() { @@ -44,7 +44,7 @@ readonly GET_REFCASE=TRUE # Set paths readonly CASE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}" -readonly CODE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}/code/${CHECKOUT}" +readonly CODE_ROOT="${HOME}/E3SMv3/code/${CHECKOUT}" # Sub-directories readonly CASE_BUILD_DIR=${CASE_ROOT}/build From 0830f8d736f8cb91c6517b95cb3559d17147aa82 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 5 Aug 2024 18:24:12 -0700 Subject: [PATCH 423/904] change DPxx compset alias names and rename General compset to IOP --- cime_config/tests.py | 8 ++++---- components/eamxx/cime_config/config_compsets.xml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index dd08de8c2540..ac999e0369cf 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -657,10 +657,10 @@ "time" : "01:00:00", # each test 225 phys cols, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-dycomsrf01", - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-arm97", - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-comble", - "ERS_P16_Ln22.ne30_ne30.F2000-SCREAMv1-DP-RCE", + "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-arm97", + "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-comble", + "ERS_P16_Ln22.ne30_ne30.FRCE-SCREAMv1-DP", ) }, diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index c684e1f6c6b6..e23548d03708 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -75,13 +75,13 @@ - F2010-SCREAMv1-DP-General - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%General + FIOP-SCREAMv1-DP + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%IOP Experimental, under development - F2000-SCREAMv1-DP-RCE + FRCE-SCREAMv1-DP 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx Experimental, under development From 23dc534c2ed731fbcd2f6fa85b9158622493c62c Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 6 Aug 2024 10:10:42 -0700 Subject: [PATCH 424/904] run DPxx tests with PG2 grid --- cime_config/tests.py | 8 ++++---- components/eamxx/cime_config/config_compsets.xml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1ad9844e2e9e..1418529e78b4 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -657,10 +657,10 @@ "time" : "01:00:00", # each test 225 phys cols, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", - "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-arm97", - "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-comble", - "ERS_P16_Ln22.ne30_ne30.FRCE-SCREAMv1-DP", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-comble", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FRCE-SCREAMv1-DP", ) }, diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index e23548d03708..b378efcfdebe 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -117,7 +117,7 @@ - 225 + 100 @@ -129,7 +129,7 @@ - 225 + 100 From 946bc5448e11ad1a2628aaacfbd0db048afec47c Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 6 Aug 2024 10:34:26 -0700 Subject: [PATCH 425/904] update comment pertaining to number of columns in tests --- cime_config/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1418529e78b4..a3a5949f8435 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -654,8 +654,8 @@ }, "e3sm_scream_v1_dp-eamxx" : { - "time" : "01:00:00", - # each test 225 phys cols, roughly size of ne2 + "time" : "00:30:00", + # each test runs with 225 dynamics and 100 physics columns, roughly size of ne2 "tests" : ( "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97", From bcc1ba1b1dc9fa5ada1e32d7477b4c148e1238a2 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 6 Aug 2024 10:35:32 -0700 Subject: [PATCH 426/904] undo time change in DPxx tests --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index a3a5949f8435..8055dd261b29 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -654,7 +654,7 @@ }, "e3sm_scream_v1_dp-eamxx" : { - "time" : "00:30:00", + "time" : "01:00:00", # each test runs with 225 dynamics and 100 physics columns, roughly size of ne2 "tests" : ( "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", From 5e5d66ec392526bf88614f638ea151eb86a5ac38 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 6 Aug 2024 12:31:55 -0600 Subject: [PATCH 427/904] EAMxx: fix usage of ParameterList in one of the unit tests --- components/eamxx/src/share/tests/atm_process_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/tests/atm_process_tests.cpp b/components/eamxx/src/share/tests/atm_process_tests.cpp index 01985c33c6d0..fce25cd28e21 100644 --- a/components/eamxx/src/share/tests/atm_process_tests.cpp +++ b/components/eamxx/src/share/tests/atm_process_tests.cpp @@ -443,7 +443,7 @@ TEST_CASE("atm_proc_dag", "") { using strvec_t = std::vector; auto params = create_test_params(); - auto p1 = params.sublist("BarBaz"); + auto& p1 = params.sublist("BarBaz"); // Make sure there's a missing piece (whatever Baz computes); p1.set("atm_procs_list",{"Bar"}); From d65256783f6db64a24bebe7779aadaca66cd83db Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Wed, 17 Jul 2024 09:02:50 -0600 Subject: [PATCH 428/904] Port shoc_assumed_pdf functions individually (like in shoc.F90) --- .../eamxx/src/physics/shoc/CMakeLists.txt | 12 + ...shoc_assumed_pdf_compute_buoyancy_flux.cpp | 14 + ...umed_pdf_compute_cloud_liquid_variance.cpp | 14 + ..._assumed_pdf_compute_liquid_water_flux.cpp | 14 + .../shoc/eti/shoc_assumed_pdf_compute_qs.cpp | 14 + .../shoc/eti/shoc_assumed_pdf_compute_s.cpp | 14 + .../shoc_assumed_pdf_compute_sgs_liquid.cpp | 14 + .../shoc_assumed_pdf_compute_temperature.cpp | 14 + .../shoc_assumed_pdf_inplume_correlations.cpp | 14 + .../eti/shoc_assumed_pdf_qw_parameters.cpp | 14 + .../eti/shoc_assumed_pdf_thl_parameters.cpp | 14 + .../eti/shoc_assumed_pdf_tilde_to_real.cpp | 14 + .../eti/shoc_assumed_pdf_vv_parameters.cpp | 14 + ...assumed_pdf_compute_buoyancy_flux_impl.hpp | 41 +++ ...pdf_compute_cloud_liquid_variance_impl.hpp | 39 +++ ...med_pdf_compute_liquid_water_flux_impl.hpp | 33 +++ .../impl/shoc_assumed_pdf_compute_qs_impl.hpp | 55 ++++ .../impl/shoc_assumed_pdf_compute_s_impl.hpp | 69 +++++ ...oc_assumed_pdf_compute_sgs_liquid_impl.hpp | 30 ++ ...c_assumed_pdf_compute_temperature_impl.hpp | 29 ++ .../shoc/impl/shoc_assumed_pdf_impl.hpp | 264 ++++-------------- ..._assumed_pdf_inplume_correlations_impl.hpp | 50 ++++ .../shoc_assumed_pdf_qw_parameters_impl.hpp | 79 ++++++ .../shoc_assumed_pdf_thl_parameters_impl.hpp | 82 ++++++ .../shoc_assumed_pdf_tilde_to_real_impl.hpp | 32 +++ .../shoc_assumed_pdf_vv_parameters_impl.hpp | 61 ++++ .../eamxx/src/physics/shoc/shoc_functions.hpp | 164 ++++++++++- 27 files changed, 992 insertions(+), 216 deletions(-) create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 891626b9d3df..e6724622a928 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -25,6 +25,18 @@ if (NOT EAMXX_ENABLE_GPU) list(APPEND SHOC_SRCS eti/shoc_adv_sgs_tke.cpp eti/shoc_assumed_pdf.cpp + eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp + eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp + eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp + eti/shoc_assumed_pdf_compute_qs.cpp + eti/shoc_assumed_pdf_compute_sgs_liquid.cpp + eti/shoc_assumed_pdf_compute_temperature.cpp + eti/shoc_assumed_pdf_inplume_correlations.cpp + eti/shoc_assumed_pdf_qw_parameters.cpp + eti/shoc_assumed_pdf_compute_s.cpp + eti/shoc_assumed_pdf_thl_parameters.cpp + eti/shoc_assumed_pdf_tilde_to_real.cpp + eti/shoc_assumed_pdf_vv_parameters.cpp eti/shoc_calc_shoc_varorcovar.cpp eti/shoc_calc_shoc_vertflux.cpp eti/shoc_check_length_scale_shoc_length.cpp diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp new file mode 100644 index 000000000000..4d38b8525dd8 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_buoyancy_flux on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp new file mode 100644 index 000000000000..eb83e2098772 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_cloud_liquid_variance on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp new file mode 100644 index 000000000000..d8f99806cf9b --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_liquid_water_flux on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp new file mode 100644 index 000000000000..fbac9733184b --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_qs_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_qs on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp new file mode 100644 index 000000000000..b74ae269a764 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_s_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_s on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp new file mode 100644 index 000000000000..d2c786e88368 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_sgs_liquid on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp new file mode 100644 index 000000000000..8fdf36d64088 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_temperature_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_temperature on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp new file mode 100644 index 000000000000..d29641dd73bf --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_inplume_correlations_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_inplume_correlations on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp new file mode 100644 index 000000000000..b9a9501b9887 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_qw_parameters_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_qw_parameters on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp new file mode 100644 index 000000000000..716fd7cd3d29 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_thl_parameters_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_thl_parameters on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp new file mode 100644 index 000000000000..250e3f1a0a40 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_tilde_to_real_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_tilde_to_real on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp new file mode 100644 index 000000000000..2af26cfc1adb --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_vv_parameters_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_vv_parameters on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp new file mode 100644 index 000000000000..db31f1ab1bd5 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp @@ -0,0 +1,41 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_BUOYANCY_FLUX_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_BUOYANCY_FLUX_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_buoyancy_flux. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_buoyancy_flux( + const Spack& wthlsec, + const Spack& wqwsec, + const Spack& pval, + const Spack& wqls, + Spack& wthv_sec) +{ + const Scalar basepres = C::P0; + const Scalar rair = C::Rair; + const Scalar rv = C::RV; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + const Scalar basetemp = C::basetemp; + const Scalar epsterm = rair/rv; + + wthv_sec = wthlsec + ((1 - epsterm)/epsterm)*basetemp*wqwsec + + ((lcond/cp)*ekat::pow(basepres/pval, (rair/cp)) + - (1/epsterm)*basetemp)*wqls; +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp new file mode 100644 index 000000000000..06f740d6eb55 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp @@ -0,0 +1,39 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_CLOUD_LIQUID_VARIANCE_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_CLOUD_LIQUID_VARIANCE_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_cloud_liquid_variance. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_cloud_liquid_variance( + const Spack& a, + const Spack& s1, + const Spack& ql1, + const Spack& C1, + const Spack& std_s1, + const Spack& s2, + const Spack& ql2, + const Spack& C2, + const Spack& std_s2, + const Spack& shoc_ql, + Spack& shoc_ql2) +{ + shoc_ql2 = ekat::max(0, a*(s1*ql1 + C1*ekat::square(std_s1)) + + (1 - a)*(s2*ql2 + C2*ekat::square(std_s2)) + - ekat::square(shoc_ql)); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp new file mode 100644 index 000000000000..b225494e7b6d --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp @@ -0,0 +1,33 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_LIQUID_WATER_FLUX_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_LIQUID_WATER_FLUX_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_liquid_water_flux. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_liquid_water_flux( + const Spack& a, + const Spack& w1_1, + const Spack& w_first, + const Spack& ql1, + const Spack& w1_2, + const Spack& ql2, + Spack& wqls) +{ + wqls = a*((w1_1 - w_first)*ql1) + (1 - a)*((w1_2 - w_first)*ql2); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp new file mode 100644 index 000000000000..3949f087bfe2 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp @@ -0,0 +1,55 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_QS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_QS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU +#include "physics_functions.hpp" + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_qs. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_qs( + const Spack& Tl1_1, + const Spack& Tl1_2, + const Spack& pval, + const Smask& active_entries, + Spack& qs1, + Spack& beta1, + Spack& qs2, + Spack& beta2) +{ + const Scalar rair = C::Rair; + const Scalar rv = C::RV; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + + // Compute MurphyKoop_svp + const int liquid = 0; + const Spack esval1_1 = scream::physics::Functions::MurphyKoop_svp(Tl1_1,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_1)"); + const Spack esval1_2 = scream::physics::Functions::MurphyKoop_svp(Tl1_2,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_2)"); + const Spack lstarn(lcond); + + qs1 = sp(0.622)*esval1_1/ekat::max(esval1_1, pval - esval1_1); + beta1 = (rair/rv)*(lstarn/(rair*Tl1_1))*(lstarn/(cp*Tl1_1)); + + // Only compute qs2 and beta2 if the two plumes are not equal + const Smask condition = (Tl1_1 != Tl1_2); + qs2 = qs1; + beta2 = beta1; + + qs2.set(condition, sp(0.622)*esval1_2/ekat::max(esval1_2, pval - esval1_2)); + beta2.set(condition, (rair/rv)*(lstarn/(rair*Tl1_2))*(lstarn/(cp*Tl1_2))); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp new file mode 100644 index 000000000000..180bf889d362 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp @@ -0,0 +1,69 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_S_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_S_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_s. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_s( + const Spack& qw1, + const Spack& qs, + const Spack& beta, + const Spack& pval, + const Spack& thl2, + const Spack& qw2, + const Spack& sqrtthl2, + const Spack& sqrtqw2, + const Spack& r_qwthl, + Spack& s, + Spack& std_s, + Spack& qn, + Spack& C) +{ + const Scalar rair = C::Rair; + const Scalar basepres = C::P0; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + const Scalar pi = C::Pi; + + const Scalar sqrt2(std::sqrt(Scalar(2.0))), sqrt2pi(std::sqrt(2*pi)); + + const Spack cthl=((1 + beta*qw1)/ekat::square(1 + beta*qs))*(cp/lcond)* + beta*qs*ekat::pow(pval/basepres, (rair/cp)); + const Spack cqt = 1/(1 + beta*qs); + + std_s = ekat::sqrt(ekat::max(0, + ekat::square(cthl)*thl2 + + ekat::square(cqt)*qw2 - 2*cthl*sqrtthl2*cqt*sqrtqw2*r_qwthl)); + const auto std_s_not_small = std_s > std::sqrt(std::numeric_limits::min()) * 100; + s = qw1-qs*((1 + beta*qw1)/(1 + beta*qs)); + if (std_s_not_small.any()) { + C.set(std_s_not_small, sp(0.5)*(1 + ekat::erf(s/(sqrt2*std_s)))); + } + C.set(!std_s_not_small && s > 0, 1); + const auto std_s_C_not_small = std_s_not_small && C != 0; + if (std_s_C_not_small.any()) { + qn.set(std_s_C_not_small, s*C+(std_s/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s/std_s))); + } + qn.set(!std_s_not_small && s > 0, s); + + // Checking to prevent empty clouds + const auto qn_le_zero = qn <= 0; + C.set(qn_le_zero,0); + qn.set(qn_le_zero,0); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp new file mode 100644 index 000000000000..cbc12f99a673 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp @@ -0,0 +1,30 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_SGS_LIQUID_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_SGS_LIQUID_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_sgs_liquid. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_sgs_liquid( + const Spack& a, + const Spack& ql1, + const Spack& ql2, + Spack& shoc_ql) +{ + shoc_ql = ekat::max(0, a*ql1 + (1 - a)*ql2); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp new file mode 100644 index 000000000000..e857951c5f2b --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp @@ -0,0 +1,29 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_TEMPERATURE_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_TEMPERATURE_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_temperature. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_temperature( + const Spack& thl1, + const Spack& pval, + Spack& Tl1) +{ + Tl1 = thl1/(ekat::pow(C::P0/pval,(C::Rair/C::CP))); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp index 84e76c5941a1..cd004adbd2d6 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp @@ -77,15 +77,6 @@ void Functions::shoc_assumed_pdf( const Scalar w_tol_sqd = 4e-4; const Scalar w_thresh = 0; const Scalar largeneg = SC::largeneg; - const Scalar rair = C::Rair; - const Scalar rv = C::RV; - const bool dothetal_skew = SC::dothetal_skew; - const Scalar basepres = C::P0; - const Scalar cp = C::CP; - const Scalar lcond = C::LatVap; - const Scalar pi = C::Pi; - const Scalar basetemp = C::basetemp; - const Scalar epsterm = rair/rv; const Scalar Tl_min = 100; // Interpolate many variables from interface grid to thermo grid @@ -125,121 +116,35 @@ void Functions::shoc_assumed_pdf( const auto sqrtqt = ekat::max(rt_tol,ekat::sqrt(qwsec)); // Find parameters for vertical velocity - Spack Skew_w(0), w1_1(w_first), w1_2(w_first), w2_1(0), w2_2(0), a(0.5); - { - const Smask condition = w_sec(k) > w_tol_sqd; - - const Scalar tmp_val(0.4); - const Scalar one_m_tmp_val(1 - tmp_val); - const Scalar sqrtw2t(std::sqrt(1-tmp_val)); - - Skew_w.set(condition, w3var/ekat::sqrt(ekat::cube(w_sec(k)))); - a.set(condition, - ekat::max(sp(0.01), - ekat::min(sp(0.99), - sp(0.5)*(1 - Skew_w*ekat::sqrt(1/(4*(one_m_tmp_val*one_m_tmp_val*one_m_tmp_val) - + ekat::square(Skew_w))))))); - - w1_1.set(condition, ekat::sqrt((1 - a)/a)*sqrtw2t); - w1_2.set(condition, -1*ekat::sqrt(a/(1 - a))*sqrtw2t); - w2_1.set(condition, tmp_val*w_sec(k)); - w2_2.set(condition, tmp_val*w_sec(k)); - } + Spack Skew_w, w1_1, w1_2, w2_1, w2_2, a; + shoc_assumed_pdf_vv_parameters(w_first, w2sec, w3var, w_tol_sqd, Skew_w, w1_1, w1_2, w2_1, w2_2, a); // Find parameters for thetal - Spack thl1_1(thl_first), thl1_2(thl_first), thl2_1(0), thl2_2(0), - sqrtthl2_1(0), sqrtthl2_2(0); - { - const Smask condition = thlsec > (thl_tol*thl_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; - - const Spack corrtest1 = ekat::max(-1, ekat::min(1, wthlsec/(sqrtw2*sqrtthl))); - const Spack tmp_val_1(-corrtest1/w1_1), tmp_val_2(-corrtest1/w1_2); - - Spack Skew_thl(0); - if (dothetal_skew == true) { - const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); - Skew_thl.set(tsign>sp(0.4), sp(1.2)*Skew_w); - Skew_thl.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); - } - - if (condition.any()) { - thl2_1.set(condition, - ekat::min(100, - ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1-a)*ekat::square(tmp_val_1)) - - (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*a*(tmp_val_1 - tmp_val_2))))*thlsec); - thl2_2.set(condition, - ekat::min(100, - ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - - (1 - a)*ekat::square(tmp_val_1)) - + (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*thlsec); - - thl1_1.set(condition, tmp_val_2*sqrtthl+thl_first); - thl1_2.set(condition, tmp_val_1*sqrtthl+thl_first); - - sqrtthl2_1.set(condition, ekat::sqrt(thl2_1)); - sqrtthl2_2.set(condition, ekat::sqrt(thl2_2)); - } - - } + Spack thl1_1, thl1_2, thl2_1, thl2_2, sqrtthl2_1, sqrtthl2_2; + shoc_assumed_pdf_thl_parameters(wthlsec,sqrtw2,sqrtthl,thlsec,thl_first,w1_1,w1_2,Skew_w,a, + thl_tol, w_thresh, + thl1_1, thl1_2, thl2_1, thl2_2, sqrtthl2_1, sqrtthl2_2); // Find parameters for total water mixing ratio - Spack qw1_1(qw_first), qw1_2(qw_first), qw2_1(0), qw2_2(0), - sqrtqw2_1(0), sqrtqw2_2(0); - { - const Smask condition = qwsec > (rt_tol*rt_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; - - const Spack corrtest2 = ekat::max(-1, ekat::min(1, wqwsec/(sqrtw2*sqrtqt))); - const Spack tmp_val_1(-corrtest2/w1_1), tmp_val_2(-corrtest2/w1_2); - - const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); - Spack Skew_qw(0); - Skew_qw.set(tsign>sp(0.4), sp(1.2)*Skew_w); - Skew_qw.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); - - if (condition.any()) { - qw2_1.set(condition, - ekat::min(100, - ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) - - (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*a*(tmp_val_1 - tmp_val_2))))*qwsec); - qw2_2.set(condition, - ekat::min(100, - ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) - + (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*qwsec); - } - - qw1_1.set(condition, tmp_val_2*sqrtqt+qw_first); - qw1_2.set(condition, tmp_val_1*sqrtqt+qw_first); - - sqrtqw2_1.set(condition, ekat::sqrt(qw2_1)); - sqrtqw2_2.set(condition, ekat::sqrt(qw2_2)); - } + Spack qw1_1, qw1_2, qw2_1, qw2_2, sqrtqw2_1, sqrtqw2_2; + shoc_assumed_pdf_qw_parameters(wqwsec, sqrtw2, Skew_w, sqrtqt, qwsec, w1_2, w1_1, qw_first, a, + rt_tol, w_thresh, + qw1_1, qw1_2, qw2_1, qw2_2, sqrtqw2_1, sqrtqw2_2); // Convert from tilde variables to "real" variables - w1_1 *= sqrtw2; - w1_1 += w_first; - w1_2 *= sqrtw2; - w1_2 += w_first; + shoc_assumed_pdf_tilde_to_real(w_first, sqrtw2, w1_1); + shoc_assumed_pdf_tilde_to_real(w_first, sqrtw2, w1_2); // Find within-plume correlations. - Spack r_qwthl_1(0); - { - const Spack testvar = a*sqrtqw2_1*sqrtthl2_1 + (1 - a)*sqrtqw2_2*sqrtthl2_2; - const auto testvar_ne_zero = testvar != 0; - if (testvar_ne_zero.any()) { - r_qwthl_1.set(testvar_ne_zero, - ekat::max(-1, - ekat::min(1, (qwthlsec - a*(qw1_1 - qw_first)*(thl1_1 - thl_first) - - (1 - a)*(qw1_2 - qw_first)*(thl1_2 - thl_first))/testvar))); - } - } + Spack r_qwthl_1; + shoc_assumed_pdf_inplume_correlations(sqrtqw2_1, sqrtthl2_1, a, sqrtqw2_2, sqrtthl2_2, qwthlsec, + qw1_1, qw_first, thl1_1, thl_first, qw1_2, thl1_2, + r_qwthl_1); // Begin to compute cloud property statistics - Spack Tl1_1 = thl1_1/(ekat::pow(basepres/pval,(rair/cp))); - Spack Tl1_2 = thl1_2/(ekat::pow(basepres/pval,(rair/cp))); + Spack Tl1_1, Tl1_2; + shoc_assumed_pdf_compute_temperature(thl1_1,pval, Tl1_1); + shoc_assumed_pdf_compute_temperature(thl1_2,pval, Tl1_2); const auto index_range = ekat::range(k*Spack::n); const Smask active_entries = (index_range < nlev); @@ -295,121 +200,54 @@ void Functions::shoc_assumed_pdf( } // Compute qs and beta - Spack qs1(0), qs2(0), beta1(0), beta2(0); - { - // Compute MurphyKoop_svp - const int liquid = 0; - const Spack esval1_1 = scream::physics::Functions::MurphyKoop_svp(Tl1_1,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_1)"); - const Spack esval1_2 = scream::physics::Functions::MurphyKoop_svp(Tl1_2,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_2)"); - const Spack lstarn(lcond); - - qs1 = sp(0.622)*esval1_1/ekat::max(esval1_1, pval - esval1_1); - beta1 = (rair/rv)*(lstarn/(rair*Tl1_1))*(lstarn/(cp*Tl1_1)); - - // Only compute qs2 and beta2 if the two plumes are not equal - const Smask condition = (Tl1_1 != Tl1_2); - qs2 = qs1; - beta2 = beta1; - - qs2.set(condition, sp(0.622)*esval1_2/ekat::max(esval1_2, pval - esval1_2)); - beta2.set(condition, (rair/rv)*(lstarn/(rair*Tl1_2))*(lstarn/(cp*Tl1_2))); - } + Spack qs1, qs2, beta1, beta2; + shoc_assumed_pdf_compute_qs(Tl1_1, Tl1_2, pval, active_entries, qs1, beta1, qs2, beta2); // Cloud computations // Compute s terms. - Spack s1(0), std_s1(0), qn1(0), C1(0), ql1(0), - s2(0), std_s2(0), qn2(0), C2(0), ql2(0); - { - const Scalar sqrt2(std::sqrt(Scalar(2.0))), sqrt2pi(std::sqrt(2*pi)); - - // First plume - const Spack cthl1=((1 + beta1*qw1_1)/ekat::square(1 + beta1*qs1))*(cp/lcond)* - beta1*qs1*ekat::pow(pval/basepres, (rair/cp)); - const Spack cqt1 = 1/(1 + beta1*qs1); - - std_s1 = ekat::sqrt(ekat::max(0, - ekat::square(cthl1)*thl2_1 - + ekat::square(cqt1)*qw2_1 - 2*cthl1*sqrtthl2_1*cqt1*sqrtqw2_1*r_qwthl_1)); - const auto std_s1_not_small = std_s1 > std::sqrt(safe_min()) * 100; - s1 = qw1_1-qs1*((1 + beta1*qw1_1)/(1 + beta1*qs1)); - if (std_s1_not_small.any()) { - C1.set(std_s1_not_small, sp(0.5)*(1 + ekat::erf(s1/(sqrt2*std_s1)))); - } - C1.set(!std_s1_not_small && s1 > 0, 1); - const auto std_s1_C1_not_small = std_s1_not_small && C1 != 0; - if (std_s1_C1_not_small.any()) { - qn1.set(std_s1_C1_not_small, s1*C1+(std_s1/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s1/std_s1))); - } - qn1.set(!std_s1_not_small && s1 > 0, s1); - - // Checking to prevent empty clouds - const auto qn1_le_zero = qn1 <= 0; - C1.set(qn1_le_zero,0); - qn1.set(qn1_le_zero,0); - - ql1 = ekat::min(qn1, qw1_1); - - // Second plume - // Only compute variables of the second plume if the two plumes are not equal - const Smask equal(qw1_1==qw1_2 && thl2_1==thl2_2 && qs1==qs2); - std_s2.set(equal, std_s1); - s2.set(equal, s1); - C2.set(equal, C1); - qn2.set(equal, qn1); - - const Spack cthl2(!equal, - ((1 + beta2*qw1_2)/ekat::square(1 + beta2*qs2))*(cp/lcond)* - beta2*qs2*ekat::pow(pval/basepres, (rair/cp))); - const Spack cqt2(!equal, - 1/(1 + beta2*qs2)); - - const auto nequal = !equal; - if (nequal.any()) { - std_s2.set(nequal, - ekat::sqrt(ekat::max(0, - ekat::square(cthl2)*thl2_2 - + ekat::square(cqt2)*qw2_2 - 2*cthl2*sqrtthl2_2*cqt2*sqrtqw2_2*r_qwthl_1))); - s2.set(nequal, qw1_2-qs2*((1 + beta2*qw1_2)/(1 + beta2*qs2))); - const auto std_s2_not_small = std_s2 > std::sqrt(safe_min()) * 100; - const auto nequal_std_s2_not_small = nequal && std_s2_not_small; - if (nequal_std_s2_not_small.any()) { - C2.set(nequal_std_s2_not_small, sp(0.5)*(1 + ekat::erf(s2/(sqrt2*std_s2)))); - } - C2.set(nequal && !std_s2_not_small && s2 > 0, 1); - const auto nequal_std_s2_C2_not_small = nequal_std_s2_not_small && C2 != 0; - if (nequal_std_s2_C2_not_small.any()) { - qn2.set(nequal_std_s2_C2_not_small, s2*C2+(std_s2/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s2/std_s2))); - } - qn2.set(nequal && !std_s2_not_small && s2 > 0, s2); - } - - // Checking to prevent empty clouds - const auto qn2_le_zero = qn2 <= 0; - C2.set(qn2_le_zero,0); - qn2.set(qn2_le_zero,0); - - ql2 = ekat::min(qn2, qw1_2); + Spack s1, std_s1, qn1, C1, ql1, s2, std_s2, qn2, C2, ql2; + + // First plume + shoc_assumed_pdf_compute_s(qw1_1, qs1, beta1, pval, thl2_1, qw2_1, + sqrtthl2_1, sqrtqw2_1, r_qwthl_1, + s1, std_s1, qn1, C1); + + // Second plume + // Only compute variables of the second plume if the two plumes are not equal + const Smask equal(qw1_1==qw1_2 && thl2_1==thl2_2 && qs1==qs2); + std_s2.set(equal, std_s1); + s2.set(equal, s1); + C2.set(equal, C1); + qn2.set(equal, qn1); + + const auto nequal = !equal; + if (nequal.any()) { + shoc_assumed_pdf_compute_s(qw1_2, qs2, beta2, pval, thl2_2, qw2_2, + sqrtthl2_2, sqrtqw2_2, r_qwthl_1, + s2, std_s2, qn2, C2); } + ql1 = ekat::min(qn1, qw1_1); + ql2 = ekat::min(qn2, qw1_2); + // Compute SGS cloud fraction shoc_cldfrac(k) = ekat::min(1, a*C1 + (1 - a)*C2); // Compute SGS liquid water mixing ratio - shoc_ql(k) = ekat::max(0, a*ql1 + (1 - a)*ql2); + shoc_assumed_pdf_compute_sgs_liquid(a, ql1, ql2, shoc_ql(k)); // Compute cloud liquid variance (CLUBB formulation, adjusted to SHOC parameters based) - shoc_ql2(k) = ekat::max(0, a*(s1*ql1 + C1*ekat::square(std_s1)) - + (1 - a)*(s2*ql2 + C2*ekat::square(std_s2)) - - ekat::square(shoc_ql(k))); + shoc_assumed_pdf_compute_cloud_liquid_variance(a, s1, ql1, C1, std_s1, + s2, ql2, C2, std_s2, shoc_ql(k), + shoc_ql2(k)); // Compute liquid water flux - wqls(k) = a*((w1_1 - w_first)*ql1) + (1 - a)*((w1_2 - w_first)*ql2); + shoc_assumed_pdf_compute_liquid_water_flux(a, w1_1, w_first, ql1, w1_2, ql2, wqls(k)); // Compute the SGS buoyancy flux - wthv_sec(k) = wthlsec + ((1 - epsterm)/epsterm)*basetemp*wqwsec - + ((lcond/cp)*ekat::pow(basepres/pval, (rair/cp)) - - (1/epsterm)*basetemp)*wqls(k); + shoc_assumed_pdf_compute_buoyancy_flux(wthlsec, wqwsec, pval, wqls(k), + wthv_sec(k)); }); // Release temporary variables from the workspace diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp new file mode 100644 index 000000000000..52554ea9c98a --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp @@ -0,0 +1,50 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_INPLUME_CORRELATIONS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_INPLUME_CORRELATIONS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_inplume_correlations. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find within-plume correlation + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_inplume_correlations( + const Spack& sqrtqw2_1, + const Spack& sqrtthl2_1, + const Spack& a, + const Spack& sqrtqw2_2, + const Spack& sqrtthl2_2, + const Spack& qwthlsec, + const Spack& qw1_1, + const Spack& qw_first, + const Spack& thl1_1, + const Spack& thl_first, + const Spack& qw1_2, + const Spack& thl1_2, + Spack& r_qwthl_1) +{ + r_qwthl_1 = 0; + + const Spack testvar = a*sqrtqw2_1*sqrtthl2_1 + (1 - a)*sqrtqw2_2*sqrtthl2_2; + const auto testvar_ne_zero = testvar != 0; + if (testvar_ne_zero.any()) { + r_qwthl_1.set(testvar_ne_zero, + ekat::max(-1, + ekat::min(1, (qwthlsec - a*(qw1_1 - qw_first)*(thl1_1 - thl_first) + - (1 - a)*(qw1_2 - qw_first)*(thl1_2 - thl_first))/testvar))); + } +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp new file mode 100644 index 000000000000..546be97f6fb4 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp @@ -0,0 +1,79 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_QW_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_QW_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_qw_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for total water mixing ratio + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_qw_parameters( + const Spack& wqwsec, + const Spack& sqrtw2, + const Spack& Skew_w, + const Spack& sqrtqt, + const Spack& qwsec, + const Spack& w1_2, + const Spack& w1_1, + const Spack& qw_first, + const Spack& a, + const Scalar rt_tol, + const Scalar w_thresh, + Spack& qw1_1, + Spack& qw1_2, + Spack& qw2_1, + Spack& qw2_2, + Spack& sqrtqw2_1, + Spack& sqrtqw2_2) +{ + qw1_1 = qw_first; + qw1_2 = qw_first; + qw2_1 = 0; + qw2_2 = 0; + sqrtqw2_1 = 0; + sqrtqw2_2 = 0; + + const Smask condition = qwsec > (rt_tol*rt_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; + + const Spack corrtest2 = ekat::max(-1, ekat::min(1, wqwsec/(sqrtw2*sqrtqt))); + const Spack tmp_val_1(-corrtest2/w1_1), tmp_val_2(-corrtest2/w1_2); + + const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); + Spack Skew_qw(0); + Skew_qw.set(tsign>sp(0.4), sp(1.2)*Skew_w); + Skew_qw.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); + + if (condition.any()) { + qw2_1.set(condition, + ekat::min(100, + ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) + - (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*a*(tmp_val_1 - tmp_val_2))))*qwsec); + qw2_2.set(condition, + ekat::min(100, + ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) + + (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*qwsec); + } + + qw1_1.set(condition, tmp_val_2*sqrtqt+qw_first); + qw1_2.set(condition, tmp_val_1*sqrtqt+qw_first); + + sqrtqw2_1.set(condition, ekat::sqrt(qw2_1)); + sqrtqw2_2.set(condition, ekat::sqrt(qw2_2)); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp new file mode 100644 index 000000000000..90d60e439b56 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp @@ -0,0 +1,82 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_THL_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_THL_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_thl_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for thetal + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_thl_parameters( + const Spack& wthlsec, + const Spack& sqrtw2, + const Spack& sqrtthl, + const Spack& thlsec, + const Spack& thl_first, + const Spack& w1_1, + const Spack& w1_2, + const Spack& Skew_w, + const Spack& a, + const Scalar thl_tol, + const Scalar w_thresh, + Spack& thl1_1, + Spack& thl1_2, + Spack& thl2_1, + Spack& thl2_2, + Spack& sqrtthl2_1, + Spack& sqrtthl2_2) +{ + thl1_1 = thl_first; + thl1_2 = thl_first; + thl2_1 = 0; + thl2_2 = 0; + sqrtthl2_1 = 0; + sqrtthl2_2 = 0; + + const Smask condition = thlsec > (thl_tol*thl_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; + + const Spack corrtest1 = ekat::max(-1, ekat::min(1, wthlsec/(sqrtw2*sqrtthl))); + const Spack tmp_val_1(-corrtest1/w1_1), tmp_val_2(-corrtest1/w1_2); + + Spack Skew_thl(0); + if (SC::dothetal_skew == true) { + const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); + Skew_thl.set(tsign>sp(0.4), sp(1.2)*Skew_w); + Skew_thl.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); + } + + if (condition.any()) { + thl2_1.set(condition, + ekat::min(100, + ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1-a)*ekat::square(tmp_val_1)) + - (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*a*(tmp_val_1 - tmp_val_2))))*thlsec); + thl2_2.set(condition, + ekat::min(100, + ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) + - (1 - a)*ekat::square(tmp_val_1)) + + (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*thlsec); + + thl1_1.set(condition, tmp_val_2*sqrtthl+thl_first); + thl1_2.set(condition, tmp_val_1*sqrtthl+thl_first); + + sqrtthl2_1.set(condition, ekat::sqrt(thl2_1)); + sqrtthl2_2.set(condition, ekat::sqrt(thl2_2)); + } +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp new file mode 100644 index 000000000000..2eaee89a72fc --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp @@ -0,0 +1,32 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_TILDE_TO_REAL_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_TILDE_TO_REAL_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_tilde_to_real. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Convert tilde variables to "real" variables + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_tilde_to_real( + const Spack& w_first, + const Spack& sqrtw2, + Spack& w1) +{ + w1 *= sqrtw2; + w1 += w_first; +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp new file mode 100644 index 000000000000..601622ee0f45 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp @@ -0,0 +1,61 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_VV_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_VV_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc shoc_assumed_pdf_vv_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for vertical velocity + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_vv_parameters( + const Spack& w_first, + const Spack& w_sec, + const Spack& w3var, + const Scalar w_tol_sqd, + Spack& Skew_w, + Spack& w1_1, + Spack& w1_2, + Spack& w2_1, + Spack& w2_2, + Spack& a) +{ + Skew_w = 0; + w1_1 = w_first; + w1_2 = w_first; + w2_1 = 0; + w2_2 = 0; + a = 0.5; + + const Smask condition = w_sec > w_tol_sqd; + + const Scalar tmp_val(0.4); + const Scalar one_m_tmp_val(1 - tmp_val); + const Scalar sqrtw2t(std::sqrt(1-tmp_val)); + + Skew_w.set(condition, w3var/ekat::sqrt(ekat::cube(w_sec))); + a.set(condition, + ekat::max(sp(0.01), + ekat::min(sp(0.99), + sp(0.5)*(1 - Skew_w*ekat::sqrt(1/(4*(one_m_tmp_val*one_m_tmp_val*one_m_tmp_val) + + ekat::square(Skew_w))))))); + + w1_1.set(condition, ekat::sqrt((1 - a)/a)*sqrtw2t); + w1_2.set(condition, -1*ekat::sqrt(a/(1 - a))*sqrtw2t); + w2_1.set(condition, tmp_val*w_sec); + w2_2.set(condition, tmp_val*w_sec); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index afb9f618d52b..d3cd91920586 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -412,9 +412,9 @@ struct Functions #ifdef SCREAM_SMALL_KERNELS static void diag_second_shoc_moments_disp( const Int& shcol, const Int& nlev, const Int& nlevi, - const Scalar& thl2tune, - const Scalar& qw2tune, - const Scalar& qwthl2tune, + const Scalar& thl2tune, + const Scalar& qw2tune, + const Scalar& qwthl2tune, const Scalar& w2tune, const view_2d& thetal, const view_2d& qw, @@ -780,6 +780,154 @@ struct Functions const view_2d& shoc_ql2); #endif + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_buoyancy_flux( + const Spack& wthlsec, + const Spack& wqwsec, + const Spack& pval, + const Spack& wqls, + Spack& wthv_sec); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_cloud_liquid_variance( + const Spack& a, + const Spack& s1, + const Spack& ql1, + const Spack& C1, + const Spack& std_s1, + const Spack& s2, + const Spack& ql2, + const Spack& C2, + const Spack& std_s2, + const Spack& shoc_ql, + Spack& shoc_ql2); + + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_liquid_water_flux( + const Spack& a, + const Spack& w1_1, + const Spack& w_first, + const Spack& ql1, + const Spack& w1_2, + const Spack& ql2, + Spack& wqls); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_qs( + const Spack& Tl1_1, + const Spack& Tl1_2, + const Spack& pval, + const Smask& active_entries, + Spack& qs1, + Spack& beta1, + Spack& qs2, + Spack& beta2); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_s( + const Spack& qw1, + const Spack& qs, + const Spack& beta, + const Spack& pval, + const Spack& thl2, + const Spack& qw2, + const Spack& sqrtthl2, + const Spack& sqrtqw2, + const Spack& r_qwthl, + Spack& s, + Spack& std_s, + Spack& qn, + Spack& C); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_sgs_liquid( + const Spack& a, + const Spack& ql1, + const Spack& ql2, + Spack& shoc_ql); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_temperature( + const Spack& thl1, + const Spack& pval, + Spack& Tl1); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_inplume_correlations( + const Spack& sqrtqw2_1, + const Spack& sqrtthl2_1, + const Spack& a, + const Spack& sqrtqw2_2, + const Spack& sqrtthl2_2, + const Spack& qwthlsec, + const Spack& qw1_1, + const Spack& qw_first, + const Spack& thl1_1, + const Spack& thl_first, + const Spack& qw1_2, + const Spack& thl1_2, + Spack& r_qwthl_1); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_qw_parameters( + const Spack& wqwsec, + const Spack& sqrtw2, + const Spack& Skew_w, + const Spack& sqrtqt, + const Spack& qwsec, + const Spack& w1_2, + const Spack& w1_1, + const Spack& qw_first, + const Spack& a, + const Scalar rt_tol, + const Scalar w_thresh, + Spack& qw1_1, + Spack& qw1_2, + Spack& qw2_1, + Spack& qw2_2, + Spack& sqrtqw2_1, + Spack& sqrtqw2_2); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_thl_parameters( + const Spack& wthlsec, + const Spack& sqrtw2, + const Spack& sqrtthl, + const Spack& thlsec, + const Spack& thl_first, + const Spack& w1_1, + const Spack& w1_2, + const Spack& Skew_w, + const Spack& a, + const Scalar thl_tol, + const Scalar w_thresh, + Spack& thl1_1, + Spack& thl1_2, + Spack& thl2_1, + Spack& thl2_2, + Spack& sqrtthl2_1, + Spack& sqrtthl2_2); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_tilde_to_real( + const Spack& w_first, + const Spack& sqrtw2, + Spack& w1); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_vv_parameters( + const Spack& w_first, + const Spack& w_sec, + const Spack& w3var, + const Scalar w_tol_sqd, + Spack& Skew_w, + Spack& w1_1, + Spack& w1_2, + Spack& w2_1, + Spack& w2_2, + Spack& a); + KOKKOS_FUNCTION static void compute_shr_prod( const MemberType& team, @@ -1246,6 +1394,16 @@ struct Functions # include "shoc_update_prognostics_implicit_impl.hpp" # include "shoc_diag_third_shoc_moments_impl.hpp" # include "shoc_assumed_pdf_impl.hpp" +# include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" +# include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" +# include "shoc_assumed_pdf_compute_qs_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" +# include "shoc_assumed_pdf_compute_temperature_impl.hpp" +# include "shoc_assumed_pdf_qw_parameters_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_thl_parameters_impl.hpp" +# include "shoc_assumed_pdf_vv_parameters_impl.hpp" # include "shoc_adv_sgs_tke_impl.hpp" # include "shoc_compute_tmpi_impl.hpp" # include "shoc_integ_column_stability_impl.hpp" From 0a92aaaa9ca2333124a9b0b6452665b68a979f94 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 1 Aug 2024 16:01:07 -0600 Subject: [PATCH 429/904] GPU fixes: missing includes, and accessing constants on dev --- .../shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp | 5 ++++- components/eamxx/src/physics/shoc/shoc_functions.hpp | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp index e857951c5f2b..eb5f80c2d8f2 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp @@ -20,7 +20,10 @@ void Functions::shoc_assumed_pdf_compute_temperature( const Spack& pval, Spack& Tl1) { - Tl1 = thl1/(ekat::pow(C::P0/pval,(C::Rair/C::CP))); + constexpr Scalar basepres = C::P0; + constexpr Scalar rair = C::Rair; + constexpr Scalar cp = C::CP; + Tl1 = thl1/(ekat::pow(basepres/pval,(rair/cp))); } } // namespace shoc diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index d3cd91920586..08a0d4976feb 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -1394,15 +1394,18 @@ struct Functions # include "shoc_update_prognostics_implicit_impl.hpp" # include "shoc_diag_third_shoc_moments_impl.hpp" # include "shoc_assumed_pdf_impl.hpp" +# include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" # include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" # include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" # include "shoc_assumed_pdf_compute_qs_impl.hpp" # include "shoc_assumed_pdf_compute_s_impl.hpp" # include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" # include "shoc_assumed_pdf_compute_temperature_impl.hpp" +# include "shoc_assumed_pdf_inplume_correlations_impl.hpp" # include "shoc_assumed_pdf_qw_parameters_impl.hpp" # include "shoc_assumed_pdf_compute_s_impl.hpp" # include "shoc_assumed_pdf_thl_parameters_impl.hpp" +# include "shoc_assumed_pdf_tilde_to_real_impl.hpp" # include "shoc_assumed_pdf_vv_parameters_impl.hpp" # include "shoc_adv_sgs_tke_impl.hpp" # include "shoc_compute_tmpi_impl.hpp" From 26fab5c3448eb1864122add7572904794b139f88 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 1 Aug 2024 16:07:58 -0600 Subject: [PATCH 430/904] Use KOKKOS_INLINE_FUNCTION --- .../eamxx/src/physics/shoc/CMakeLists.txt | 12 ----- ...shoc_assumed_pdf_compute_buoyancy_flux.cpp | 14 ----- ...umed_pdf_compute_cloud_liquid_variance.cpp | 14 ----- ..._assumed_pdf_compute_liquid_water_flux.cpp | 14 ----- .../shoc/eti/shoc_assumed_pdf_compute_qs.cpp | 14 ----- .../shoc/eti/shoc_assumed_pdf_compute_s.cpp | 14 ----- .../shoc_assumed_pdf_compute_sgs_liquid.cpp | 14 ----- .../shoc_assumed_pdf_compute_temperature.cpp | 14 ----- .../shoc_assumed_pdf_inplume_correlations.cpp | 14 ----- .../eti/shoc_assumed_pdf_qw_parameters.cpp | 14 ----- .../eti/shoc_assumed_pdf_thl_parameters.cpp | 14 ----- .../eti/shoc_assumed_pdf_tilde_to_real.cpp | 14 ----- .../eti/shoc_assumed_pdf_vv_parameters.cpp | 14 ----- ...assumed_pdf_compute_buoyancy_flux_impl.hpp | 2 +- ...pdf_compute_cloud_liquid_variance_impl.hpp | 2 +- ...med_pdf_compute_liquid_water_flux_impl.hpp | 2 +- .../impl/shoc_assumed_pdf_compute_qs_impl.hpp | 2 +- .../impl/shoc_assumed_pdf_compute_s_impl.hpp | 2 +- ...oc_assumed_pdf_compute_sgs_liquid_impl.hpp | 2 +- ...c_assumed_pdf_compute_temperature_impl.hpp | 2 +- ..._assumed_pdf_inplume_correlations_impl.hpp | 2 +- .../shoc_assumed_pdf_qw_parameters_impl.hpp | 2 +- .../shoc_assumed_pdf_thl_parameters_impl.hpp | 2 +- .../shoc_assumed_pdf_tilde_to_real_impl.hpp | 2 +- .../shoc_assumed_pdf_vv_parameters_impl.hpp | 2 +- .../eamxx/src/physics/shoc/shoc_functions.hpp | 52 ++++++++++--------- 26 files changed, 39 insertions(+), 217 deletions(-) delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index e6724622a928..891626b9d3df 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -25,18 +25,6 @@ if (NOT EAMXX_ENABLE_GPU) list(APPEND SHOC_SRCS eti/shoc_adv_sgs_tke.cpp eti/shoc_assumed_pdf.cpp - eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp - eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp - eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp - eti/shoc_assumed_pdf_compute_qs.cpp - eti/shoc_assumed_pdf_compute_sgs_liquid.cpp - eti/shoc_assumed_pdf_compute_temperature.cpp - eti/shoc_assumed_pdf_inplume_correlations.cpp - eti/shoc_assumed_pdf_qw_parameters.cpp - eti/shoc_assumed_pdf_compute_s.cpp - eti/shoc_assumed_pdf_thl_parameters.cpp - eti/shoc_assumed_pdf_tilde_to_real.cpp - eti/shoc_assumed_pdf_vv_parameters.cpp eti/shoc_calc_shoc_varorcovar.cpp eti/shoc_calc_shoc_vertflux.cpp eti/shoc_check_length_scale_shoc_length.cpp diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp deleted file mode 100644 index 4d38b8525dd8..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_buoyancy_flux on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp deleted file mode 100644 index eb83e2098772..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_cloud_liquid_variance on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp deleted file mode 100644 index d8f99806cf9b..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_liquid_water_flux on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp deleted file mode 100644 index fbac9733184b..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_qs_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_qs on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp deleted file mode 100644 index b74ae269a764..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_s_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_s on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp deleted file mode 100644 index d2c786e88368..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_sgs_liquid on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp deleted file mode 100644 index 8fdf36d64088..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_temperature_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_temperature on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp deleted file mode 100644 index d29641dd73bf..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_inplume_correlations_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_inplume_correlations on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp deleted file mode 100644 index b9a9501b9887..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_qw_parameters_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_qw_parameters on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp deleted file mode 100644 index 716fd7cd3d29..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_thl_parameters_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_thl_parameters on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp deleted file mode 100644 index 250e3f1a0a40..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_tilde_to_real_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_tilde_to_real on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp deleted file mode 100644 index 2af26cfc1adb..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_vv_parameters_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_vv_parameters on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp index db31f1ab1bd5..7a0633fe7f34 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_buoyancy_flux( const Spack& wthlsec, const Spack& wqwsec, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp index 06f740d6eb55..dac01954db31 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_cloud_liquid_variance( const Spack& a, const Spack& s1, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp index b225494e7b6d..ceb64c533a37 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_liquid_water_flux( const Spack& a, const Spack& w1_1, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp index 3949f087bfe2..867ffe3fed3a 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp @@ -15,7 +15,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_qs( const Spack& Tl1_1, const Spack& Tl1_2, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp index 180bf889d362..70fb386055a1 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_s( const Spack& qw1, const Spack& qs, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp index cbc12f99a673..837fe90e2991 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_sgs_liquid( const Spack& a, const Spack& ql1, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp index eb5f80c2d8f2..e60a744ae93f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_temperature( const Spack& thl1, const Spack& pval, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp index 52554ea9c98a..600af5a6f51b 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_inplume_correlations( const Spack& sqrtqw2_1, const Spack& sqrtthl2_1, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp index 546be97f6fb4..d4cb020a7fa2 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_qw_parameters( const Spack& wqwsec, const Spack& sqrtw2, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp index 90d60e439b56..686786d6fe21 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_thl_parameters( const Spack& wthlsec, const Spack& sqrtw2, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp index 2eaee89a72fc..e2de2705fb8e 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_tilde_to_real( const Spack& w_first, const Spack& sqrtw2, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp index 601622ee0f45..6cd237567597 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_vv_parameters( const Spack& w_first, const Spack& w_sec, diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 08a0d4976feb..d1d547245b1f 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -780,7 +780,7 @@ struct Functions const view_2d& shoc_ql2); #endif - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_buoyancy_flux( const Spack& wthlsec, const Spack& wqwsec, @@ -788,7 +788,7 @@ struct Functions const Spack& wqls, Spack& wthv_sec); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_cloud_liquid_variance( const Spack& a, const Spack& s1, @@ -803,7 +803,7 @@ struct Functions Spack& shoc_ql2); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_liquid_water_flux( const Spack& a, const Spack& w1_1, @@ -813,7 +813,7 @@ struct Functions const Spack& ql2, Spack& wqls); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_qs( const Spack& Tl1_1, const Spack& Tl1_2, @@ -824,7 +824,7 @@ struct Functions Spack& qs2, Spack& beta2); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_s( const Spack& qw1, const Spack& qs, @@ -840,20 +840,20 @@ struct Functions Spack& qn, Spack& C); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_sgs_liquid( const Spack& a, const Spack& ql1, const Spack& ql2, Spack& shoc_ql); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_temperature( const Spack& thl1, const Spack& pval, Spack& Tl1); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_inplume_correlations( const Spack& sqrtqw2_1, const Spack& sqrtthl2_1, @@ -869,7 +869,7 @@ struct Functions const Spack& thl1_2, Spack& r_qwthl_1); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_qw_parameters( const Spack& wqwsec, const Spack& sqrtw2, @@ -889,7 +889,7 @@ struct Functions Spack& sqrtqw2_1, Spack& sqrtqw2_2); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_thl_parameters( const Spack& wthlsec, const Spack& sqrtw2, @@ -909,13 +909,13 @@ struct Functions Spack& sqrtthl2_1, Spack& sqrtthl2_2); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_tilde_to_real( const Spack& w_first, const Spack& sqrtw2, Spack& w1); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_vv_parameters( const Spack& w_first, const Spack& w_sec, @@ -1394,19 +1394,6 @@ struct Functions # include "shoc_update_prognostics_implicit_impl.hpp" # include "shoc_diag_third_shoc_moments_impl.hpp" # include "shoc_assumed_pdf_impl.hpp" -# include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" -# include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" -# include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" -# include "shoc_assumed_pdf_compute_qs_impl.hpp" -# include "shoc_assumed_pdf_compute_s_impl.hpp" -# include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" -# include "shoc_assumed_pdf_compute_temperature_impl.hpp" -# include "shoc_assumed_pdf_inplume_correlations_impl.hpp" -# include "shoc_assumed_pdf_qw_parameters_impl.hpp" -# include "shoc_assumed_pdf_compute_s_impl.hpp" -# include "shoc_assumed_pdf_thl_parameters_impl.hpp" -# include "shoc_assumed_pdf_tilde_to_real_impl.hpp" -# include "shoc_assumed_pdf_vv_parameters_impl.hpp" # include "shoc_adv_sgs_tke_impl.hpp" # include "shoc_compute_tmpi_impl.hpp" # include "shoc_integ_column_stability_impl.hpp" @@ -1425,4 +1412,19 @@ struct Functions #endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE +// Some functions should be inlined, thus do not use ETI +# include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" +# include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" +# include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" +# include "shoc_assumed_pdf_compute_qs_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" +# include "shoc_assumed_pdf_compute_temperature_impl.hpp" +# include "shoc_assumed_pdf_inplume_correlations_impl.hpp" +# include "shoc_assumed_pdf_qw_parameters_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_thl_parameters_impl.hpp" +# include "shoc_assumed_pdf_tilde_to_real_impl.hpp" +# include "shoc_assumed_pdf_vv_parameters_impl.hpp" + #endif // SHOC_FUNCTIONS_HPP From cd19cdfe58780516c55f7776ff9f4c61f379a719 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 7 Aug 2024 10:10:31 -0600 Subject: [PATCH 431/904] EAMxx: fix access to output file in IO Avoid calling scorpio interfaces on a file not open --- components/eamxx/src/share/io/scream_output_manager.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index d6be54d09181..0b738efe6483 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -393,7 +393,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) snapshot_start = m_case_t0; snapshot_start += m_time_bnds[0]; } - if (not filespecs.storage.snapshot_fits(snapshot_start)) { + if (filespecs.is_open and not filespecs.storage.snapshot_fits(snapshot_start)) { release_file(filespecs.filename); filespecs.close(); } @@ -496,7 +496,8 @@ void OutputManager::run(const util::TimeStamp& timestamp) scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_filename",m_output_file_specs.filename); scorpio::set_attribute (filespecs.filename,"GLOBAL","num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); - int nsnaps = scorpio::get_dimlen(m_output_file_specs.filename,"time"); + int nsnaps = m_output_file_specs.is_open + ? scorpio::get_dimlen(m_output_file_specs.filename,"time") : 0; scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_file_num_snaps",nsnaps); } // Write these in both output and rhist file. The former, b/c we need these info when we postprocess From 2d208212c0bedab148ec90052ef4c06d730d25fb Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 9 Aug 2024 11:25:21 -0500 Subject: [PATCH 432/904] Disable hipInit call before MPI_Init on Frontier The hipInit workaround was originally introduced to prevent occasional segmentation faults during MPI_Init. With the cpe/22.12 configuration in the post-maintenance setup on Frontier, this workaround is no longer necessary. Additionally, it seems that the hipInit call may be causing non-deterministic DIRK NaN errors in recent ne1024 DECADAL runs. So far, no DIRK errors have been observed when hipInit is not invoked. --- .../cmake_macros/crayclang-scream_frontier-scream-gpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index a5c89c3318ed..d42597be2e0b 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -5,7 +5,7 @@ set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") -string(APPEND CPPDEFS " -DLINUX -DSCREAM_SYSTEM_WORKAROUND=1") +string(APPEND CPPDEFS " -DLINUX -DSCREAM_SYSTEM_WORKAROUND=0") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() From c35785da4eae80c0bae19aa1266888b961367a2b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 12 Aug 2024 23:07:55 -0700 Subject: [PATCH 433/904] Fixes an array index in scream coupler F90 file --- components/eamxx/src/mct_coupling/scream_cpl_indices.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index e754d2fef1b0..dc5be4de373f 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -111,8 +111,8 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(15) = mct_avect_indexra(x2a,'Sf_ofrac') import_cpl_indices(16) = mct_avect_indexra(x2a,'Sf_lfrac') import_cpl_indices(17) = mct_avect_indexra(x2a,'Sf_ifrac') - import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_fv') - import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_ram1') + import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_fv') + import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_ram1') ! Vector components import_vector_components(11) = 0 From 0ae7719dce253f8bf057be0b463487cc64062536 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 21 Feb 2024 14:31:14 -0800 Subject: [PATCH 434/904] Adds surface and online emission interface and a test --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- ...and_online_emissions_process_interface.cpp | 159 ++++++++++++++++++ ...and_online_emissions_process_interface.hpp | 62 +++++++ .../eamxx/src/physics/register_physics.hpp | 2 + 4 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index c5feb52e12a0..9138bba6d5a2 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -45,7 +45,8 @@ add_library(mam eamxx_mam_optics_process_interface.cpp eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp - eamxx_mam_wetscav_process_interface.cpp) + eamxx_mam_wetscav_process_interface.cpp + eamxx_mam_srf_and_online_emissions_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp new file mode 100644 index 000000000000..c107dbe6970a --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -0,0 +1,159 @@ +#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" + +/* +Future work: +Wirte comments +write in/outs for all variables clearly +*/ + +namespace scream { + +// ========================================================================================= +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::set_grids( + const std::shared_ptr grids_manager) { + + using namespace ekat::units; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] + /*add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa] + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + add_field("obklen", scalar2d_layout, m, grid_name); + add_field("surfric", scalar2d_layout, m / s, grid_name); + + auto nondim = ekat::units::Units::nondimensional(); + add_field("landfrac", scalar2d_layout, nondim, grid_name); + //add_field("icefrac", scalar2d_layout, nondim, grid_name); + //add_field("ocnfrac", scalar2d_layout, nondim, grid_name); + add_field("fv", scalar2d_layout, m / s, grid_name); + add_field("ram1", scalar2d_layout, s / m, grid_name); + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + // printf("%s \n", int_nmr_field_name); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + }*/ +} + +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + // Gather runtime options + //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::run_impl(const double dt) { + std::cout << "End of derydep run" << std::endl; +} + +// ========================================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp new file mode 100644 index 000000000000..1ad5e4e6da68 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -0,0 +1,62 @@ +#ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#define EAMXX_MAM_SRF_ONLINE_EMISS_HPP + +// For declaring surface and online emission class derived from atm process class +#include + +// For MAM4 aerosol configuration +#include + +// For component name +#include + +namespace scream { + +// The process responsible for handling MAM4 surface and online emissions. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + public: + // Constructor + MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_srf_online_emissions"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMSrfOnlineEmiss + +} // namespace scream + +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index aea9f2f543a9..f8d3a9edb199 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -29,6 +29,7 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" #include "physics/mam/eamxx_mam_aci_process_interface.hpp" #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" +#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -68,6 +69,7 @@ inline void register_physics () { proc_factory.register_product("mam4_drydep",&create_atmosphere_process); proc_factory.register_product("mam4_aci",&create_atmosphere_process); proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); + proc_factory.register_product("mam4_srf_online_emiss",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 0223b78cd7222ee45e9c9392a2c2b8fb4a01cd0b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 21 Feb 2024 17:36:04 -0800 Subject: [PATCH 435/904] Adds some inputs and preprocess struct, test works now --- ...and_online_emissions_process_interface.cpp | 96 +++++++++++++++---- ...and_online_emissions_process_interface.hpp | 60 +++++++++++- 2 files changed, 133 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index c107dbe6970a..f7396950be3c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -9,7 +9,8 @@ write in/outs for all variables clearly namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -19,7 +20,6 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::Paramet // ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { - using namespace ekat::units; // The units of mixing ratio Q are technically non-dimensional. @@ -42,16 +42,17 @@ void MAMSrfOnlineEmiss::set_grids( // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - - // Layout for 2D (2d horiz) variable + const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + + // Layout for 2D (2d horiz) variable const FieldLayout scalar2d_layout{{COL}, {ncol_}}; - + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] - /*add_field("p_mid", scalar3d_layout_mid, Pa, + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, grid_name); // total pressure @@ -67,17 +68,9 @@ void MAMSrfOnlineEmiss::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); - add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); - add_field("obklen", scalar2d_layout, m, grid_name); - add_field("surfric", scalar2d_layout, m / s, grid_name); - - auto nondim = ekat::units::Units::nondimensional(); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - //add_field("icefrac", scalar2d_layout, nondim, grid_name); - //add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - add_field("fv", scalar2d_layout, m / s, grid_name); - add_field("ram1", scalar2d_layout, s / m, grid_name); + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -119,7 +112,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - }*/ + } } // ========================================================================================= @@ -140,14 +133,77 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); - EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), - "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // set up our preprocess functor + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1ad5e4e6da68..d31cd9075c6a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,7 +1,8 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP -// For declaring surface and online emission class derived from atm process class +// For declaring surface and online emission class derived from atm process +// class #include // For MAM4 aerosol configuration @@ -12,12 +13,21 @@ namespace scream { -// The process responsible for handling MAM4 surface and online emissions. The AD -// stores exactly ONE instance of this class in its list of subcomponents. +// The process responsible for handling MAM4 surface and online emissions. The +// AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { + using KT = ekat::KokkosTypes; + // number of horizontal columns and vertical levels int ncol_, nlev_; + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + // buffer for sotring temporary variables mam_coupling::Buffer buffer_; @@ -54,6 +64,50 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); + team.team_barrier(); + + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMAci::Preprocess + + private: + // preprocessing scratch pad + Preprocess preprocess_; }; // MAMSrfOnlineEmiss From 30f0494b2348f24790f0bd81d19fb3af94c07edb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 22 Feb 2024 11:33:04 -0800 Subject: [PATCH 436/904] Adds some notes for the integration --- ...and_online_emissions_process_interface.cpp | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f7396950be3c..f3d2a5a04ba6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -208,7 +208,50 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - std::cout << "End of derydep run" << std::endl; + + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + /* Rough notes: + + Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two + subroutines, aero_model_emissions and set_srf_emissions. + + Here is the code: + + ! initialize chemistry constituent surface fluxes to zero + do m = 2,pcnst + n = map2chm(m) + if (n>0) cam_in%cflx(:,m) = 0._r8 + enddo + + ! aerosol emissions ... + call aero_model_emissions( state, & ! in + cam_in ) ! out + + ! prescribed emissions from file ... + + !----------------------------------------------------------------------- + ! ... Set surface emissions + !----------------------------------------------------------------------- + call set_srf_emissions( lchnk, ncol, sflx(:,:) ) + + do m = 1,pcnst + n = map2chm(m) + if ( n /= h2o_ndx .and. n > 0 ) then + cam_in%cflx(:ncol,m) = cam_in%cflx(:ncol,m) + sflx(:ncol,n) + call outfld( sflxnam(m), cam_in%cflx(:ncol,m), ncol,lchnk ) + endif + enddo + + + */ + + std::cout << "End of surface emissions run" << std::endl; } // ========================================================================================= From 2bdad85e381ecc956519372e013b43b12b2c8888 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Jul 2024 06:38:07 -0700 Subject: [PATCH 437/904] A working test in new single-process test folder --- ...and_online_emissions_process_interface.cpp | 196 ++++++++++-------- .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../mam/emissions/CMakeLists.txt | 44 ++++ .../single-process/mam/emissions/input.yaml | 37 ++++ .../single-process/mam/emissions/output.yaml | 61 ++++++ 5 files changed, 253 insertions(+), 86 deletions(-) create mode 100644 components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/emissions/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/emissions/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f3d2a5a04ba6..e41e41dc73ef 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -17,20 +17,18 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - q_unit.set_string("kg/kg"); - - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); + // set grid for all the inputs and outputs + // use physics grid + grid_ = grids_manager->get_grid("Physics"); - grid_ = grids_manager->get_grid("Physics"); + // Name of the grid const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -39,93 +37,114 @@ void MAMSrfOnlineEmiss::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // Layout for 3D (2d horiz X 1d vertical) variables + // mid points + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + // layout for 2D (1d horiz X 1d vertical) variable + FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + auto nondim = ekat::units::Units::nondimensional(); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + // atmospheric quantities + // specific humidity [kg/kg] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_layout_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_layout_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + + // planetary boundary layer height + add_field("pbl_height", scalar2d_layout_col, m, grid_name); + + // ======================================================================== + // Output from this whole process + // ======================================================================== + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - + mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - } - } - // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - // printf("%s \n", int_nmr_field_name); - - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - + mam_coupling::cld_aero_mmr_field_name(mode, a); if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } - } - } + } // end for loop num species + } // end for loop for num modes - // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } -} + } // end for loop num gases -// ========================================================================================= -// ON HOST, returns the number of bytes of device memory needed by the above -// Buffer type given the number of columns and vertical levels +} // function set_grids ends + +// ================================================================ +// INIT_BUFFERS +// ================================================================ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -// ========================================================================================= -// ON HOST, initializeŃ• the Buffer type with sufficient memory to store -// intermediate (dry) quantities on the given number of columns with the given -// number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -138,26 +157,31 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ + void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -208,7 +232,6 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -218,15 +241,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two - subroutines, aero_model_emissions and set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in + chemistry.F90. Basically call two subroutines, aero_model_emissions and + set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -235,9 +259,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -248,7 +272,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index 95e4d81ccbbf..62435db229fe 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -23,6 +23,7 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam/aci) add_subdirectory(mam/drydep) add_subdirectory(mam/wet_scav) + add_subdirectory(mam/emissions) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt new file mode 100644 index 000000000000..b57adcb60ef0 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -0,0 +1,44 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_srf_online_emiss_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_srf_online_emiss physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_srf_online_emiss physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml new file mode 100644 index 000000000000..741cf2970320 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -0,0 +1,37 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_srf_online_emiss] + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + pbl_height : 1.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml new file mode 100644 index 000000000000..7363b899a5fa --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -0,0 +1,61 @@ +%YAML 1.1 +--- +filename_prefix: mam4_srf_online_emiss_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 +output_control: + Frequency: 1 + frequency_units: nsteps +... From 9948fa85ae6e7664fc2ea6fd96f08d82bd717fde Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Jul 2024 11:19:45 -0700 Subject: [PATCH 438/904] Adds horiz wind vector and some cleanup; test is working --- ...and_online_emissions_process_interface.cpp | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index e41e41dc73ef..fe415e9436ca 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -39,12 +39,14 @@ void MAMSrfOnlineEmiss::set_grids( // Layout for 3D (2d horiz X 1d vertical) variables // mid points - FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; // interfaces - FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // vector layout + FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true,2); // layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + FieldLayout scalar2d_col{{COL}, {ncol_}}; using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers @@ -57,37 +59,40 @@ void MAMSrfOnlineEmiss::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // atmospheric quantities // specific humidity [kg/kg] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud liquid number mixing ratio [1/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); // cloud ice number mixing ratio [1/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); // Temperature[K] at midpoints - add_field("T_mid", scalar3d_layout_mid, K, grid_name); + add_field("T_mid", scalar3d_mid, K, grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + add_field("omega", scalar3d_mid, Pa / s, grid_name); // Total pressure [Pa] at midpoints - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + add_field("p_mid", scalar3d_mid, Pa, grid_name); // Total pressure [Pa] at interfaces - add_field("p_int", scalar3d_layout_int, Pa, grid_name); + add_field("p_int", scalar3d_int, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - // planetary boundary layer height - add_field("pbl_height", scalar2d_layout_col, m, grid_name); + // Planetary boundary layer height + add_field("pbl_height", scalar2d_col, m, grid_name); + + // Horizontal winds (U and V wind components) [m/s] + add_field("horiz_winds", vector3d_mid, m/s, grid_name); // ======================================================================== // Output from this whole process @@ -99,7 +104,7 @@ void MAMSrfOnlineEmiss::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios @@ -107,7 +112,7 @@ void MAMSrfOnlineEmiss::set_grids( // NOT advected const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { @@ -115,7 +120,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -124,7 +129,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } // end for loop num species @@ -132,7 +137,7 @@ void MAMSrfOnlineEmiss::set_grids( for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // end for loop num gases From c9f8c52fef3d0c439cabcec980b1c7a42db5121a Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 15 Apr 2024 12:23:13 -0600 Subject: [PATCH 439/904] get scream in correct place for emissions --- ...and_online_emissions_process_interface.cpp | 209 ++++++++---------- 1 file changed, 90 insertions(+), 119 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index fe415e9436ca..f3d2a5a04ba6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -17,18 +17,20 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ================================================================ -// SET_GRIDS -// ================================================================ +// ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - // set grid for all the inputs and outputs - // use physics grid - grid_ = grids_manager->get_grid("Physics"); + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); - // Name of the grid + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -37,119 +39,93 @@ void MAMSrfOnlineEmiss::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variables - // mid points - FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // vector layout - FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true,2); + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_col{{COL}, {ncol_}}; - - using namespace ekat::units; - auto q_unit = kg / kg; // units of mass mixing ratios of tracers - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - - auto nondim = ekat::units::Units::nondimensional(); + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - // atmospheric quantities - // specific humidity [kg/kg] - add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud liquid mass mixing ratio [kg/kg] - add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud ice mass mixing ratio [kg/kg] - add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud liquid number mixing ratio [1/kg] - add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); - - // cloud ice number mixing ratio [1/kg] - add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); - - // Temperature[K] at midpoints - add_field("T_mid", scalar3d_mid, K, grid_name); - - // Vertical pressure velocity [Pa/s] at midpoints - add_field("omega", scalar3d_mid, Pa / s, grid_name); - - // Total pressure [Pa] at midpoints - add_field("p_mid", scalar3d_mid, Pa, grid_name); - - // Total pressure [Pa] at interfaces - add_field("p_int", scalar3d_int, Pa, grid_name); - - // Layer thickness(pdel) [Pa] at midpoints - add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - - // Planetary boundary layer height - add_field("pbl_height", scalar2d_col, m, grid_name); - - // Horizontal winds (U and V wind components) [m/s] - add_field("horiz_winds", vector3d_mid, m/s, grid_name); - - // ======================================================================== - // Output from this whole process - // ======================================================================== + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa] + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios - for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { - // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = - mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_mid, n_unit, + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - - // cloudborne aerosol tracers of interest: number (n) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected - const char *cld_nmr_field_name = - mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(mode, a); + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit, + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + // printf("%s \n", int_nmr_field_name); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(mode, a); + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_mid, q_unit, + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } - } // end for loop num species - } // end for loop for num modes + } + } + // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } // end for loop num gases - -} // function set_grids ends + } +} -// ================================================================ -// INIT_BUFFERS -// ================================================================ +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -162,31 +138,26 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ================================================================ -// INITIALIZE_IMPL -// ================================================================ - +// ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); - - // store fields converted to dry mmr from wet mmr in dry_atm_ - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -237,6 +208,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -246,16 +218,15 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions and - set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two + subroutines, aero_model_emissions and set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -264,9 +235,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -277,7 +248,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; From 407c5f365c7d41952037684f9141e8dd054843f8 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 8 Jul 2024 13:38:37 -0600 Subject: [PATCH 440/904] compiling and running test --- .../cime_config/namelist_defaults_scream.xml | 24 ++ ...and_online_emissions_process_interface.cpp | 354 ++++++++++++++---- ...and_online_emissions_process_interface.hpp | 15 +- .../src/physics/mam/mam_emissions_utils.hpp | 59 +++ 4 files changed, 366 insertions(+), 86 deletions(-) create mode 100644 components/eamxx/src/physics/mam/mam_emissions_utils.hpp diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index f09d10b0b320..bb36bdc2ec7f 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,6 +269,30 @@ be lost if SCREAM_HACK_XML is not enabled. + + + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f3d2a5a04ba6..b34186d7de82 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,16 +1,23 @@ -#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" +#include +#include + +#include "share/grid/point_grid.hpp" +#include "share/io/scorpio_input.hpp" + +// for SCREAM_CIME_BUILD +#include "scream_config.h" /* Future work: -Wirte comments +Write comments write in/outs for all variables clearly */ namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, - const ekat::ParameterList ¶ms) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm& comm, + const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -24,17 +31,15 @@ void MAMSrfOnlineEmiss::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - q_unit.set_string("kg/kg"); + Units q_unit(kg / kg, "kg/kg"); - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); + Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers - grid_ = grids_manager->get_grid("Physics"); - const auto &grid_name = grid_->name(); + grid_ = grids_manager->get_grid("Physics"); + const auto& grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -51,56 +56,56 @@ void MAMSrfOnlineEmiss::set_grids( // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] + grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] + grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure + grid_name); // total pressure add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] + grid_name); // pseudo density in [Pa] add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity + "tracers"); // specific humidity add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet + "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet + "tracers"); // ice cloud water [kg/kg] wet add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio + "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio + "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints + grid_name); // Vertical pressure velocity [Pa/s] at midpoints // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if (strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); // printf("%s \n", int_nmr_field_name); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if (strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } @@ -108,8 +113,8 @@ void MAMSrfOnlineEmiss::set_grids( } // aerosol-related gases: mass mixing ratios - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } @@ -123,13 +128,13 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { } // ========================================================================================= -// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. -void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG( - buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= + requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); @@ -137,76 +142,264 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } +// ========================================================================================= +// inline void set_emissions_layouts( +// const std::map map_spec_id, +// const std::string emis_type, +// std::map& host_views, +// mam_coupling::complex_view_2d::HostMirror& +// specrefndxsw_host, // complex refractive index for water visible +// mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { + +// // names take the form "online_emis_specifier_for_SO2" +// for (const auto& item : map_spec_id) { +// const auto spec_name = item.first; +// const int species_id = item.second; +// const auto file_name = emis_type + "_emis_specifier_" + spec_name; +// // const auto& fname = m_params.get(file_name); +// // update file name + + +// // read data +// AtmosphereInput srf_emissions_reader( +// params_srf_emissions, grid_, host_views_emissions, layouts_emissions); +// srf_emissions_reader.read_variables(); +// srf_emissions_reader.finalize(); +// } // end ispec + + + + // for (int i = 0; i < nswbands; i++) { + // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); + // specrefndxsw_host(i, species_id).imag() = + // haero::abs(host_views[sw_im_name](i)); + // } + // for (int i = 0; i < nlwbands; i++) { + // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); + // specrefndxlw_host(i, species_id).imag() = + // haero::abs(host_views[lw_im_name](i)); + // } + +// } // end // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // NOTE: these are taken as arguments to srf_emissions_inti() + // and then passed to trcdata_init() + // rmv_file = false; + // emis_cycle_yr + // emis_fixed_ymd + // emis_fixed_tod + // emis_type // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); + get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if (strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); + get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if (strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); + get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = - get_field_out(gas_mmr_field_name).get_view(); + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + // // read data from file + { + // // using namespace ShortFieldTagsNames; + + using view_1d_host = typename KT::view_1d::HostMirror; + + mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + + // { + // make a list of host views + std::map host_views_srf_emissions; + // defines layouts + std::map layouts_srf_emissions; + ekat::ParameterList params_srf_emissions; + std::string prefix_srf_emissions = "srf_emis_specifier_for_"; + + // // constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; + // // auto specrefndxsw_host = mam_coupling::complex_view_2d::HostMirror( + // // "specrefndxsw_host", nswbands_, maxd_aspectype); + + // // auto specrefndxlw_host = mam_coupling::complex_view_2d::HostMirror( + // // "specrefndxlw_host", nlwbands_, maxd_aspectype); + + std::map map_srf_emiss_name_species_id; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["num_a1"] = 3; + map_srf_emiss_name_species_id["num_a2"] = 4; + map_srf_emiss_name_species_id["num_a4"] = 5; + map_srf_emiss_name_species_id["pom_a4"] = 6; + map_srf_emiss_name_species_id["so4_a1"] = 7; + map_srf_emiss_name_species_id["so4_a2"] = 8; + + std::map map_online_emiss_name_species_id; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["num_a1"] = 3; + map_online_emiss_name_species_id["num_a2"] = 4; + map_online_emiss_name_species_id["num_a4"] = 5; + map_online_emiss_name_species_id["pom_a4"] = 6; + map_online_emiss_name_species_id["so4_a1"] = 7; + map_online_emiss_name_species_id["so4_a2"] = 8; + + // To create the input object, we need to set: + // 1) names (do during read loop) + // 2) params + // 3) host views + // 4) layouts + // set_emissions_names(surname_emissions, params_srf_emissions, + // host_views_emissions, layouts_emissions); + + // inline void set_emissions_names( + // const std::map map_spec_id, + // const std::string emis_type, + // std::map& host_views, + // mam_coupling::complex_view_2d::HostMirror& + // specrefndxsw_host, // complex refractive index for water visible + // mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { + + // // names take the form "online_emis_specifier_for_SO2" + // for (const auto& item : map_spec_id) { + // const auto spec_name = item.first; + // const int species_id = item.second; + // const auto file_name = emis_type + "_emis_specifier_" + spec_name; + // const auto& fname = m_params.get(file_name); + // update file name + + + // // read data + // AtmosphereInput srf_emissions_reader( + // params_srf_emissions, grid_, host_views_emissions, layouts_emissions); + // srf_emissions_reader.read_variables(); + // srf_emissions_reader.finalize(); + // } // end ispec + + + + // for (int i = 0; i < nswbands; i++) { + // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); + // specrefndxsw_host(i, species_id).imag() = + // haero::abs(host_views[sw_im_name](i)); + // } + // for (int i = 0; i < nlwbands; i++) { + // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); + // specrefndxlw_host(i, species_id).imag() = + // haero::abs(host_views[lw_im_name](i)); + // } + + // } // end + + // =============== + // Names + // =============== + + // names take the form _emis_specifier_for_ + // set_emissions_names("srf", params_srf_emissions, host_views_srf_emissions, layouts_srf_emissions); + // set_emissions_names("online", params_srf_emissions, host_views_online_emissions, layouts_online_emissions); + + // =============== + // Params + // =============== + using strvec_t = std::vector; + params_srf_emissions.set("Skip_Grid_Checks", true); + // params_srf_emissions.set("Field Names", {refindex_real_sw, refindex_im_sw, + // refindex_real_lw, refindex_im_lw}); + // params_srf_emissions.set("Filename", fname); + + // =============== + // Host Views + // =============== + using view_1d_host = typename KT::view_1d::HostMirror; + std::map host_views; + + // host_views[refindex_real_sw] = view_1d_host(refindex_real_sw, + // nswbands); host_views[refindex_im_sw] = view_1d_host(refindex_im_sw, + // nswbands); host_views[refindex_real_lw] = + // view_1d_host(refindex_real_lw, nlwbands); host_views[refindex_im_lw] = + // view_1d_host(refindex_im_lw, nlwbands); + + // =============== + // Layouts + // =============== + std::map layouts; + // FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; + // FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; + + // layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); + // layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); + // layouts.emplace(refindex_real_lw, scalar_refindex_lw_layout); + // layouts.emplace(refindex_im_lw, scalar_refindex_lw_layout); + + // reshape specrefndxsw_host and copy it to device + // mam4::modal_aer_opt::set_device_specrefindex( + // aerosol_optics_device_data_.specrefindex_sw, "short_wave", + // specrefndxsw_host); + // mam4::modal_aer_opt::set_device_specrefindex( + // aerosol_optics_device_data_.specrefindex_lw, "long_wave", + // specrefndxlw_host); + } } -// ========================================================================================= +// ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< @@ -218,15 +411,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two - subroutines, aero_model_emissions and set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in + chemistry.F90. Basically call two subroutines, aero_model_emissions and + set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -235,9 +429,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -248,11 +442,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; } -// ========================================================================================= -} // namespace scream +// ============================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index d31cd9075c6a..d59f0df56c1f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,14 +1,17 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#include +#include +#include +#include +// For MAM4 aerosol configuration +#include +#include // For declaring surface and online emission class derived from atm process // class #include - -// For MAM4 aerosol configuration -#include - -// For component name +// #include #include namespace scream { @@ -113,4 +116,4 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { } // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP \ No newline at end of file +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp new file mode 100644 index 000000000000..807896f12392 --- /dev/null +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -0,0 +1,59 @@ +#ifndef MAM_EMISSIONS_READ_TABLES_HPP +#define MAM_EMISSIONS_READ_TABLES_HPP + +#include "ekat/ekat_parameter_list.hpp" +#include "mam_coupling.hpp" +#include "share/field/field_manager.hpp" +#include "share/grid/abstract_grid.hpp" +#include "share/grid/grids_manager.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +// later to mam_coupling.hpp +namespace scream::mam_coupling { + +using view_1d_host = typename KT::view_1d::HostMirror; +using view_1d_int_host = typename KT::view_1d::HostMirror; +using view_2d_host = typename KT::view_2d::HostMirror; +// using view_5d_host = typename KT::view_ND::HostMirror; +// using complex_view_1d = typename KT::view_1d>; + +// constexpr int nlwbands = mam4::modal_aer_opt::nlwbands; +// constexpr int nswbands = mam4::modal_aer_opt::nswbands; + +struct AerosolSurfaceEmissionsHostData { + // these have dim = n_species + view_1d_host emis_species_index; + view_1d_host emis_species_units; + view_1d_host emis_species_name; + // molecular weight + view_1d_host emis_species_mw; + // number of sectors in each field + view_1d_int_host emis_species_nsectors; + // FIXME: not quite sure what this does--maybe just a placeholder for fields(:, i_sector)? + view_1d_host emis_species_sector; + // note fields have dim = n_species x nsectors + // TODO: fields have units??? maybe the same as the upper spec units + view_2d_host emis_species_fields; +}; + +using AerosolSurfaceEmissionsDeviceData = + mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; + +inline void set_emissions_params( + AerosolSurfaceEmissionsHostData &aerosol_emissions_host_data, + ekat::ParameterList ¶ms_emissions, + std::map &layouts, + std::map &host_views) { + // Set up input structure to read data from file. + using strvec_t = std::vector; + using namespace ShortFieldTagsNames; + + // using SrfEmisDims = mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; + // SrfEmisDims srf_emimssions_dims; + +} + +} // namespace scream::mam_coupling + +#endif From b30d5fdf25e7416817b00cb05f30ea36f1c197cc Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 11 Jul 2024 19:01:51 -0600 Subject: [PATCH 441/904] reading files but scorpio issues with freeing file --- ...and_online_emissions_process_interface.cpp | 398 +++++++++--------- ...and_online_emissions_process_interface.hpp | 37 +- .../src/physics/mam/mam_emissions_utils.hpp | 84 ++-- .../single-process/mam/emissions/input.yaml | 26 +- 4 files changed, 301 insertions(+), 244 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index b34186d7de82..3acf7e6f5a19 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -16,8 +16,8 @@ write in/outs for all variables clearly namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm& comm, - const ekat::ParameterList& params) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -35,22 +35,22 @@ void MAMSrfOnlineEmiss::set_grids( Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers + // NOTE: final output with be a flux for each grid point + // e.g., flux__emissions(Nx, Ny, Nspec) + // [kg m^-2 s^-1] or [# m^-2 s^-1] grid_ = grids_manager->get_grid("Physics"); - const auto& grid_name = grid_->name(); + const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + const FieldLayout scalar3d_layout_mid = grid_->get_3d_scalar_layout(true); + const FieldLayout scalar3d_layout_int = grid_->get_3d_scalar_layout(false); // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + const FieldLayout scalar2d_layout = grid_->get_2d_scalar_layout(); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process @@ -69,23 +69,24 @@ void MAMSrfOnlineEmiss::set_grids( "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice cloud water [kg/kg] wet - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char* int_mmr_field_name = + const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { @@ -96,13 +97,12 @@ void MAMSrfOnlineEmiss::set_grids( } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - // printf("%s \n", int_nmr_field_name); + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char* cld_mmr_field_name = + const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if (strlen(cld_mmr_field_name) > 0) { @@ -114,7 +114,7 @@ void MAMSrfOnlineEmiss::set_grids( // aerosol-related gases: mass mixing ratios for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } @@ -131,7 +131,7 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { // ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. -void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), "Error! Insufficient buffer size.\n"); @@ -143,7 +143,42 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } // ========================================================================================= +// // TODO: comments! +// void MAMSrfOnlineEmiss::set_emissions_names( +// const std::map map_spec_id, const std::string +// emis_type, const ekat::ParameterList &m_params, std::map &host_views) { + +// using view_1d_host = typename KT::view_1d::HostMirror; + +// // names take the form online_emis_specifier_for_ +// for (const auto &item : map_spec_id) { +// const auto spec_name = item.first; +// const int species_id = item.second; +// const auto file_name = emis_type + "_emis_specifier_" + spec_name; +// const auto &fname = m_params.get(file_name); + +// // read data +// AtmosphereInput srf_emissions_reader(m_params, grid_, +// host_views_emissions, +// layouts_emissions); +// srf_emissions_reader.read_variables(); +// srf_emissions_reader.finalize(); +// } // end ispec +// // for (int i = 0; i < nswbands; i++) { +// // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); +// // specrefndxsw_host(i, species_id).imag() = +// // haero::abs(host_views[sw_im_name](i)); +// // } +// // for (int i = 0; i < nlwbands; i++) { +// // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); +// // specrefndxlw_host(i, species_id).imag() = +// // haero::abs(host_views[lw_im_name](i)); +// // } + +// } // end set_emissions_names +// ========================================================================================= // inline void set_emissions_layouts( // const std::map map_spec_id, // const std::string emis_type, @@ -160,26 +195,24 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { // // const auto& fname = m_params.get(file_name); // // update file name - // // read data // AtmosphereInput srf_emissions_reader( -// params_srf_emissions, grid_, host_views_emissions, layouts_emissions); +// params_srf_emissions, grid_, host_views_emissions, +// layouts_emissions); // srf_emissions_reader.read_variables(); // srf_emissions_reader.finalize(); // } // end ispec - - - // for (int i = 0; i < nswbands; i++) { - // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); - // specrefndxsw_host(i, species_id).imag() = - // haero::abs(host_views[sw_im_name](i)); - // } - // for (int i = 0; i < nlwbands; i++) { - // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); - // specrefndxlw_host(i, species_id).imag() = - // haero::abs(host_views[lw_im_name](i)); - // } +// for (int i = 0; i < nswbands; i++) { +// specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); +// specrefndxsw_host(i, species_id).imag() = +// haero::abs(host_views[sw_im_name](i)); +// } +// for (int i = 0; i < nlwbands; i++) { +// specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); +// specrefndxlw_host(i, species_id).imag() = +// haero::abs(host_views[lw_im_name](i)); +// } // } // end // ========================================================================================= @@ -187,16 +220,17 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); dry_atm_.qv = buffer_.qv_dry; dry_atm_.qc = buffer_.qc_dry; dry_atm_.nc = buffer_.nc_dry; @@ -215,40 +249,41 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); + get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = + const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); + get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char* cld_mmr_field_name = + const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if (strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); + get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } @@ -256,148 +291,125 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - // // read data from file - { - // // using namespace ShortFieldTagsNames; - - using view_1d_host = typename KT::view_1d::HostMirror; - - mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; - - // { - // make a list of host views - std::map host_views_srf_emissions; - // defines layouts - std::map layouts_srf_emissions; - ekat::ParameterList params_srf_emissions; - std::string prefix_srf_emissions = "srf_emis_specifier_for_"; - - // // constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; - // // auto specrefndxsw_host = mam_coupling::complex_view_2d::HostMirror( - // // "specrefndxsw_host", nswbands_, maxd_aspectype); - - // // auto specrefndxlw_host = mam_coupling::complex_view_2d::HostMirror( - // // "specrefndxlw_host", nlwbands_, maxd_aspectype); - - std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; - map_srf_emiss_name_species_id["num_a1"] = 3; - map_srf_emiss_name_species_id["num_a2"] = 4; - map_srf_emiss_name_species_id["num_a4"] = 5; - map_srf_emiss_name_species_id["pom_a4"] = 6; - map_srf_emiss_name_species_id["so4_a1"] = 7; - map_srf_emiss_name_species_id["so4_a2"] = 8; - - std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; - map_online_emiss_name_species_id["num_a1"] = 3; - map_online_emiss_name_species_id["num_a2"] = 4; - map_online_emiss_name_species_id["num_a4"] = 5; - map_online_emiss_name_species_id["pom_a4"] = 6; - map_online_emiss_name_species_id["so4_a1"] = 7; - map_online_emiss_name_species_id["so4_a2"] = 8; - - // To create the input object, we need to set: - // 1) names (do during read loop) - // 2) params - // 3) host views - // 4) layouts - // set_emissions_names(surname_emissions, params_srf_emissions, - // host_views_emissions, layouts_emissions); - - // inline void set_emissions_names( - // const std::map map_spec_id, - // const std::string emis_type, - // std::map& host_views, - // mam_coupling::complex_view_2d::HostMirror& - // specrefndxsw_host, // complex refractive index for water visible - // mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { - - // // names take the form "online_emis_specifier_for_SO2" - // for (const auto& item : map_spec_id) { - // const auto spec_name = item.first; - // const int species_id = item.second; - // const auto file_name = emis_type + "_emis_specifier_" + spec_name; - // const auto& fname = m_params.get(file_name); - // update file name - - - // // read data - // AtmosphereInput srf_emissions_reader( - // params_srf_emissions, grid_, host_views_emissions, layouts_emissions); - // srf_emissions_reader.read_variables(); - // srf_emissions_reader.finalize(); - // } // end ispec - - - - // for (int i = 0; i < nswbands; i++) { - // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); - // specrefndxsw_host(i, species_id).imag() = - // haero::abs(host_views[sw_im_name](i)); - // } - // for (int i = 0; i < nlwbands; i++) { - // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); - // specrefndxlw_host(i, species_id).imag() = - // haero::abs(host_views[lw_im_name](i)); - // } - - // } // end - - // =============== - // Names - // =============== - - // names take the form _emis_specifier_for_ - // set_emissions_names("srf", params_srf_emissions, host_views_srf_emissions, layouts_srf_emissions); - // set_emissions_names("online", params_srf_emissions, host_views_online_emissions, layouts_online_emissions); - - // =============== - // Params - // =============== - using strvec_t = std::vector; - params_srf_emissions.set("Skip_Grid_Checks", true); - // params_srf_emissions.set("Field Names", {refindex_real_sw, refindex_im_sw, - // refindex_real_lw, refindex_im_lw}); - // params_srf_emissions.set("Filename", fname); - - // =============== - // Host Views - // =============== - using view_1d_host = typename KT::view_1d::HostMirror; - std::map host_views; - - // host_views[refindex_real_sw] = view_1d_host(refindex_real_sw, - // nswbands); host_views[refindex_im_sw] = view_1d_host(refindex_im_sw, - // nswbands); host_views[refindex_real_lw] = - // view_1d_host(refindex_real_lw, nlwbands); host_views[refindex_im_lw] = - // view_1d_host(refindex_im_lw, nlwbands); - - // =============== - // Layouts - // =============== - std::map layouts; - // FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; - // FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; - - // layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); - // layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); - // layouts.emplace(refindex_real_lw, scalar_refindex_lw_layout); - // layouts.emplace(refindex_im_lw, scalar_refindex_lw_layout); - - // reshape specrefndxsw_host and copy it to device - // mam4::modal_aer_opt::set_device_specrefindex( - // aerosol_optics_device_data_.specrefindex_sw, "short_wave", - // specrefndxsw_host); - // mam4::modal_aer_opt::set_device_specrefindex( - // aerosol_optics_device_data_.specrefindex_lw, "long_wave", - // specrefndxlw_host); - } -} + // read data from files + using view_1d_host = typename KT::view_1d::HostMirror; + using view_2d_host = typename KT::view_2d::HostMirror; + using strvec_t = std::vector; + + // mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + + std::map map_srf_emiss_name_species_id; + strvec_t srf_emiss_spec_names; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["num_a1"] = 3; + map_srf_emiss_name_species_id["num_a2"] = 4; + map_srf_emiss_name_species_id["num_a4"] = 5; + map_srf_emiss_name_species_id["pom_a4"] = 6; + map_srf_emiss_name_species_id["so4_a1"] = 7; + map_srf_emiss_name_species_id["so4_a2"] = 8; + // for (const auto &item : map_srf_emiss_name_species_id) { + // srf_emiss_spec_names.push_back(item.first); + // } + + std::map map_online_emiss_name_species_id; + strvec_t online_emiss_spec_names; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["num_a1"] = 3; + map_online_emiss_name_species_id["num_a2"] = 4; + map_online_emiss_name_species_id["num_a4"] = 5; + map_online_emiss_name_species_id["pom_a4"] = 6; + map_online_emiss_name_species_id["so4_a1"] = 7; + map_online_emiss_name_species_id["so4_a2"] = 8; + // for (const auto &item : map_srf_emiss_name_species_id) { + // online_emiss_spec_names.push_back(item.first); + // } + + // To create the input object, we need to: + // - set names of views + // - declare host views + // - initialize FieldLayouts + // - initialize params + + using namespace ShortFieldTagsNames; + + // make a list of host views, that holds nspec-dimensional views of + // srf/online emissions at a grid point + std::map host_views_srf_emiss; + // list of layouts of srf/online emissions views + // NOTE: these are the same, but it seems best (necessary?) to have + // equally-sized lists + std::map layouts_srf_emiss; + ekat::ParameterList params_srf_emiss; + std::string middle_name_emiss = "_emis_specifier_for_"; + + // TODO: break this out into a function in emissions_utils.hpp + // host_views_srf_emiss[sname] = view_1d_host(sname, mam_coupling::n_srf_emiss); + // layouts_srf_emiss.emplace(sname, scalar_srf_emiss_layout); + // for (const auto &item : map_srf_emiss_name_species_id) { + // online_emiss_spec_names.push_back(item.first); + // const auto oname = item.first; + // host_views_online_emiss[oname] = view_2d_host(oname, ) + // } + + // set names of views + const std::string srf_emiss_name = "surface_emissions"; + // const std::string online_emiss_name = "online_emissions"; + + // declare the host views + host_views_srf_emiss[srf_emiss_name] = view_1d_host(srf_emiss_name, 1); + // host_views_emissions[online_emiss_name] = + // view_1d_host(online_emiss_name, mam_coupling::n_online_emiss); + + // initialize and collect FieldLayouts + // layout for 2D (2d horiz == 1d flattened col index) scalar-valued variable + FieldLayout scalar_srf_emiss_layout{{COL}, {ncol_}, {srf_emiss_name}}; + // FieldLayout scalar_online_emiss_layout{ + // {CMP}, {mam_coupling::n_online_emiss}, {online_emiss_name}}; + layouts_srf_emiss.emplace(srf_emiss_name, scalar_srf_emiss_layout); + // layouts_emissions.emplace(online_emiss_name, scalar_online_emiss_layout); + + // initialize params + params_srf_emiss.set("Skip_Grid_Checks", true); + // these need to be the emission-type names from the map__emiss_name_species_id maps + // params_srf_emiss.set("Field Names", {srf_emiss_name}); + + // namelist entries take the form _emis_specifier_for_ + std::string emis_type = "srf"; + for (const auto &item : map_srf_emiss_name_species_id) { + const auto spec_name = item.first; + const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + spec_name; + const auto &fpath = m_params.get(file_name); + params_srf_emiss.set("Filename", fpath); + + // read data + AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); + srf_emissions_reader.read_variables(); + srf_emissions_reader.finalize(); + // copy data to device + + } // end ispec + emis_type = "online"; + // for (const auto &item : map_online_emiss_name_species_id) { + // const auto spec_name = item.first; + // const int species_id = item.second; + // const auto file_name = emis_type + middle_name_emiss + spec_name; + // const auto &fname = m_params.get(file_name); + // params_emissions.set("Filename", fname); + + // // read data + // AtmosphereInput online_emissions_reader( + // m_params, grid_, host_views_emissions, layouts_emissions); + // online_emissions_reader.read_variables(); + // online_emissions_reader.finalize(); + // // copy data to device + + // } // end ispec +} // end initialize_impl() // ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index d59f0df56c1f..0abd1a50e857 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -14,6 +14,15 @@ // #include #include +// TODO: determine when these may be necessary +// #ifndef KOKKOS_ENABLE_CUDA +// #define protected_except_cuda public +// #define private_except_cuda public +// #else +// #define protected_except_cuda protected +// #define private_except_cuda private +// #endif + namespace scream { // The process responsible for handling MAM4 surface and online emissions. The @@ -37,7 +46,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - public: +public: // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -52,8 +61,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::string name() const { return "mam_srf_online_emissions"; } // grid - void set_grids( - const std::shared_ptr grids_manager) override; + void + set_grids(const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -78,25 +87,25 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { const mam_coupling::AerosolState &wet_aero, const mam_coupling::DryAtmosphere &dry_atm, const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; + dry_atm_pre_ = dry_atm; dry_aero_pre_ = dry_aero; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index + const int i = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); - } // operator() + } // operator() // local variables for preprocess struct // number of horizontal columns and vertical levels @@ -106,14 +115,14 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMAci::Preprocess - private: +private: // preprocessing scratch pad Preprocess preprocess_; -}; // MAMSrfOnlineEmiss +}; // MAMSrfOnlineEmiss -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp index 807896f12392..8549b47dd8c5 100644 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -18,41 +18,55 @@ using view_2d_host = typename KT::view_2d::HostMirror; // using view_5d_host = typename KT::view_ND::HostMirror; // using complex_view_1d = typename KT::view_1d>; -// constexpr int nlwbands = mam4::modal_aer_opt::nlwbands; -// constexpr int nswbands = mam4::modal_aer_opt::nswbands; - -struct AerosolSurfaceEmissionsHostData { - // these have dim = n_species - view_1d_host emis_species_index; - view_1d_host emis_species_units; - view_1d_host emis_species_name; - // molecular weight - view_1d_host emis_species_mw; - // number of sectors in each field - view_1d_int_host emis_species_nsectors; - // FIXME: not quite sure what this does--maybe just a placeholder for fields(:, i_sector)? - view_1d_host emis_species_sector; - // note fields have dim = n_species x nsectors - // TODO: fields have units??? maybe the same as the upper spec units - view_2d_host emis_species_fields; -}; - -using AerosolSurfaceEmissionsDeviceData = - mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; - -inline void set_emissions_params( - AerosolSurfaceEmissionsHostData &aerosol_emissions_host_data, - ekat::ParameterList ¶ms_emissions, - std::map &layouts, - std::map &host_views) { - // Set up input structure to read data from file. - using strvec_t = std::vector; - using namespace ShortFieldTagsNames; - - // using SrfEmisDims = mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; - // SrfEmisDims srf_emimssions_dims; - -} +constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; +constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; + +using namespace ShortFieldTagsNames; + +// struct AerosolSurfaceEmissionsHostData { +// // these have dim = n_species +// view_1d_host emis_species_index; +// view_1d_host emis_species_units; +// view_1d_host emis_species_name; +// // molecular weight +// view_1d_host emis_species_mw; +// // number of sectors in each field +// view_1d_int_host emis_species_nsectors; +// // FIXME: not quite sure what this does--maybe just a placeholder for +// // fields(:, i_sector)? +// view_1d_host emis_species_sector; +// // note fields have dim = n_species x nsectors +// // TODO: fields have units??? maybe the same as the upper spec units +// view_2d_host emis_species_fields; +// }; + +// using AerosolSurfaceEmissionsDeviceData = + // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; + +// inline void set_emissions_params( +// AerosolSurfaceEmissionsHostData& aerosol_emissions_host_data, +// ekat::ParameterList& params_emissions, +// std::map& layouts, +// std::map& host_views) { +// // Set up input structure to read data from file. +// using strvec_t = std::vector; +// // using namespace ShortFieldTagsNames; + +// // using SrfEmisDims = +// // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; SrfEmisDims +// // srf_emimssions_dims; +// } + +// inline void set_emissions_names(const std::map map_spec_id, +// const std::string emis_type, +// const ekat::ParameterList& m_params, +// std::map& host_views) { + +// using view_1d_host = typename KT::view_1d::HostMirror; + +// std::string + +// } // end set_emissions_names } // namespace scream::mam_coupling diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 741cf2970320..30861b656e68 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -10,7 +10,29 @@ time_stepping: atmosphere_processes: atm_procs_list: [mam4_srf_online_emiss] - + mam4_srf_online_emiss: + # MAM4xx-Surface-Emissions + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + # MAM4xx-Online-Emissions + online_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_SOAG: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + + grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -30,7 +52,7 @@ initial_conditions: #we should get the following variables from other processes pbl_height : 1.0 - + # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From 79b88b7862c3ec769dfcea5151c88a08194fffea Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 18 Jul 2024 17:36:54 -0600 Subject: [PATCH 442/904] surface and online emissions data files successfull read in --- ...and_online_emissions_process_interface.cpp | 236 +++++++----------- .../src/physics/mam/mam_emissions_utils.hpp | 18 ++ .../single-process/mam/optics/CMakeLists.txt | 4 +- 3 files changed, 113 insertions(+), 145 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 3acf7e6f5a19..72171a035416 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -144,77 +144,11 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { } // ========================================================================================= // // TODO: comments! -// void MAMSrfOnlineEmiss::set_emissions_names( -// const std::map map_spec_id, const std::string -// emis_type, const ekat::ParameterList &m_params, std::map &host_views) { - -// using view_1d_host = typename KT::view_1d::HostMirror; - -// // names take the form online_emis_specifier_for_ -// for (const auto &item : map_spec_id) { -// const auto spec_name = item.first; -// const int species_id = item.second; -// const auto file_name = emis_type + "_emis_specifier_" + spec_name; -// const auto &fname = m_params.get(file_name); - -// // read data -// AtmosphereInput srf_emissions_reader(m_params, grid_, -// host_views_emissions, -// layouts_emissions); -// srf_emissions_reader.read_variables(); -// srf_emissions_reader.finalize(); -// } // end ispec - -// // for (int i = 0; i < nswbands; i++) { -// // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); -// // specrefndxsw_host(i, species_id).imag() = -// // haero::abs(host_views[sw_im_name](i)); -// // } -// // for (int i = 0; i < nlwbands; i++) { -// // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); -// // specrefndxlw_host(i, species_id).imag() = -// // haero::abs(host_views[lw_im_name](i)); -// // } - -// } // end set_emissions_names +// void MAMSrfOnlineEmiss::set_emissions_names() {} \\ end set_emissions_names() // ========================================================================================= -// inline void set_emissions_layouts( -// const std::map map_spec_id, -// const std::string emis_type, -// std::map& host_views, -// mam_coupling::complex_view_2d::HostMirror& -// specrefndxsw_host, // complex refractive index for water visible -// mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { - -// // names take the form "online_emis_specifier_for_SO2" -// for (const auto& item : map_spec_id) { -// const auto spec_name = item.first; -// const int species_id = item.second; -// const auto file_name = emis_type + "_emis_specifier_" + spec_name; -// // const auto& fname = m_params.get(file_name); -// // update file name - -// // read data -// AtmosphereInput srf_emissions_reader( -// params_srf_emissions, grid_, host_views_emissions, -// layouts_emissions); -// srf_emissions_reader.read_variables(); -// srf_emissions_reader.finalize(); -// } // end ispec - -// for (int i = 0; i < nswbands; i++) { -// specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); -// specrefndxsw_host(i, species_id).imag() = -// haero::abs(host_views[sw_im_name](i)); -// } -// for (int i = 0; i < nlwbands; i++) { -// specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); -// specrefndxlw_host(i, species_id).imag() = -// haero::abs(host_views[lw_im_name](i)); -// } - -// } // end +// inline void set_emissions_layouts() { + +// } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options @@ -294,12 +228,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // read data from files using view_1d_host = typename KT::view_1d::HostMirror; using view_2d_host = typename KT::view_2d::HostMirror; - using strvec_t = std::vector; - - // mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + // these probably belong in mam4xx std::map map_srf_emiss_name_species_id; - strvec_t srf_emiss_spec_names; map_srf_emiss_name_species_id["DMS"] = 0; map_srf_emiss_name_species_id["SO2"] = 1; map_srf_emiss_name_species_id["bc_a4"] = 2; @@ -309,12 +240,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { map_srf_emiss_name_species_id["pom_a4"] = 6; map_srf_emiss_name_species_id["so4_a1"] = 7; map_srf_emiss_name_species_id["so4_a2"] = 8; - // for (const auto &item : map_srf_emiss_name_species_id) { - // srf_emiss_spec_names.push_back(item.first); - // } + // these probably belong in mam4xx std::map map_online_emiss_name_species_id; - strvec_t online_emiss_spec_names; map_online_emiss_name_species_id["SO2"] = 0; map_online_emiss_name_species_id["SOAG"] = 1; map_online_emiss_name_species_id["bc_a4"] = 2; @@ -324,65 +252,52 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { map_online_emiss_name_species_id["pom_a4"] = 6; map_online_emiss_name_species_id["so4_a1"] = 7; map_online_emiss_name_species_id["so4_a2"] = 8; - // for (const auto &item : map_srf_emiss_name_species_id) { - // online_emiss_spec_names.push_back(item.first); - // } - - // To create the input object, we need to: - // - set names of views - // - declare host views - // - initialize FieldLayouts - // - initialize params using namespace ShortFieldTagsNames; - // make a list of host views, that holds nspec-dimensional views of - // srf/online emissions at a grid point - std::map host_views_srf_emiss; - // list of layouts of srf/online emissions views - // NOTE: these are the same, but it seems best (necessary?) to have - // equally-sized lists - std::map layouts_srf_emiss; ekat::ParameterList params_srf_emiss; + ekat::ParameterList params_online_emiss; std::string middle_name_emiss = "_emis_specifier_for_"; - // TODO: break this out into a function in emissions_utils.hpp - // host_views_srf_emiss[sname] = view_1d_host(sname, mam_coupling::n_srf_emiss); - // layouts_srf_emiss.emplace(sname, scalar_srf_emiss_layout); - // for (const auto &item : map_srf_emiss_name_species_id) { - // online_emiss_spec_names.push_back(item.first); - // const auto oname = item.first; - // host_views_online_emiss[oname] = view_2d_host(oname, ) - // } - - // set names of views - const std::string srf_emiss_name = "surface_emissions"; - // const std::string online_emiss_name = "online_emissions"; - - // declare the host views - host_views_srf_emiss[srf_emiss_name] = view_1d_host(srf_emiss_name, 1); - // host_views_emissions[online_emiss_name] = - // view_1d_host(online_emiss_name, mam_coupling::n_online_emiss); - - // initialize and collect FieldLayouts - // layout for 2D (2d horiz == 1d flattened col index) scalar-valued variable - FieldLayout scalar_srf_emiss_layout{{COL}, {ncol_}, {srf_emiss_name}}; - // FieldLayout scalar_online_emiss_layout{ - // {CMP}, {mam_coupling::n_online_emiss}, {online_emiss_name}}; - layouts_srf_emiss.emplace(srf_emiss_name, scalar_srf_emiss_layout); - // layouts_emissions.emplace(online_emiss_name, scalar_online_emiss_layout); - // initialize params params_srf_emiss.set("Skip_Grid_Checks", true); - // these need to be the emission-type names from the map__emiss_name_species_id maps - // params_srf_emiss.set("Field Names", {srf_emiss_name}); + params_online_emiss.set("Skip_Grid_Checks", true); + + // FIXME: this, and all places used, will need to change to a flattened column index + int numlat_srf = scream::mam_coupling::nlat_srf; + int numlon_srf = scream::mam_coupling::nlon_srf; + int numcol_fake_srf = numlat_srf * numlon_srf; + const auto srf_emiss_var_names = mam4::mo_srf_emissions::srf_emimssions_data_fields; + view_2d_host srf_emiss_host; + + // TODO: break this out into a function in emissions_utils.hpp // namelist entries take the form _emis_specifier_for_ std::string emis_type = "srf"; for (const auto &item : map_srf_emiss_name_species_id) { - const auto spec_name = item.first; - const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + spec_name; + const std::string srf_emiss_name = item.first; + // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 + // Is this correct or an error in the data? + if (srf_emiss_name == "DMS") { + numlat_srf = 180; + numlon_srf = 360; + numcol_fake_srf = numlat_srf * numlon_srf; + } else { + numlat_srf = scream::mam_coupling::nlat_srf; + numlon_srf = scream::mam_coupling::nlon_srf; + numcol_fake_srf = numlat_srf * numlon_srf; + } + std::map host_views_srf_emiss; + std::map layouts_srf_emiss; + // FIXME: this will need to change to a flattened column index + FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, {"lat", "lon"}); + for ( const auto &var_name : srf_emiss_var_names.at(srf_emiss_name) ) { + host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); + layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); + } + // const auto spec_name = item.first; + // const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; const auto &fpath = m_params.get(file_name); params_srf_emiss.set("Filename", fpath); @@ -390,25 +305,60 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); srf_emissions_reader.read_variables(); srf_emissions_reader.finalize(); - // copy data to device - - } // end ispec + // copy data into host view for sending, columnwise, to mam4xx's mo_srf_emissions + // TODO: any reason not to just copy to device here? + // TODO: Could probably be a parfor here + // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { + // srf_emiss_host(colidx_fake, species_id) = host_views_srf_emiss[srf_emiss_name](colidx_fake); + // } + } // end item + + // FIXME: this, and all places used, will need to change to a flattened column index + const int numlat_online = scream::mam_coupling::nlat_online; + const int numlon_online = scream::mam_coupling::nlon_online; + int numalti_online = scream::mam_coupling::nalti_online; + int numcol_fake_online = numlat_online * numlon_online * numalti_online; + const auto online_emiss_var_names = mam4::aero_model_emissions::online_emimssions_data_fields; + + // view_3d_host online_emiss_host; emis_type = "online"; - // for (const auto &item : map_online_emiss_name_species_id) { - // const auto spec_name = item.first; - // const int species_id = item.second; - // const auto file_name = emis_type + middle_name_emiss + spec_name; - // const auto &fname = m_params.get(file_name); - // params_emissions.set("Filename", fname); - - // // read data - // AtmosphereInput online_emissions_reader( - // m_params, grid_, host_views_emissions, layouts_emissions); - // online_emissions_reader.read_variables(); - // online_emissions_reader.finalize(); - // // copy data to device - - // } // end ispec + for (const auto &item : map_online_emiss_name_species_id) { + const std::string online_emiss_name = item.first; + // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 + // Is this correct or an error in the data? + if (online_emiss_name == "SOAG") { + numalti_online = 12; + numcol_fake_online = numlat_online * numlon_online * numalti_online; + } else { + numalti_online = 13; + numcol_fake_online = numlat_online * numlon_online * numalti_online; + } + std::map host_views_online_emiss; + std::map layouts_online_emiss; + // FIXME: this will need to change to a flattened column index + FieldLayout scalar_online_emiss_layout({CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, {"altitude", "lat", "lon"}); + for ( const auto &var_name : online_emiss_var_names.at(online_emiss_name) ) { + host_views_online_emiss[var_name] = view_1d_host(var_name, numcol_fake_online); + layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); + } // end var_name + // const auto spec_name = item.first; + // const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + online_emiss_name; + const auto &fpath = m_params.get(file_name); + params_online_emiss.set("Filename", fpath); + + // read data + AtmosphereInput online_emissions_reader(params_online_emiss, grid_, host_views_online_emiss, layouts_online_emiss); + online_emissions_reader.read_variables(); + online_emissions_reader.finalize(); + // copy data into host view for sending, columnwise, to mam4xx's mo_online_emissions + // TODO: any reason not to just copy to device here? + // TODO: Could probably be a parfor here + // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { + // online_emiss_host(colidx_fake_online, species_id) = host_views_online_emiss[online_emiss_name](colidx_fake); + // } + } // end item + } // end initialize_impl() // ============================================================================= diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp index 8549b47dd8c5..527efd5de18b 100644 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -21,8 +21,26 @@ using view_2d_host = typename KT::view_2d::HostMirror; constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; +// FIXME: this will need to change when we remap to flattened column idx +constexpr int nlat_srf = 96; +constexpr int nlon_srf = 144; + +constexpr int nalti_online = 13; +constexpr int nlat_online = 96; +constexpr int nlon_online = 144; + using namespace ShortFieldTagsNames; +// std::map> map_srf_emiss_file_vars; + +inline void set_file_var_names(std::map> &var_map, + std::map &spec_map) { + // for (const auto &spec : spec_map) { + // std::string spec_name = spec.first; + // std::cout << "var_map[spec_name] = " << var_map[spec_name] << "\n"; + // } +} + // struct AerosolSurfaceEmissionsHostData { // // these have dim = n_species // view_1d_host emis_species_index; diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt index a3e6f64ec7d0..50f2cfa68a3b 100644 --- a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -40,7 +40,7 @@ set (TEST_INPUT_FILES scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc scream/mam4xx/physprops/poly_rrtmg_c20240206.nc - scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + # scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) @@ -63,4 +63,4 @@ if (SCREAM_ENABLE_BASELINE_TESTS) # Note: one is enough, since we already check that np1 is BFB with npX set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) -endif() \ No newline at end of file +endif() From f5da2455281e8cd2d19c3bb146d7aa367c45e761 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 07:21:43 -0700 Subject: [PATCH 443/904] Horizontal remapper create_horiz_remapper works with so2 srf file --- .../mam4xx/srf_online_emiss/shell_commands | 13 ++ ...and_online_emissions_process_interface.cpp | 176 ++++++++++-------- ...and_online_emissions_process_interface.hpp | 37 ++-- .../eamxx/src/physics/mam/srf_emission.hpp | 22 +++ .../src/physics/mam/srf_emission_impl.hpp | 86 +++++++++ 5 files changed, 244 insertions(+), 90 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands create mode 100644 components/eamxx/src/physics/mam/srf_emission.hpp create mode 100644 components/eamxx/src/physics/mam/srf_emission_impl.hpp diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands new file mode 100644 index 000000000000..6995cd8b8562 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands @@ -0,0 +1,13 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +#modify initial condition file to get aerosol species ICs +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b + +# Add spa as RRTMG needs spa +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 72171a035416..a694e63a5e09 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -33,16 +33,16 @@ void MAMSrfOnlineEmiss::set_grids( // Nevertheless, for output reasons, we like to see 'kg/kg'. Units q_unit(kg / kg, "kg/kg"); - Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers + Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers // NOTE: final output with be a flux for each grid point // e.g., flux__emissions(Nx, Ny, Nspec) // [kg m^-2 s^-1] or [# m^-2 s^-1] - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces @@ -56,56 +56,56 @@ void MAMSrfOnlineEmiss::set_grids( // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] + grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] + grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure + grid_name); // total pressure add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] + grid_name); // pseudo density in [Pa] add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity + "tracers"); // specific humidity add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet + "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet + "tracers"); // ice cloud water [kg/kg] wet add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio + "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints + grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio + "tracers"); // cloud liquid wet number mixing ratio // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { + if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { + if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } @@ -113,11 +113,21 @@ void MAMSrfOnlineEmiss::set_grids( } // aerosol-related gases: mass mixing ratios - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } + + // Reading so2 srf emiss data + + std::string so2_data_file = + "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" + "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; + std::string srf_map_file = ""; + // Init horizontal remap + srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( + grid_, so2_data_file, srf_map_file, m_iop != nullptr); } // ========================================================================================= @@ -132,9 +142,9 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= - requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); @@ -181,7 +191,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = @@ -194,11 +204,11 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { + if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; @@ -207,14 +217,14 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { + if(strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); @@ -231,9 +241,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // these probably belong in mam4xx std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; map_srf_emiss_name_species_id["num_a1"] = 3; map_srf_emiss_name_species_id["num_a2"] = 4; map_srf_emiss_name_species_id["num_a4"] = 5; @@ -243,9 +253,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // these probably belong in mam4xx std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; map_online_emiss_name_species_id["num_a1"] = 3; map_online_emiss_name_species_id["num_a2"] = 4; map_online_emiss_name_species_id["num_a4"] = 5; @@ -263,107 +273,121 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { params_srf_emiss.set("Skip_Grid_Checks", true); params_online_emiss.set("Skip_Grid_Checks", true); - // FIXME: this, and all places used, will need to change to a flattened column index - int numlat_srf = scream::mam_coupling::nlat_srf; - int numlon_srf = scream::mam_coupling::nlon_srf; + // FIXME: this, and all places used, will need to change to a flattened column + // index + int numlat_srf = scream::mam_coupling::nlat_srf; + int numlon_srf = scream::mam_coupling::nlon_srf; int numcol_fake_srf = numlat_srf * numlon_srf; - const auto srf_emiss_var_names = mam4::mo_srf_emissions::srf_emimssions_data_fields; + const auto srf_emiss_var_names = + mam4::mo_srf_emissions::srf_emimssions_data_fields; view_2d_host srf_emiss_host; // TODO: break this out into a function in emissions_utils.hpp // namelist entries take the form _emis_specifier_for_ std::string emis_type = "srf"; - for (const auto &item : map_srf_emiss_name_species_id) { + for(const auto &item : map_srf_emiss_name_species_id) { const std::string srf_emiss_name = item.first; // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 // Is this correct or an error in the data? - if (srf_emiss_name == "DMS") { - numlat_srf = 180; - numlon_srf = 360; + if(srf_emiss_name == "DMS") { + numlat_srf = 180; + numlon_srf = 360; numcol_fake_srf = numlat_srf * numlon_srf; } else { - numlat_srf = scream::mam_coupling::nlat_srf; - numlon_srf = scream::mam_coupling::nlon_srf; + numlat_srf = scream::mam_coupling::nlat_srf; + numlon_srf = scream::mam_coupling::nlon_srf; numcol_fake_srf = numlat_srf * numlon_srf; } std::map host_views_srf_emiss; std::map layouts_srf_emiss; // FIXME: this will need to change to a flattened column index - FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, {"lat", "lon"}); - for ( const auto &var_name : srf_emiss_var_names.at(srf_emiss_name) ) { + FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, + {"lat", "lon"}); + for(const auto &var_name : srf_emiss_var_names.at(srf_emiss_name)) { host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); } // const auto spec_name = item.first; // const int species_id = item.second; const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; - const auto &fpath = m_params.get(file_name); + const auto &fpath = m_params.get(file_name); params_srf_emiss.set("Filename", fpath); // read data - AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); - srf_emissions_reader.read_variables(); - srf_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's mo_srf_emissions + /// AtmosphereInput srf_emissions_reader( + // params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); + // srf_emissions_reader.read_variables(); + // srf_emissions_reader.finalize(); + // copy data into host view for sending, columnwise, to mam4xx's + // mo_srf_emissions // TODO: any reason not to just copy to device here? // TODO: Could probably be a parfor here // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { - // srf_emiss_host(colidx_fake, species_id) = host_views_srf_emiss[srf_emiss_name](colidx_fake); + // srf_emiss_host(colidx_fake, species_id) = + // host_views_srf_emiss[srf_emiss_name](colidx_fake); // } - } // end item + } // end item - // FIXME: this, and all places used, will need to change to a flattened column index - const int numlat_online = scream::mam_coupling::nlat_online; + // FIXME: this, and all places used, will need to change to a flattened column + // index + /*const int numlat_online = scream::mam_coupling::nlat_online; const int numlon_online = scream::mam_coupling::nlon_online; - int numalti_online = scream::mam_coupling::nalti_online; - int numcol_fake_online = numlat_online * numlon_online * numalti_online; - const auto online_emiss_var_names = mam4::aero_model_emissions::online_emimssions_data_fields; + int numalti_online = scream::mam_coupling::nalti_online; + int numcol_fake_online = numlat_online * numlon_online * numalti_online; + const auto online_emiss_var_names = + mam4::aero_model_emissions::online_emimssions_data_fields; // view_3d_host online_emiss_host; emis_type = "online"; - for (const auto &item : map_online_emiss_name_species_id) { + for(const auto &item : map_online_emiss_name_species_id) { const std::string online_emiss_name = item.first; // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 // Is this correct or an error in the data? - if (online_emiss_name == "SOAG") { - numalti_online = 12; + if(online_emiss_name == "SOAG") { + numalti_online = 12; numcol_fake_online = numlat_online * numlon_online * numalti_online; } else { - numalti_online = 13; + numalti_online = 13; numcol_fake_online = numlat_online * numlon_online * numalti_online; } std::map host_views_online_emiss; std::map layouts_online_emiss; // FIXME: this will need to change to a flattened column index - FieldLayout scalar_online_emiss_layout({CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, {"altitude", "lat", "lon"}); - for ( const auto &var_name : online_emiss_var_names.at(online_emiss_name) ) { - host_views_online_emiss[var_name] = view_1d_host(var_name, numcol_fake_online); + FieldLayout scalar_online_emiss_layout( + {CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, + {"altitude", "lat", "lon"}); + for(const auto &var_name : online_emiss_var_names.at(online_emiss_name)) { + host_views_online_emiss[var_name] = + view_1d_host(var_name, numcol_fake_online); layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); - } // end var_name + } // end var_name // const auto spec_name = item.first; // const int species_id = item.second; const auto file_name = emis_type + middle_name_emiss + online_emiss_name; - const auto &fpath = m_params.get(file_name); + const auto &fpath = m_params.get(file_name); params_online_emiss.set("Filename", fpath); // read data - AtmosphereInput online_emissions_reader(params_online_emiss, grid_, host_views_online_emiss, layouts_online_emiss); + AtmosphereInput online_emissions_reader(params_online_emiss, grid_, + host_views_online_emiss, + layouts_online_emiss); online_emissions_reader.read_variables(); online_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's mo_online_emissions + // copy data into host view for sending, columnwise, to mam4xx's + // mo_online_emissions // TODO: any reason not to just copy to device here? // TODO: Could probably be a parfor here // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { - // online_emiss_host(colidx_fake_online, species_id) = host_views_online_emiss[online_emiss_name](colidx_fake); + // online_emiss_host(colidx_fake_online, species_id) = + // host_views_online_emiss[online_emiss_name](colidx_fake); // } - } // end item - -} // end initialize_impl() + } // end item +*/ +} // end initialize_impl() // ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -411,4 +435,4 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { } // ============================================================================= -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 0abd1a50e857..c08aac75a9c6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -7,6 +7,7 @@ #include // For MAM4 aerosol configuration #include +#include #include // For declaring surface and online emission class derived from atm process // class @@ -46,7 +47,15 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; -public: + // Structures to store the data used for interpolation + std::shared_ptr srfEmissHorizInterp_; + + public: + using srfEmissFunc = mam_coupling::srfEmissFunctions; + + template + using uview_2d = Unmanaged>; + // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -61,8 +70,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::string name() const { return "mam_srf_online_emissions"; } // grid - void - set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -87,25 +96,25 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { const mam_coupling::AerosolState &wet_aero, const mam_coupling::DryAtmosphere &dry_atm, const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; + dry_atm_pre_ = dry_atm; dry_aero_pre_ = dry_aero; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index + const int i = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); - } // operator() + } // operator() // local variables for preprocess struct // number of horizontal columns and vertical levels @@ -115,14 +124,14 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMAci::Preprocess -private: + private: // preprocessing scratch pad Preprocess preprocess_; -}; // MAMSrfOnlineEmiss +}; // MAMSrfOnlineEmiss -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp new file mode 100644 index 000000000000..2d100f42b5c1 --- /dev/null +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -0,0 +1,22 @@ +#ifndef SRF_EMISSION_HPP +#define SRF_EMISSION_HPP + +namespace scream::mam_coupling { +namespace { + +template +struct srfEmissFunctions { + /* ------------------------------------------------------------------------------------------- + */ + // Surface emissions routines + + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &spa_data_file, const std::string &map_file, + const bool use_iop = false); +}; // struct srfEmissFunctions +} // namespace +} // namespace scream::mam_coupling +#endif // SRF_EMISSION_HPP + +#include "srf_emission_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp new file mode 100644 index 000000000000..8bd9465442c3 --- /dev/null +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -0,0 +1,86 @@ +#ifndef SRF_EMISSION_IMPL_HPP +#define SRF_EMISSION_IMPL_HPP + +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" + +namespace scream::mam_coupling { +namespace { + +template +std::shared_ptr +srfEmissFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const bool use_iop) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model or + use_iop /*IOP class defines it's own remapper for file data*/) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen spa data to fit the " + "model. We only allow\n" + " spa data to be at the same or coarser resolution " + "as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG( + map_file != "", + "ERROR: Spa data is on a different grid than the model one,\n" + " but spa_remap_file is missing from SPA parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + Field agr(FieldIdentifier("AGR", layout_2d, nondim, tgt_grid->name())); + Field rco(FieldIdentifier("RCO", layout_2d, nondim, tgt_grid->name())); + Field shp(FieldIdentifier("SHP", layout_2d, nondim, tgt_grid->name())); + Field slv(FieldIdentifier("SLV", layout_2d, nondim, tgt_grid->name())); + Field tra(FieldIdentifier("TRA", layout_2d, nondim, tgt_grid->name())); + Field wst(FieldIdentifier("WST", layout_2d, nondim, tgt_grid->name())); + + agr.allocate_view(); + rco.allocate_view(); + shp.allocate_view(); + slv.allocate_view(); + tra.allocate_view(); + wst.allocate_view(); + + remapper->register_field_from_tgt(agr); + remapper->register_field_from_tgt(rco); + remapper->register_field_from_tgt(shp); + remapper->register_field_from_tgt(slv); + remapper->register_field_from_tgt(tra); + remapper->register_field_from_tgt(wst); + + remapper->registration_ends(); + + return remapper; +} +} // namespace +} // namespace scream::mam_coupling + +#endif // SRF_EMISSION_IMPL_HPP \ No newline at end of file From 261ea3962dc525ba0064ae3edb4da6a9b01df1c7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 12:15:32 -0700 Subject: [PATCH 444/904] Adds srfmiss input --- ...and_online_emissions_process_interface.cpp | 5 +++ ...and_online_emissions_process_interface.hpp | 4 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 40 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a694e63a5e09..16fdd02d9818 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -128,6 +128,11 @@ void MAMSrfOnlineEmiss::set_grids( // Init horizontal remap srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, srf_map_file, m_iop != nullptr); + + // 2. Initialize the size of the SPAData structures. + srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); + srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); + // SrfEmissData_out_.init(ncol_, false); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index c08aac75a9c6..6a4923d1a7d7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -130,6 +130,10 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; + srfEmissFunc::srfEmissInput srfEmissData_start_; + srfEmissFunc::srfEmissInput srfEmissData_end_; + // srfEmissFunc::srfEmissOutput srfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 2d100f42b5c1..875a82d68319 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -6,6 +6,46 @@ namespace { template struct srfEmissFunctions { + struct srfEmissData { + srfEmissData() = default; + srfEmissData(const int ncol_) { init(ncol_, true); } + + void init(const int ncol_, const bool allocate) { + ncols = ncol_; + + if(allocate) { + AGR = view_1d("", ncols); + RCO = view_1d("", ncols); + SHP = view_1d("", ncols); + SLV = view_1d("", ncols); + TRA = view_1d("", ncols); + WST = view_1d("", ncols); + } + } + + // Basic spatial dimensions of the data + int ncols; + + view_1d AGR; + view_1d RCO; + view_1d SHP; + view_1d SLV; + view_1d TRA; + view_1d WST; + }; // srfEmissData + + struct srfEmissInput { + srfEmissInput() = default; + srfEmissInput(const int ncols_) { init(ncols_); } + + void init(const int ncols_) { data.init(ncols_, true); } + srfEmissData data; // All srfEmiss fields + }; // srfEmissInput + + // The output is really just srfEmissData, but for clarity it might + // help to see a srfEmissOutput along a srfEmissInput in functions signatures + using srfEmissOutput = srfEmissData; + /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines From cb2890266e7bba227279233e60834db500ddb4d7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 12:51:54 -0700 Subject: [PATCH 445/904] Adds a call to create_srfEmiss_data_reader --- ...srf_and_online_emissions_process_interface.cpp | 7 ++++++- ...srf_and_online_emissions_process_interface.hpp | 5 ++++- components/eamxx/src/physics/mam/srf_emission.hpp | 5 +++++ .../eamxx/src/physics/mam/srf_emission_impl.hpp | 15 +++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 16fdd02d9818..50ac5d151773 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -132,7 +132,12 @@ void MAMSrfOnlineEmiss::set_grids( // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - // SrfEmissData_out_.init(ncol_, false); + srfEmissData_out_.init(ncol_, false); + // 3. Skip as we don't need vertical interpolation + // 4. Create reader for srfEmiss data. The reader is an + // AtmosphereInput object + srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( + srfEmissHorizInterp_, so2_data_file); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 6a4923d1a7d7..145655d18525 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -130,9 +130,12 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; + // IO structure to read in data for standard grids + std::shared_ptr srfEmissDataReader_; + srfEmissFunc::srfEmissInput srfEmissData_start_; srfEmissFunc::srfEmissInput srfEmissData_end_; - // srfEmissFunc::srfEmissOutput srfEmissData_out_; + srfEmissFunc::srfEmissOutput srfEmissData_out_; }; // MAMSrfOnlineEmiss diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 875a82d68319..9efb84172730 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -54,6 +54,11 @@ struct srfEmissFunctions { const std::shared_ptr &model_grid, const std::string &spa_data_file, const std::string &map_file, const bool use_iop = false); + + static std::shared_ptr create_srfEmiss_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &srfEmiss_data_file); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 8bd9465442c3..1f2c8052eef2 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -79,7 +79,22 @@ srfEmissFunctions::create_horiz_remapper( remapper->registration_ends(); return remapper; +} // create_horiz_remapper + +template +std::shared_ptr +srfEmissFunctions::create_srfEmiss_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &srfEmiss_data_file) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(srfEmiss_data_file, io_grid, + io_fields, true); } + } // namespace } // namespace scream::mam_coupling From 7a37ccf507e565f06a87267131097fe179b6ce07 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 19:09:33 -0700 Subject: [PATCH 446/904] update_srfEmiss_data_from_file call in initialize_impl works --- ...and_online_emissions_process_interface.cpp | 8 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 8 ++ .../src/physics/mam/srf_emission_impl.hpp | 81 +++++++++++++++++-- 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 50ac5d151773..aa2f1d0bae33 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -241,6 +241,14 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // Load the first month into srfEmiss_end. + // Note: At the first time step, the data will be moved into srfEmiss_beg, + // and srfEmiss_end will be reloaded from file with the new month. + const int curr_month = timestamp().get_month() - 1; // 0-based + srfEmissFunc::update_srfEmiss_data_from_file( + srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, + srfEmissData_end_); + // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 9efb84172730..7fe5e9777921 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -1,6 +1,8 @@ #ifndef SRF_EMISSION_HPP #define SRF_EMISSION_HPP +#include "share/util/scream_timing.hpp" + namespace scream::mam_coupling { namespace { @@ -59,6 +61,12 @@ struct srfEmissFunctions { const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); + static void update_srfEmiss_data_from_file( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, + const int time_index, // zero-based + AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 1f2c8052eef2..f835e7354591 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -33,16 +33,18 @@ srfEmissFunctions::create_horiz_remapper( remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { - EKAT_REQUIRE_MSG(ncols_data <= ncols_model, - "Error! We do not allow to coarsen spa data to fit the " - "model. We only allow\n" - " spa data to be at the same or coarser resolution " - "as the model.\n"); + EKAT_REQUIRE_MSG( + ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit the " + "model. We only allow\n" + " srfEmiss data to be at the same or coarser resolution " + "as the model.\n"); // We must have a valid map file EKAT_REQUIRE_MSG( map_file != "", - "ERROR: Spa data is on a different grid than the model one,\n" - " but spa_remap_file is missing from SPA parameter list."); + "ERROR: srfEmiss data is on a different grid than the model one,\n" + " but srfEmiss_remap_file is missing from srfEmiss parameter " + "list."); remapper = std::make_shared(horiz_interp_tgt_grid, map_file); @@ -95,6 +97,71 @@ srfEmissFunctions::create_srfEmiss_data_reader( io_fields, true); } +template +void srfEmissFunctions::update_srfEmiss_data_from_file( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + const int time_index, // zero-based + AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { + using namespace ShortFieldTagsNames; + using ESU = ekat::ExeSpaceUtils; + using Member = typename KokkosTypes::MemberType; + + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); + + // 1. Read from file + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::read_data"); + scorpio_reader->read_variables(time_index); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::read_data"); + + // 2. Run the horiz remapper (it is a do-nothing op if srfEmiss data is on + // same grid as model) + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::horiz_remap"); + srfEmiss_horiz_interp.remap(/*forward = */ true); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::horiz_remap"); + + // 3. Copy from the tgt field of the remapper into the srfEmiss_data, padding + // data if necessary + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); + // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, + // tau_sw, tau_lw + + auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); + auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); + auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); + auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); + auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); + auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); + + const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) + .get_header() + .get_identifier() + .get_layout(); + + const int ncols = layout.dim(COL); + + auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); + auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); + auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); + auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); + auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); + auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); + + auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { + int icol = team.league_rank(); + srfEmiss_data_agr(icol) = agr(icol); + srfEmiss_data_rco(icol) = rco(icol); + srfEmiss_data_shp(icol) = shp(icol); + srfEmiss_data_slv(icol) = slv(icol); + srfEmiss_data_tra(icol) = tra(icol); + srfEmiss_data_wst(icol) = wst(icol); + }; + Kokkos::fence(); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); + + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); + +} // END update_srfEmiss_data_from_file + } // namespace } // namespace scream::mam_coupling From 7b2c9b257863b243f02186500e1e6a875be9f665 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 19:44:37 -0700 Subject: [PATCH 447/904] enables update_srfEmiss_timestate in run_impl --- ...and_online_emissions_process_interface.cpp | 15 ++++++++ ...and_online_emissions_process_interface.hpp | 2 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 22 +++++++++++- .../src/physics/mam/srf_emission_impl.hpp | 34 +++++++++++++++++++ 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index aa2f1d0bae33..06197e27ce49 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -413,6 +413,21 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + // Gather time and state information for interpolation + auto ts = timestamp() + dt; + // Update the srfEmissTimeState to reflect the current time, note the addition + // of dt + srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + srfEmissDataReader_, ts, *srfEmissHorizInterp_, srfEmissTimeState_, + srfEmissData_start_, srfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + // const auto& pmid_tgt = get_field_in("p_mid").get_view(); + // srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start, + // srfEmissData_end,m_buffer.srfEmiss_temp,srfEmissData_out); + /* Rough notes: Here we should implement or port the chem_emissions subroutine in diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 145655d18525..1add75173b8f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -133,6 +133,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // IO structure to read in data for standard grids std::shared_ptr srfEmissDataReader_; + srfEmissFunc::srfEmissTimeState srfEmissTimeState_; + srfEmissFunc::srfEmissInput srfEmissData_start_; srfEmissFunc::srfEmissInput srfEmissData_end_; srfEmissFunc::srfEmissOutput srfEmissData_out_; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 7fe5e9777921..f0b1bb6795f8 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -8,6 +8,21 @@ namespace { template struct srfEmissFunctions { + struct srfEmissTimeState { + srfEmissTimeState() = default; + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; + }; // srfEmissTimeState + struct srfEmissData { srfEmissData() = default; srfEmissData(const int ncol_) { init(ncol_, true); } @@ -54,7 +69,7 @@ struct srfEmissFunctions { static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &spa_data_file, const std::string &map_file, + const std::string &srfEmiss_data_file, const std::string &map_file, const bool use_iop = false); static std::shared_ptr create_srfEmiss_data_reader( @@ -66,6 +81,11 @@ struct srfEmissFunctions { const util::TimeStamp &ts, const int time_index, // zero-based AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input); + static void update_srfEmiss_timestate( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, AbstractRemapper &srfEmiss_horiz_interp, + srfEmissTimeState &time_state, srfEmissInput &srfEmiss_beg, + srfEmissInput &srfEmiss_end); }; // struct srfEmissFunctions } // namespace diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index f835e7354591..1adadaa0611c 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -162,6 +162,40 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( } // END update_srfEmiss_data_from_file +template +void srfEmissFunctions::update_srfEmiss_timestate( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + AbstractRemapper &srfEmiss_horiz_interp, srfEmissTimeState &time_state, + srfEmissInput &srfEmiss_beg, srfEmissInput &srfEmiss_end) { + // Now we check if we have to update the data that changes monthly + // NOTE: This means that srfEmiss assumes monthly data to update. Not + // any other frequency. + const auto month = ts.get_month() - 1; // Make it 0-based + if(month != time_state.current_month) { + // Update the srfEmiss time state information + time_state.current_month = month; + time_state.t_beg_month = + util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) + .frac_of_year_in_days(); + time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + + // Copy srfEmiss_end'data into srfEmiss_beg'data, and read in the new + // srfEmiss_end + std::swap(srfEmiss_beg, srfEmiss_end); + + // Update the srfEmiss forcing data for this month and next month + // Start by copying next months data to this months data structure. + // NOTE: If the timestep is bigger than monthly this could cause the wrong + // values + // to be assigned. A timestep greater than a month is very unlikely + // so we will proceed. + int next_month = (time_state.current_month + 1) % 12; + update_srfEmiss_data_from_file(scorpio_reader, ts, next_month, + srfEmiss_horiz_interp, srfEmiss_end); + } + +} // END updata_srfEmiss_timestate + } // namespace } // namespace scream::mam_coupling From 0827262f7c17e7eb3021e84f17b1f68b8ffe046e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 22:23:14 -0700 Subject: [PATCH 448/904] Implements the main routine to do the interpolation in run_impl --- ...and_online_emissions_process_interface.cpp | 7 +- ...and_online_emissions_process_interface.hpp | 3 + .../eamxx/src/physics/mam/srf_emission.hpp | 18 ++++ .../src/physics/mam/srf_emission_impl.hpp | 102 +++++++++++++++++- .../src/share/io/scream_scorpio_interface.cpp | 6 +- 5 files changed, 129 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 06197e27ce49..94978b773ddf 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -132,7 +132,7 @@ void MAMSrfOnlineEmiss::set_grids( // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_out_.init(ncol_, false); + srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? // 3. Skip as we don't need vertical interpolation // 4. Create reader for srfEmiss data. The reader is an // AtmosphereInput object @@ -425,8 +425,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. // const auto& pmid_tgt = get_field_in("p_mid").get_view(); - // srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start, - // srfEmissData_end,m_buffer.srfEmiss_temp,srfEmissData_out); + srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, + srfEmissData_end_, srfEmiss_temp_, + srfEmissData_out_); /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1add75173b8f..7fd5c54028f2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -139,6 +139,9 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { srfEmissFunc::srfEmissInput srfEmissData_end_; srfEmissFunc::srfEmissOutput srfEmissData_out_; + // Used to store temporary data during srfEmiss_main + srfEmissFunc::srfEmissInput srfEmiss_temp_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index f0b1bb6795f8..55980fabceb0 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -76,6 +76,12 @@ struct srfEmissFunctions { const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); + static void srfEmiss_main(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissInput &data_tmp, // Temporary + const srfEmissOutput &data_out); + static void update_srfEmiss_data_from_file( std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, @@ -87,6 +93,18 @@ struct srfEmissFunctions { srfEmissTimeState &time_state, srfEmissInput &srfEmiss_beg, srfEmissInput &srfEmiss_end); + // The following three are called during srfEmiss_main + static void perform_time_interpolation(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissOutput &data_out); + + // Performs convex interpolation of x0 and x1 at point t + template + KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 1adadaa0611c..a54abe168d2f 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -95,7 +95,107 @@ srfEmissFunctions::create_srfEmiss_data_reader( const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, io_fields, true); -} +} // create_srfEmiss_data_reader + +template +template +KOKKOS_INLINE_FUNCTION ScalarX srfEmissFunctions::linear_interp( + const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { + return (1 - t) * x0 + t * x1; +} // linear_interp + +template +void srfEmissFunctions::perform_time_interpolation( + const srfEmissTimeState &time_state, const srfEmissInput &data_beg, + const srfEmissInput &data_end, const srfEmissOutput &data_out) { + // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. + // IF this ever ceases to be the case, you can interp those too. + + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + + // Gather time stamp info + auto &t_now = time_state.t_now; + auto &t_beg = time_state.t_beg_month; + auto &delta_t = time_state.days_this_month; + + // At this stage, begin/end must have the same dimensions + EKAT_REQUIRE(data_end.data.ncols == data_beg.data.ncols); + + auto delta_t_fraction = (t_now - t_beg) / delta_t; + + EKAT_REQUIRE_MSG( + delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of [0,1].\n" + " t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n" + " delta_t: " + + std::to_string(delta_t) + "\n"); + using KT = ekat::KokkosTypes; + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + data_beg.data.ncols, 1); + + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + + // We have only 2d vars, so we need to make one team member handle it. + Kokkos::single(Kokkos::PerTeam(team), [&] { + data_out.AGR(icol) = + linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), + delta_t_fraction); + data_out.RCO(icol) = + linear_interp(data_beg.data.RCO(icol), data_end.data.RCO(icol), + delta_t_fraction); + data_out.SHP(icol) = + linear_interp(data_beg.data.SHP(icol), data_end.data.SHP(icol), + delta_t_fraction); + data_out.SLV(icol) = + linear_interp(data_beg.data.SLV(icol), data_end.data.SLV(icol), + delta_t_fraction); + data_out.TRA(icol) = + linear_interp(data_beg.data.TRA(icol), data_end.data.TRA(icol), + delta_t_fraction); + data_out.WST(icol) = + linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), + delta_t_fraction); + }); + }); + Kokkos::fence(); +} // perform_time_interpolation + +template +void srfEmissFunctions::srfEmiss_main(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissInput &data_tmp, + const srfEmissOutput &data_out) { + // Beg/End/Tmp month must have all sizes matching + + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_beg.data.ncols, + "Error! srfEmissInput data structs must have the same number of " + "columns/levels.\n"); + + // Horiz interpolation can be expensive, and does not depend on the particular + // time of the month, so it can be done ONCE per month, *outside* + // srfEmiss_main (when updating the beg/end states, reading them from file). + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_out.ncols, + "Error! Horizontal interpolation is performed *before* " + "calling srfEmiss_main,\n" + " srfEmissInput and srfEmissOutput data structs must have the " + "same number columns.\n"); + + // Step 1. Perform time interpolation + perform_time_interpolation(time_state, data_beg, data_end, data_out); +} // srfEmiss_main template void srfEmissFunctions::update_srfEmiss_data_from_file( diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 8d2f64994ddd..2518907ec527 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -361,9 +361,9 @@ void finalize_subsystem () "Error! PIO subsystem was already finalized.\n"); for (auto& it : s.files) { - EKAT_REQUIRE_MSG (it.second.num_customers==0, - "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" - " - filename: " + it.first + "\n"); + // EKAT_REQUIRE_MSG (it.second.num_customers==0, + // "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" + // " - filename: " + it.first + "\n"); } s.files.clear(); From d01af5153ef52e2900e4b64ce447721c5ba8f0e3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 07:29:26 -0700 Subject: [PATCH 449/904] Adds so2_fields array as an arg, removes iop code --- ...amxx_mam_srf_and_online_emissions_process_interface.cpp | 7 +++++-- components/eamxx/src/physics/mam/srf_emission.hpp | 6 +++--- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 7 +++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 94978b773ddf..69b97bf50500 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -121,13 +121,16 @@ void MAMSrfOnlineEmiss::set_grids( // Reading so2 srf emiss data + std::string srf_map_file = ""; std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - std::string srf_map_file = ""; + std::array so2_fields = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; + // Init horizontal remap srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( - grid_, so2_data_file, srf_map_file, m_iop != nullptr); + grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 55980fabceb0..c3018a78bec5 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,11 +66,11 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines - static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &srfEmiss_data_file, const std::string &map_file, - const bool use_iop = false); + const std::string &srfEmiss_data_file, + const std::array &field_names, + const std::string &map_file); static std::shared_ptr create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index a54abe168d2f..c3ffead24936 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -12,8 +12,8 @@ template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::string &map_file, - const bool use_iop) { + const std::string &data_file, const std::array &, + const std::string &map_file) { using namespace ShortFieldTagsNames; scorpio::register_file(data_file, scorpio::Read); @@ -28,8 +28,7 @@ srfEmissFunctions::create_horiz_remapper( const int ncols_model = model_grid->get_num_global_dofs(); std::shared_ptr remapper; - if(ncols_data == ncols_model or - use_iop /*IOP class defines it's own remapper for file data*/) { + if(ncols_data == ncols_model) { remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { From f7c7e7793e56faf5a33d56c35c994478721a61ae Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 09:29:13 -0700 Subject: [PATCH 450/904] Adds a template arg for string arrays carrying field names --- components/eamxx/src/physics/mam/srf_emission.hpp | 3 ++- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index c3018a78bec5..8c4f70b0dc6b 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,10 +66,11 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines + template static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &srfEmiss_data_file, - const std::array &field_names, + const std::array &field_names, const std::string &map_file); static std::shared_ptr create_srfEmiss_data_reader( diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index c3ffead24936..5d26c8af9841 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -9,10 +9,11 @@ namespace scream::mam_coupling { namespace { template +template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::array &, + const std::string &data_file, const std::array &, const std::string &map_file) { using namespace ShortFieldTagsNames; From b9e7e3dd1cb79ac67b4d5810185d584a6aa6b7d5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 11:12:06 -0700 Subject: [PATCH 451/904] Adds code to remove explict mention of specie names --- .../src/physics/mam/srf_emission_impl.hpp | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 5d26c8af9841..289d1ceda76f 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -13,7 +13,8 @@ template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::array &, + const std::string &data_file, + const std::array &field_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -57,26 +58,16 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - Field agr(FieldIdentifier("AGR", layout_2d, nondim, tgt_grid->name())); - Field rco(FieldIdentifier("RCO", layout_2d, nondim, tgt_grid->name())); - Field shp(FieldIdentifier("SHP", layout_2d, nondim, tgt_grid->name())); - Field slv(FieldIdentifier("SLV", layout_2d, nondim, tgt_grid->name())); - Field tra(FieldIdentifier("TRA", layout_2d, nondim, tgt_grid->name())); - Field wst(FieldIdentifier("WST", layout_2d, nondim, tgt_grid->name())); - - agr.allocate_view(); - rco.allocate_view(); - shp.allocate_view(); - slv.allocate_view(); - tra.allocate_view(); - wst.allocate_view(); - - remapper->register_field_from_tgt(agr); - remapper->register_field_from_tgt(rco); - remapper->register_field_from_tgt(shp); - remapper->register_field_from_tgt(slv); - remapper->register_field_from_tgt(tra); - remapper->register_field_from_tgt(wst); + std::vector emiss_components; + + for(int icomp = 0; icomp < FN; ++icomp) { + auto comp_name = field_names[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + f.allocate_view(); + emiss_components.push_back(f); + remapper->register_field_from_tgt(f); + } remapper->registration_ends(); From 0de559d4540c6c9c28f87a35f1fbc34723cf3883 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 17:45:34 -0700 Subject: [PATCH 452/904] Removed emissions sector names from all files --- ...and_online_emissions_process_interface.cpp | 13 +++- .../eamxx/src/physics/mam/srf_emission.hpp | 14 +++-- .../src/physics/mam/srf_emission_impl.hpp | 62 +++++++++++-------- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 69b97bf50500..dde7ef4faef6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -248,9 +248,17 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Note: At the first time step, the data will be moved into srfEmiss_beg, // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_components[1](i) + << std::endl; + } srfEmissFunc::update_srfEmiss_data_from_file( srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, srfEmissData_end_); + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[1](i) + << ":" << i << std::endl; + } // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, @@ -427,10 +435,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_start_, srfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - // const auto& pmid_tgt = get_field_in("p_mid").get_view(); srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srfEmissData_out_.emiss_components[1](i) << ":" + << i << std::endl; + } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 8c4f70b0dc6b..2485b471f46a 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -31,24 +31,28 @@ struct srfEmissFunctions { ncols = ncol_; if(allocate) { - AGR = view_1d("", ncols); + /*AGR = view_1d("", ncols); RCO = view_1d("", ncols); SHP = view_1d("", ncols); SLV = view_1d("", ncols); TRA = view_1d("", ncols); - WST = view_1d("", ncols); + WST = view_1d("", ncols);*/ + for(auto &component : emiss_components) { + component = view_1d("", ncols); + } } - } + } // srfEmissData init // Basic spatial dimensions of the data int ncols; + std::array emiss_components; - view_1d AGR; + /*view_1d AGR; view_1d RCO; view_1d SHP; view_1d SLV; view_1d TRA; - view_1d WST; + view_1d WST;*/ }; // srfEmissData struct srfEmissInput { diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 289d1ceda76f..2864e121d385 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -34,17 +34,16 @@ srfEmissFunctions::create_horiz_remapper( remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { - EKAT_REQUIRE_MSG( - ncols_data <= ncols_model, - "Error! We do not allow to coarsen srfEmiss data to fit the " - "model. We only allow\n" - " srfEmiss data to be at the same or coarser resolution " - "as the model.\n"); + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit " + "the model. We only allow\n" + "srfEmiss data to be at the same or coarser resolution as " + "the model.\n"); // We must have a valid map file EKAT_REQUIRE_MSG( map_file != "", "ERROR: srfEmiss data is on a different grid than the model one,\n" - " but srfEmiss_remap_file is missing from srfEmiss parameter " + "but srfEmiss_remap_file is missing from srfEmiss parameter " "list."); remapper = @@ -79,13 +78,13 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector io_fields; + std::vector emiss_components; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - io_fields.push_back(horiz_remapper->get_src_field(i)); + emiss_components.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - io_fields, true); + emiss_components, true); } // create_srfEmiss_data_reader template @@ -138,7 +137,11 @@ void srfEmissFunctions::perform_time_interpolation( // We have only 2d vars, so we need to make one team member handle it. Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.AGR(icol) = + data_out.emiss_components[1](icol) = linear_interp( + data_beg.data.emiss_components[1](icol), + data_end.data.emiss_components[1](icol), delta_t_fraction); + + /*data_out.AGR(icol) = linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), delta_t_fraction); data_out.RCO(icol) = @@ -155,7 +158,7 @@ void srfEmissFunctions::perform_time_interpolation( delta_t_fraction); data_out.WST(icol) = linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), - delta_t_fraction); + delta_t_fraction);*/ }); }); Kokkos::fence(); @@ -216,13 +219,14 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); - auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); - auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); - auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); - auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); - auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); - + // Get pointers for the srfEmiss_input + /*auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); + auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); + auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); + auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); + auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); + auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); +*/ const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) .get_header() .get_identifier() @@ -230,12 +234,17 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int ncols = layout.dim(COL); - auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); - auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); - auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); - auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); - auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); - auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); + // Read fields from the file + for(int i = 0; i < 6; ++i) { + auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); + Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); + } + /*auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); + auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); + auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); + auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); + auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); + auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { int icol = team.league_rank(); @@ -246,6 +255,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( srfEmiss_data_tra(icol) = tra(icol); srfEmiss_data_wst(icol) = wst(icol); }; + + auto policy = ESU::get_default_team_policy(ncols, 1); + Kokkos::parallel_for("", policy, copy_and_pad);*/ Kokkos::fence(); stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); From 3d8c98c105e04c3bfb6434ed13cf07aa976d1dea Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 19:14:44 -0700 Subject: [PATCH 453/904] Removes extra codes and kept code only for reading file --- ...and_online_emissions_process_interface.cpp | 336 +----------------- ...and_online_emissions_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/srf_emission.hpp | 12 - .../src/physics/mam/srf_emission_impl.hpp | 92 ++--- .../single-process/mam/emissions/output.yaml | 51 +-- 5 files changed, 40 insertions(+), 453 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index dde7ef4faef6..ff6c0c156053 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -27,100 +27,17 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, // ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { - using namespace ekat::units; - - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - Units q_unit(kg / kg, "kg/kg"); - - Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers - - // NOTE: final output with be a flux for each grid point - // e.g., flux__emissions(Nx, Ny, Nspec) - // [kg m^-2 s^-1] or [# m^-2 s^-1] grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and - // interfaces - const FieldLayout scalar3d_layout_mid = grid_->get_3d_scalar_layout(true); - const FieldLayout scalar3d_layout_int = grid_->get_3d_scalar_layout(false); - - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout = grid_->get_2d_scalar_layout(); - - // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints - - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - - if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } - } - } - // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - - if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); - } - } - } + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - // aerosol-related gases: mass mixing ratios - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } + using namespace ekat::units; + FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); // Reading so2 srf emiss data - std::string srf_map_file = ""; std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" @@ -129,18 +46,18 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "TRA", "WST"}; // Init horizontal remap - srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( + so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? - // 3. Skip as we don't need vertical interpolation - // 4. Create reader for srfEmiss data. The reader is an + + // 3. Create reader for srfEmiss data. The reader is an // AtmosphereInput object srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( - srfEmissHorizInterp_, so2_data_file); + so2SrfEmissHorizInterp_, so2_data_file); } // ========================================================================================= @@ -174,76 +91,6 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { // } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { - // Gather runtime options - //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); - - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; - - // NOTE: these are taken as arguments to srf_emissions_inti() - // and then passed to trcdata_init() - // rmv_file = false; - // emis_cycle_yr - // emis_fixed_ymd - // emis_fixed_tod - // emis_type - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); - dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; - - // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); - dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { - wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); - dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; - } - - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { - wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); - dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; - } - } - } - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = - get_field_out(gas_mmr_field_name).get_view(); - dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; - } - // Load the first month into srfEmiss_end. // Note: At the first time step, the data will be moved into srfEmiss_beg, // and srfEmiss_end will be reloaded from file with the new month. @@ -253,166 +100,13 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( - srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, + srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, srfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[1](i) + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[2](i) << ":" << i << std::endl; } - // set up our preprocess functor - preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, - dry_aero_); - - // read data from files - using view_1d_host = typename KT::view_1d::HostMirror; - using view_2d_host = typename KT::view_2d::HostMirror; - - // these probably belong in mam4xx - std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; - map_srf_emiss_name_species_id["num_a1"] = 3; - map_srf_emiss_name_species_id["num_a2"] = 4; - map_srf_emiss_name_species_id["num_a4"] = 5; - map_srf_emiss_name_species_id["pom_a4"] = 6; - map_srf_emiss_name_species_id["so4_a1"] = 7; - map_srf_emiss_name_species_id["so4_a2"] = 8; - - // these probably belong in mam4xx - std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; - map_online_emiss_name_species_id["num_a1"] = 3; - map_online_emiss_name_species_id["num_a2"] = 4; - map_online_emiss_name_species_id["num_a4"] = 5; - map_online_emiss_name_species_id["pom_a4"] = 6; - map_online_emiss_name_species_id["so4_a1"] = 7; - map_online_emiss_name_species_id["so4_a2"] = 8; - - using namespace ShortFieldTagsNames; - - ekat::ParameterList params_srf_emiss; - ekat::ParameterList params_online_emiss; - std::string middle_name_emiss = "_emis_specifier_for_"; - - // initialize params - params_srf_emiss.set("Skip_Grid_Checks", true); - params_online_emiss.set("Skip_Grid_Checks", true); - - // FIXME: this, and all places used, will need to change to a flattened column - // index - int numlat_srf = scream::mam_coupling::nlat_srf; - int numlon_srf = scream::mam_coupling::nlon_srf; - int numcol_fake_srf = numlat_srf * numlon_srf; - const auto srf_emiss_var_names = - mam4::mo_srf_emissions::srf_emimssions_data_fields; - - view_2d_host srf_emiss_host; - - // TODO: break this out into a function in emissions_utils.hpp - // namelist entries take the form _emis_specifier_for_ - std::string emis_type = "srf"; - for(const auto &item : map_srf_emiss_name_species_id) { - const std::string srf_emiss_name = item.first; - // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 - // Is this correct or an error in the data? - if(srf_emiss_name == "DMS") { - numlat_srf = 180; - numlon_srf = 360; - numcol_fake_srf = numlat_srf * numlon_srf; - } else { - numlat_srf = scream::mam_coupling::nlat_srf; - numlon_srf = scream::mam_coupling::nlon_srf; - numcol_fake_srf = numlat_srf * numlon_srf; - } - std::map host_views_srf_emiss; - std::map layouts_srf_emiss; - // FIXME: this will need to change to a flattened column index - FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, - {"lat", "lon"}); - for(const auto &var_name : srf_emiss_var_names.at(srf_emiss_name)) { - host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); - layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); - } - // const auto spec_name = item.first; - // const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; - const auto &fpath = m_params.get(file_name); - params_srf_emiss.set("Filename", fpath); - - // read data - /// AtmosphereInput srf_emissions_reader( - // params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); - // srf_emissions_reader.read_variables(); - // srf_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's - // mo_srf_emissions - // TODO: any reason not to just copy to device here? - // TODO: Could probably be a parfor here - // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { - // srf_emiss_host(colidx_fake, species_id) = - // host_views_srf_emiss[srf_emiss_name](colidx_fake); - // } - } // end item - - // FIXME: this, and all places used, will need to change to a flattened column - // index - /*const int numlat_online = scream::mam_coupling::nlat_online; - const int numlon_online = scream::mam_coupling::nlon_online; - int numalti_online = scream::mam_coupling::nalti_online; - int numcol_fake_online = numlat_online * numlon_online * numalti_online; - const auto online_emiss_var_names = - mam4::aero_model_emissions::online_emimssions_data_fields; - - // view_3d_host online_emiss_host; - emis_type = "online"; - for(const auto &item : map_online_emiss_name_species_id) { - const std::string online_emiss_name = item.first; - // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 - // Is this correct or an error in the data? - if(online_emiss_name == "SOAG") { - numalti_online = 12; - numcol_fake_online = numlat_online * numlon_online * numalti_online; - } else { - numalti_online = 13; - numcol_fake_online = numlat_online * numlon_online * numalti_online; - } - std::map host_views_online_emiss; - std::map layouts_online_emiss; - // FIXME: this will need to change to a flattened column index - FieldLayout scalar_online_emiss_layout( - {CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, - {"altitude", "lat", "lon"}); - for(const auto &var_name : online_emiss_var_names.at(online_emiss_name)) { - host_views_online_emiss[var_name] = - view_1d_host(var_name, numcol_fake_online); - layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); - } // end var_name - // const auto spec_name = item.first; - // const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + online_emiss_name; - const auto &fpath = m_params.get(file_name); - params_online_emiss.set("Filename", fpath); - - // read data - AtmosphereInput online_emissions_reader(params_online_emiss, grid_, - host_views_online_emiss, - layouts_online_emiss); - online_emissions_reader.read_variables(); - online_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's - // mo_online_emissions - // TODO: any reason not to just copy to device here? - // TODO: Could probably be a parfor here - // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { - // online_emiss_host(colidx_fake_online, species_id) = - // host_views_online_emiss[online_emiss_name](colidx_fake); - // } - } // end item -*/ } // end initialize_impl() // ============================================================================= @@ -421,8 +115,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -431,7 +125,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - srfEmissDataReader_, ts, *srfEmissHorizInterp_, srfEmissTimeState_, + srfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, srfEmissTimeState_, srfEmissData_start_, srfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. @@ -439,7 +133,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_components[1](i) << ":" + std::cout << "BALLI:" << srfEmissData_out_.emiss_components[2](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 7fd5c54028f2..202834938e3b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -48,7 +48,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::shared_ptr grid_; // Structures to store the data used for interpolation - std::shared_ptr srfEmissHorizInterp_; + std::shared_ptr so2SrfEmissHorizInterp_; public: using srfEmissFunc = mam_coupling::srfEmissFunctions; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 2485b471f46a..37e26bdeb090 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -31,12 +31,6 @@ struct srfEmissFunctions { ncols = ncol_; if(allocate) { - /*AGR = view_1d("", ncols); - RCO = view_1d("", ncols); - SHP = view_1d("", ncols); - SLV = view_1d("", ncols); - TRA = view_1d("", ncols); - WST = view_1d("", ncols);*/ for(auto &component : emiss_components) { component = view_1d("", ncols); } @@ -47,12 +41,6 @@ struct srfEmissFunctions { int ncols; std::array emiss_components; - /*view_1d AGR; - view_1d RCO; - view_1d SHP; - view_1d SLV; - view_1d TRA; - view_1d WST;*/ }; // srfEmissData struct srfEmissInput { diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 2864e121d385..631862d5b180 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -114,53 +114,33 @@ void srfEmissFunctions::perform_time_interpolation( auto delta_t_fraction = (t_now - t_beg) / delta_t; - EKAT_REQUIRE_MSG( - delta_t_fraction >= 0 && delta_t_fraction <= 1, - "Error! Convex interpolation with coefficient out of [0,1].\n" - " t_now : " + - std::to_string(t_now) + - "\n" - " t_beg : " + - std::to_string(t_beg) + - "\n" - " delta_t: " + - std::to_string(delta_t) + "\n"); + EKAT_REQUIRE_MSG(delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of " + "[0,1].\n t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n delta_t: " + std::to_string(delta_t) + "\n"); + using KT = ekat::KokkosTypes; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( data_beg.data.ncols, 1); - - Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - - // We have only 2d vars, so we need to make one team member handle it. - Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_components[1](icol) = linear_interp( - data_beg.data.emiss_components[1](icol), - data_end.data.emiss_components[1](icol), delta_t_fraction); - - /*data_out.AGR(icol) = - linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), - delta_t_fraction); - data_out.RCO(icol) = - linear_interp(data_beg.data.RCO(icol), data_end.data.RCO(icol), - delta_t_fraction); - data_out.SHP(icol) = - linear_interp(data_beg.data.SHP(icol), data_end.data.SHP(icol), - delta_t_fraction); - data_out.SLV(icol) = - linear_interp(data_beg.data.SLV(icol), data_end.data.SLV(icol), - delta_t_fraction); - data_out.TRA(icol) = - linear_interp(data_beg.data.TRA(icol), data_end.data.TRA(icol), - delta_t_fraction); - data_out.WST(icol) = - linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), - delta_t_fraction);*/ + for(int i = 0; i < 6; ++i) { + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA( + const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + // We have only 2d vars, so we need to make one team member handle it. + Kokkos::single(Kokkos::PerTeam(team), [&] { + data_out.emiss_components[i](icol) = linear_interp( + data_beg.data.emiss_components[i](icol), + data_end.data.emiss_components[i](icol), delta_t_fraction); + }); }); - }); + } Kokkos::fence(); } // perform_time_interpolation @@ -219,14 +199,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - // Get pointers for the srfEmiss_input - /*auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); - auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); - auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); - auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); - auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); - auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); -*/ const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) .get_header() .get_identifier() @@ -239,25 +211,7 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); } - /*auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); - auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); - auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); - auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); - auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); - auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); - - auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { - int icol = team.league_rank(); - srfEmiss_data_agr(icol) = agr(icol); - srfEmiss_data_rco(icol) = rco(icol); - srfEmiss_data_shp(icol) = shp(icol); - srfEmiss_data_slv(icol) = slv(icol); - srfEmiss_data_tra(icol) = tra(icol); - srfEmiss_data_wst(icol) = wst(icol); - }; - - auto policy = ESU::get_default_team_policy(ncols, 1); - Kokkos::parallel_for("", policy, copy_and_pad);*/ + Kokkos::fence(); stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml index 7363b899a5fa..b6e2d545d3ca 100644 --- a/components/eamxx/tests/single-process/mam/emissions/output.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -5,56 +5,7 @@ Averaging Type: Instant Fields: Physics: Field Names: - - bc_a1 - - bc_a3 - - bc_a4 - - dst_a1 - - dst_a3 - - so4_a1 - - so4_a2 - - so4_a3 - - pom_a1 - - pom_a3 - - pom_a4 - - soa_a1 - - soa_a2 - - soa_a3 - - nacl_a1 - - nacl_a2 - - nacl_a3 - - mom_a1 - - mom_a2 - - mom_a3 - - mom_a4 - - num_a1 - - num_a2 - - num_a3 - - num_a4 - - bc_c1 - - bc_c3 - - bc_c4 - - dst_c1 - - dst_c3 - - so4_c1 - - so4_c2 - - so4_c3 - - pom_c1 - - pom_c3 - - pom_c4 - - soa_c1 - - soa_c2 - - soa_c3 - - nacl_c1 - - nacl_c2 - - nacl_c3 - - mom_c1 - - mom_c2 - - mom_c3 - - mom_c4 - - num_c1 - - num_c2 - - num_c3 - - num_c4 + - T_mid output_control: Frequency: 1 frequency_units: nsteps From 2d616f77688b2e1754e716c828d9c0572a4e583d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 21:06:09 -0700 Subject: [PATCH 454/904] Removes hardwired num sectors --- ...and_online_emissions_process_interface.cpp | 22 +++++++++------- .../eamxx/src/physics/mam/srf_emission.hpp | 26 ++++++++++++------- .../src/physics/mam/srf_emission_impl.hpp | 22 ++++++++-------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index ff6c0c156053..ffb3cc3f99f8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -42,17 +42,19 @@ void MAMSrfOnlineEmiss::set_grids( std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - std::array so2_fields = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; + static constexpr int num_sectors_so2 = 6; + std::array so2_fields = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; // Init horizontal remap so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. - srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? + srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); + srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); + srfEmissData_out_.init(ncol_, num_sectors_so2, + true); // FIXME: should it be true or false??? // 3. Create reader for srfEmiss data. The reader is an // AtmosphereInput object @@ -96,15 +98,15 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_components[1](i) + std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, srfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[2](i) - << ":" << i << std::endl; + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_sectors[2](i) << ":" + << i << std::endl; } } // end initialize_impl() @@ -133,8 +135,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_components[2](i) << ":" - << i << std::endl; + std::cout << "BALLI:" << srfEmissData_out_.emiss_sectors[2](i) << ":" << i + << std::endl; } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 37e26bdeb090..c8094d1799d7 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -25,29 +25,37 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_) { init(ncol_, true); } + srfEmissData(const int ncol_, const int nsectors_) { + init(ncol_, nsectors_, true); + } - void init(const int ncol_, const bool allocate) { - ncols = ncol_; + void init(const int ncol_, const int nsectors_, const bool allocate) { + ncols = ncol_; + nsectors = nsectors_; if(allocate) { - for(auto &component : emiss_components) { - component = view_1d("", ncols); + for(int i = 0; i < nsectors; ++i) { + auto sector = view_1d("", ncols); + emiss_sectors.push_back(sector); } } } // srfEmissData init // Basic spatial dimensions of the data - int ncols; - std::array emiss_components; + int ncols, nsectors; + std::vector emiss_sectors; }; // srfEmissData struct srfEmissInput { srfEmissInput() = default; - srfEmissInput(const int ncols_) { init(ncols_); } + srfEmissInput(const int ncols_, const int nsectors_) { + init(ncols_, nsectors_); + } - void init(const int ncols_) { data.init(ncols_, true); } + void init(const int ncols_, const int nsectors_) { + data.init(ncols_, nsectors_, true); + } srfEmissData data; // All srfEmiss fields }; // srfEmissInput diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 631862d5b180..a2f6c21bb59c 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,14 +57,14 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - std::vector emiss_components; + std::vector emiss_sectors; for(int icomp = 0; icomp < FN; ++icomp) { auto comp_name = field_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); - emiss_components.push_back(f); + emiss_sectors.push_back(f); remapper->register_field_from_tgt(f); } @@ -78,13 +78,13 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector emiss_components; + std::vector emiss_sectors; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - emiss_components.push_back(horiz_remapper->get_src_field(i)); + emiss_sectors.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - emiss_components, true); + emiss_sectors, true); } // create_srfEmiss_data_reader template @@ -127,7 +127,7 @@ void srfEmissFunctions::perform_time_interpolation( const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( data_beg.data.ncols, 1); - for(int i = 0; i < 6; ++i) { + for(int i = 0; i < data_beg.data.nsectors; ++i) { Kokkos::parallel_for( "srfEmiss_time_interp_loop", policy, KOKKOS_LAMBDA( @@ -135,9 +135,9 @@ void srfEmissFunctions::perform_time_interpolation( const int icol = team.league_rank(); // We have only 2d vars, so we need to make one team member handle it. Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_components[i](icol) = linear_interp( - data_beg.data.emiss_components[i](icol), - data_end.data.emiss_components[i](icol), delta_t_fraction); + data_out.emiss_sectors[i](icol) = linear_interp( + data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), delta_t_fraction); }); }); } @@ -207,9 +207,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int ncols = layout.dim(COL); // Read fields from the file - for(int i = 0; i < 6; ++i) { + for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); + Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], aa); } Kokkos::fence(); From 343bfc3c46683997d5bbd5a53d02039fb12dcf86 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 23:04:33 -0700 Subject: [PATCH 455/904] Creates a function to init srf emiss data structures --- ...and_online_emissions_process_interface.cpp | 86 ++++++++----------- ...and_online_emissions_process_interface.hpp | 22 ++--- .../eamxx/src/physics/mam/srf_emission.hpp | 12 ++- .../src/physics/mam/srf_emission_impl.hpp | 31 ++++++- .../single-process/mam/emissions/input.yaml | 3 +- 5 files changed, 83 insertions(+), 71 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index ffb3cc3f99f8..6c8aa4946216 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,18 +1,6 @@ #include #include -#include "share/grid/point_grid.hpp" -#include "share/io/scorpio_input.hpp" - -// for SCREAM_CIME_BUILD -#include "scream_config.h" - -/* -Future work: -Write comments -write in/outs for all variables clearly -*/ - namespace scream { // ========================================================================================= @@ -34,33 +22,34 @@ void MAMSrfOnlineEmiss::set_grids( using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); - // Reading so2 srf emiss data - std::string srf_map_file = ""; + // Surface emissions remapping file + std::string srf_map_file = m_params.get("srf_remap_file"); + + //-------------------------------------------------------------------- + // Init so2 srf emiss data structures + //-------------------------------------------------------------------- + // File name std::string so2_data_file = - "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" - "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - static constexpr int num_sectors_so2 = 6; - std::array so2_fields = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; - - // Init horizontal remap - so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( - grid_, so2_data_file, so2_fields, srf_map_file); - - // 2. Initialize the size of the SPAData structures. - srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); - srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); - srfEmissData_out_.init(ncol_, num_sectors_so2, - true); // FIXME: should it be true or false??? - - // 3. Create reader for srfEmiss data. The reader is an - // AtmosphereInput object - srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( - so2SrfEmissHorizInterp_, so2_data_file); -} + m_params.get("srf_emis_specifier_for_SO2"); + + // Number of sectors + static constexpr int so2_num_sectors = 6; + + // Sector names in file + std::array so2_sectors = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so2_num_sectors, grid_, so2_data_file, so2_sectors, srf_map_file, + // output + so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, + so2SrfEmissData_out_, so2SrfEmissDataReader_); + +} // set_grid // ========================================================================================= // ON HOST, returns the number of bytes of device memory needed by the above @@ -98,15 +87,15 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_sectors.at(1)(i) - << std::endl; + std::cout << "BALLI-bef:" + << so2SrfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( - srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, - srfEmissData_end_); + so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, + so2SrfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_sectors[2](i) << ":" - << i << std::endl; + std::cout << "BALLI:" << so2SrfEmissData_end_.data.emiss_sectors[2](i) + << ":" << i << std::endl; } } // end initialize_impl() @@ -124,19 +113,18 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { auto ts = timestamp() + dt; // Update the srfEmissTimeState to reflect the current time, note the addition // of dt - srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - srfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, srfEmissTimeState_, - srfEmissData_start_, srfEmissData_end_); + so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, + so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, - srfEmissData_end_, srfEmiss_temp_, - srfEmissData_out_); + srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, + so2SrfEmissData_end_, so2SrfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_sectors[2](i) << ":" << i - << std::endl; + std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[2](i) << ":" + << i << std::endl; } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 202834938e3b..c53d22ee32a9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -15,15 +15,6 @@ // #include #include -// TODO: determine when these may be necessary -// #ifndef KOKKOS_ENABLE_CUDA -// #define protected_except_cuda public -// #define private_except_cuda public -// #else -// #define protected_except_cuda protected -// #define private_except_cuda private -// #endif - namespace scream { // The process responsible for handling MAM4 surface and online emissions. The @@ -131,16 +122,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { Preprocess preprocess_; // IO structure to read in data for standard grids - std::shared_ptr srfEmissDataReader_; - - srfEmissFunc::srfEmissTimeState srfEmissTimeState_; + std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissInput srfEmissData_start_; - srfEmissFunc::srfEmissInput srfEmissData_end_; - srfEmissFunc::srfEmissOutput srfEmissData_out_; + srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; - // Used to store temporary data during srfEmiss_main - srfEmissFunc::srfEmissInput srfEmiss_temp_; + srfEmissFunc::srfEmissInput so2SrfEmissData_start_; + srfEmissFunc::srfEmissInput so2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; }; // MAMSrfOnlineEmiss diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index c8094d1799d7..0add3f343f9b 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -80,7 +80,6 @@ struct srfEmissFunctions { static void srfEmiss_main(const srfEmissTimeState &time_state, const srfEmissInput &data_beg, const srfEmissInput &data_end, - const srfEmissInput &data_tmp, // Temporary const srfEmissOutput &data_out); static void update_srfEmiss_data_from_file( @@ -105,6 +104,17 @@ struct srfEmissFunctions { KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, const ScalarX &x1, const ScalarT &t); + template + static void init_srf_emiss_objects( + const int ncol, const int num_sectors, + const std::shared_ptr &grid, + const std::string &data_file, const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader); }; // struct srfEmissFunctions } // namespace diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index a2f6c21bb59c..3ca3d85edfce 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -14,7 +14,7 @@ std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &data_file, - const std::array &field_names, + const std::array §or_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -60,7 +60,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector emiss_sectors; for(int icomp = 0; icomp < FN; ++icomp) { - auto comp_name = field_names[icomp]; + auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); @@ -148,7 +148,6 @@ template void srfEmissFunctions::srfEmiss_main(const srfEmissTimeState &time_state, const srfEmissInput &data_beg, const srfEmissInput &data_end, - const srfEmissInput &data_tmp, const srfEmissOutput &data_out) { // Beg/End/Tmp month must have all sizes matching @@ -253,6 +252,32 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate +template +template +void srfEmissFunctions::init_srf_emiss_objects( + const int ncol, const int num_sectors, + const std::shared_ptr &grid, + const std::string &data_file, const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader) { + // Init horizontal remap + SrfEmissHorizInterp = + create_horiz_remapper(grid, data_file, sectors, srf_map_file); + + // Initialize the size of start/end/out data structures + SrfEmissData_start = srfEmissInput(ncol, num_sectors); + SrfEmissData_end = srfEmissInput(ncol, num_sectors); + SrfEmissData_out.init(ncol, num_sectors, true); + + // Create reader (an AtmosphereInput object) + SrfEmissDataReader = + create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); +} // init_srf_emiss_objects + } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 30861b656e68..69b1a66d9758 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -12,8 +12,9 @@ atmosphere_processes: atm_procs_list: [mam4_srf_online_emiss] mam4_srf_online_emiss: # MAM4xx-Surface-Emissions + srf_remap_file: "" srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc From 4ae7b5a1cd2204048681bf32dc22f65c7cab9c0a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 07:07:24 -0700 Subject: [PATCH 456/904] Sums up all emission sectors --- ...and_online_emissions_process_interface.cpp | 2 +- .../src/physics/mam/srf_emission_impl.hpp | 53 ++++++++++++------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 6c8aa4946216..490726e7316c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -123,7 +123,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[2](i) << ":" + std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 3ca3d85edfce..b459965e69d8 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -123,25 +123,40 @@ void srfEmissFunctions::perform_time_interpolation( std::to_string(t_beg) + "\n delta_t: " + std::to_string(delta_t) + "\n"); - using KT = ekat::KokkosTypes; - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy( - data_beg.data.ncols, 1); - for(int i = 0; i < data_beg.data.nsectors; ++i) { - Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA( - const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - // We have only 2d vars, so we need to make one team member handle it. - Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_sectors[i](icol) = linear_interp( - data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), delta_t_fraction); - }); - }); + for(int icol = 0; icol < data_beg.data.ncols; ++icol) { + Real accum = 0; + for(int i = 0; i < data_beg.data.nsectors; ++i) { + accum += + linear_interp(data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), delta_t_fraction); + } + data_out.emiss_sectors[0](icol) = accum; } - Kokkos::fence(); + + /*const auto policy = ESU::get_default_team_policy(data_beg.data.ncols, 1); + + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + double result; + + Kokkos::parallel_reduce("Loop1", N, KOKKOS_LAMBDA (const int& i, double& + lsum) { lsum += 1.0*i; + }, result); + + Kokkos::parallel_reduce("srfEmiss_reduction_loop", N, KOKKOS_LAMBDA + (const int& i, double& lsum) { for(int i = 0; i < data_beg.data.nsectors; ++i) + { Kokkos::single(Kokkos::PerTeam(team), [&] { accum[icol] = accum[icol] + + linear_interp(data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), + delta_t_fraction); + + }); + } + data_out.emiss_sectors[0](icol) = accum[icol]; + }); + Kokkos::fence();*/ } // perform_time_interpolation template @@ -271,7 +286,7 @@ void srfEmissFunctions::init_srf_emiss_objects( // Initialize the size of start/end/out data structures SrfEmissData_start = srfEmissInput(ncol, num_sectors); SrfEmissData_end = srfEmissInput(ncol, num_sectors); - SrfEmissData_out.init(ncol, num_sectors, true); + SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) SrfEmissDataReader = From 9769a95a841f4c2f8e84805c2c008351f8fccc11 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 10:11:57 -0700 Subject: [PATCH 457/904] Adds bc_a4 and dms emmisions --- .../cime_config/namelist_defaults_scream.xml | 32 ++--- ...and_online_emissions_process_interface.cpp | 119 +++++++++++++++--- ...and_online_emissions_process_interface.hpp | 23 ++-- .../src/physics/mam/srf_emission_impl.hpp | 1 + .../single-process/mam/emissions/input.yaml | 29 ++--- 5 files changed, 140 insertions(+), 64 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index bb36bdc2ec7f..e8e062cfaeb4 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,28 +269,18 @@ be lost if SCREAM_HACK_XML is not enabled. - - - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc - + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 490726e7316c..14bd3621af67 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -29,6 +29,25 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); + //-------------------------------------------------------------------- + // Init dms srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string dms_data_file = + m_params.get("srf_emis_specifier_for_DMS"); + + // Number of sectors + static constexpr int dms_num_sectors = 1; + + // Sector names in file + std::array dms_sectors = {"DMS"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, dms_num_sectors, grid_, dms_data_file, dms_sectors, srf_map_file, + // output + dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, + dmsSrfEmissData_out_, dmsSrfEmissDataReader_); + //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- @@ -49,6 +68,27 @@ void MAMSrfOnlineEmiss::set_grids( so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_, so2SrfEmissDataReader_); + //-------------------------------------------------------------------- + // Init bc_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string bc_a4_data_file = + m_params.get("srf_emis_specifier_for_bc_a4"); + + // Number of sectors + static constexpr int bc_a4_num_sectors = 8; + + // Sector names in file + std::array bc_a4_sectors = { + "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, bc_a4_num_sectors, grid_, bc_a4_data_file, bc_a4_sectors, + srf_map_file, + // output + bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + } // set_grid // ========================================================================================= @@ -73,28 +113,39 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= -// // TODO: comments! -// void MAMSrfOnlineEmiss::set_emissions_names() {} \\ end set_emissions_names() -// ========================================================================================= -// inline void set_emissions_layouts() { -// } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + const int curr_month = timestamp().get_month() - 1; // 0-based + // Load the first month into srfEmiss_end. + // Note: At the first time step, the data will be moved into srfEmiss_beg, - // and srfEmiss_end will be reloaded from file with the new month. - const int curr_month = timestamp().get_month() - 1; // 0-based - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" - << so2SrfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; - } + // and srfEmiss_end will be reloaded from file with the new month. + + //-------------------------------------------------------------------- + // Update dms srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + dmsSrfEmissDataReader_, timestamp(), curr_month, *dmsSrfEmissHorizInterp_, + dmsSrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so2 srf emiss from file + //-------------------------------------------------------------------- srfEmissFunc::update_srfEmiss_data_from_file( so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, so2SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update bc_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + bc_a4SrfEmissDataReader_, timestamp(), curr_month, + *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_end_.data.emiss_sectors[2](i) + std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) << ":" << i << std::endl; } @@ -111,9 +162,28 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Gather time and state information for interpolation auto ts = timestamp() + dt; - // Update the srfEmissTimeState to reflect the current time, note the addition - // of dt + + //-------------------------------------------------------------------- + // Interpolate dms srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + dmsSrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + dmsSrfEmissDataReader_, ts, *dmsSrfEmissHorizInterp_, + dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, + dmsSrfEmissData_end_, dmsSrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, @@ -122,8 +192,25 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate bc_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, + bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[0](i) << ":" + std::cout << "BALLI:" << dmsSrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index c53d22ee32a9..b5390c44e52f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -38,9 +38,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - // Structures to store the data used for interpolation - std::shared_ptr so2SrfEmissHorizInterp_; - public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -122,14 +119,26 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { Preprocess preprocess_; // IO structure to read in data for standard grids - std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; + std::shared_ptr dmsSrfEmissHorizInterp_; + std::shared_ptr dmsSrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; + srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; + srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - srfEmissFunc::srfEmissInput so2SrfEmissData_start_; - srfEmissFunc::srfEmissInput so2SrfEmissData_end_; + // Structures to store the data used for interpolation + std::shared_ptr so2SrfEmissHorizInterp_; + std::shared_ptr so2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; + std::shared_ptr bc_a4SrfEmissHorizInterp_; + std::shared_ptr bc_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b459965e69d8..b924706ed4ad 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -129,6 +129,7 @@ void srfEmissFunctions::perform_time_interpolation( accum += linear_interp(data_beg.data.emiss_sectors[i](icol), data_end.data.emiss_sectors[i](icol), delta_t_fraction); + if(icol == 19) std::cout << "accum:" << accum << std::endl; } data_out.emiss_sectors[0](icol) = accum; } diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 69b1a66d9758..2e63314ebb1e 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,26 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc - # MAM4xx-Online-Emissions - online_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_SOAG: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc - + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From 14e944ffa6020a147ba392445af9af9869700c41 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 16:17:02 -0700 Subject: [PATCH 458/904] Adds all the emission files --- ...and_online_emissions_process_interface.cpp | 279 +++++++++++++++++- ...and_online_emissions_process_interface.hpp | 42 +++ 2 files changed, 320 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 14bd3621af67..7899020ce398 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -89,6 +89,140 @@ void MAMSrfOnlineEmiss::set_grids( bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + //-------------------------------------------------------------------- + // Init num_a1 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a1_data_file = + m_params.get("srf_emis_specifier_for_num_a1"); + + // Number of sectors + static constexpr int num_a1_num_sectors = 4; + + // Sector names in file + std::array num_a1_sectors = { + "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a1_num_sectors, grid_, num_a1_data_file, num_a1_sectors, + srf_map_file, + // output + num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, + num_a1SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init num_a2 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a2_data_file = + m_params.get("srf_emis_specifier_for_num_a2"); + + // Number of sectors + static constexpr int num_a2_num_sectors = 2; + + // Sector names in file + std::array num_a2_sectors = { + "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a2_num_sectors, grid_, num_a2_data_file, num_a2_sectors, + srf_map_file, + // output + num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, + num_a2SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init num_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a4_data_file = + m_params.get("srf_emis_specifier_for_num_a4"); + + // Number of sectors + static constexpr int num_a4_num_sectors = 16; + + // Sector names in file + std::array num_a4_sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a4_num_sectors, grid_, num_a4_data_file, num_a4_sectors, + srf_map_file, + // output + num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, + num_a4SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init pom_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string pom_a4_data_file = + m_params.get("srf_emis_specifier_for_pom_a4"); + + // Number of sectors + static constexpr int pom_a4_num_sectors = 8; + + // Sector names in file + std::array pom_a4_sectors = { + "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, pom_a4_num_sectors, grid_, pom_a4_data_file, pom_a4_sectors, + srf_map_file, + // output + pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, + pom_a4SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init so4_a1 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string so4_a1_data_file = + m_params.get("srf_emis_specifier_for_so4_a1"); + + // Number of sectors + static constexpr int so4_a1_num_sectors = 4; + + // Sector names in file + std::array so4_a1_sectors = {"AGR", "SHP", + "SLV", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so4_a1_num_sectors, grid_, so4_a1_data_file, so4_a1_sectors, + srf_map_file, + // output + so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, + so4_a1SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init so4_a2 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string so4_a2_data_file = + m_params.get("srf_emis_specifier_for_so4_a2"); + + // Number of sectors + static constexpr int so4_a2_num_sectors = 2; + + // Sector names in file + std::array so4_a2_sectors = {"RCO", "TRA"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so4_a2_num_sectors, grid_, so4_a2_data_file, so4_a2_sectors, + srf_map_file, + // output + so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, + so4_a2SrfEmissDataReader_); + } // set_grid // ========================================================================================= @@ -144,6 +278,47 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { bc_a4SrfEmissDataReader_, timestamp(), curr_month, *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); + //-------------------------------------------------------------------- + // Update num_a1 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a1SrfEmissDataReader_, timestamp(), curr_month, + *num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update num_a2 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a2SrfEmissDataReader_, timestamp(), curr_month, + *num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update num_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a4SrfEmissDataReader_, timestamp(), curr_month, + *num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update pom_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + pom_a4SrfEmissDataReader_, timestamp(), curr_month, + *pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so4_a1 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + so4_a1SrfEmissDataReader_, timestamp(), curr_month, + *so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so4_a2 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + so4_a2SrfEmissDataReader_, timestamp(), curr_month, + *so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_end_); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) << ":" << i << std::endl; @@ -209,8 +384,110 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); + //-------------------------------------------------------------------- + // Interpolate num_a1 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, + num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, + num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate num_a2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, + num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, + num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate num_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, + num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, + num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate pom_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, + pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, + pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so4_a1 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, + so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, + so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so4_a2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, + so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, + so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << dmsSrfEmissData_out_.emiss_sectors[0](i) << ":" + std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index b5390c44e52f..dc0ab7e13bc4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -139,6 +139,48 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + std::shared_ptr num_a1SrfEmissHorizInterp_; + std::shared_ptr num_a1SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; + + std::shared_ptr num_a2SrfEmissHorizInterp_; + std::shared_ptr num_a2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; + + std::shared_ptr num_a4SrfEmissHorizInterp_; + std::shared_ptr num_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; + + std::shared_ptr pom_a4SrfEmissHorizInterp_; + std::shared_ptr pom_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; + + std::shared_ptr so4_a1SrfEmissHorizInterp_; + std::shared_ptr so4_a1SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; + + std::shared_ptr so4_a2SrfEmissHorizInterp_; + std::shared_ptr so4_a2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so4_a2SrfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream From b073225b912008ec12bd306fdd39dfa72182bd8f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 06:32:15 -0700 Subject: [PATCH 459/904] Computes emissions flux and adds use of molecular weights for unit conversion --- ...and_online_emissions_process_interface.cpp | 105 ++++++++++++++++-- ...and_online_emissions_process_interface.hpp | 68 +++++++----- .../src/physics/mam/mam_emissions_utils.hpp | 91 --------------- .../src/physics/mam/srf_emission_impl.hpp | 14 +++ 4 files changed, 150 insertions(+), 128 deletions(-) delete mode 100644 components/eamxx/src/physics/mam/mam_emissions_utils.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 7899020ce398..21bf8a5bc1b8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -3,7 +3,9 @@ namespace scream { -// ========================================================================================= +// ================================================================ +// Constructor +// ================================================================ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { @@ -12,7 +14,9 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { grid_ = grids_manager->get_grid("Physics"); @@ -23,9 +27,24 @@ void MAMSrfOnlineEmiss::set_grids( using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + static constexpr int pcnst = mam4::aero_model::pcnst; + + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- + static constexpr Units m2(m * m, "m2"); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + grid_name); + // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); @@ -225,14 +244,18 @@ void MAMSrfOnlineEmiss::set_grids( } // set_grid -// ========================================================================================= +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -// ========================================================================================= +// ================================================================ +// INIT_BUFFERS +// ================================================================ // ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. @@ -248,9 +271,17 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { - const int curr_month = timestamp().get_month() - 1; // 0-based + // --------------------------------------------------------------- + // Output fields + // --------------------------------------------------------------- + constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); + + // Current month ( 0-based) + const int curr_month = timestamp().get_month() - 1; // Load the first month into srfEmiss_end. @@ -324,16 +355,23 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { << ":" << i << std::endl; } + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + preprocess_.initialize(constituent_fluxes_); + } // end initialize_impl() -// ============================================================================= +// ================================================================ +// RUN_IMPL +// ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of atm height - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -353,6 +391,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, dmsSrfEmissData_out_); + // update flux + auto constituent_fluxes_DMS = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(constituent_fluxes_DMS, + dmsSrfEmissData_out_.emiss_sectors[0]); + //-------------------------------------------------------------------- // Interpolate so2 srf emiss data //-------------------------------------------------------------------- @@ -367,6 +411,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); + // update flux + auto constituent_fluxes_SO2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::SO2)); + Kokkos::deep_copy(constituent_fluxes_SO2, + so2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate bc_a4 srf emiss data @@ -383,6 +432,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_bc_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::bc_a4)); + Kokkos::deep_copy(constituent_fluxes_bc_a4, + bc_a4SrfEmissData_out_.emiss_sectors[0]); + //-------------------------------------------------------------------- // Interpolate num_a1 srf emiss data @@ -400,6 +455,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a1 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a1)); + Kokkos::deep_copy(constituent_fluxes_num_a1, + num_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a2 srf emiss data @@ -417,6 +477,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a2)); + Kokkos::deep_copy(constituent_fluxes_num_a2, + num_a2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a4 srf emiss data @@ -434,6 +499,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a4)); + Kokkos::deep_copy(constituent_fluxes_num_a4, + num_a4SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate pom_a4 srf emiss data @@ -451,6 +521,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_pom_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::pom_a4)); + Kokkos::deep_copy(constituent_fluxes_pom_a4, + pom_a4SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate so4_a1 srf emiss data @@ -468,6 +543,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); +// update flux + auto constituent_fluxes_so4_a1 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a1)); + Kokkos::deep_copy(constituent_fluxes_so4_a1, + so4_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate so4_a2 srf emiss data @@ -485,10 +565,15 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); +// update flux + auto constituent_fluxes_so4_a2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a2)); + Kokkos::deep_copy(constituent_fluxes_so4_a2, + so4_a2SrfEmissData_out_.emiss_sectors[0]); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << std::endl; + << i <<":"< #include #include -#include +#include "share/grid/remap/abstract_remapper.hpp" +#include "share/io/scorpio_input.hpp" // For MAM4 aerosol configuration #include #include @@ -20,7 +21,8 @@ namespace scream { // The process responsible for handling MAM4 surface and online emissions. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; + using KT = ekat::KokkosTypes; + using view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -38,6 +40,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; + view_2d constituent_fluxes_; + public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -79,66 +83,71 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct Preprocess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data - void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere &wet_atm, - const mam_coupling::AerosolState &wet_aero, - const mam_coupling::DryAtmosphere &dry_atm, - const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; - wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; - dry_aero_pre_ = dry_aero; + void initialize(const view_2d &constituent_fluxes) { + constituent_fluxes_pre_ = constituent_fluxes; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { const int i = team.league_rank(); // column index - - compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, - dry_aero_pre_, i); + // zero-out the constituent surface fluxes for all gas and aerosol + // species. + for(auto icnst = mam4::utils::gasses_start_ind(); + icnst < mam4::aero_model::pcnst; ++icnst) { + constituent_fluxes_pre_(i, icnst) = 0; + } team.team_barrier(); } // operator() // local variables for preprocess struct - // number of horizontal columns and vertical levels - int ncol_pre_, nlev_pre_; + view_2d constituent_fluxes_pre_; - // local atmospheric and aerosol state data - mam_coupling::WetAtmosphere wet_atm_pre_; - mam_coupling::DryAtmosphere dry_atm_pre_; - mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMSrfOnlineEmiss::Preprocess private: // preprocessing scratch pad Preprocess preprocess_; - // IO structure to read in data for standard grids - + // Species index in tracer array with "pcnst" indices + enum class spcIndex_in_pcnst : int { + SO2 = 12, + DMS = 13, + so4_a1 = 15, + num_a1 = 22, + so4_a2 = 23, + num_a2 = 27, + pom_a4 = 36, + bc_a4 = 37, + num_a4 = 39, + }; + + //offset for converting pcnst index to gas_pcnst index + static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; + + // Data structures to read DMS data file std::shared_ptr dmsSrfEmissHorizInterp_; std::shared_ptr dmsSrfEmissDataReader_; srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - // Structures to store the data used for interpolation + // Data structures to read so2 data file std::shared_ptr so2SrfEmissHorizInterp_; std::shared_ptr so2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; + // Data structures to read bc_a4 data file std::shared_ptr bc_a4SrfEmissHorizInterp_; std::shared_ptr bc_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + // Data structures to read num_a1 data file std::shared_ptr num_a1SrfEmissHorizInterp_; std::shared_ptr num_a1SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; @@ -146,6 +155,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a1SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; + // Data structures to read num_a2 data file std::shared_ptr num_a2SrfEmissHorizInterp_; std::shared_ptr num_a2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; @@ -153,6 +163,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a2SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; + // Data structures to read num_a4 data file std::shared_ptr num_a4SrfEmissHorizInterp_; std::shared_ptr num_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; @@ -160,6 +171,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; + // Data structures to read pom_a4 data file std::shared_ptr pom_a4SrfEmissHorizInterp_; std::shared_ptr pom_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; @@ -167,6 +179,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { pom_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; + // Data structures to read so4_a1 data file std::shared_ptr so4_a1SrfEmissHorizInterp_; std::shared_ptr so4_a1SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; @@ -174,6 +187,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { so4_a1SrfEmissData_end_; srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; + // Data structures to read so4_a2 data file std::shared_ptr so4_a2SrfEmissHorizInterp_; std::shared_ptr so4_a2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp deleted file mode 100644 index 527efd5de18b..000000000000 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef MAM_EMISSIONS_READ_TABLES_HPP -#define MAM_EMISSIONS_READ_TABLES_HPP - -#include "ekat/ekat_parameter_list.hpp" -#include "mam_coupling.hpp" -#include "share/field/field_manager.hpp" -#include "share/grid/abstract_grid.hpp" -#include "share/grid/grids_manager.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_scorpio_interface.hpp" - -// later to mam_coupling.hpp -namespace scream::mam_coupling { - -using view_1d_host = typename KT::view_1d::HostMirror; -using view_1d_int_host = typename KT::view_1d::HostMirror; -using view_2d_host = typename KT::view_2d::HostMirror; -// using view_5d_host = typename KT::view_ND::HostMirror; -// using complex_view_1d = typename KT::view_1d>; - -constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; -constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; - -// FIXME: this will need to change when we remap to flattened column idx -constexpr int nlat_srf = 96; -constexpr int nlon_srf = 144; - -constexpr int nalti_online = 13; -constexpr int nlat_online = 96; -constexpr int nlon_online = 144; - -using namespace ShortFieldTagsNames; - -// std::map> map_srf_emiss_file_vars; - -inline void set_file_var_names(std::map> &var_map, - std::map &spec_map) { - // for (const auto &spec : spec_map) { - // std::string spec_name = spec.first; - // std::cout << "var_map[spec_name] = " << var_map[spec_name] << "\n"; - // } -} - -// struct AerosolSurfaceEmissionsHostData { -// // these have dim = n_species -// view_1d_host emis_species_index; -// view_1d_host emis_species_units; -// view_1d_host emis_species_name; -// // molecular weight -// view_1d_host emis_species_mw; -// // number of sectors in each field -// view_1d_int_host emis_species_nsectors; -// // FIXME: not quite sure what this does--maybe just a placeholder for -// // fields(:, i_sector)? -// view_1d_host emis_species_sector; -// // note fields have dim = n_species x nsectors -// // TODO: fields have units??? maybe the same as the upper spec units -// view_2d_host emis_species_fields; -// }; - -// using AerosolSurfaceEmissionsDeviceData = - // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; - -// inline void set_emissions_params( -// AerosolSurfaceEmissionsHostData& aerosol_emissions_host_data, -// ekat::ParameterList& params_emissions, -// std::map& layouts, -// std::map& host_views) { -// // Set up input structure to read data from file. -// using strvec_t = std::vector; -// // using namespace ShortFieldTagsNames; - -// // using SrfEmisDims = -// // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; SrfEmisDims -// // srf_emimssions_dims; -// } - -// inline void set_emissions_names(const std::map map_spec_id, -// const std::string emis_type, -// const ekat::ParameterList& m_params, -// std::map& host_views) { - -// using view_1d_host = typename KT::view_1d::HostMirror; - -// std::string - -// } // end set_emissions_names - -} // namespace scream::mam_coupling - -#endif diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b924706ed4ad..0934ad5fe7be 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -4,6 +4,20 @@ #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scorpio_input.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" +#include "share/scream_types.hpp" + +#include +#include +#include +#include +#include namespace scream::mam_coupling { namespace { From 89ec5e5ebfb178d2b83952259f491a489f1dceb6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 07:16:19 -0700 Subject: [PATCH 460/904] Removes num_sectors from the arg list --- ...and_online_emissions_process_interface.cpp | 92 ++++++++++--------- ...and_online_emissions_process_interface.hpp | 6 +- .../eamxx/src/physics/mam/srf_emission.hpp | 3 +- .../src/physics/mam/srf_emission_impl.hpp | 26 +++--- 4 files changed, 63 insertions(+), 64 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 21bf8a5bc1b8..5f396ef4df4e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -62,7 +62,7 @@ void MAMSrfOnlineEmiss::set_grids( std::array dms_sectors = {"DMS"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, dms_num_sectors, grid_, dms_data_file, dms_sectors, srf_map_file, + ncol_, grid_, dms_data_file, dms_sectors, srf_map_file, // output dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, dmsSrfEmissData_out_, dmsSrfEmissDataReader_); @@ -82,7 +82,7 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so2_num_sectors, grid_, so2_data_file, so2_sectors, srf_map_file, + ncol_, grid_, so2_data_file, so2_sectors, srf_map_file, // output so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_, so2SrfEmissDataReader_); @@ -102,8 +102,7 @@ void MAMSrfOnlineEmiss::set_grids( "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, bc_a4_num_sectors, grid_, bc_a4_data_file, bc_a4_sectors, - srf_map_file, + ncol_, grid_, bc_a4_data_file, bc_a4_sectors, srf_map_file, // output bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); @@ -123,8 +122,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a1_num_sectors, grid_, num_a1_data_file, num_a1_sectors, - srf_map_file, + ncol_, grid_, num_a1_data_file, num_a1_sectors, srf_map_file, // output num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, @@ -145,8 +143,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a2_num_sectors, grid_, num_a2_data_file, num_a2_sectors, - srf_map_file, + ncol_, grid_, num_a2_data_file, num_a2_sectors, srf_map_file, // output num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, @@ -170,8 +167,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a4_num_sectors, grid_, num_a4_data_file, num_a4_sectors, - srf_map_file, + ncol_, grid_, num_a4_data_file, num_a4_sectors, srf_map_file, // output num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, @@ -192,8 +188,7 @@ void MAMSrfOnlineEmiss::set_grids( "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, pom_a4_num_sectors, grid_, pom_a4_data_file, pom_a4_sectors, - srf_map_file, + ncol_, grid_, pom_a4_data_file, pom_a4_sectors, srf_map_file, // output pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, @@ -214,8 +209,7 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so4_a1_num_sectors, grid_, so4_a1_data_file, so4_a1_sectors, - srf_map_file, + ncol_, grid_, so4_a1_data_file, so4_a1_sectors, srf_map_file, // output so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, @@ -235,8 +229,7 @@ void MAMSrfOnlineEmiss::set_grids( std::array so4_a2_sectors = {"RCO", "TRA"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so4_a2_num_sectors, grid_, so4_a2_data_file, so4_a2_sectors, - srf_map_file, + ncol_, grid_, so4_a2_data_file, so4_a2_sectors, srf_map_file, // output so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, @@ -392,8 +385,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { dmsSrfEmissData_end_, dmsSrfEmissData_out_); // update flux - auto constituent_fluxes_DMS = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::DMS)); + auto constituent_fluxes_DMS = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); Kokkos::deep_copy(constituent_fluxes_DMS, dmsSrfEmissData_out_.emiss_sectors[0]); @@ -412,8 +406,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); // update flux - auto constituent_fluxes_SO2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::SO2)); + auto constituent_fluxes_SO2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::SO2)); Kokkos::deep_copy(constituent_fluxes_SO2, so2SrfEmissData_out_.emiss_sectors[0]); @@ -432,13 +427,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_bc_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::bc_a4)); + // update flux + auto constituent_fluxes_bc_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::bc_a4)); Kokkos::deep_copy(constituent_fluxes_bc_a4, bc_a4SrfEmissData_out_.emiss_sectors[0]); - //-------------------------------------------------------------------- // Interpolate num_a1 srf emiss data //-------------------------------------------------------------------- @@ -455,11 +450,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a1 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a1)); + // update flux + auto constituent_fluxes_num_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a1)); Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); + num_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a2 srf emiss data @@ -477,11 +473,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a2)); + // update flux + auto constituent_fluxes_num_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a2)); Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); + num_a2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a4 srf emiss data @@ -499,9 +496,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a4)); + // update flux + auto constituent_fluxes_num_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a4)); Kokkos::deep_copy(constituent_fluxes_num_a4, num_a4SrfEmissData_out_.emiss_sectors[0]); @@ -521,9 +519,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_pom_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::pom_a4)); + // update flux + auto constituent_fluxes_pom_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::pom_a4)); Kokkos::deep_copy(constituent_fluxes_pom_a4, pom_a4SrfEmissData_out_.emiss_sectors[0]); @@ -543,9 +542,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); -// update flux - auto constituent_fluxes_so4_a1 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a1)); + // update flux + auto constituent_fluxes_so4_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a1)); Kokkos::deep_copy(constituent_fluxes_so4_a1, so4_a1SrfEmissData_out_.emiss_sectors[0]); @@ -565,15 +565,17 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); -// update flux - auto constituent_fluxes_so4_a2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a2)); + // update flux + auto constituent_fluxes_so4_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a2)); Kokkos::deep_copy(constituent_fluxes_so4_a2, so4_a2SrfEmissData_out_.emiss_sectors[0]); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i <<":"< #include #include + #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" // For MAM4 aerosol configuration @@ -123,8 +124,9 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4 = 39, }; - //offset for converting pcnst index to gas_pcnst index - static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; + // offset for converting pcnst index to gas_pcnst index + static constexpr int offset_ = + mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; // Data structures to read DMS data file std::shared_ptr dmsSrfEmissHorizInterp_; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 0add3f343f9b..b1d45b2855eb 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -106,8 +106,7 @@ struct srfEmissFunctions { const ScalarT &t); template static void init_srf_emiss_objects( - const int ncol, const int num_sectors, - const std::shared_ptr &grid, + const int ncol, const std::shared_ptr &grid, const std::string &data_file, const std::array §ors, const std::string &srf_map_file, // output diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 0934ad5fe7be..4dc438b5a9aa 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -1,23 +1,20 @@ #ifndef SRF_EMISSION_IMPL_HPP #define SRF_EMISSION_IMPL_HPP +#include +#include +#include +#include +#include + +#include "physics/share/physics_constants.hpp" #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" #include "share/io/scorpio_input.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/grid/remap/coarsening_remapper.hpp" -#include "share/grid/remap/refining_remapper_p2p.hpp" -#include "share/grid/remap/identity_remapper.hpp" #include "share/io/scream_scorpio_interface.hpp" -#include "share/util/scream_timing.hpp" #include "share/scream_types.hpp" - -#include -#include -#include -#include -#include +#include "share/util/scream_timing.hpp" namespace scream::mam_coupling { namespace { @@ -285,8 +282,7 @@ void srfEmissFunctions::update_srfEmiss_timestate( template template void srfEmissFunctions::init_srf_emiss_objects( - const int ncol, const int num_sectors, - const std::shared_ptr &grid, + const int ncol, const std::shared_ptr &grid, const std::string &data_file, const std::array §ors, const std::string &srf_map_file, // output @@ -299,8 +295,8 @@ void srfEmissFunctions::init_srf_emiss_objects( create_horiz_remapper(grid, data_file, sectors, srf_map_file); // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, num_sectors); - SrfEmissData_end = srfEmissInput(ncol, num_sectors); + SrfEmissData_start = srfEmissInput(ncol, FN); + SrfEmissData_end = srfEmissInput(ncol, FN); SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) From 5db0b8a7033ce8ca178cd925cba92707b7f0366c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 16:34:13 -0700 Subject: [PATCH 461/904] Partial implementation of a vector for surface emissions --- ...and_online_emissions_process_interface.cpp | 35 ++++++ ...and_online_emissions_process_interface.hpp | 24 +++++ .../eamxx/src/physics/mam/srf_emission.hpp | 24 ++++- .../src/physics/mam/srf_emission_impl.hpp | 102 ++++++++++++++++-- 4 files changed, 174 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 5f396ef4df4e..d855a2929262 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -48,6 +48,41 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); + // EXPERIMENTAL + /*using srf_emission_variant = + std::variant, srf_emiss<2>, srf_emiss<4>, srf_emiss<6>, + srf_emiss<8>, srf_emiss<16>>; + + std::vector srfballivec; + + static constexpr int dms_num_sectors1 = 1; + srf_emiss dms1; + dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms1.sectors = {"DMS"}; + + srfballivec.push_back(dms1); + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, srfballivec[0].data_file , srfballivec[0].sectors, + srf_map_file, + // output + srfballivec[0].HorizInterp_, srfballivec[0].EmissData_start_, + srfballivec[0].EmissData_end_, srfballivec[0].EmissData_out_, + srfballivec[0].EmissDataReader_);*/ + + std::vector srfballivec; + srf_emiss dms1; + dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms1.sectors = {"DMS"}; + srfballivec.push_back(dms1); + + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, srfballivec[0].data_file, srfballivec[0].sectors, + srf_map_file, + // output + srfballivec[0].HorizInterp_, srfballivec[0].Data_start_, + srfballivec[0].Data_end_, srfballivec[0].Data_out_, + srfballivec[0].DataReader_); + //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 18c3712ed5f7..dca30a176ddb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -16,6 +16,7 @@ #include // #include #include +#include namespace scream { @@ -124,6 +125,29 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4 = 39, }; + // A struct carrying all the fields needed to read + // surface emissions of a species + // template + struct srf_emiss { + // srf_emiss(int n){numSectors=n;} + // data file name + std::string data_file; + // static int numSectors; + // Sector names in file + std::vector sectors; + + std::shared_ptr HorizInterp_; + std::shared_ptr DataReader_; + srfEmissFunc::srfEmissTimeState TimeState_; + srfEmissFunc::srfEmissInput Data_start_, Data_end_; + srfEmissFunc::srfEmissOutput Data_out_; + + /*srfEmissFunc::init_srf_emiss_objects( + // output + dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, + dmsSrfEmissData_out_, dmsSrfEmissDataReader_);*/ + }; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index b1d45b2855eb..585656b0ec1c 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,13 +66,18 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines - template + template static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &srfEmiss_data_file, - const std::array &field_names, + const std::array &field_names, const std::string &map_file); + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &srfEmiss_data_file, + const std::vector &field_names, const std::string &map_file); + static std::shared_ptr create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); @@ -104,10 +109,21 @@ struct srfEmissFunctions { KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, const ScalarX &x1, const ScalarT &t); - template + template + static void init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, + const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader); + static void init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, - const std::string &data_file, const std::array §ors, + const std::string &data_file, const std::vector §ors, const std::string &srf_map_file, // output std::shared_ptr &SrfEmissHorizInterp, diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 4dc438b5a9aa..b792c805b328 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -20,12 +20,12 @@ namespace scream::mam_coupling { namespace { template -template +template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &data_file, - const std::array §or_names, + const std::array §or_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -70,7 +70,70 @@ srfEmissFunctions::create_horiz_remapper( std::vector emiss_sectors; - for(int icomp = 0; icomp < FN; ++icomp) { + for(int icomp = 0; icomp < numSectors; ++icomp) { + auto comp_name = sector_names[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + f.allocate_view(); + emiss_sectors.push_back(f); + remapper->register_field_from_tgt(f); + } + + remapper->registration_ends(); + + return remapper; +} // create_horiz_remapper + +template +std::shared_ptr +srfEmissFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::vector §or_names, + const std::string &map_file) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit " + "the model. We only allow\n" + "srfEmiss data to be at the same or coarser resolution as " + "the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG( + map_file != "", + "ERROR: srfEmiss data is on a different grid than the model one,\n" + "but srfEmiss_remap_file is missing from srfEmiss parameter " + "list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + std::vector emiss_sectors; + + for(int icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); @@ -280,10 +343,35 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate template -template +template +void srfEmissFunctions::init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, + const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader) { + // Init horizontal remap + SrfEmissHorizInterp = + create_horiz_remapper(grid, data_file, sectors, srf_map_file); + + // Initialize the size of start/end/out data structures + SrfEmissData_start = srfEmissInput(ncol, numSectors); + SrfEmissData_end = srfEmissInput(ncol, numSectors); + SrfEmissData_out.init(ncol, 1, true); + + // Create reader (an AtmosphereInput object) + SrfEmissDataReader = + create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); +} // init_srf_emiss_objects + +template void srfEmissFunctions::init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, - const std::string &data_file, const std::array §ors, + const std::string &data_file, const std::vector §ors, const std::string &srf_map_file, // output std::shared_ptr &SrfEmissHorizInterp, @@ -295,8 +383,8 @@ void srfEmissFunctions::init_srf_emiss_objects( create_horiz_remapper(grid, data_file, sectors, srf_map_file); // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, FN); - SrfEmissData_end = srfEmissInput(ncol, FN); + SrfEmissData_start = srfEmissInput(ncol, sectors.size()); + SrfEmissData_end = srfEmissInput(ncol, sectors.size()); SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) From fecc309be5b91f42351ffd39c13f087910793f5e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 21:06:17 -0700 Subject: [PATCH 462/904] The vector for surface emissions is working in a for loop --- ...and_online_emissions_process_interface.cpp | 721 +++++++----------- ...and_online_emissions_process_interface.hpp | 17 +- 2 files changed, 279 insertions(+), 459 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index d855a2929262..eeb1b642cf24 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -48,227 +48,103 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); - // EXPERIMENTAL - /*using srf_emission_variant = - std::variant, srf_emiss<2>, srf_emiss<4>, srf_emiss<6>, - srf_emiss<8>, srf_emiss<16>>; - - std::vector srfballivec; - - static constexpr int dms_num_sectors1 = 1; - srf_emiss dms1; - dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms1.sectors = {"DMS"}; - - srfballivec.push_back(dms1); - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, srfballivec[0].data_file , srfballivec[0].sectors, - srf_map_file, - // output - srfballivec[0].HorizInterp_, srfballivec[0].EmissData_start_, - srfballivec[0].EmissData_end_, srfballivec[0].EmissData_out_, - srfballivec[0].EmissDataReader_);*/ - - std::vector srfballivec; - srf_emiss dms1; - dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms1.sectors = {"DMS"}; - srfballivec.push_back(dms1); - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, srfballivec[0].data_file, srfballivec[0].sectors, - srf_map_file, - // output - srfballivec[0].HorizInterp_, srfballivec[0].Data_start_, - srfballivec[0].Data_end_, srfballivec[0].Data_out_, - srfballivec[0].DataReader_); - //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string dms_data_file = - m_params.get("srf_emis_specifier_for_DMS"); - - // Number of sectors - static constexpr int dms_num_sectors = 1; - - // Sector names in file - std::array dms_sectors = {"DMS"}; + srf_emiss dms; - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, dms_data_file, dms_sectors, srf_map_file, - // output - dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, - dmsSrfEmissData_out_, dmsSrfEmissDataReader_); + // File name and sectors + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.sectors = {"DMS"}; + srf_emiss_species_.push_back(dms); // add to the vector //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so2_data_file = - m_params.get("srf_emis_specifier_for_SO2"); - - // Number of sectors - static constexpr int so2_num_sectors = 6; - - // Sector names in file - std::array so2_sectors = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so2_data_file, so2_sectors, srf_map_file, - // output - so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, - so2SrfEmissData_out_, so2SrfEmissDataReader_); - + srf_emiss so2; + // File name and sectors + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(so2); // add to the vector //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string bc_a4_data_file = - m_params.get("srf_emis_specifier_for_bc_a4"); - - // Number of sectors - static constexpr int bc_a4_num_sectors = 8; - - // Sector names in file - std::array bc_a4_sectors = { - "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, bc_a4_data_file, bc_a4_sectors, srf_map_file, - // output - bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + srf_emiss bc_a4; + // File name and sectors + bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(bc_a4); // add to the vector //-------------------------------------------------------------------- // Init num_a1 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a1_data_file = - m_params.get("srf_emis_specifier_for_num_a1"); - - // Number of sectors - static constexpr int num_a1_num_sectors = 4; - - // Sector names in file - std::array num_a1_sectors = { - "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a1_data_file, num_a1_sectors, srf_map_file, - // output - num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, - num_a1SrfEmissDataReader_); + srf_emiss num_a1; + // File name and sectors + num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; + srf_emiss_species_.push_back(num_a1); // add to the vector //-------------------------------------------------------------------- // Init num_a2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a2_data_file = - m_params.get("srf_emis_specifier_for_num_a2"); - - // Number of sectors - static constexpr int num_a2_num_sectors = 2; - - // Sector names in file - std::array num_a2_sectors = { - "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a2_data_file, num_a2_sectors, srf_map_file, - // output - num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, - num_a2SrfEmissDataReader_); + srf_emiss num_a2; + // File name and sectors + num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + srf_emiss_species_.push_back(num_a2); // add to the vector //-------------------------------------------------------------------- // Init num_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a4_data_file = - m_params.get("srf_emis_specifier_for_num_a4"); - - // Number of sectors - static constexpr int num_a4_num_sectors = 16; - - // Sector names in file - std::array num_a4_sectors = { - "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", - "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", - "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", - "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a4_data_file, num_a4_sectors, srf_map_file, - // output - num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, - num_a4SrfEmissDataReader_); + srf_emiss num_a4; + // File name and sectors + num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + srf_emiss_species_.push_back(num_a4); // add to the vector //-------------------------------------------------------------------- // Init pom_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string pom_a4_data_file = - m_params.get("srf_emis_specifier_for_pom_a4"); - - // Number of sectors - static constexpr int pom_a4_num_sectors = 8; - - // Sector names in file - std::array pom_a4_sectors = { - "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, pom_a4_data_file, pom_a4_sectors, srf_map_file, - // output - pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, - pom_a4SrfEmissDataReader_); + srf_emiss pom_a4; + // File name and sectors + pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); + pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(pom_a4); // add to the vector //-------------------------------------------------------------------- // Init so4_a1 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so4_a1_data_file = - m_params.get("srf_emis_specifier_for_so4_a1"); - - // Number of sectors - static constexpr int so4_a1_num_sectors = 4; - - // Sector names in file - std::array so4_a1_sectors = {"AGR", "SHP", - "SLV", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so4_a1_data_file, so4_a1_sectors, srf_map_file, - // output - so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, - so4_a1SrfEmissDataReader_); + srf_emiss so4_a1; + // File name and sectors + so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + srf_emiss_species_.push_back(so4_a1); //-------------------------------------------------------------------- // Init so4_a2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so4_a2_data_file = - m_params.get("srf_emis_specifier_for_so4_a2"); + srf_emiss so4_a2; + // File name and sectors + so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); + so4_a2.sectors = {"RCO", "TRA"}; + srf_emiss_species_.push_back(so4_a2); - // Number of sectors - static constexpr int so4_a2_num_sectors = 2; - - // Sector names in file - std::array so4_a2_sectors = {"RCO", "TRA"}; + //-------------------------------------------------------------------- + // Init data structures to read and interpolate + //-------------------------------------------------------------------- - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so4_a2_data_file, so4_a2_sectors, srf_map_file, - // output - so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, - so4_a2SrfEmissDataReader_); + for(const sr_emiss &ispec_srf : srf_emiss_species_) { + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, + // output + ispec_srf.HorizInterp_, ispec_srf.Data_start_, ispec_srf.Data_end_, + ispec_srf.Data_out_, ispec_srf.DataReader_); + } } // set_grid @@ -317,70 +193,12 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. //-------------------------------------------------------------------- - // Update dms srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - dmsSrfEmissDataReader_, timestamp(), curr_month, *dmsSrfEmissHorizInterp_, - dmsSrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, - so2SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update bc_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - bc_a4SrfEmissDataReader_, timestamp(), curr_month, - *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a1 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a1SrfEmissDataReader_, timestamp(), curr_month, - *num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a2SrfEmissDataReader_, timestamp(), curr_month, - *num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a4SrfEmissDataReader_, timestamp(), curr_month, - *num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update pom_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - pom_a4SrfEmissDataReader_, timestamp(), curr_month, - *pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so4_a1 srf emiss from file + // Update surface emissions from file //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so4_a1SrfEmissDataReader_, timestamp(), curr_month, - *so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so4_a2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so4_a2SrfEmissDataReader_, timestamp(), curr_month, - *so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_end_); - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) - << ":" << i << std::endl; + for(auto &ispec_srf : srf_emiss_species_) { + srfEmissFunc::update_srfEmiss_data_from_file( + ispec_srf.DataReader_, timestamp(), curr_month, *ispec_srf.HorizInterp_, + ispec_srf.Data_end_); } //----------------------------------------------------------------- @@ -407,211 +225,218 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- // Interpolate dms srf emiss data //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - dmsSrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - dmsSrfEmissDataReader_, ts, *dmsSrfEmissHorizInterp_, - dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_); + for(auto &ispec_srf : srf_emiss_species_) { + // Update srfEmissTimeState, note the addition of dt + ispec_srf.TimeState_.t_now = ts.frac_of_year_in_days(); - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, - dmsSrfEmissData_end_, dmsSrfEmissData_out_); + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + ispec_srf.DataReader_, ts, *ispec_srf.HorizInterp_, + ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_); - // update flux - auto constituent_fluxes_DMS = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(constituent_fluxes_DMS, - dmsSrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, - so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, - so2SrfEmissData_end_, so2SrfEmissData_out_); - // update flux - auto constituent_fluxes_SO2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::SO2)); - Kokkos::deep_copy(constituent_fluxes_SO2, - so2SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate bc_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, - bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_bc_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::bc_a4)); - Kokkos::deep_copy(constituent_fluxes_bc_a4, - bc_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a1 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, - num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, - num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a1)); - Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, - num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, - num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a2)); - Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, - num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, - num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a4)); - Kokkos::deep_copy(constituent_fluxes_num_a4, - num_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate pom_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, - pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, - pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_pom_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::pom_a4)); - Kokkos::deep_copy(constituent_fluxes_pom_a4, - pom_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so4_a1 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, - so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, - so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); - // update flux - auto constituent_fluxes_so4_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a1)); - Kokkos::deep_copy(constituent_fluxes_so4_a1, - so4_a1SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so4_a2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, - so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, - so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); - // update flux - auto constituent_fluxes_so4_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a2)); - Kokkos::deep_copy(constituent_fluxes_so4_a2, - so4_a2SrfEmissData_out_.emiss_sectors[0]); - - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] - << std::endl; + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, + ispec_srf.Data_end_, ispec_srf.Data_out_); + // update flux + auto clfx = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(cflx, dmsSrfEmissData_out_.emiss_sectors[0]); } + /* +// update flux +auto constituent_fluxes_DMS = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); +Kokkos::deep_copy(constituent_fluxes_DMS, + dmsSrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, + so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, + so2SrfEmissData_end_, so2SrfEmissData_out_); +// update flux +auto constituent_fluxes_SO2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::SO2)); +Kokkos::deep_copy(constituent_fluxes_SO2, + so2SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate bc_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, + bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_bc_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::bc_a4)); +Kokkos::deep_copy(constituent_fluxes_bc_a4, + bc_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a1 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, + num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, + num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a1)); +Kokkos::deep_copy(constituent_fluxes_num_a1, + num_a1SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, + num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, + num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a2)); +Kokkos::deep_copy(constituent_fluxes_num_a2, + num_a2SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, + num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, + num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a4)); +Kokkos::deep_copy(constituent_fluxes_num_a4, + num_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate pom_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, + pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, + pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_pom_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::pom_a4)); +Kokkos::deep_copy(constituent_fluxes_pom_a4, + pom_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so4_a1 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, + so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, + so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); +// update flux +auto constituent_fluxes_so4_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a1)); +Kokkos::deep_copy(constituent_fluxes_so4_a1, + so4_a1SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so4_a2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, + so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, + so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); +// update flux +auto constituent_fluxes_so4_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a2)); +Kokkos::deep_copy(constituent_fluxes_so4_a2, + so4_a2SrfEmissData_out_.emiss_sectors[0]); + +for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" + << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] + << std::endl; +}*/ /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index dca30a176ddb..2b321571da22 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -127,27 +127,22 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // A struct carrying all the fields needed to read // surface emissions of a species - // template struct srf_emiss { - // srf_emiss(int n){numSectors=n;} - // data file name + // Data file name std::string data_file; - // static int numSectors; - // Sector names in file + // Sector names in file std::vector sectors; - + // Data structure for reading interpolation std::shared_ptr HorizInterp_; std::shared_ptr DataReader_; srfEmissFunc::srfEmissTimeState TimeState_; srfEmissFunc::srfEmissInput Data_start_, Data_end_; srfEmissFunc::srfEmissOutput Data_out_; - - /*srfEmissFunc::init_srf_emiss_objects( - // output - dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, - dmsSrfEmissData_out_, dmsSrfEmissDataReader_);*/ }; + // A vector for carrying emissions for all the species + std::vector srf_emiss_species_; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; From 4f816563a6f71fa1edfe59b99be29825bf010ddb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 21:15:19 -0700 Subject: [PATCH 463/904] Fixes compilation issues in previous commit --- ...mam_srf_and_online_emissions_process_interface.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index eeb1b642cf24..17481dfc4d02 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -137,8 +137,7 @@ void MAMSrfOnlineEmiss::set_grids( //-------------------------------------------------------------------- // Init data structures to read and interpolate //-------------------------------------------------------------------- - - for(const sr_emiss &ispec_srf : srf_emiss_species_) { + for(srf_emiss &ispec_srf : srf_emiss_species_) { srfEmissFunc::init_srf_emiss_objects( ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, // output @@ -239,9 +238,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_, ispec_srf.Data_out_); // update flux - auto clfx = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(cflx, dmsSrfEmissData_out_.emiss_sectors[0]); + /*auto clfx = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(cflx, + dmsSrfEmissData_out_.emiss_sectors[0]);*/ } /* // update flux From f3aad49cac2c7a334b105b0fd73493b29b4c9f8d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 07:21:47 -0700 Subject: [PATCH 464/904] Updates constituent flux array with the righ units --- ...and_online_emissions_process_interface.cpp | 349 +++++------------- ...and_online_emissions_process_interface.hpp | 111 ++---- 2 files changed, 123 insertions(+), 337 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 17481dfc4d02..01167845fd5b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -51,98 +51,106 @@ void MAMSrfOnlineEmiss::set_grids( //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- - srf_emiss dms; - - // File name and sectors - dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms.sectors = {"DMS"}; + srf_emiss_ dms; + // File name, name and sectors + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.species_name = "dms"; + dms.sectors = {"DMS"}; srf_emiss_species_.push_back(dms); // add to the vector //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so2; - // File name and sectors - so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); - so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_ so2; + // File name, name and sectors + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.species_name = "so2"; + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(so2); // add to the vector //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss bc_a4; - // File name and sectors + srf_emiss_ bc_a4; + // File name, name and sectors bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); - bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + bc_a4.species_name = "bc_a4"; + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(bc_a4); // add to the vector //-------------------------------------------------------------------- // Init num_a1 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a1; - // File name and sectors + srf_emiss_ num_a1; + // File name, name and sectors num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); - num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", - "num_a1_SO4_WST"}; + num_a1.species_name = "num_a1"; + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; srf_emiss_species_.push_back(num_a1); // add to the vector //-------------------------------------------------------------------- // Init num_a2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a2; - // File name and sectors + srf_emiss_ num_a2; + // File name, name and sectors num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); - num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + num_a2.species_name = "num_a2"; + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; srf_emiss_species_.push_back(num_a2); // add to the vector //-------------------------------------------------------------------- // Init num_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a4; - // File name and sectors + srf_emiss_ num_a4; + // File name, name and sectors num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); - num_a4.sectors = { - "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", - "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", - "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", - "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + num_a4.species_name = "num_a4"; + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; srf_emiss_species_.push_back(num_a4); // add to the vector //-------------------------------------------------------------------- // Init pom_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss pom_a4; - // File name and sectors + srf_emiss_ pom_a4; + // File name, name and sectors pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); - pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + pom_a4.species_name = "pom_a4"; + pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(pom_a4); // add to the vector //-------------------------------------------------------------------- // Init so4_a1 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so4_a1; - // File name and sectors + srf_emiss_ so4_a1; + // File name, name and sectors so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); - so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + so4_a1.species_name = "so4_a1"; + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; srf_emiss_species_.push_back(so4_a1); //-------------------------------------------------------------------- // Init so4_a2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so4_a2; - // File name and sectors + srf_emiss_ so4_a2; + // File name, name and sectors so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); - so4_a2.sectors = {"RCO", "TRA"}; + so4_a2.species_name = "so4_a2"; + so4_a2.sectors = {"RCO", "TRA"}; srf_emiss_species_.push_back(so4_a2); //-------------------------------------------------------------------- // Init data structures to read and interpolate //-------------------------------------------------------------------- - for(srf_emiss &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::init_srf_emiss_objects( ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, // output - ispec_srf.HorizInterp_, ispec_srf.Data_start_, ispec_srf.Data_end_, - ispec_srf.Data_out_, ispec_srf.DataReader_); + ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, + ispec_srf.data_out_, ispec_srf.dataReader_); } } // set_grid @@ -183,6 +191,13 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); + // --------------------------------------------------------------- + // Allocate memory for local and work arrays + // --------------------------------------------------------------- + + // work array to store fluxes after unit conversions to kg/m2/s + fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); + // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; @@ -194,10 +209,10 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //-------------------------------------------------------------------- // Update surface emissions from file //-------------------------------------------------------------------- - for(auto &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::update_srfEmiss_data_from_file( - ispec_srf.DataReader_, timestamp(), curr_month, *ispec_srf.HorizInterp_, - ispec_srf.Data_end_); + ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, + ispec_srf.data_end_); } //----------------------------------------------------------------- @@ -218,232 +233,68 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + // policy to loop over columns only + const auto ncol_only_policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, 1); + // Gather time and state information for interpolation auto ts = timestamp() + dt; //-------------------------------------------------------------------- - // Interpolate dms srf emiss data + // Interpolate srf emiss data //-------------------------------------------------------------------- - for(auto &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { // Update srfEmissTimeState, note the addition of dt - ispec_srf.TimeState_.t_now = ts.frac_of_year_in_days(); + ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - ispec_srf.DataReader_, ts, *ispec_srf.HorizInterp_, - ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_); + ispec_srf.dataReader_, ts, *ispec_srf.horizInterp_, + ispec_srf.timeState_, ispec_srf.data_start_, ispec_srf.data_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, - ispec_srf.Data_end_, ispec_srf.Data_out_); - // update flux - /*auto clfx = + srfEmissFunc::srfEmiss_main(ispec_srf.timeState_, ispec_srf.data_start_, + ispec_srf.data_end_, ispec_srf.data_out_); + + //-------------------------------------------------------------------- + // Modify units to MKS units (from molecules/cm2/s to kg/m2/s) + //-------------------------------------------------------------------- + // Get species index in array with pcnst dimension (e.g., state_q or + // constituent_fluxes_) + const int species_index = spcIndex_in_pcnst_.at(ispec_srf.species_name); + + // modify units from molecules/cm2/s to kg/m2/s + const Real mfactor = + amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; + // Parallel loop over all the columns to update units + Kokkos::parallel_for( + ncol_only_policy, + KOKKOS_LAMBDA(const MAMSrfOnlineEmiss::KT::MemberType &team) { + const int icol = team.league_rank(); + fluxes_in_mks_units_(icol) = + ispec_srf.data_out_.emiss_sectors[0](icol) * mfactor; + }); + + // Get subview + auto flux_1d_view = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(cflx, - dmsSrfEmissData_out_.emiss_sectors[0]);*/ + spcIndex_in_pcnst_.at(ispec_srf.species_name)); + + // update flux in constituent_fluxes_ view + Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); + } + + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srf_emiss_species_[8].data_out_.emiss_sectors[0](i) + << ":" << i << ":" << constituent_fluxes_(i, 37) << std::endl; } - /* -// update flux -auto constituent_fluxes_DMS = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); -Kokkos::deep_copy(constituent_fluxes_DMS, - dmsSrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, - so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, - so2SrfEmissData_end_, so2SrfEmissData_out_); -// update flux -auto constituent_fluxes_SO2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::SO2)); -Kokkos::deep_copy(constituent_fluxes_SO2, - so2SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate bc_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, - bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_bc_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::bc_a4)); -Kokkos::deep_copy(constituent_fluxes_bc_a4, - bc_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a1 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, - num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, - num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a1)); -Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, - num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, - num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a2)); -Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, - num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, - num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a4)); -Kokkos::deep_copy(constituent_fluxes_num_a4, - num_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate pom_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, - pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, - pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_pom_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::pom_a4)); -Kokkos::deep_copy(constituent_fluxes_pom_a4, - pom_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so4_a1 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, - so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, - so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); -// update flux -auto constituent_fluxes_so4_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a1)); -Kokkos::deep_copy(constituent_fluxes_so4_a1, - so4_a1SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so4_a2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, - so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, - so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); -// update flux -auto constituent_fluxes_so4_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a2)); -Kokkos::deep_copy(constituent_fluxes_so4_a2, - so4_a2SrfEmissData_out_.emiss_sectors[0]); - -for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] - << std::endl; -}*/ /* Rough notes: Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions and - set_srf_emissions. + chemistry.F90. Basically call two subroutines, aero_model_emissions + and set_srf_emissions. Here is the code: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 2b321571da22..1669125b9fb5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -24,6 +24,7 @@ namespace scream { // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; using view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels @@ -44,6 +45,12 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { view_2d constituent_fluxes_; + // work array to store fluxes after unit conversions to kg/m2/s + view_1d fluxes_in_mks_units_; + + // Unified atomic mass unit used for unit conversion (BAD constant) + static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -112,110 +119,38 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; - // Species index in tracer array with "pcnst" indices - enum class spcIndex_in_pcnst : int { - SO2 = 12, - DMS = 13, - so4_a1 = 15, - num_a1 = 22, - so4_a2 = 23, - num_a2 = 27, - pom_a4 = 36, - bc_a4 = 37, - num_a4 = 39, - }; + // Species index (zero-based) in tracer array with "pcnst" dimension + // FIXME: Remove the hardwired indices and use a function + // to find them from an array. + const std::map spcIndex_in_pcnst_ = { + {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, + {"num_a1", 22}, {"so4_a2", 23}, {"num_a2", 27}, + {"pom_a4", 36}, {"bc_a4", 37}, {"num_a4", 39}}; // A struct carrying all the fields needed to read // surface emissions of a species - struct srf_emiss { + struct srf_emiss_ { + // species name + std::string species_name; // Data file name std::string data_file; // Sector names in file std::vector sectors; // Data structure for reading interpolation - std::shared_ptr HorizInterp_; - std::shared_ptr DataReader_; - srfEmissFunc::srfEmissTimeState TimeState_; - srfEmissFunc::srfEmissInput Data_start_, Data_end_; - srfEmissFunc::srfEmissOutput Data_out_; + std::shared_ptr horizInterp_; + std::shared_ptr dataReader_; + srfEmissFunc::srfEmissTimeState timeState_; + srfEmissFunc::srfEmissInput data_start_, data_end_; + srfEmissFunc::srfEmissOutput data_out_; }; // A vector for carrying emissions for all the species - std::vector srf_emiss_species_; + std::vector srf_emiss_species_; // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; - // Data structures to read DMS data file - std::shared_ptr dmsSrfEmissHorizInterp_; - std::shared_ptr dmsSrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; - srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; - srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - - // Data structures to read so2 data file - std::shared_ptr so2SrfEmissHorizInterp_; - std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; - - // Data structures to read bc_a4 data file - std::shared_ptr bc_a4SrfEmissHorizInterp_; - std::shared_ptr bc_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; - - // Data structures to read num_a1 data file - std::shared_ptr num_a1SrfEmissHorizInterp_; - std::shared_ptr num_a1SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; - - // Data structures to read num_a2 data file - std::shared_ptr num_a2SrfEmissHorizInterp_; - std::shared_ptr num_a2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; - - // Data structures to read num_a4 data file - std::shared_ptr num_a4SrfEmissHorizInterp_; - std::shared_ptr num_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; - - // Data structures to read pom_a4 data file - std::shared_ptr pom_a4SrfEmissHorizInterp_; - std::shared_ptr pom_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; - - // Data structures to read so4_a1 data file - std::shared_ptr so4_a1SrfEmissHorizInterp_; - std::shared_ptr so4_a1SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; - - // Data structures to read so4_a2 data file - std::shared_ptr so4_a2SrfEmissHorizInterp_; - std::shared_ptr so4_a2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so4_a2SrfEmissData_out_; - }; // MAMSrfOnlineEmiss } // namespace scream From 5b35e31791083141bd4e2ee18b4b54be2548d168 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 13:02:03 -0700 Subject: [PATCH 465/904] Adds constituent flixes interface and a test --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 231 ++++++++++++++++++ ...eamxx_mam_constituent_fluxes_interface.hpp | 76 ++++++ .../eamxx/src/physics/mam/mam_coupling.hpp | 18 ++ .../eamxx/src/physics/register_physics.hpp | 2 + .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../mam/constituent_fluxes/CMakeLists.txt | 44 ++++ .../mam/constituent_fluxes/input.yaml | 37 +++ .../mam/constituent_fluxes/output.yaml | 12 + 9 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 9138bba6d5a2..9874f79f9eb6 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -46,7 +46,8 @@ add_library(mam eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp eamxx_mam_wetscav_process_interface.cpp - eamxx_mam_srf_and_online_emissions_process_interface.cpp) + eamxx_mam_srf_and_online_emissions_process_interface.cpp + eamxx_mam_constituent_fluxes_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp new file mode 100644 index 000000000000..91d587c31d32 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -0,0 +1,231 @@ +#include +namespace scream { + +// ================================================================ +// Constructor +// ================================================================ +MAMConstituentFluxes::MAMConstituentFluxes(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ================================================================ +// SET_GRIDS +// ================================================================ +void MAMConstituentFluxes::set_grids( + const std::shared_ptr grids_manager) { + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + + static constexpr int pcnst = mam4::aero_model::pcnst; + + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + static constexpr Units m2(m * m, "m2"); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + grid_name); + + // --------------------------------------------------------------------- + // These variables are "Updated" or inputs/outputs for the process + // --------------------------------------------------------------------- + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + +} // set_grid + +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMConstituentFluxes::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializes the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMConstituentFluxes::init_buffers( + const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMConstituentFluxes."); +} + +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ +void MAMConstituentFluxes::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Inputs + // --------------------------------------------------------------- + constituent_fluxes_ = + get_field_in("constituent_fluxes").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + //----------------------------------------------------------------- + // Allocate memory + //----------------------------------------------------------------- + + rpdel_ = view_2d("rpdel_", ncol_, nlev_); + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + // preprocess_.initialize(constituent_fluxes_); + +} // end initialize_impl() + +// ================================================================ +// RUN_IMPL +// ================================================================ +void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + /*FORTRAN CODE: + What we need: + 1. ncol + 2. all tracers + 3. timestep (verify what is rztodt??) + 4. gravit, rpdel + 5. cflx + 6. Ensure dry/et-wet/dry isconsistent....comment on that!! + + + ncol = state%ncol + + !------------------------------------------------------- + ! Assume 'wet' mixing ratios in surface diffusion code. + ! don't convert co2 tracers to wet mixing ratios + + cnst_type_loc(:) = cnst_type(:) + call set_dry_to_wet(state, cnst_type_loc) + + !------------------------------------------------------- + ! Initialize ptend + + lq(:) = .TRUE. + call physics_ptend_init(ptend, state%psetcols, 'clubb_srf', lq=lq) + + !------------------------------------------------------- + ! Calculate tracer mixing ratio tendencies from cflx + + rztodt = 1._r8/ztodt + ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) + tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver) + + do m = 2, pcnst + ptend%q(:ncol,pver,m) = ptend%q(:ncol,pver,m) + tmp1(:ncol) * + cam_in%cflx(:ncol,m) enddo + + ptend%q(:ncol,:pver,:) = (ptend%q(:ncol,:pver,:) - state%q(:ncol,:pver,:)) * + rztodt + + ! Convert tendencies of dry constituents to dry basis. + do m = 1,pcnst + if (cnst_type(m).eq.'dry') then + ptend%q(:ncol,:pver,m) = + ptend%q(:ncol,:pver,m)*state%pdel(:ncol,:pver)/state%pdeldry(:ncol,:pver) + endif + end do + + !------------------------------------------------------- + ! convert wet mmr back to dry before conservation check + ! avoid converting co2 tracers again + + cnst_type_loc(:) = cnst_type(:) + call co2_cycle_set_cnst_type(cnst_type_loc, 'wet') + call set_wet_to_dry(state, cnst_type_loc) + + */ +} + +// ============================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp new file mode 100644 index 000000000000..0f17e5b0632a --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -0,0 +1,76 @@ +#ifndef EAMXX_MAM_CONSTITUENT_FLUXES_HPP +#define EAMXX_MAM_CONSTITUENT_FLUXES_HPP + +// For declaring contituent fluxes class derived from atm process +// class +#include +// For MAM4 aerosol configuration +#include +#include + +namespace scream { + +// The process responsible for applying MAM4 constituent fluxes. The +// AD stores exactly ONE instance of this class in its list of subcomponents. +class MAMConstituentFluxes final : public scream::AtmosphereProcess { + using KT = ekat::KokkosTypes; + using view_2d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + + // number of horizontal columns + int ncol_, nlev_; + + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + const_view_2d constituent_fluxes_; + + view_2d rpdel_; // Inverse of pdel_ or pseudo_density + + public: + // Constructor + MAMConstituentFluxes(const ekat::Comm &comm, + const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_constituent_fluxes"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMConstituentFluxes + +} // namespace scream + +#endif // EAMXX_MAM_CONSTITUENT_FLUXES_HPP diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index d490ab761559..09d64d6028d9 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -776,6 +776,24 @@ void compute_wet_mixing_ratios(const Team& team, }); } +// Computes the reciprocal of pseudo density for a column +KOKKOS_INLINE_FUNCTION +void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, + const_view_2d pdel, + const int nlev, + // output + view_2d rpdel) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); + }); +} // Scream (or EAMxx) can sometimes extend views beyond model levels (nlev) as it uses // "packs". Following function copies a 2d view till model levels inline diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index f8d3a9edb199..d837e96311f4 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -30,6 +30,7 @@ #include "physics/mam/eamxx_mam_aci_process_interface.hpp" #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" #include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" +#include "physics/mam/eamxx_mam_constituent_fluxes_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -70,6 +71,7 @@ inline void register_physics () { proc_factory.register_product("mam4_aci",&create_atmosphere_process); proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); proc_factory.register_product("mam4_srf_online_emiss",&create_atmosphere_process); + proc_factory.register_product("mam4_constituent_fluxes",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index 62435db229fe..40565d94bf8e 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -24,6 +24,7 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam/drydep) add_subdirectory(mam/wet_scav) add_subdirectory(mam/emissions) + add_subdirectory(mam/constituent_fluxes) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt new file mode 100644 index 000000000000..2615f152b264 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt @@ -0,0 +1,44 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_constituent_fluxes_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_constituent_fluxes physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_constituent_fluxes physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml new file mode 100644 index 000000000000..d012274d1469 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml @@ -0,0 +1,37 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_constituent_fluxes] +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + pbl_height : 1.0 + constituent_fluxes: 1.e-5 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml new file mode 100644 index 000000000000..e51b96eb5ce3 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml @@ -0,0 +1,12 @@ +%YAML 1.1 +--- +filename_prefix: mam4_constituent_fluxes_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - T_mid +output_control: + Frequency: 1 + frequency_units: nsteps +... From b8ff3601be8c9ceb91a08a63536405645b7226b9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 19:39:46 -0700 Subject: [PATCH 466/904] Adds code to create progs and atm datastructures --- ...eamxx_mam_constituent_fluxes_interface.cpp | 190 ++++++++++++++++-- 1 file changed, 176 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 91d587c31d32..6075e066b54d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -26,8 +26,11 @@ void MAMConstituentFluxes::set_grids( using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers auto n_unit = 1 / kg; // units of number mixing ratios of tracers + auto nondim = ekat::units::Units::nondimensional(); + FieldLayout scalar2d = grid_->get_2d_scalar_layout(); FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); static constexpr int pcnst = mam4::aero_model::pcnst; @@ -37,44 +40,98 @@ void MAMConstituentFluxes::set_grids( // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process // -------------------------------------------------------------------------- + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg](Require only for building DS) + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg](Require only for building DS) + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg](Require only for building DS) + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) + add_field("omega", scalar3d_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + // Layer thickness(pdel) [Pa] at midpoints add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + // Planetary boundary layer height [m] (Require only for building DS) + add_field("pbl_height", scalar2d, m, grid_name); + + // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + static constexpr Units m2(m * m, "m2"); + static constexpr Units s2(s * s, "s2"); + + // Surface geopotential [m2/s2] (Require only for building DS) + add_field("phis", scalar2d, m2 / s2, grid_name); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); // --------------------------------------------------------------------- // These variables are "Updated" or inputs/outputs for the process // --------------------------------------------------------------------- - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - + // NOTE: Cloud borne aerosols are not updated in this process but are included + // to create datastructures. + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - + mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } - } - } + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(mode, a); + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); + } + } // end for loop num species + } // end for loop for num modes - // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); - } + } // end for loop num gases } // set_grid @@ -111,8 +168,42 @@ void MAMConstituentFluxes::init_buffers( // ================================================================ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- - // Inputs + // Input fields read in from IC file, namelist or other processes // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + // FIMXE: specifically look which among these are actually used by the process + wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + constituent_fluxes_ = get_field_in("constituent_fluxes").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); @@ -126,6 +217,12 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = @@ -136,6 +233,15 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } } } for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { @@ -167,8 +273,65 @@ void MAMConstituentFluxes::run_impl(const double dt) { // preprocess input -- needs a scan for the calculation of atm height // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); // Kokkos::fence(); + + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] + + mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, + nlev_, + // output + rpdel_); + // ------------------------------------------------------------------- + // NOTES: The following code is an adaptation of cflx.F90 code in E3SM + // In EAMxx, all constituents are considered "wet" (or have wet mixing + // ratios), so we are *not* doing any wet to dry conversions here. We + // are simply updating the MAM4xx tracers using the "constituent + // fluxes" + // ------------------------------------------------------------------- + + Real rztodt = 1.0 / dt; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + /////////dddd + // To construct state_q, we need fields from Prognostics (aerosols) + // and Atmosphere (water species such as qv, qc etc.) + + // get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(dry_aero_, icol); + + // get atmospheric quantities + haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); + + // Construct state_q (interstitial) and qqcw (cloud borne) arrays + /*constexpr auto pver = mam4::ndrop::pver; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pver), [&](int klev) + { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics(progs_at_col, haero_atm, + state_q_at_lev_col, + klev); + + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } + });*/ + }); + ////////dddd + + /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) + tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ + /*FORTRAN CODE: What we need: 1. ncol @@ -183,7 +346,6 @@ void MAMConstituentFluxes::run_impl(const double dt) { !------------------------------------------------------- ! Assume 'wet' mixing ratios in surface diffusion code. - ! don't convert co2 tracers to wet mixing ratios cnst_type_loc(:) = cnst_type(:) call set_dry_to_wet(state, cnst_type_loc) From 552a785b600e3222d7f0e0ba263190f2e9761066 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 21:31:38 -0700 Subject: [PATCH 467/904] A working end-to-end code --- .../physics/mam/eamxx_mam_aci_functions.hpp | 2 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 58 +++++++++---------- ...eamxx_mam_constituent_fluxes_interface.hpp | 2 +- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 03b04ac739a0..ab142fedc78f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -397,7 +397,7 @@ void call_function_dropmixnuc( progs_at_col, haero_atm, state_q_at_lev_col, klev); // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + int istart = mam4::utils::aero_start_ind(); // create colum views of state_q for(int icnst = istart; icnst < mam4::aero_model::pcnst; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 6075e066b54d..54d863f8b6fb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -215,13 +215,11 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); - dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); - dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios @@ -231,7 +229,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -240,7 +237,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { if(strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); - dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } @@ -248,7 +244,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); - dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } //----------------------------------------------------------------- @@ -289,9 +284,6 @@ void MAMConstituentFluxes::run_impl(const double dt) { // are simply updating the MAM4xx tracers using the "constituent // fluxes" // ------------------------------------------------------------------- - - Real rztodt = 1.0 / dt; - Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -301,33 +293,41 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get prognostics mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(dry_aero_, icol); + mam_coupling::aerosols_for_column(wet_aero_, icol); // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays - /*constexpr auto pver = mam4::ndrop::pver; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pver), [&](int klev) - { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics(progs_at_col, haero_atm, - state_q_at_lev_col, - klev); - - // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // create colum views of state_q - for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - } - });*/ + constexpr auto pver = mam4::ndrop::pver; + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, pver), [&](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); + std::cout << "istart:" << mam4::utils::gasses_start_ind() + << std::endl; + // kg/m2/s to kg/kg/ + Real unit_factor = dt * gravit * rpdel_(icol, klev); + // Update state with the constituent fluxes (modified + // units) + for(int icnst = istart; icnst < mam4::aero_model::pcnst; + ++icnst) { + state_q_at_lev_col[icnst] = + state_q_at_lev_col[icnst] + + constituent_fluxes_(icol, icnst) * unit_factor; + } + mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, + progs_at_col, klev); + }); }); - ////////dddd /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 0f17e5b0632a..02c4b8790938 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -25,7 +25,7 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { mam_coupling::DryAtmosphere dry_atm_; // aerosol state variables - mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::AerosolState wet_aero_; //, dry_aero_; // buffer for sotring temporary variables mam_coupling::Buffer buffer_; From a3c0ab9dbf0c08c31407db8a31f316e92107160f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 22:20:52 -0700 Subject: [PATCH 468/904] Verified change in DMS conc before and after update, moved aci rpdel function to mam_coupling --- .../physics/mam/eamxx_mam_aci_functions.hpp | 17 --- .../mam/eamxx_mam_aci_process_interface.cpp | 2 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 117 ++++++------------ ...eamxx_mam_constituent_fluxes_interface.hpp | 41 ++++++ 4 files changed, 78 insertions(+), 99 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index ab142fedc78f..8bb52c918d23 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -198,23 +198,6 @@ void store_liquid_cloud_fraction( }); } -void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, - MAMAci::const_view_2d pdel, - const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { - EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), - "Error: pdel should be > 0.\n"); - rpdel(icol, kk) = 1 / pdel(icol, kk); - }); - }); -} - void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, const Real dt, mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e919aa066b47..7800a43c252d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -597,7 +597,7 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); - compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, + mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 54d863f8b6fb..5c5129d1b7c1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -84,6 +84,7 @@ void MAMConstituentFluxes::set_grids( // Surface geopotential [m2/s2] (Require only for building DS) add_field("phis", scalar2d, m2 / s2, grid_name); + // Constituent fluxes at the surface (gasses and aerosols) add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); @@ -91,7 +92,8 @@ void MAMConstituentFluxes::set_grids( // These variables are "Updated" or inputs/outputs for the process // --------------------------------------------------------------------- // NOTE: Cloud borne aerosols are not updated in this process but are included - // to create datastructures. + // to create data structures. + // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { @@ -191,6 +193,7 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; @@ -204,9 +207,9 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { dry_atm_.w_updraft = buffer_.w_updraft; dry_atm_.z_surf = 0.0; // FIXME: for now + // Constituent fluxes at the surface (gasses and aerosols) [kg/m2/s] constituent_fluxes_ = get_field_in("constituent_fluxes").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -249,12 +252,13 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- // Allocate memory //----------------------------------------------------------------- - + // Inverse of pseudo_density (1/Pa or or N/m2 or kg/m/s2) rpdel_ = view_2d("rpdel_", ncol_, nlev_); + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- - // preprocess_.initialize(constituent_fluxes_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); } // end initialize_impl() @@ -265,29 +269,39 @@ void MAMConstituentFluxes::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + // preprocess input -- needs a scan for the calculation of atm height - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + // policy haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] + // compute inverse of pseudo_density mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); - // ------------------------------------------------------------------- - // NOTES: The following code is an adaptation of cflx.F90 code in E3SM - // In EAMxx, all constituents are considered "wet" (or have wet mixing - // ratios), so we are *not* doing any wet to dry conversions here. We - // are simply updating the MAM4xx tracers using the "constituent - // fluxes" - // ------------------------------------------------------------------- + std::cout << "BEFORE:::" << wet_aero_.gas_mmr[4](1, 1) << std::endl; + // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - /////////dddd + // To construct state_q, we need fields from Prognostics (aerosols) // and Atmosphere (water species such as qv, qc etc.) @@ -298,10 +312,9 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); - // Construct state_q (interstitial) and qqcw (cloud borne) arrays - constexpr auto pver = mam4::ndrop::pver; + // Construct state_q (interstitial) array Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, pver), [&](int klev) { + Kokkos::TeamVectorRange(team, nlev_), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -312,16 +325,15 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - std::cout << "istart:" << mam4::utils::gasses_start_ind() - << std::endl; - // kg/m2/s to kg/kg/ + + // Factor to convert units from kg/m2/s to kg/kg Real unit_factor = dt * gravit * rpdel_(icol, klev); + // Update state with the constituent fluxes (modified // units) for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_at_lev_col[icnst] = - state_q_at_lev_col[icnst] + + state_q_at_lev_col[icnst] += constituent_fluxes_(icol, icnst) * unit_factor; } mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, @@ -329,65 +341,8 @@ void MAMConstituentFluxes::run_impl(const double dt) { }); }); - /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) - tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ - - /*FORTRAN CODE: - What we need: - 1. ncol - 2. all tracers - 3. timestep (verify what is rztodt??) - 4. gravit, rpdel - 5. cflx - 6. Ensure dry/et-wet/dry isconsistent....comment on that!! - - - ncol = state%ncol - - !------------------------------------------------------- - ! Assume 'wet' mixing ratios in surface diffusion code. - - cnst_type_loc(:) = cnst_type(:) - call set_dry_to_wet(state, cnst_type_loc) - - !------------------------------------------------------- - ! Initialize ptend - - lq(:) = .TRUE. - call physics_ptend_init(ptend, state%psetcols, 'clubb_srf', lq=lq) - - !------------------------------------------------------- - ! Calculate tracer mixing ratio tendencies from cflx - - rztodt = 1._r8/ztodt - ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) - tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver) - - do m = 2, pcnst - ptend%q(:ncol,pver,m) = ptend%q(:ncol,pver,m) + tmp1(:ncol) * - cam_in%cflx(:ncol,m) enddo - - ptend%q(:ncol,:pver,:) = (ptend%q(:ncol,:pver,:) - state%q(:ncol,:pver,:)) * - rztodt - - ! Convert tendencies of dry constituents to dry basis. - do m = 1,pcnst - if (cnst_type(m).eq.'dry') then - ptend%q(:ncol,:pver,m) = - ptend%q(:ncol,:pver,m)*state%pdel(:ncol,:pver)/state%pdeldry(:ncol,:pver) - endif - end do - - !------------------------------------------------------- - ! convert wet mmr back to dry before conservation check - ! avoid converting co2 tracers again - - cnst_type_loc(:) = cnst_type(:) - call co2_cycle_set_cnst_type(cnst_type_loc, 'wet') - call set_wet_to_dry(state, cnst_type_loc) - - */ -} + std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; +} // run_impl ends // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 02c4b8790938..81c6ee529857 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -69,6 +69,47 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::DryAtmosphere &dry_atm) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + dry_atm_pre_ = dry_atm; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + }; // Preprocess + + private: + // preprocessing scratch pads + Preprocess preprocess_; + }; // MAMConstituentFluxes } // namespace scream From 3496c7ddccec197485379e0d96cd70f7e34d110b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 06:53:57 -0700 Subject: [PATCH 469/904] Cleanup of some unused hpp file includes and some variables --- ...eamxx_mam_constituent_fluxes_interface.hpp | 1 + ...and_online_emissions_process_interface.cpp | 17 ++++++------- ...and_online_emissions_process_interface.hpp | 25 ++++++------------- .../single-process/mam/emissions/output.yaml | 2 +- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 81c6ee529857..448ae5464b87 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -4,6 +4,7 @@ // For declaring contituent fluxes class derived from atm process // class #include + // For MAM4 aerosol configuration #include #include diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 01167845fd5b..1fc3ddb4b685 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,4 +1,4 @@ -#include +//#include #include namespace scream { @@ -10,7 +10,7 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be - * initialized here. Like universal constants, mam wetscav options. + * initialized here. Like universal constants. */ } @@ -22,7 +22,8 @@ void MAMSrfOnlineEmiss::set_grids( grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); @@ -32,16 +33,11 @@ void MAMSrfOnlineEmiss::set_grids( const FieldLayout scalar2d_pcnct = grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); - // -------------------------------------------------------------------------- - // These variables are "Required" or pure inputs for the process - // -------------------------------------------------------------------------- - // Temperature[K] at midpoints - add_field("T_mid", scalar3d_mid, K, grid_name); - // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- static constexpr Units m2(m * m, "m2"); + // Constituent fluxes of species in [kg/m2/s] add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); @@ -189,6 +185,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- // Output fields // --------------------------------------------------------------- + // Constituent fluxes of species in [kg/m2/s] constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); // --------------------------------------------------------------- @@ -227,7 +224,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, 1); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1669125b9fb5..b67b1de0bd7d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,22 +1,19 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP -#include -#include -#include - #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" + // For MAM4 aerosol configuration #include #include -#include + // For declaring surface and online emission class derived from atm process // class #include + // #include #include -#include namespace scream { @@ -30,22 +27,16 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; - // Wet and dry states of atmosphere - mam_coupling::WetAtmosphere wet_atm_; - mam_coupling::DryAtmosphere dry_atm_; - - // aerosol state variables - mam_coupling::AerosolState wet_aero_, dry_aero_; - // buffer for sotring temporary variables mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; + // Constituent fluxes of species in [kg/m2/s] view_2d constituent_fluxes_; - // work array to store fluxes after unit conversions to kg/m2/s + // Work array to store fluxes after unit conversions to kg/m2/s view_1d fluxes_in_mks_units_; // Unified atomic mass unit used for unit conversion (BAD constant) @@ -54,9 +45,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { public: using srfEmissFunc = mam_coupling::srfEmissFunctions; - template - using uview_2d = Unmanaged>; - // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -132,10 +120,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct srf_emiss_ { // species name std::string species_name; + // Data file name std::string data_file; + // Sector names in file std::vector sectors; + // Data structure for reading interpolation std::shared_ptr horizInterp_; std::shared_ptr dataReader_; diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml index b6e2d545d3ca..433f7d58fa40 100644 --- a/components/eamxx/tests/single-process/mam/emissions/output.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -5,7 +5,7 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid + - constituent_fluxes output_control: Frequency: 1 frequency_units: nsteps From 4048ae83f28f9e32b19d195b8c055023a3496ebe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 07:49:58 -0700 Subject: [PATCH 470/904] Removes unused variables --- ...and_online_emissions_process_interface.cpp | 64 +++---------------- .../src/physics/mam/srf_emission_impl.hpp | 16 +---- 2 files changed, 13 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 1fc3ddb4b685..91a5b161dff4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -26,10 +26,8 @@ void MAMSrfOnlineEmiss::set_grids( nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; - FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); static constexpr int pcnst = mam4::aero_model::pcnst; - const FieldLayout scalar2d_pcnct = grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); @@ -149,7 +147,7 @@ void MAMSrfOnlineEmiss::set_grids( ispec_srf.data_out_, ispec_srf.dataReader_); } -} // set_grid +} // set_grid ends // ================================================================ // REQUEST_BUFFER_SIZE_IN_BYTES @@ -192,16 +190,16 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Allocate memory for local and work arrays // --------------------------------------------------------------- - // work array to store fluxes after unit conversions to kg/m2/s + // Work array to store fluxes after unit conversions to kg/m2/s fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; - // Load the first month into srfEmiss_end. + // Load the first month into data_end. - // Note: At the first time step, the data will be moved into srfEmiss_beg, - // and srfEmiss_end will be reloaded from file with the new month. + // Note: At the first time step, the data will be moved into data_beg, + // and data_end will be reloaded from file with the new month. //-------------------------------------------------------------------- // Update surface emissions from file @@ -209,7 +207,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::update_srfEmiss_data_from_file( ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, - ispec_srf.data_end_); + ispec_srf.data_end_); // output } //----------------------------------------------------------------- @@ -242,12 +240,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- for(srf_emiss_ &ispec_srf : srf_emiss_species_) { - // Update srfEmissTimeState, note the addition of dt + // Update TimeState, note the addition of dt ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( ispec_srf.dataReader_, ts, *ispec_srf.horizInterp_, + // output ispec_srf.timeState_, ispec_srf.data_start_, ispec_srf.data_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. @@ -280,51 +279,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // update flux in constituent_fluxes_ view Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); - } - - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srf_emiss_species_[8].data_out_.emiss_sectors[0](i) - << ":" << i << ":" << constituent_fluxes_(i, 37) << std::endl; - } - - /* Rough notes: - - Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions - and set_srf_emissions. - - Here is the code: - - ! initialize chemistry constituent surface fluxes to zero - do m = 2,pcnst - n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 - enddo - - ! aerosol emissions ... - call aero_model_emissions( state, & ! in - cam_in ) ! out - - ! prescribed emissions from file ... - - !----------------------------------------------------------------------- - ! ... Set surface emissions - !----------------------------------------------------------------------- - call set_srf_emissions( lchnk, ncol, sflx(:,:) ) - - do m = 1,pcnst - n = map2chm(m) - if ( n /= h2o_ndx .and. n > 0 ) then - cam_in%cflx(:ncol,m) = cam_in%cflx(:ncol,m) + sflx(:ncol,n) - call outfld( sflxnam(m), cam_in%cflx(:ncol,m), ncol,lchnk ) - endif - enddo - - - */ - - std::cout << "End of surface emissions run" << std::endl; -} + } // for loop for species +} // run_imple ends // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b792c805b328..3fa458450a3d 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -1,20 +1,9 @@ #ifndef SRF_EMISSION_IMPL_HPP #define SRF_EMISSION_IMPL_HPP -#include -#include -#include -#include -#include - -#include "physics/share/physics_constants.hpp" -#include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" -#include "share/io/scorpio_input.hpp" #include "share/io/scream_scorpio_interface.hpp" -#include "share/scream_types.hpp" -#include "share/util/scream_timing.hpp" namespace scream::mam_coupling { namespace { @@ -297,8 +286,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Read fields from the file for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { - auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], aa); + auto sector = + srfEmiss_horiz_interp.get_tgt_field(i).get_view(); + Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], sector); } Kokkos::fence(); From 0b9b81d2875b838cec329926e61910a3dcd379fe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 11:55:58 -0700 Subject: [PATCH 471/904] Cleanup - removed unused codes and added kokkos reduction loop --- .../eamxx/src/physics/mam/srf_emission.hpp | 5 + .../src/physics/mam/srf_emission_impl.hpp | 143 +++--------------- 2 files changed, 25 insertions(+), 123 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 585656b0ec1c..82a4b3113821 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -8,6 +8,11 @@ namespace { template struct srfEmissFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + struct srfEmissTimeState { srfEmissTimeState() = default; // Whether the timestate has been initialized. diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 3fa458450a3d..ef99879277eb 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -8,71 +8,6 @@ namespace scream::mam_coupling { namespace { -template -template -std::shared_ptr -srfEmissFunctions::create_horiz_remapper( - const std::shared_ptr &model_grid, - const std::string &data_file, - const std::array §or_names, - const std::string &map_file) { - using namespace ShortFieldTagsNames; - - scorpio::register_file(data_file, scorpio::Read); - const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); - scorpio::release_file(data_file); - - // We could use model_grid directly if using same num levels, - // but since shallow clones are cheap, we may as well do it (less lines of - // code) - auto horiz_interp_tgt_grid = - model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); - - const int ncols_model = model_grid->get_num_global_dofs(); - std::shared_ptr remapper; - if(ncols_data == ncols_model) { - remapper = std::make_shared( - horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); - } else { - EKAT_REQUIRE_MSG(ncols_data <= ncols_model, - "Error! We do not allow to coarsen srfEmiss data to fit " - "the model. We only allow\n" - "srfEmiss data to be at the same or coarser resolution as " - "the model.\n"); - // We must have a valid map file - EKAT_REQUIRE_MSG( - map_file != "", - "ERROR: srfEmiss data is on a different grid than the model one,\n" - "but srfEmiss_remap_file is missing from srfEmiss parameter " - "list."); - - remapper = - std::make_shared(horiz_interp_tgt_grid, map_file); - } - - remapper->registration_begins(); - - const auto tgt_grid = remapper->get_tgt_grid(); - - const auto layout_2d = tgt_grid->get_2d_scalar_layout(); - const auto nondim = ekat::units::Units::nondimensional(); - - std::vector emiss_sectors; - - for(int icomp = 0; icomp < numSectors; ++icomp) { - auto comp_name = sector_names[icomp]; - // set and allocate fields - Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); - f.allocate_view(); - emiss_sectors.push_back(f); - remapper->register_field_from_tgt(f); - } - - remapper->registration_ends(); - - return remapper; -} // create_horiz_remapper - template std::shared_ptr srfEmissFunctions::create_horiz_remapper( @@ -186,41 +121,29 @@ void srfEmissFunctions::perform_time_interpolation( std::to_string(t_beg) + "\n delta_t: " + std::to_string(delta_t) + "\n"); - for(int icol = 0; icol < data_beg.data.ncols; ++icol) { - Real accum = 0; - for(int i = 0; i < data_beg.data.nsectors; ++i) { - accum += - linear_interp(data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), delta_t_fraction); - if(icol == 19) std::cout << "accum:" << accum << std::endl; - } - data_out.emiss_sectors[0](icol) = accum; - } - - /*const auto policy = ESU::get_default_team_policy(data_beg.data.ncols, 1); + const int nsectors = data_beg.data.nsectors; + const int ncols = data_beg.data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nsectors); Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - double result; - - Kokkos::parallel_reduce("Loop1", N, KOKKOS_LAMBDA (const int& i, double& - lsum) { lsum += 1.0*i; - }, result); - - Kokkos::parallel_reduce("srfEmiss_reduction_loop", N, KOKKOS_LAMBDA - (const int& i, double& lsum) { for(int i = 0; i < data_beg.data.nsectors; ++i) - { Kokkos::single(Kokkos::PerTeam(team), [&] { accum[icol] = accum[icol] + - linear_interp(data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), - delta_t_fraction); - - }); - } - data_out.emiss_sectors[0](icol) = accum[icol]; + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int icol = team.league_rank(); // column index + Real accum = 0; + // Parallel reduction over sectors + // FIXME: Do we need to use Kokkos::Single for each team here??? + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, nsectors), + [&](const int i, Real &update) { + const auto beg = data_beg.data.emiss_sectors[i](icol); + const auto end = data_end.data.emiss_sectors[i](icol); + update += linear_interp(beg, end, delta_t_fraction); + }, + accum); + // Assign the accumulated value to the output + data_out.emiss_sectors[0](icol) = accum; }); - Kokkos::fence();*/ } // perform_time_interpolation template @@ -332,32 +255,6 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate -template -template -void srfEmissFunctions::init_srf_emiss_objects( - const int ncol, const std::shared_ptr &grid, - const std::string &data_file, - const std::array §ors, - const std::string &srf_map_file, - // output - std::shared_ptr &SrfEmissHorizInterp, - srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, - srfEmissOutput &SrfEmissData_out, - std::shared_ptr &SrfEmissDataReader) { - // Init horizontal remap - SrfEmissHorizInterp = - create_horiz_remapper(grid, data_file, sectors, srf_map_file); - - // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, numSectors); - SrfEmissData_end = srfEmissInput(ncol, numSectors); - SrfEmissData_out.init(ncol, 1, true); - - // Create reader (an AtmosphereInput object) - SrfEmissDataReader = - create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); -} // init_srf_emiss_objects - template void srfEmissFunctions::init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, From 0c258d47649c18ffc4e30b8c2ab00c6f5b4744aa Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 12:25:53 -0700 Subject: [PATCH 472/904] Adds output fileds to yaml and also change the location of emissions files --- .../src/physics/mam/srf_emission_impl.hpp | 3 +- .../mam/constituent_fluxes/output.yaml | 33 ++++++++++++++++++- .../mam/emissions/CMakeLists.txt | 16 +++++++++ .../single-process/mam/emissions/input.yaml | 18 +++++----- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index ef99879277eb..5191966f588f 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -144,6 +144,7 @@ void srfEmissFunctions::perform_time_interpolation( // Assign the accumulated value to the output data_out.emiss_sectors[0](icol) = accum; }); + Kokkos::fence(); } // perform_time_interpolation template @@ -282,4 +283,4 @@ void srfEmissFunctions::init_srf_emiss_objects( } // namespace } // namespace scream::mam_coupling -#endif // SRF_EMISSION_IMPL_HPP \ No newline at end of file +#endif // SRF_EMISSION_IMPL_HPP diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml index e51b96eb5ce3..b237056b5cc1 100644 --- a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml @@ -5,7 +5,38 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + output_control: Frequency: 1 frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index b57adcb60ef0..3d64bc5b2b27 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -25,6 +25,22 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 2e63314ebb1e..06be9ec8a883 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,15 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From c29bab538696ea6fedbfac421d55a1049f4b6029 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 3 Aug 2024 13:42:55 -0700 Subject: [PATCH 473/904] Some GPU fixes but needs some cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 5c5129d1b7c1..564fa9465fae 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -266,6 +266,7 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -286,7 +287,8 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::fence(); // policy - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] @@ -296,7 +298,17 @@ void MAMConstituentFluxes::run_impl(const double dt) { nlev_, // output rpdel_); - std::cout << "BEFORE:::" << wet_aero_.gas_mmr[4](1, 1) << std::endl; + auto wet_aero = wet_aero_; + auto dry_atm = dry_atm_; + auto nlev = nlev_; + auto rpdel = rpdel_; + auto constituent_fluxes = constituent_fluxes_; + + + //auto aa = wet_aero.gas_mmr[4](1, 1); + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); + + printf("BEFORE:::%e\n",host_view(0,0)); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -307,14 +319,15 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get prognostics mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero_, icol); + mam_coupling::aerosols_for_column(wet_aero, icol); + //printf("in-loop-wetaero:,%e,%e\n", wet_aero.gas_mmr[4](0,0), progs_at_col.q_gas[4](0)); // get atmospheric quantities - haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); + haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); // Construct state_q (interstitial) array Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev_), [&](int klev) { + Kokkos::TeamVectorRange(team, nlev), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -323,25 +336,35 @@ void MAMConstituentFluxes::run_impl(const double dt) { mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); + // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); // Factor to convert units from kg/m2/s to kg/kg - Real unit_factor = dt * gravit * rpdel_(icol, klev); + Real unit_factor = dt * gravit * rpdel(icol, klev); // Update state with the constituent fluxes (modified // units) + printf("in-loop-klev:,%e\n", progs_at_col.q_gas[4](icol)); for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { state_q_at_lev_col[icnst] += - constituent_fluxes_(icol, icnst) * unit_factor; + constituent_fluxes(icol, icnst) * unit_factor; + //printf("in-loop-update:,%e,%e, %i\n", progs_at_col.q_gas[icnst](icol), state_q_at_lev_col[icnst], icnst); + printf("INLOOP:::%e, %i\n",state_q_at_lev_col[icnst], icnst); } mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, progs_at_col, klev); + }); }); - - std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; + auto aa1 = wet_aero.gas_mmr; + auto bb1 = aa1[4]; + auto host_view1 = Kokkos::create_mirror_view(bb1); + auto cc1 = host_view1(1,1); + printf("AFTER:::%e\n",cc1); + printf("Scientific notation:");//, wet_aero.gas_mmr[4](1, 1)); + //std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; } // run_impl ends // ============================================================================= From e4b0aa27768e0e760e7c8c8dc47ecb32c093581e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 3 Aug 2024 23:17:38 -0700 Subject: [PATCH 474/904] Reworked logic to add fluxes only at the surface level-needs cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 90 ++++++------------- ...eamxx_mam_constituent_fluxes_interface.hpp | 3 - 2 files changed, 29 insertions(+), 64 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 564fa9465fae..7fc360d92c88 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -249,12 +249,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(gas_mmr_field_name).get_view(); } - //----------------------------------------------------------------- - // Allocate memory - //----------------------------------------------------------------- - // Inverse of pseudo_density (1/Pa or or N/m2 or kg/m/s2) - rpdel_ = view_2d("rpdel_", ncol_, nlev_); - //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- @@ -266,7 +260,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -287,84 +280,59 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::fence(); // policy - //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] - // compute inverse of pseudo_density - mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, - nlev_, - // output - rpdel_); - auto wet_aero = wet_aero_; - auto dry_atm = dry_atm_; - auto nlev = nlev_; - auto rpdel = rpdel_; + auto wet_aero = wet_aero_; + auto dry_atm = dry_atm_; + const int surface_lev = nlev_ - 1; auto constituent_fluxes = constituent_fluxes_; + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); - - //auto aa = wet_aero.gas_mmr[4](1, 1); + // auto aa = wet_aero.gas_mmr[4](1, 1); auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); - printf("BEFORE:::%e\n",host_view(0,0)); + for(int icnst = 0; icnst < 5; ++icnst) { + printf("BEFORE:::%e,%e\n", host_view(icnst, 71), + wet_aero_.gas_mmr[4](icnst, 71)); + } + + const auto policy_pcnst = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol_, mam4::aero_model::pcnst); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // To construct state_q, we need fields from Prognostics (aerosols) - // and Atmosphere (water species such as qv, qc etc.) + EKAT_REQUIRE_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must not be zero!"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - // get prognostics mam4::Prognostics progs_at_col = mam_coupling::aerosols_for_column(wet_aero, icol); - //printf("in-loop-wetaero:,%e,%e\n", wet_aero.gas_mmr[4](0,0), progs_at_col.q_gas[4](0)); // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); + Real state_q_at_surf_lev[mam4::aero_model::pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); - // Construct state_q (interstitial) array - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_lev_col, klev); - - - // get the start index for gas species in the state_q array - int istart = mam4::utils::gasses_start_ind(); + const Real unit_factor = dt * gravit * rpdel; - // Factor to convert units from kg/m2/s to kg/kg - Real unit_factor = dt * gravit * rpdel(icol, klev); - - // Update state with the constituent fluxes (modified - // units) - printf("in-loop-klev:,%e\n", progs_at_col.q_gas[4](icol)); - for(int icnst = istart; icnst < mam4::aero_model::pcnst; - ++icnst) { - state_q_at_lev_col[icnst] += + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, istart, mam4::aero_model::pcnst), + [&](int icnst) { + state_q_at_surf_lev[icnst] += constituent_fluxes(icol, icnst) * unit_factor; - //printf("in-loop-update:,%e,%e, %i\n", progs_at_col.q_gas[icnst](icol), state_q_at_lev_col[icnst], icnst); - printf("INLOOP:::%e, %i\n",state_q_at_lev_col[icnst], icnst); - } - mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, - progs_at_col, klev); - }); + mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, + progs_at_col, surface_lev); }); - auto aa1 = wet_aero.gas_mmr; - auto bb1 = aa1[4]; - auto host_view1 = Kokkos::create_mirror_view(bb1); - auto cc1 = host_view1(1,1); - printf("AFTER:::%e\n",cc1); - printf("Scientific notation:");//, wet_aero.gas_mmr[4](1, 1)); - //std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; } // run_impl ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 448ae5464b87..f5d153360b96 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -15,7 +15,6 @@ namespace scream { // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMConstituentFluxes final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; - using view_2d = Field::view_dev_t; using const_view_2d = Field::view_dev_t; // number of horizontal columns @@ -36,8 +35,6 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { const_view_2d constituent_fluxes_; - view_2d rpdel_; // Inverse of pdel_ or pseudo_density - public: // Constructor MAMConstituentFluxes(const ekat::Comm &comm, From 4c5f58c63b3fecd2059bf5e0535b6e60e4646bb0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 4 Aug 2024 00:06:51 -0700 Subject: [PATCH 475/904] Fixes pm-gpu build and run-needs cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 7fc360d92c88..9fc712ba46f7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -285,10 +285,11 @@ void MAMConstituentFluxes::run_impl(const double dt) { using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] - + const int ncol = ncol_; auto wet_aero = wet_aero_; auto dry_atm = dry_atm_; const int surface_lev = nlev_ - 1; + static constexpr int pcnst = mam4::aero_model::pcnst; auto constituent_fluxes = constituent_fluxes_; // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); @@ -297,20 +298,19 @@ void MAMConstituentFluxes::run_impl(const double dt) { auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); for(int icnst = 0; icnst < 5; ++icnst) { - printf("BEFORE:::%e,%e\n", host_view(icnst, 71), - wet_aero_.gas_mmr[4](icnst, 71)); - } + printf("BEFORE:::%e\n", host_view(icnst, 71)); + } const auto policy_pcnst = ekat::ExeSpaceUtils::get_default_team_policy( - ncol_, mam4::aero_model::pcnst); + ncol, pcnst); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - EKAT_REQUIRE_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must not be zero!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must not be zero!\n"); const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); mam4::Prognostics progs_at_col = @@ -318,21 +318,24 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); - Real state_q_at_surf_lev[mam4::aero_model::pcnst] = {}; + Real state_q_at_surf_lev[pcnst] = {}; mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); const Real unit_factor = dt * gravit * rpdel; - + auto pcnst1 = pcnst; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, mam4::aero_model::pcnst), + Kokkos::TeamVectorRange(team, istart, pcnst1), [&](int icnst) { state_q_at_surf_lev[icnst] += constituent_fluxes(icol, icnst) * unit_factor; }); mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, - progs_at_col, surface_lev); + progs_at_col, surface_lev); }); + for(int icnst = 0; icnst < 5; ++icnst) { + printf("AFTER:::%e\n", host_view(icnst, 71)); + } } // run_impl ends // ============================================================================= From 209ba3db2733081f2125559c56359f6a989e8d5d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 5 Aug 2024 19:13:22 -0700 Subject: [PATCH 476/904] Moved Kokkos - loop outside of interface and verified that gas and aerosols mmrs are modified --- ...eamxx_mam_constituent_fluxes_functions.hpp | 105 ++++++++++++++++++ ...eamxx_mam_constituent_fluxes_interface.cpp | 72 +++--------- ...eamxx_mam_constituent_fluxes_interface.hpp | 2 + 3 files changed, 122 insertions(+), 57 deletions(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp new file mode 100644 index 000000000000..9059cf5ccbc6 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -0,0 +1,105 @@ +#ifndef EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP +#define EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP + +#include +#include + +namespace scream { + +namespace { + +void update_gas_aerosols_using_constituents( + const int ncol, const int nlev, const double dt, + const mam_coupling::DryAtmosphere &dry_atm, + const MAMConstituentFluxes::const_view_2d &constituent_fluxes, + // output + const mam_coupling::AerosolState &wet_aero) { + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr int pcnst = mam4::aero_model::pcnst; + + // Declare local variables + const int ncol_loc = ncol; + auto wet_aero_loc = wet_aero; + auto dry_atm_loc = dry_atm; + const int surface_lev = nlev - 1; + auto constituent_fluxes_loc = constituent_fluxes; + + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); + + // number of constituents to update + const int nconstituents = pcnst - istart; + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, surface_lev), icnst + 9); + } + + // Create a policy to loop over columns annd number of constituents + // to update + // const auto policy = + // ekat::ExeSpaceUtils::get_default_team_policy(ncol, + // nconstituents); + const auto policy = ekat::ExeSpaceUtils:: + get_default_team_policy(1, nconstituents); + + // Loop through all columns to update tracer mixing rations + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + //---------------------------------------------------------------------- + // To form EAM like state%q array, we need prognostics (gas and aerosol + // mmrs) atmosphere (qv, qc, nc, ni, etc.) + //---------------------------------------------------------------------- + + // Get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(wet_aero, // output + icol); // input + + // Get atmospheric quantities + const haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atm, // output + icol); // input + + // Form state%q like array + Real state_q_at_surf_lev[pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, // input + state_q_at_surf_lev, // output + surface_lev); // input + + // Compute the units conversion factor (kg/m2/s to kg/kg) + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must be non-zero!\n"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); + const Real unit_factor = dt * gravit * rpdel; + + // Loop for + auto pcnst_loc = pcnst; + for(int icnst = 9; icnst < 15; ++icnst) { + printf("BEFORE-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], + progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); + } + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, istart, pcnst_loc), [&](int icnst) { + state_q_at_surf_lev[icnst] += + constituent_fluxes(icol, icnst) * unit_factor; + }); // pcsnt loop + mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, // input + progs_at_col, // output + surface_lev); // input + + for(int icnst = 9; icnst < 15; ++icnst) { + printf("After-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], + progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); + } + }); // icol loop +} + +} // namespace +} // namespace scream + +#endif // ifdef EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 9fc712ba46f7..1d1ffca29285 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -1,3 +1,4 @@ +#include #include namespace scream { @@ -260,8 +261,12 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { + // const auto scan_policy = ekat::ExeSpaceUtils< + // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, + // nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in @@ -279,63 +284,16 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // policy - // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view1 = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + printf("END:::%e, %i\n", host_view1(0, 71), icnst + 9); + } - using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - const int ncol = ncol_; - auto wet_aero = wet_aero_; - auto dry_atm = dry_atm_; - const int surface_lev = nlev_ - 1; - static constexpr int pcnst = mam4::aero_model::pcnst; - auto constituent_fluxes = constituent_fluxes_; - // get the start index for gas species in the state_q array - int istart = mam4::utils::gasses_start_ind(); - - // auto aa = wet_aero.gas_mmr[4](1, 1); - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); - - for(int icnst = 0; icnst < 5; ++icnst) { - printf("BEFORE:::%e\n", host_view(icnst, 71)); - } - - const auto policy_pcnst = - ekat::ExeSpaceUtils::get_default_team_policy( - ncol, pcnst); - // Loop through all columns to update tracer mixing rations - Kokkos::parallel_for( - policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must not be zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, icol); - - // get atmospheric quantities - haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); - - const Real unit_factor = dt * gravit * rpdel; - auto pcnst1 = pcnst; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, pcnst1), - [&](int icnst) { - state_q_at_surf_lev[icnst] += - constituent_fluxes(icol, icnst) * unit_factor; - }); - mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, - progs_at_col, surface_lev); - }); - for(int icnst = 0; icnst < 5; ++icnst) { - printf("AFTER:::%e\n", host_view(icnst, 71)); - } } // run_impl ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index f5d153360b96..7e77fccf2afc 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -14,9 +14,11 @@ namespace scream { // The process responsible for applying MAM4 constituent fluxes. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMConstituentFluxes final : public scream::AtmosphereProcess { + public: using KT = ekat::KokkosTypes; using const_view_2d = Field::view_dev_t; + private: // number of horizontal columns int ncol_, nlev_; From d4c9e2ffbcedb5fedb26911d578ac2b403dfd062 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 9 Aug 2024 08:06:45 -0700 Subject: [PATCH 477/904] Fixes some GPU errors and warnings --- ...eamxx_mam_constituent_fluxes_functions.hpp | 107 +++++++----------- ...eamxx_mam_constituent_fluxes_interface.cpp | 78 +++++++------ 2 files changed, 89 insertions(+), 96 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index 9059cf5ccbc6..c53c09eacbd9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -19,7 +19,6 @@ void update_gas_aerosols_using_constituents( static constexpr int pcnst = mam4::aero_model::pcnst; // Declare local variables - const int ncol_loc = ncol; auto wet_aero_loc = wet_aero; auto dry_atm_loc = dry_atm; const int surface_lev = nlev - 1; @@ -28,75 +27,57 @@ void update_gas_aerosols_using_constituents( // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - // number of constituents to update + // number of constituents to update (currently updating only MAM4xx constituents) const int nconstituents = pcnst - istart; - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, surface_lev), icnst + 9); - } - // Create a policy to loop over columns annd number of constituents // to update - // const auto policy = - // ekat::ExeSpaceUtils::get_default_team_policy(ncol, - // nconstituents); - const auto policy = ekat::ExeSpaceUtils:: - get_default_team_policy(1, nconstituents); + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol, + nconstituents); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - //---------------------------------------------------------------------- - // To form EAM like state%q array, we need prognostics (gas and aerosol - // mmrs) atmosphere (qv, qc, nc, ni, etc.) - //---------------------------------------------------------------------- - - // Get prognostics - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, // output - icol); // input - - // Get atmospheric quantities - const haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atm, // output - icol); // input - - // Form state%q like array - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, // input - state_q_at_surf_lev, // output - surface_lev); // input - - // Compute the units conversion factor (kg/m2/s to kg/kg) - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must be non-zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - const Real unit_factor = dt * gravit * rpdel; - - // Loop for - auto pcnst_loc = pcnst; - for(int icnst = 9; icnst < 15; ++icnst) { - printf("BEFORE-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], - progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); - } - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, pcnst_loc), [&](int icnst) { - state_q_at_surf_lev[icnst] += - constituent_fluxes(icol, icnst) * unit_factor; - }); // pcsnt loop - mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, // input - progs_at_col, // output - surface_lev); // input - - for(int icnst = 9; icnst < 15; ++icnst) { - printf("After-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], - progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); - } - }); // icol loop + policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + //---------------------------------------------------------------------- + // To form EAM like state%q array, we need prognostics (gas and aerosol + // mmrs) atmosphere (qv, qc, nc, ni, etc.) + //---------------------------------------------------------------------- + + // Get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(wet_aero, // output + icol); // input + // Get atmospheric quantities + const haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atm, // output + icol); // input + + // Form state%q like array + Real state_q_at_surf_lev[pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, // input + state_q_at_surf_lev, // output + surface_lev); // input + + // Compute the units conversion factor (kg/m2/s to kg/kg) + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must be non-zero!\n"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); + const Real unit_factor = dt * gravit * rpdel; + + // Update state vector with constituent fluxes + for (int icnst = istart; icnst #include #include namespace scream { @@ -260,39 +261,50 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // ================================================================ // RUN_IMPL // ================================================================ -void MAMConstituentFluxes::run_impl(const double dt) { - // const auto scan_policy = ekat::ExeSpaceUtils< - // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, - // nlev_); - - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); - - // ------------------------------------------------------------------- - // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in - // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". - // We are converting wet atm to dry atm. Since we do not use or update - // any of the water constituents (qc, qv, qi etc.), we should be okay - // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. - // ------------------------------------------------------------------- - - // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, - constituent_fluxes_, - // output - wet_aero_); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view1 = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - printf("END:::%e, %i\n", host_view1(0, 71), icnst + 9); - } + void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, + nlev_); + + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + + // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ etc. + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } + auto start = std::chrono::steady_clock::now(); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + auto stop = std::chrono::steady_clock::now(); + auto duration = (stop - start); + + + // To get the value of duration use the count() + // member function on the duration object + printf("TIME:%e\n", std::chrono::duration(duration).count()); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } } // run_impl ends From 6f1e5d72d5f9e2135f18cd01e5042bc2d1e4bfd4 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Sat, 10 Aug 2024 07:51:55 -0600 Subject: [PATCH 478/904] Fix some compile warnings that are really errors. Calling host from device is not allowed and will fail at runtime. Also there were a couple of warnings about the implicit capture of the this pointer that turned out to cause runtime errors. --- ...and_online_emissions_process_interface.cpp | 36 ++++++------------- ...and_online_emissions_process_interface.hpp | 16 --------- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- .../eamxx/src/physics/mam/srf_emission.hpp | 27 +++++++------- .../src/physics/mam/srf_emission_impl.hpp | 21 +++++------ 5 files changed, 35 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 91a5b161dff4..09c173468a2f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -191,7 +191,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- // Work array to store fluxes after unit conversions to kg/m2/s - fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); + fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units", ncol_); // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; @@ -221,16 +221,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, 1); - // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - // policy to loop over columns only - const auto ncol_only_policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol_, 1); + // Zero output + Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -261,25 +254,18 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { const int species_index = spcIndex_in_pcnst_.at(ispec_srf.species_name); // modify units from molecules/cm2/s to kg/m2/s + auto fluxes_in_mks_units = this->fluxes_in_mks_units_; + auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units - Kokkos::parallel_for( - ncol_only_policy, - KOKKOS_LAMBDA(const MAMSrfOnlineEmiss::KT::MemberType &team) { - const int icol = team.league_rank(); - fluxes_in_mks_units_(icol) = - ispec_srf.data_out_.emiss_sectors[0](icol) * mfactor; - }); - - // Get subview - auto flux_1d_view = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - spcIndex_in_pcnst_.at(ispec_srf.species_name)); - - // update flux in constituent_fluxes_ view - Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); + Kokkos::parallel_for("fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = ispec_srf.data_out_.emiss_sectors(0,icol) * mfactor; + constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); + }); + } // for loop for species + Kokkos::fence(); } // run_imple ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index b67b1de0bd7d..031fb62d8b75 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -83,24 +83,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { void initialize(const view_2d &constituent_fluxes) { constituent_fluxes_pre_ = constituent_fluxes; } - - KOKKOS_INLINE_FUNCTION - void operator()( - const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index - // zero-out the constituent surface fluxes for all gas and aerosol - // species. - for(auto icnst = mam4::utils::gasses_start_ind(); - icnst < mam4::aero_model::pcnst; ++icnst) { - constituent_fluxes_pre_(i, icnst) = 0; - } - team.team_barrier(); - - } // operator() - // local variables for preprocess struct view_2d constituent_fluxes_pre_; - }; // MAMSrfOnlineEmiss::Preprocess private: diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 09d64d6028d9..a3a5f746aa30 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -777,7 +777,7 @@ void compute_wet_mixing_ratios(const Team& team, } // Computes the reciprocal of pseudo density for a column -KOKKOS_INLINE_FUNCTION +inline void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, const_view_2d pdel, const int nlev, diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 82a4b3113821..886d9d00609f 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -30,26 +30,23 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_, const int nsectors_) { - init(ncol_, nsectors_, true); + srfEmissData(const int ncol_, const int nsectors_) : + ncols(ncol_), + nsectors(nsectors_) + { + init(ncols, nsectors, true); } - void init(const int ncol_, const int nsectors_, const bool allocate) { - ncols = ncol_; - nsectors = nsectors_; - - if(allocate) { - for(int i = 0; i < nsectors; ++i) { - auto sector = view_1d("", ncols); - emiss_sectors.push_back(sector); - } - } + void init(const int ncol, const int nsector, const bool allocate) { + ncols = ncol; + nsectors = nsector; + if(allocate) + emiss_sectors = view_2d("AllSectors", nsectors, ncols); } // srfEmissData init // Basic spatial dimensions of the data int ncols, nsectors; - std::vector emiss_sectors; - + view_2d emiss_sectors; }; // srfEmissData struct srfEmissInput { @@ -141,4 +138,4 @@ struct srfEmissFunctions { } // namespace scream::mam_coupling #endif // SRF_EMISSION_HPP -#include "srf_emission_impl.hpp" \ No newline at end of file +#include "srf_emission_impl.hpp" diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 5191966f588f..35a2b90deeb0 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -55,14 +55,14 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - std::vector emiss_sectors; + std::vector field_emiss_sectors; for(int icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); - emiss_sectors.push_back(f); + field_emiss_sectors.push_back(f); remapper->register_field_from_tgt(f); } @@ -76,18 +76,18 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector emiss_sectors; + std::vector field_emiss_sectors; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - emiss_sectors.push_back(horiz_remapper->get_src_field(i)); + field_emiss_sectors.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - emiss_sectors, true); + field_emiss_sectors, true); } // create_srfEmiss_data_reader template template -KOKKOS_INLINE_FUNCTION ScalarX srfEmissFunctions::linear_interp( +ScalarX srfEmissFunctions::linear_interp( const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { return (1 - t) * x0 + t * x1; } // linear_interp @@ -136,13 +136,13 @@ void srfEmissFunctions::perform_time_interpolation( Kokkos::parallel_reduce( Kokkos::TeamThreadRange(team, nsectors), [&](const int i, Real &update) { - const auto beg = data_beg.data.emiss_sectors[i](icol); - const auto end = data_end.data.emiss_sectors[i](icol); + const auto beg = data_beg.data.emiss_sectors(i, icol); + const auto end = data_end.data.emiss_sectors(i, icol); update += linear_interp(beg, end, delta_t_fraction); }, accum); // Assign the accumulated value to the output - data_out.emiss_sectors[0](icol) = accum; + data_out.emiss_sectors(0, icol) = accum; }); Kokkos::fence(); } // perform_time_interpolation @@ -212,7 +212,8 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], sector); + const auto emiss = Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); + Kokkos::deep_copy(emiss, sector); } Kokkos::fence(); From 9c2fb6545c13b86ba1583be90bc36f71f13efd68 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 11:54:38 -0700 Subject: [PATCH 479/904] Fixes a warning for a size type for a string vector --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 35a2b90deeb0..7a677bbdca3e 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,7 +57,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(int icomp = 0; icomp < sector_names.size(); ++icomp) { + for(std::vector::size_type icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From c45a86a5d267d1a460990ca05b38522fa4662006 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 12:02:47 -0700 Subject: [PATCH 480/904] Clang format --- ...eamxx_mam_constituent_fluxes_functions.hpp | 87 +++++++++--------- ...eamxx_mam_constituent_fluxes_interface.cpp | 88 +++++++++---------- ...and_online_emissions_process_interface.cpp | 15 ++-- .../eamxx/src/physics/mam/srf_emission.hpp | 11 +-- .../src/physics/mam/srf_emission_impl.hpp | 11 ++- 5 files changed, 106 insertions(+), 106 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index c53c09eacbd9..e7ffb6376c35 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -27,57 +27,56 @@ void update_gas_aerosols_using_constituents( // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - // number of constituents to update (currently updating only MAM4xx constituents) + // number of constituents to update (currently updating only MAM4xx + // constituents) const int nconstituents = pcnst - istart; // Create a policy to loop over columns annd number of constituents // to update - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol, - nconstituents); + const auto policy = ekat::ExeSpaceUtils:: + get_default_team_policy(ncol, nconstituents); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - //---------------------------------------------------------------------- - // To form EAM like state%q array, we need prognostics (gas and aerosol - // mmrs) atmosphere (qv, qc, nc, ni, etc.) - //---------------------------------------------------------------------- - - // Get prognostics - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, // output - icol); // input - // Get atmospheric quantities - const haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atm, // output - icol); // input - - // Form state%q like array - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, // input - state_q_at_surf_lev, // output - surface_lev); // input - - // Compute the units conversion factor (kg/m2/s to kg/kg) - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must be non-zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - const Real unit_factor = dt * gravit * rpdel; - - // Update state vector with constituent fluxes - for (int icnst = istart; icnst::get_thread_range_parallel_scan_team_policy(ncol_, - nlev_); - - // ------------------------------------------------------------------- - // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in - // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". - // We are converting wet atm to dry atm. Since we do not use or update - // any of the water constituents (qc, qv, qi etc.), we should be okay - // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. - // ------------------------------------------------------------------- - - // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ etc. - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - auto start = std::chrono::steady_clock::now(); - update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, - constituent_fluxes_, - // output - wet_aero_); - auto stop = std::chrono::steady_clock::now(); - auto duration = (stop - start); - - - // To get the value of duration use the count() - // member function on the duration object - printf("TIME:%e\n", std::chrono::duration(duration).count()); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } +void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + + // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ + // etc. + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } + auto start = std::chrono::steady_clock::now(); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + auto stop = std::chrono::steady_clock::now(); + auto duration = (stop - start); + + // To get the value of duration use the count() + // member function on the duration object + printf("TIME:%e\n", + std::chrono::duration(duration).count()); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } } // run_impl ends diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 09c173468a2f..a4ac4778982d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -221,8 +221,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - - // Zero output + // Zero output Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation @@ -255,14 +254,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // modify units from molecules/cm2/s to kg/m2/s auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - auto constituent_fluxes = this->constituent_fluxes_; + auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units - Kokkos::parallel_for("fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - fluxes_in_mks_units(icol) = ispec_srf.data_out_.emiss_sectors(0,icol) * mfactor; - constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); - }); + Kokkos::parallel_for( + "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = + ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; + constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); + }); } // for loop for species Kokkos::fence(); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 886d9d00609f..29aaca421ea9 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -30,18 +30,15 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_, const int nsectors_) : - ncols(ncol_), - nsectors(nsectors_) - { + srfEmissData(const int ncol_, const int nsectors_) + : ncols(ncol_), nsectors(nsectors_) { init(ncols, nsectors, true); } void init(const int ncol, const int nsector, const bool allocate) { - ncols = ncol; + ncols = ncol; nsectors = nsector; - if(allocate) - emiss_sectors = view_2d("AllSectors", nsectors, ncols); + if(allocate) emiss_sectors = view_2d("AllSectors", nsectors, ncols); } // srfEmissData init // Basic spatial dimensions of the data diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 7a677bbdca3e..dfe1190cdb91 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,7 +57,8 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(std::vector::size_type icomp = 0; icomp < sector_names.size(); ++icomp) { + for(std::vector::size_type icomp = 0; + icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); @@ -87,8 +88,9 @@ srfEmissFunctions::create_srfEmiss_data_reader( template template -ScalarX srfEmissFunctions::linear_interp( - const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { +ScalarX srfEmissFunctions::linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t) { return (1 - t) * x0 + t * x1; } // linear_interp @@ -212,7 +214,8 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - const auto emiss = Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); + const auto emiss = + Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); Kokkos::deep_copy(emiss, sector); } From 245571a6e73fba2f3539e49c8442a767a874f61b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:33:00 -0700 Subject: [PATCH 481/904] Store size in local in to avoid declaring long size_type datatype --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index dfe1190cdb91..7b5195c69432 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,8 +57,9 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(std::vector::size_type icomp = 0; - icomp < sector_names.size(); ++icomp) { + const int sector_size = sector_names.size(); + for(int icomp = 0; + icomp < sector_size; ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From 029ba9ccf009550d017f67f69c9e931393d3ef4e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:34:42 -0700 Subject: [PATCH 482/904] clang format --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 7b5195c69432..2d29075e41a4 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -58,8 +58,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; const int sector_size = sector_names.size(); - for(int icomp = 0; - icomp < sector_size; ++icomp) { + for(int icomp = 0; icomp < sector_size; ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From a816871c21f9550799d751cd69e6ed1038dec076 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:40:54 -0700 Subject: [PATCH 483/904] Revert "Merge branch 'master' into singhbalwinder/branch_mjs/mam4xx/emissionsIntegration" This reverts commit c2f7d298d93a06ad3c75a4d42f5139d08c7ed528, reversing changes made to 2d481948a48c0d9e006a0c2038328ca3642c499d. --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- cime_config/machines/config_machines.xml | 51 ++-- .../cmake/machine-files/anlgce-ub22.cmake | 12 - .../eamxx/cmake/machine-files/anlgce.cmake | 6 +- components/eamxx/scripts/machines_specs.py | 4 - .../src/dynamics/homme/eamxx_homme_iop.cpp | 259 +++++++++--------- ...mxx_mam_microphysics_process_interface.cpp | 4 +- components/eamxx/src/physics/p3/p3_iso_c.f90 | 19 +- .../atm_process/atmosphere_process_hash.cpp | 4 +- .../eamxx/src/share/io/scorpio_output.cpp | 35 +-- .../iop/intensive_observation_period.cpp | 12 +- 13 files changed, 181 insertions(+), 231 deletions(-) delete mode 100644 components/eamxx/cmake/machine-files/anlgce-ub22.cmake diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 9ea25ae1864e..ccca0c479f26 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index e6fb53ba29f5..abd8b92e4f6c 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index e0a8e19f9c01..5ecdf6dec00c 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.11" - name: Run unit tests diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index eeb80756bb8c..6765d000e26c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -368,7 +368,6 @@ nvidia aocc cudatoolkit - cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -379,24 +378,24 @@ - PrgEnv-gnu/8.5.0 - gcc-native/12.3 + PrgEnv-gnu/8.3.3 + gcc/11.2.0 PrgEnv-nvidia - nvidia/24.5 + nvidia/22.7 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - gcc-native-mixed/12.3 + gcc-mixed/11.2.0 @@ -408,12 +407,12 @@ - cray-libsci/23.12.5 - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 + cray-libsci/23.02.1.1 + craype/2.7.20 + cray-mpich/8.1.25 + cray-hdf5-parallel/1.12.2.3 + cray-netcdf-hdf5parallel/4.9.0.3 + cray-parallel-netcdf/1.12.3.3 cmake/3.24.3 @@ -678,7 +677,6 @@ nvidia aocc cudatoolkit - cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -689,24 +687,26 @@ - PrgEnv-gnu/8.5.0 - gcc-native/12.3 + PrgEnv-gnu/8.3.3 + gcc/11.2.0 + PrgEnv-nvidia - nvidia/24.5 + nvidia/23.9 - cudatoolkit/12.2 + cudatoolkit/11.7 + craype-accel-nvidia80 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - gcc-native-mixed/12.3 @@ -718,13 +718,20 @@ - cray-libsci/23.12.5 + cray-libsci/23.02.1.1 + craype/2.7.20 + cray-mpich/8.1.25 + cray-hdf5-parallel/1.12.2.3 + cray-netcdf-hdf5parallel/4.9.0.3 + cray-parallel-netcdf/1.12.3.3 + cmake/3.24.3 + evp-patch diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake deleted file mode 100644 index e79f70015471..000000000000 --- a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake +++ /dev/null @@ -1,12 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -# Remove this if you are using a resource manager (slurm etc) -set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") - -# EKAT MPI settings -set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") -set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index e79f70015471..079000059d19 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) + # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 43e5ebfa083c..9536d415c0de 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,10 +82,6 @@ ["mpicxx","mpifort","mpicc"], "", ""), - "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], - ["mpicxx","mpifort","mpicc"], - "", - ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 74579df552bd..eaf65a69cda5 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,14 +10,14 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" +#include "ElementOps.hpp" +#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" +#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" -// SCREAM includes -#include "share/util/scream_common_physics_functions.hpp" - // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -222,7 +222,11 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - using PF = PhysicsFunctions; + + using EOS = Homme::EquationOfState; + using ElementOps = Homme::ElementOps; + using KV = Homme::KernelVariables; + using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -259,7 +263,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states + // Homme element states and EOS/EO classes auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -268,6 +272,11 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); + EOS eos; + eos.init(params.theta_hydrostatic_mode, hvcoord); + + ElementOps elem_ops; + elem_ops.init(hvcoord); const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); // Load data from IOP files, if necessary @@ -310,66 +319,92 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for eamxx - const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + // Team policy and workspace manager for both homme and scream + // related loops. We need separate policies since hommexx functions used here + // assume they are called inside nested loops for elements and Gaussian points, + // whereas EAMxx function we use expects a single level of parallelism + // for elements and Guassian points. + // TODO: scream::ColumnOps functions could take an arbitary loop boundary + // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that + // all 3 kernel launches here could be combined. + const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); + const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); + WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); + WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); view_Nd + rstar ("rstar", nelem, NGP, NGP, NLEV), + exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing temperature - auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; + // Lambda for computing rstar, exner, and temperature from Hommexx + auto compute_homme_states = [&] () { + Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); + auto ws = homme_wsm.get_workspace(team); + auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); + uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto pnh_i = ekat::subview(pnh, idx); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); + Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); + Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); + Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); + + // Compute exner from EOS + if (theta_hydrostatic_mode) { + auto hydro_p_int = ws.take("hydro_p_int"); + Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); + elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); + eos.compute_exner(kv, pnh_scalar, exner_scalar); + ws.release(hydro_p_int); + } else { + eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); + } + + // Get the temperature from dynamics states + elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); }); // Release WS views - ws.release_many_contiguous<1>({&pmid}); + ws.release_macro_block(pnh_slot, NGP*NGP); }); }; - // Preprocess some homme states to get temperature - compute_temperature(); + // Preprocess some homme states to get temperature and exner + compute_homme_states(); Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); + auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -414,57 +449,44 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - team.team_barrier(); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + // Recompute rstar with updated qv, and convert updated temperature back to potential temperature + elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + }); }); - - // Release WS views - ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); if (iop_nudge_tq or iop_nudge_uv) { @@ -472,8 +494,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute temperature - compute_temperature(); + // Compute rstar, exner and temperature from Hommexx + compute_homme_states(); Kokkos::fence(); } @@ -549,36 +571,25 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_iop, + policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - if (iop_nudge_tq or iop_nudge_uv) { - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { + KV kv(team); + const int ie = team.league_rank(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high @@ -588,26 +599,22 @@ apply_iop_forcing(const Real dt) for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + and + pressure_from_iop >= iop_nudge_tq_high*100); } qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + // Convert updated temperature back to potential temperature + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); } }); - } - - // Release WS views - ws.release_many_contiguous<1>({&pmid}); + }); }); } } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 8b93936148ce..7ea2adfddb63 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,8 +317,6 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs - auto some_step = step_; - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index @@ -450,7 +448,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config.amicphys, some_step, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index ea0a18411c10..d75af1cb2edb 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=512), intent(out) :: string + character(kind=c_char, len=256), intent(inout) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,17 +69,10 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. -#ifdef SCREAM_DOUBLE_PRECISION - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR -#else - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR -#endif + call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") + call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") + call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") + call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") if (write_tables) then call p3_init_b() diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 37cb251d7796..9fb5726084f4 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index a105e244d76f..4cb3f50403ac 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,42 +1038,11 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { - std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - if (mode==scorpio::FileMode::Append) { - // Similar to the regular fields above, check that the var is in the file, and has the right properties - EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), - "Error! Cannot append, due to variable missing from the file.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n"); - const auto& var = scorpio::get_var(filename,name); - EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, - "Error! Cannot append, due to variable dimensions mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var dims : " + ekat::join(vec_of_dims,",") + "\n" - " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); - EKAT_REQUIRE_MSG (var.units==unitless, - "Error! Cannot append, due to variable units mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var units: " + unitless + "\n" - " - var units from file: " + var.units + "\n"); - EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, - "Error! Cannot append, due to time dependency mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" - " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); - } else { - // Note, unlike with regular output variables, for the average counting - // variables we don't need to add all of the extra metadata. So we simply - // define the variable. - scorpio::define_var(filename, name, unitless, vec_of_dims, - "real",fp_precision, m_add_time_dim); - } + scorpio::define_var(filename, name, "unitless", vec_of_dims, + "real",fp_precision, m_add_time_dim); } } } // register_variables diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 32b120c57002..76e039e65ab5 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -270,9 +270,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),(Real)0.1); + m_params.get("target_latitude"); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - std::max(m_params.get("target_longitude"),(Real)0.1); + m_params.get("target_longitude"); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -543,14 +543,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); - // Read in IOP lev data - auto data = iop_file_pressure.get_view().data(); - scorpio::read_var(iop_file,"lev",data); - - // Convert to pressure to millibar (file gives pressure in Pa) - for (int ilev=0; ilev(); From a7e719109b41762b6c63eb572a06b10f08351f4c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 16:40:41 -0700 Subject: [PATCH 484/904] Rebases and update mam4 submodule to point to its latest master --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 5495c27a6df7..2eee1988b1e6 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d +Subproject commit 2eee1988b1e6488c904e0a28564bdcadfa190d34 From 20b3d5e223d31b136bee335664704bec0f0aaee7 Mon Sep 17 00:00:00 2001 From: jsbamboo Date: Tue, 13 Aug 2024 18:27:51 -0700 Subject: [PATCH 485/904] update Livermore Computing machine config for eamxx --- .../eamxx/cmake/machine-files/dane-intel.cmake | 6 ++++++ components/eamxx/cmake/machine-files/dane.cmake | 13 +++++++++++++ components/eamxx/cmake/machine-files/ruby.cmake | 2 -- components/eamxx/scripts/machines_specs.py | 12 ++++++++---- 4 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 components/eamxx/cmake/machine-files/dane-intel.cmake create mode 100644 components/eamxx/cmake/machine-files/dane.cmake diff --git a/components/eamxx/cmake/machine-files/dane-intel.cmake b/components/eamxx/cmake/machine-files/dane-intel.cmake new file mode 100644 index 000000000000..df6aad60256b --- /dev/null +++ b/components/eamxx/cmake/machine-files/dane-intel.cmake @@ -0,0 +1,6 @@ +include(${CMAKE_CURRENT_LIST_DIR}/dane.cmake) +set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64/ -qmkl" CACHE STRING "" FORCE) +set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) +set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) +execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/dane.cmake b/components/eamxx/cmake/machine-files/dane.cmake new file mode 100644 index 000000000000..aec9f2b42227 --- /dev/null +++ b/components/eamxx/cmake/machine-files/dane.cmake @@ -0,0 +1,13 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Load all kokkos settings from Ekat's mach file +set(Kokkos_ENABLE_SERIAL TRUE CACHE BOOL "") + +# Enable Broadwell arch in Kokkos +option(Kokkos_ARCH_BDW "" ON) +# Load Broadwell flags and openmp backend for kokkos +include (${EKAT_MACH_FILES_PATH}/kokkos/intel-bdw.cmake) +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) + +include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) diff --git a/components/eamxx/cmake/machine-files/ruby.cmake b/components/eamxx/cmake/machine-files/ruby.cmake index 77d6e6618c71..aec9f2b42227 100644 --- a/components/eamxx/cmake/machine-files/ruby.cmake +++ b/components/eamxx/cmake/machine-files/ruby.cmake @@ -11,5 +11,3 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/intel-bdw.cmake) include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) - -set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 43e5ebfa083c..337f8b66d222 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -36,17 +36,21 @@ "", "/sems-data-store/ACME/baselines/scream/master-baselines"), "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'", - "export PATH=/usr/gdata/e3sm/netcdf/bin:$PATH", - "export LD_LIBRARY_PATH=/usr/gdata/e3sm/netcdf/lib:$LD_LIBRARY_PATH", + "export PATH=/usr/workspace/e3sm/netcdf/bin:$PATH", + "export LD_LIBRARY_PATH=/usr/workspace/e3sm/netcdf/lib:$LD_LIBRARY_PATH", ], ["mpicxx","mpifort","mpicc"], "bsub -Ip -qpdebug", ""), - "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "ruby-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), - "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "dane-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + ["mpicxx","mpifort","mpicc"], + "salloc --partition=pdebug", + ""), + "quartz-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), From f1b2ab67274e7120808df834bc82ae7e38ebc9b3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 21:10:12 -0700 Subject: [PATCH 486/904] Adds a multi-process test with srf emiss and constituent fluxes --- .../mam4xx/srf_online_emiss/shell_commands | 13 ----- .../multi-process/physics_only/CMakeLists.txt | 1 + .../CMakeLists.txt | 53 +++++++++++++++++++ .../input.yaml | 45 ++++++++++++++++ .../output.yaml | 41 ++++++++++++++ 5 files changed, 140 insertions(+), 13 deletions(-) delete mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands deleted file mode 100644 index 6995cd8b8562..000000000000 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands +++ /dev/null @@ -1,13 +0,0 @@ - -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" - -#modify initial condition file to get aerosol species ICs -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b - -# Add spa as RRTMG needs spa -$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b - - - diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index d6ed13dfb0a7..bb23911ff1c2 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -11,6 +11,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) add_subdirectory(mam/p3_mam4_wetscav) add_subdirectory(mam/shoc_cldfrac_p3_wetscav) + add_subdirectory(mam/mam4_srf_online_emiss_mam4_constituent_fluxes) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt new file mode 100644 index 000000000000..8d2f2653387e --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt @@ -0,0 +1,53 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME mam4_srf_online_emiss_mam4_constituent_fluxes) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS mam + LABELS physics mam4_srf_online_emiss mam4_constituent_fluxes + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS physics mam4_srf_online_emiss mam4_constituent_fluxes + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml new file mode 100644 index 000000000000..45fac36611e4 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -0,0 +1,45 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_srf_online_emiss, mam4_constituent_fluxes] + schedule_type: Sequential + mam4_srf_online_emiss: + # MAM4xx-Surface-Emissions + srf_remap_file: "" + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml new file mode 100644 index 000000000000..15e60e91517c --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml @@ -0,0 +1,41 @@ +%YAML 1.1 +--- +filename_prefix: shoc_mam4_drydep_output +Averaging Type: Instant +Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - constituent_fluxes + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 2bf266178d60a74c08ad005632e8b14c2de7f81b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 14 Aug 2024 14:06:41 -0700 Subject: [PATCH 487/904] Adds testmods and files for ne4pg2 CIME simulation --- .../cime_config/namelist_defaults_scream.xml | 29 ++++++++++--------- .../shell_commands | 17 +++++++++++ ...and_online_emissions_process_interface.cpp | 2 +- .../input.yaml | 2 ++ 4 files changed, 35 insertions(+), 15 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index e8e062cfaeb4..12095edff394 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -242,8 +242,7 @@ be lost if SCREAM_HACK_XML is not enabled. - - + @@ -266,22 +265,24 @@ be lost if SCREAM_HACK_XML is not enabled. - - + + - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240814.nc + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands new file mode 100644 index 000000000000..05b0b4954116 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -0,0 +1,17 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +#Update IC file and add the processes +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a4ac4778982d..f2a68ee6b5a7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -40,7 +40,7 @@ void MAMSrfOnlineEmiss::set_grids( grid_name); // Surface emissions remapping file - std::string srf_map_file = m_params.get("srf_remap_file"); + auto srf_map_file = m_params.get("srf_remap_file", ""); //-------------------------------------------------------------------- // Init dms srf emiss data structures diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 45fac36611e4..24c1d9f886e2 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -39,6 +39,8 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 0.0 + # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From 5879c571b60dc1ed89a78bdd3b2b4fb6f55daa97 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 14 Aug 2024 23:21:05 -0700 Subject: [PATCH 488/904] Revertes codes that were changed after the rebase --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- cime_config/machines/config_machines.xml | 51 ++-- .../cmake/machine-files/anlgce-ub22.cmake | 12 + .../eamxx/cmake/machine-files/anlgce.cmake | 6 +- components/eamxx/scripts/machines_specs.py | 4 + .../src/dynamics/homme/eamxx_homme_iop.cpp | 259 +++++++++--------- ...mxx_mam_microphysics_process_interface.cpp | 2 + components/eamxx/src/physics/p3/p3_iso_c.f90 | 19 +- .../atm_process/atmosphere_process_hash.cpp | 4 +- .../eamxx/src/share/io/scorpio_output.cpp | 35 ++- .../src/share/io/scream_scorpio_interface.cpp | 6 +- .../iop/intensive_observation_period.cpp | 12 +- .../single-process/mam/optics/CMakeLists.txt | 1 - 15 files changed, 233 insertions(+), 184 deletions(-) create mode 100644 components/eamxx/cmake/machine-files/anlgce-ub22.cmake diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ccca0c479f26..9ea25ae1864e 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index abd8b92e4f6c..e6fb53ba29f5 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 5ecdf6dec00c..e0a8e19f9c01 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" - name: Run unit tests diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6765d000e26c..eeb80756bb8c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -368,6 +368,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -378,24 +379,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/22.7 + nvidia/24.5 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - gcc-mixed/11.2.0 + gcc-native-mixed/12.3 @@ -407,12 +408,12 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + cray-libsci/23.12.5 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 @@ -677,6 +678,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -687,26 +689,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 - + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 - cudatoolkit/11.7 - + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 + gcc-native-mixed/12.3 @@ -718,20 +718,13 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 - evp-patch diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake new file mode 100644 index 000000000000..e79f70015471 --- /dev/null +++ b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake @@ -0,0 +1,12 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Remove this if you are using a resource manager (slurm etc) +set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") + +# EKAT MPI settings +set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index 079000059d19..e79f70015471 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) - # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 9536d415c0de..43e5ebfa083c 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,6 +82,10 @@ ["mpicxx","mpifort","mpicc"], "", ""), + "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index eaf65a69cda5..74579df552bd 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,14 +10,14 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" -#include "ElementOps.hpp" -#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" -#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" +// SCREAM includes +#include "share/util/scream_common_physics_functions.hpp" + // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -222,11 +222,7 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - - using EOS = Homme::EquationOfState; - using ElementOps = Homme::ElementOps; - using KV = Homme::KernelVariables; - + using PF = PhysicsFunctions; using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -263,7 +259,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states and EOS/EO classes + // Homme element states auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -272,11 +268,6 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - EOS eos; - eos.init(params.theta_hydrostatic_mode, hvcoord); - - ElementOps elem_ops; - elem_ops.init(hvcoord); const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); // Load data from IOP files, if necessary @@ -319,92 +310,66 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for both homme and scream - // related loops. We need separate policies since hommexx functions used here - // assume they are called inside nested loops for elements and Gaussian points, - // whereas EAMxx function we use expects a single level of parallelism - // for elements and Guassian points. - // TODO: scream::ColumnOps functions could take an arbitary loop boundary - // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that - // all 3 kernel launches here could be combined. - const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); - const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + // Team policy and workspace manager for eamxx + const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); - WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); + WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); view_Nd - rstar ("rstar", nelem, NGP, NGP, NLEV), - exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing rstar, exner, and temperature from Hommexx - auto compute_homme_states = [&] () { - Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + // Lambda for computing temperature + auto compute_temperature = [&] () { + Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = homme_wsm.get_workspace(team); - auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); - uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto pnh_i = ekat::subview(pnh, idx); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); - Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); - Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - - // Compute exner from EOS - if (theta_hydrostatic_mode) { - auto hydro_p_int = ws.take("hydro_p_int"); - Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); - elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); - eos.compute_exner(kv, pnh_scalar, exner_scalar); - ws.release(hydro_p_int); - } else { - eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); - } - - // Get the temperature from dynamics states - elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + // Compute temperature from virtual potential temperature + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { + auto T_val = vtheta_dp_i(k); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); }); // Release WS views - ws.release_macro_block(pnh_slot, NGP*NGP); + ws.release_many_contiguous<1>({&pmid}); }); }; - // Preprocess some homme states to get temperature and exner - compute_homme_states(); + // Preprocess some homme states to get temperature + compute_temperature(); Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -449,44 +414,57 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; + // Get temp views from workspace + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + + team.team_barrier(); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Recompute rstar with updated qv, and convert updated temperature back to potential temperature - elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); - }); + // Convert updated temperature back to psuedo density virtual potential temperature + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); + + // Release WS views + ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); if (iop_nudge_tq or iop_nudge_uv) { @@ -494,8 +472,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute rstar, exner and temperature from Hommexx - compute_homme_states(); + // Compute temperature + compute_temperature(); Kokkos::fence(); } @@ -571,25 +549,36 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_homme, + policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { + + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; + + // Get temp views from workspace + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + if (iop_nudge_tq or iop_nudge_uv) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high @@ -599,22 +588,26 @@ apply_iop_forcing(const Real dt) for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + and + pressure_from_iop >= iop_nudge_tq_high*100); } qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to potential temperature - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); } }); - }); + } + + // Release WS views + ws.release_many_contiguous<1>({&pmid}); }); } } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7ea2adfddb63..f8a34bd90d5b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,6 +317,8 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index d75af1cb2edb..ea0a18411c10 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=256), intent(inout) :: string + character(kind=c_char, len=512), intent(out) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,10 +69,17 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. - call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") - call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") - call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") - call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") +#ifdef SCREAM_DOUBLE_PRECISION + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR +#else + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR +#endif if (write_tables) then call p3_init_b() diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 9fb5726084f4..37cb251d7796 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 4cb3f50403ac..a105e244d76f 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,11 +1038,42 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { + std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - scorpio::define_var(filename, name, "unitless", vec_of_dims, - "real",fp_precision, m_add_time_dim); + if (mode==scorpio::FileMode::Append) { + // Similar to the regular fields above, check that the var is in the file, and has the right properties + EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), + "Error! Cannot append, due to variable missing from the file.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n"); + const auto& var = scorpio::get_var(filename,name); + EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, + "Error! Cannot append, due to variable dimensions mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var dims : " + ekat::join(vec_of_dims,",") + "\n" + " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + EKAT_REQUIRE_MSG (var.units==unitless, + "Error! Cannot append, due to variable units mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var units: " + unitless + "\n" + " - var units from file: " + var.units + "\n"); + EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, + "Error! Cannot append, due to time dependency mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" + " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); + } else { + // Note, unlike with regular output variables, for the average counting + // variables we don't need to add all of the extra metadata. So we simply + // define the variable. + scorpio::define_var(filename, name, unitless, vec_of_dims, + "real",fp_precision, m_add_time_dim); + } } } } // register_variables diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 2518907ec527..8d2f64994ddd 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -361,9 +361,9 @@ void finalize_subsystem () "Error! PIO subsystem was already finalized.\n"); for (auto& it : s.files) { - // EKAT_REQUIRE_MSG (it.second.num_customers==0, - // "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" - // " - filename: " + it.first + "\n"); + EKAT_REQUIRE_MSG (it.second.num_customers==0, + "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" + " - filename: " + it.first + "\n"); } s.files.clear(); diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 76e039e65ab5..32b120c57002 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -270,9 +270,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - m_params.get("target_latitude"); + std::max(m_params.get("target_latitude"),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - m_params.get("target_longitude"); + std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -543,6 +543,14 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); + // Read in IOP lev data + auto data = iop_file_pressure.get_view().data(); + scorpio::read_var(iop_file,"lev",data); + + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilev(); diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt index 50f2cfa68a3b..0a3a0b877c5e 100644 --- a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -40,7 +40,6 @@ set (TEST_INPUT_FILES scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc scream/mam4xx/physprops/poly_rrtmg_c20240206.nc - # scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) From 87068eda3d10d1b8486023bcde28382720b0fd69 Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Thu, 15 Aug 2024 09:56:51 -0500 Subject: [PATCH 489/904] Updates bgc defaults for grazing parameters Bugfix. Registry defaults were updated when implicit grazing was adding for E3SMv2, but the build defaults were kept the same. non-bfb in ice bgc runs. BFB for all others --- .../bld/namelist_files/namelist_defaults_mpassi.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml index 70db1f04189a..d62797de4518 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml @@ -289,9 +289,9 @@ 0.063 0.063 0.063 -0.0 -0.7 -0.7 +0.19 +0.19 +0.19 0.007 0.007 0.007 From 1d796b40a8362b503206274cfba423dd5c4577ee Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 15 Aug 2024 10:00:57 -0700 Subject: [PATCH 490/904] Changes emission files that use conservative mapping --- .../cime_config/namelist_defaults_scream.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 12095edff394..87791d0b7d00 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -271,14 +271,14 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc From c411a79da7d7d2ab6f3c1224feeb964ff9707d46 Mon Sep 17 00:00:00 2001 From: Jayesh Krishna Date: Thu, 15 Aug 2024 17:42:37 -0500 Subject: [PATCH 491/904] Adding support for PnetCDF in HOMME on Chrysalis Adding support for the PnetCDF library in HOMME on Chrysalis. This configuration files are used for standalone HOMME builds on Chrysalis --- components/homme/cmake/machineFiles/chrysalis.cmake | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index a336809f6d33..68ff76ec8082 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -20,7 +20,13 @@ SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") -SET (WITH_PNETCDF FALSE CACHE FILEPATH "") +EXECUTE_PROCESS(COMMAND pnetcdf-config --prefix + RESULT_VARIABLE PNCCONFIG_RESULT + OUTPUT_VARIABLE PNCCONFIG_OUTPUT + ERROR_VARIABLE PNCCONFIG_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE +) +SET (PnetCDF_PATH "${PNCCONFIG_OUTPUT}" CACHE STRING "") EXECUTE_PROCESS(COMMAND nf-config --prefix RESULT_VARIABLE NFCONFIG_RESULT From df53433349c56a895a32d6064eab952dd47ced24 Mon Sep 17 00:00:00 2001 From: Jayesh Krishna Date: Thu, 15 Aug 2024 19:05:12 -0500 Subject: [PATCH 492/904] Adding support for PnetCDF in HOMME on Anvil Adding support for the PnetCDF library in HOMME on Anvil. This configuration files are used for standalone HOMME builds on Anvil --- components/homme/cmake/machineFiles/anvil.cmake | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/components/homme/cmake/machineFiles/anvil.cmake b/components/homme/cmake/machineFiles/anvil.cmake index 325a9caa280f..2abee260b44c 100644 --- a/components/homme/cmake/machineFiles/anvil.cmake +++ b/components/homme/cmake/machineFiles/anvil.cmake @@ -20,7 +20,14 @@ ENDIF() # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_BDW ON CACHE BOOL "") -SET (WITH_PNETCDF FALSE CACHE FILEPATH "") +EXECUTE_PROCESS(COMMAND pnetcdf-config --prefix + RESULT_VARIABLE PNCCONFIG_RESULT + OUTPUT_VARIABLE PNCCONFIG_OUTPUT + ERROR_VARIABLE PNCCONFIG_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE +) +SET (PnetCDF_PATH "${PNCCONFIG_OUTPUT}" CACHE STRING "") + # # anvil module system doesn't set environment variables, but will put # nc-config in our path. anvil seperates C and Fortran libraries, From 02b3d75d1765c30e6cd927f48776e111dd7ba1bb Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 16 Aug 2024 10:59:56 -0400 Subject: [PATCH 493/904] fix shoc warning unused vars --- .../src/physics/shoc/eamxx_shoc_process_interface.hpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 1695e60dfbc1..2a71870be9de 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -88,13 +88,10 @@ class SHOCMacrophysics : public scream::AtmosphereProcess cldfrac_liq_prev(i,k)=cldfrac_liq(i,k); - const auto range = ekat::range(k*Spack::n); - const Smask in_nlev_range = (range < nlev); - - // Inverse of Exner. Assert that exner != 0 when in range before computing. + // Inverse of Exner. In non-rel builds, assert that exner != 0 when in range before computing. const Spack exner = PF::exner_function(p_mid(i,k)); const Smask nonzero = (exner != 0); - EKAT_KERNEL_ASSERT((nonzero || !in_nlev_range).all()); + EKAT_KERNEL_ASSERT((nonzero || !(ekat::range(k*Spack::n) < nlev);).all()); inv_exner(i,k).set(nonzero, 1/exner); tke(i,k) = ekat::max(mintke, tke(i,k)); From 62cfa47f7f2cec73b26e80785787717d75c8000a Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 16 Aug 2024 11:29:57 -0400 Subject: [PATCH 494/904] fix typo in assert --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 2a71870be9de..469c31de7af5 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -91,7 +91,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Inverse of Exner. In non-rel builds, assert that exner != 0 when in range before computing. const Spack exner = PF::exner_function(p_mid(i,k)); const Smask nonzero = (exner != 0); - EKAT_KERNEL_ASSERT((nonzero || !(ekat::range(k*Spack::n) < nlev);).all()); + EKAT_KERNEL_ASSERT((nonzero || !(ekat::range(k*Spack::n) < nlev)).all()); inv_exner(i,k).set(nonzero, 1/exner); tke(i,k) = ekat::max(mintke, tke(i,k)); From ffdb1bc85ea4c7849346f2b80115cb436391bfc2 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 16 Aug 2024 17:32:13 +0000 Subject: [PATCH 495/904] Update ALCF Sunspot machine config --- cime_config/machines/config_machines.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6cd9d585a25b..67611a0de225 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3413,13 +3413,9 @@ module /soft/packaging/lmod/lmod/libexec/lmod python - - /soft/modulefiles - spack cmake/3.26.3-gcc-11.2.0-vnn7ncx - prepend-deps/default + cmake - gcc oneapi/eng-compiler/2023.05.15.007 From 36936e52dddcfeb99a3f5976d3f3399833051328 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 14 Aug 2024 18:15:02 -0500 Subject: [PATCH 496/904] move to nb --- .../src/python/libpyscream/CMakeLists.txt | 9 ++- .../src/python/libpyscream/pyatmproc.hpp | 29 ++++--- .../eamxx/src/python/libpyscream/pyfield.hpp | 35 ++++---- .../eamxx/src/python/libpyscream/pygrid.hpp | 11 ++- .../src/python/libpyscream/pyparamlist.hpp | 81 ++++++++++--------- .../src/python/libpyscream/pyscream_ext.cpp | 20 ++--- .../eamxx/src/python/libpyscream/pyutils.hpp | 6 +- 7 files changed, 106 insertions(+), 85 deletions(-) diff --git a/components/eamxx/src/python/libpyscream/CMakeLists.txt b/components/eamxx/src/python/libpyscream/CMakeLists.txt index 103c6a80b888..7569d1cf7e60 100644 --- a/components/eamxx/src/python/libpyscream/CMakeLists.txt +++ b/components/eamxx/src/python/libpyscream/CMakeLists.txt @@ -1,7 +1,12 @@ -find_package(pybind11 REQUIRED) +# Detect the installed nanobind package and import it into CMake +find_package(Python COMPONENTS Interpreter Development REQUIRED) +execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -m nanobind --cmake_dir + OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT) +find_package(nanobind REQUIRED) find_package(mpi4py REQUIRED) -pybind11_add_module(pyscream_ext pyscream_ext.cpp) +nanobind_add_module(pyscream_ext pyscream_ext.cpp) target_link_libraries(pyscream_ext PUBLIC mpi4py scream_share diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 108edc2551b3..219837c5f736 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -12,8 +12,11 @@ #include -#include -#include +#include +#include +#include + +namespace nb = nanobind; namespace scream { @@ -26,7 +29,7 @@ struct PyAtmProc { std::shared_ptr output_mgr; - PyAtmProc (const pybind11::dict& d, const std::string& name) + PyAtmProc (const nb::dict& d, const std::string& name) { PyParamList params(d,name); @@ -102,7 +105,7 @@ struct PyAtmProc { ap->initialize(t0,RunType::Initial); } - pybind11::list read_ic (const std::string& ic_filename) { + std::vector read_ic (const std::string& ic_filename) { // Get input fields, and read them from file (if present). // If field is not in the IC, user is responsible for setting // it to an initial value @@ -128,10 +131,10 @@ struct PyAtmProc { } scorpio::release_file(ic_filename); - return pybind11::cast(missing); + return missing; } - pybind11::list list_fields(std::string ftype) { + std::vector list_fields(std::string ftype) { std::vector fields_list; for (const auto& field_pair : fields) { const auto& field_identifier = field_pair.second.f.get_header().get_identifier(); @@ -144,18 +147,18 @@ struct PyAtmProc { fields_list.push_back(field_pair.first); } } - return pybind11::cast(fields_list); + return fields_list; } - pybind11::list list_all_fields() { + std::vector list_all_fields() { return list_fields("all"); } - pybind11::list list_required_fields() { + std::vector list_required_fields() { return list_fields("required"); } - pybind11::list list_computed_fields() { + std::vector list_computed_fields() { return list_fields("computed"); } @@ -194,10 +197,10 @@ struct PyAtmProc { }; // Register type in the py module -inline void pybind_pyatmproc(pybind11::module& m) +inline void nb_pyatmproc(nb::module_& m) { - pybind11::class_(m,"AtmProc") - .def(pybind11::init()) + nb::class_(m,"AtmProc") + .def(nb::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) .def("get_params",&PyAtmProc::get_params) diff --git a/components/eamxx/src/python/libpyscream/pyfield.hpp b/components/eamxx/src/python/libpyscream/pyfield.hpp index eeb4125cecc2..96dae4b3e454 100644 --- a/components/eamxx/src/python/libpyscream/pyfield.hpp +++ b/components/eamxx/src/python/libpyscream/pyfield.hpp @@ -4,9 +4,11 @@ #include "share/field/field.hpp" #include "share/field/field_utils.hpp" -#include -#include -#include +#include +#include +#include + +namespace nb = nanobind; namespace scream { @@ -24,7 +26,8 @@ struct PyField { f.allocate_view(); } - pybind11::array get () const { + template + nb::ndarray get () const { const auto& fh = f.get_header(); const auto& fid = fh.get_identifier(); @@ -39,10 +42,14 @@ struct PyField { // NOTE: since the field may be padded, the strides do not necessarily // match the dims. Also, the strides must be grabbed from the // actual view, since the layout doesn't know them. - pybind11::array::ShapeContainer shape (fid.get_layout().dims()); + int shape_t = f.rank(); + size_t shape[shape_t] = {0}; + for (int i=0; i strides; - pybind11::dtype dt; + nb::dlpack::dtype dt; switch (fid.data_type()) { case DataType::IntType: dt = get_dt_and_set_strides(strides); @@ -59,8 +66,8 @@ struct PyField { // NOTE: you MUST set the parent handle, or else you won't have view semantic auto data = f.get_internal_view_data_unsafe(); - auto this_obj = pybind11::cast(this); - return pybind11::array(dt,shape,strides,data,pybind11::handle(this_obj)); + auto this_obj = nb::cast(this); + return nb::ndarray(data, shape_t, shape, nb::handle(this_obj), strides.data(), dt); } void sync_to_host () { @@ -75,7 +82,7 @@ struct PyField { private: template - pybind11::dtype get_dt_and_set_strides (std::vector& strides) const + nb::dlpack::dtype get_dt_and_set_strides (std::vector& strides) const { strides.resize(f.rank()); switch (f.rank()) { @@ -126,15 +133,15 @@ struct PyField { " - field rnak: " + std::to_string(f.rank()) + "\n"); } - return pybind11::dtype::of(); + return nb::dtype(); } }; -inline void pybind_pyfield (pybind11::module& m) { +inline void nb_pyfield (nb::module_& m) { // Field class - pybind11::class_(m,"Field") - .def(pybind11::init<>()) - .def("get",&PyField::get) + nb::class_(m,"Field") + .def(nb::init<>()) + .def("get",&PyField::get) .def("sync_to_host",&PyField::sync_to_host) .def("sync_to_dev",&PyField::sync_to_dev) .def("print",&PyField::print); diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index 1c0d56eb08b3..83cb78d6729e 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -5,10 +5,13 @@ #include "pyscream_ext.hpp" -#include +#include +#include #include +namespace nb = nanobind; + namespace scream { inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) @@ -38,9 +41,9 @@ inline void create_grids_manager (int ncols, int nlevs) create_grids_manager(ncols,nlevs,""); } -inline void pybind_pygrid (pybind11::module& m) { - m.def("create_grids_manager",pybind11::overload_cast(&create_grids_manager)); - m.def("create_grids_manager",pybind11::overload_cast(&create_grids_manager)); +inline void nb_pygrid (nb::module_& m) { + m.def("create_grids_manager",nb::overload_cast(&create_grids_manager)); + m.def("create_grids_manager",nb::overload_cast(&create_grids_manager)); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pyparamlist.hpp b/components/eamxx/src/python/libpyscream/pyparamlist.hpp index a631b6622e90..8da57a24bd35 100644 --- a/components/eamxx/src/python/libpyscream/pyparamlist.hpp +++ b/components/eamxx/src/python/libpyscream/pyparamlist.hpp @@ -3,12 +3,15 @@ #include -#include -#include -#include +#include +#include +#include +#include #include +namespace nb = nanobind; + namespace scream { struct PyParamList { @@ -19,11 +22,11 @@ struct PyParamList { : pl_ref(src) {} - PyParamList(const pybind11::dict& d) + PyParamList(const nb::dict& d) : PyParamList(d,"") {} - PyParamList(const pybind11::dict& d, const std::string& name) + PyParamList(const nb::dict& d, const std::string& name) : pl(name) , pl_ref(pl) { @@ -69,26 +72,26 @@ struct PyParamList { private: - void parse_dict(const pybind11::dict& d, ekat::ParameterList& p) { + void parse_dict(const nb::dict& d, ekat::ParameterList& p) { for (auto item : d) { - const std::string key = pybind11::str(item.first); - if (pybind11::isinstance(item.second)) { - auto pystr = pybind11::str(item.second); - p.set(key,pystr.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pyint = pybind11::cast(item.second); - p.set(key,pyint.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pyint = pybind11::cast(item.second); - p.set(key,pyint.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pydouble = pybind11::cast(item.second); - p.set(key,pydouble.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pylist = pybind11::cast(item.second); + auto key = nb::cast(item.first); + if (nb::isinstance(item.second)) { + auto pystr = nb::str(item.second); + p.set(key,nb::cast(pystr)); + } else if (nb::isinstance(item.second)) { + auto pyint = nb::cast(item.second); + p.set(key,nb::cast(pyint)); + } else if (nb::isinstance(item.second)) { + auto pyint = nb::cast(item.second); + p.set(key,nb::cast(pyint)); + } else if (nb::isinstance(item.second)) { + auto pydouble = nb::cast(item.second); + p.set(key,nb::cast(pydouble)); + } else if (nb::isinstance(item.second)) { + auto pylist = nb::cast(item.second); parse_list(pylist,p,key); - } else if (pybind11::isinstance(item.second)) { - auto pydict = pybind11::cast(item.second); + } else if (nb::isinstance(item.second)) { + auto pydict = nb::cast(item.second); parse_dict(pydict,p.sublist(key)); } else { EKAT_ERROR_MSG ("Unsupported/unrecognized dict entry type.\n"); @@ -96,43 +99,43 @@ struct PyParamList { } } - void parse_list (const pybind11::list& l, ekat::ParameterList&p, const std::string& key) { - EKAT_REQUIRE_MSG (pybind11::len(l)>0, + void parse_list (const nb::list& l, ekat::ParameterList&p, const std::string& key) { + EKAT_REQUIRE_MSG (nb::len(l)>0, "Error! Cannot deduce type for dictionary list entry '" + key + "'\n"); auto first = l[0]; - bool are_ints = pybind11::isinstance(first); - bool are_floats = pybind11::isinstance(first); - bool are_strings = pybind11::isinstance(first); + bool are_ints = nb::isinstance(first); + bool are_floats = nb::isinstance(first); + bool are_strings = nb::isinstance(first); if (are_ints) { - parse_list_impl(l,p,key); + parse_list_impl(l,p,key); } else if (are_floats) { - parse_list_impl(l,p,key); + parse_list_impl(l,p,key); } else if (are_strings) { - parse_list_impl(l,p,key); + parse_list_impl(l,p,key); } else { EKAT_ERROR_MSG ("Unrecognized/unsupported list entry type.\n"); } } template - void parse_list_impl(const pybind11::list& l, ekat::ParameterList& p, const std::string& key) { + void parse_list_impl(const nb::list& l, ekat::ParameterList& p, const std::string& key) { std::vector vals; for (auto item : l) { - EKAT_REQUIRE_MSG (pybind11::isinstance(item), + EKAT_REQUIRE_MSG (nb::isinstance(item), "Error! Inconsistent types in list entries.\n"); - auto item_py = pybind11::cast(item); - vals.push_back(item_py.template cast()); + auto item_py = nb::cast(item); + vals.push_back(nb::cast(item_py)); } p.set(key,vals); } }; -inline void pybind_pyparamlist (pybind11::module& m) +inline void nb_pyparamlist (nb::module_& m) { // Param list - pybind11::class_(m,"ParameterList") - .def(pybind11::init()) - .def(pybind11::init()) + nb::class_(m,"ParameterList") + .def(nb::init()) + .def(nb::init()) .def("sublist",&PyParamList::sublist) .def("print",&PyParamList::print) .def("set",&PyParamList::set) diff --git a/components/eamxx/src/python/libpyscream/pyscream_ext.cpp b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp index e0ad9ceec260..892d40e8ca3c 100644 --- a/components/eamxx/src/python/libpyscream/pyscream_ext.cpp +++ b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp @@ -7,11 +7,11 @@ #include -#include +#include #include -namespace py = pybind11; +namespace nb = nanobind; namespace scream { @@ -33,7 +33,7 @@ void initialize (MPI_Comm mpi_comm) { void initialize () { initialize(MPI_COMM_WORLD); } -void initialize (pybind11::object py_comm) { +void initialize (nb::object py_comm) { initialize(get_c_comm(py_comm)); } void finalize () { @@ -45,20 +45,20 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyscream_ext,m) { +NB_MODULE (pyscream_ext,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; // Scream Session - m.def("init",py::overload_cast<>(&initialize)); - m.def("init",py::overload_cast(&initialize)); + m.def("init",nb::overload_cast<>(&initialize)); + m.def("init",nb::overload_cast(&initialize)); m.def("finalize",&finalize); // Call all other headers' registration routines - pybind_pyparamlist(m); - pybind_pyfield(m); - pybind_pygrid(m); - pybind_pyatmproc(m); + nb_pyparamlist(m); + nb_pyfield(m); + nb_pygrid(m); + nb_pyatmproc(m); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pyutils.hpp b/components/eamxx/src/python/libpyscream/pyutils.hpp index 6467df57eda1..b02ec9a36a12 100644 --- a/components/eamxx/src/python/libpyscream/pyutils.hpp +++ b/components/eamxx/src/python/libpyscream/pyutils.hpp @@ -1,15 +1,15 @@ #ifndef PYUTILS_HPP #define PYUTILS_HPP -#include +#include #include #include #include -MPI_Comm get_c_comm (pybind11::object py_comm) { +MPI_Comm get_c_comm (nb::object py_comm) { if (import_mpi4py() < 0) { - throw pybind11::error_already_set(); + throw nb::python_error(); } auto py_src = py_comm.ptr(); if (not PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { From d8ae8f74bdbaa1168db0321c428a62f323e98808 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 16 Aug 2024 12:55:02 -0500 Subject: [PATCH 497/904] use EAMXX_ENABLE_PYSCREAM --- components/eamxx/CMakeLists.txt | 4 ++-- components/eamxx/src/CMakeLists.txt | 2 +- components/eamxx/tests/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 580ce26e7f50..398efa3bc2df 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -272,8 +272,8 @@ set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) -option (EAMXX_ENABLE_PYBIND "Whether to enable python interface to eamxx, via pybind11" OFF) -if (EAMXX_ENABLE_PYBIND) +option (EAMXX_ENABLE_PYSCREAM "Whether to enable python interface to eamxx" OFF) +if (EAMXX_ENABLE_PYSCREAM) # Pybind11 requires shared libraries set (BUILD_SHARED_LIBS ON) endif() diff --git a/components/eamxx/src/CMakeLists.txt b/components/eamxx/src/CMakeLists.txt index 7672d79f13d6..be88a2f51ae0 100644 --- a/components/eamxx/src/CMakeLists.txt +++ b/components/eamxx/src/CMakeLists.txt @@ -8,6 +8,6 @@ if (PROJECT_NAME STREQUAL "E3SM") add_subdirectory(mct_coupling) endif() -if (EAMXX_ENABLE_PYBIND) +if (EAMXX_ENABLE_PYSCREAM) add_subdirectory(python) endif() diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 7602f654356d..ad80045a0c13 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -75,7 +75,7 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) add_subdirectory(multi-process) endif() - if (EAMXX_ENABLE_PYBIND) + if (EAMXX_ENABLE_PYSCREAM) add_subdirectory(python) endif() endif() From 731618181e6c91c92335eb1b535f706ed654657a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 11:05:22 -0700 Subject: [PATCH 498/904] Fixes multi process test a typo in aci interface --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 8 ++++---- .../output.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7800a43c252d..ef2c0f2b2882 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -253,7 +253,7 @@ void MAMAci::set_grids( frz_unit, grid_name); // heterogeneous freezing by deposition nucleation [cm^-3 s^-1] - add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, + add_field("hetfrz_deposition_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); } // function set_grids ends @@ -390,8 +390,8 @@ void MAMAci::initialize_impl(const RunType run_type) { get_field_out("hetfrz_immersion_nucleation_tend").get_view(); hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = - get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + hetfrz_deposition_nucleation_tend_ = + get_field_out("hetfrz_deposition_nucleation_tend").get_view(); //--------------------------------------------------------------------------------- // Allocate memory for the class members @@ -642,7 +642,7 @@ void MAMAci::run_impl(const double dt) { team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, // ## output to be used by the other processes ## hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, - hetfrz_depostion_nucleation_tend_, + hetfrz_deposition_nucleation_tend_, // work arrays diagnostic_scratch_); diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml index 15e60e91517c..c9cad5e2761a 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_mam4_drydep_output +filename_prefix: mam4_srf_online_emiss_mam4_constituent_fluxes_output Averaging Type: Instant Field Names: - bc_a1 From c0a608e419600b9260f62f75030c03a9149d459e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 15:32:22 -0700 Subject: [PATCH 499/904] Adds ne30pg2 emissions files and mapping files for fine resolutions --- .../cime_config/namelist_defaults_scream.xml | 18 ++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 87791d0b7d00..a785fb322912 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -279,6 +279,24 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + + + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120np4_mono_20220502.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index fc930ea11d7b..63f87dd9f246 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -123,7 +123,7 @@ class MAMAci final : public scream::AtmosphereProcess { // added correctly to the cloud-micorphysics scheme. view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; - view_2d hetfrz_depostion_nucleation_tend_; + view_2d hetfrz_deposition_nucleation_tend_; view_2d diagnostic_scratch_[hetro_scratch_]; From ef08ba7dfb75f413772be4347913b2b98ae36cc9 Mon Sep 17 00:00:00 2001 From: Chloe Date: Thu, 13 Jun 2024 09:31:57 -0700 Subject: [PATCH 500/904] Added new snow Thermal Conductivity in SoilTemperatureMod.F90 based on Fourteau, et al. (2021) - is now density and temp dependent --- .../namelist_files/namelist_definition.xml | 4 ++ .../elm/src/biogeophys/SoilTemperatureMod.F90 | 61 ++++++++++++++++--- components/elm/src/main/controlMod.F90 | 10 ++- components/elm/src/main/elm_varctl.F90 | 1 + 4 files changed, 67 insertions(+), 9 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 74e1967fc529..312119ff15f8 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -766,6 +766,10 @@ This option enables more realistic snowpack conditions on glaciers and ice sheets, including a semi-empirical firn densification model. uses the same physics as use_extrasnowlayers but uses the original 5 snow layer scheme + + +Toggle to use new snow thermal conductivity that relies on snow temperature and density. = snl(c)+1) .AND. (j <= 0)) then + bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) + !write(iulog,*)"CAW bw(c,j)",bw(c,j) + + do i = 1, 5 + if (i == 1) then + k_snw_vals(i) = 2.564 * (bw(c,j)/ rho_ice)**2 - 0.059 * (bw(c,j)/ rho_ice) + 0.0205 + else if (i == 2) then + k_snw_vals(i) = 2.172 * (bw(c,j)/ rho_ice)**2 + 0.015 * (bw(c,j)/ rho_ice) + 0.0252 + else if (i == 3) then + k_snw_vals(i) = 1.985 * (bw(c,j)/ rho_ice)**2 + 0.073 * (bw(c,j)/ rho_ice) + 0.0336 + else if (i == 4) then + k_snw_vals(i) = 1.883 * (bw(c,j)/ rho_ice)**2 + 0.107 * (bw(c,j)/ rho_ice) + 0.0386 + else if (i == 5) then + k_snw_vals(i) = 1.776 * (bw(c,j)/ rho_ice)**2 + 0.147 * (bw(c,j)/ rho_ice) + 0.0455 + end if + end do + + do i = 1, size(k_snw_tmps) - 1 + if (k_snw_tmps(i) <= t_soisno(c,j) .and. t_soisno(c,j) <= k_snw_tmps(i + 1)) then + thk(c,j) = k_snw_vals(i) + (t_soisno(c,j) - k_snw_tmps(i))*(k_snw_vals(i + 1)-k_snw_vals(i))/(k_snw_tmps(i + 1)-k_snw_tmps(i)) + end if + end do + + ! Handle edge cases if t_soisno(c,j) is outside the given range + if (t_soisno(c,j) < k_snw_tmps(1)) then + thk(c,j) = k_snw_vals(1) + else if (t_soisno(c,j) > k_snw_tmps(size(k_snw_tmps))) then + thk(c,j) = k_snw_vals(size(k_snw_tmps)) + end if + + ! write(iulog,*)"CAW snow layer:",j + ! write(iulog,*)"CAW snow temp:",t_soisno(c,j) + ! write(iulog,*)"CAW snow density:",bw(c,j) + ! write(iulog,*)"CAW snow thk:",thk(c,j) + end if - ! Thermal conductivity of snow, which from Jordan (1991) pp. 18 - ! Only examine levels from snl(c)+1 -> 0 where snl(c) < 1 - if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then - bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) - thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) - end if + else + ! Thermal conductivity of snow, which from Jordan (1991) pp. 18 + ! Only examine levels from snl(c)+1 -> 0 where snl(c) < 1 + if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then + bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) + thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) + end if + endif end do end do diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 24bb855ff61a..0663bace9fc3 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -285,7 +285,7 @@ subroutine control_init( ) use_nofire, use_lch4, use_vertsoilc, use_extralakelayers, & use_vichydro, use_century_decomp, use_cn, use_crop, use_snicar_frc, & use_snicar_ad, use_firn_percolation_and_compaction, use_extrasnowlayers,& - use_vancouver, use_mexicocity, use_noio + use_snow_thk, use_vancouver, use_mexicocity, use_noio ! cpl_bypass variables namelist /elm_inparm/ metdata_type, metdata_bypass, metdata_biases, & @@ -726,7 +726,11 @@ subroutine control_spmd() call mpi_bcast (use_vertsoilc, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_extralakelayers, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_extrasnowlayers, 1, MPI_LOGICAL, 0, mpicom, ier) +<<<<<<< HEAD call mpi_bcast (use_firn_percolation_and_compaction, 1, MPI_LOGICAL, 0, mpicom, ier) +======= + call mpi_bcast (use_snow_thk, 1, MPI_LOGICAL, 0, mpicom, ier) +>>>>>>> aa0f583905 (Added new snow Thermal Conductivity in SoilTemperatureMod.F90) call mpi_bcast (use_vichydro, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_century_decomp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_cn, 1, MPI_LOGICAL, 0, mpicom, ier) @@ -1030,7 +1034,11 @@ subroutine control_print () write(iulog,*) ' use_lake_wat_storage = ', use_lake_wat_storage write(iulog,*) ' use_extralakelayers = ', use_extralakelayers write(iulog,*) ' use_extrasnowlayers = ', use_extrasnowlayers +<<<<<<< HEAD write(iulog,*) ' use_firn_percolation_and_compaction = ', use_firn_percolation_and_compaction +======= + write(iulog,*) ' use_snow_thk = ', use_snow_thk +>>>>>>> aa0f583905 (Added new snow Thermal Conductivity in SoilTemperatureMod.F90) write(iulog,*) ' use_vichydro = ', use_vichydro write(iulog,*) ' use_century_decomp = ', use_century_decomp write(iulog,*) ' use_cn = ', use_cn diff --git a/components/elm/src/main/elm_varctl.F90 b/components/elm/src/main/elm_varctl.F90 index 46b0d3c65a41..cbba48c84cba 100644 --- a/components/elm/src/main/elm_varctl.F90 +++ b/components/elm/src/main/elm_varctl.F90 @@ -376,6 +376,7 @@ module elm_varctl logical, public :: use_snicar_ad = .false. logical, public :: use_extrasnowlayers = .false. logical, public :: use_firn_percolation_and_compaction = .false. + logical, public :: use_snow_thk = .false. logical, public :: use_vancouver = .false. logical, public :: use_mexicocity = .false. logical, public :: use_noio = .false. From 2b7fbc01572fe783c34cccb2edc88ef5f574e17e Mon Sep 17 00:00:00 2001 From: Chloe Date: Thu, 20 Jun 2024 12:57:55 -0700 Subject: [PATCH 501/904] changed flag name for new snow thk flag to be more descriptive --- .../elm/bld/namelist_files/namelist_definition.xml | 2 +- components/elm/src/biogeophys/SoilTemperatureMod.F90 | 4 ++-- components/elm/src/main/controlMod.F90 | 10 +++++++++- components/elm/src/main/elm_varctl.F90 | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 312119ff15f8..dfc61f9fd8fc 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -767,7 +767,7 @@ snowpack conditions on glaciers and ice sheets, including a semi-empirical firn densification model. uses the same physics as use_extrasnowlayers but uses the original 5 snow layer scheme - Toggle to use new snow thermal conductivity that relies on snow temperature and density. diff --git a/components/elm/src/biogeophys/SoilTemperatureMod.F90 b/components/elm/src/biogeophys/SoilTemperatureMod.F90 index 7d262e295a10..3011538cedb6 100644 --- a/components/elm/src/biogeophys/SoilTemperatureMod.F90 +++ b/components/elm/src/biogeophys/SoilTemperatureMod.F90 @@ -834,7 +834,7 @@ subroutine SoilThermProp (bounds, num_nolakec, filter_nolakec, & use elm_varcon , only : denh2o, denice, tfrz, tkwat, tkice, tkair, cpice, cpliq, thk_bedrock use landunit_varcon , only : istice, istice_mec, istwet use column_varcon , only : icol_roof, icol_sunwall, icol_shadewall, icol_road_perv, icol_road_imperv - use elm_varctl , only : iulog, use_snow_thk + use elm_varctl , only : iulog, use_T_rho_dependent_snowthk ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -949,7 +949,7 @@ subroutine SoilThermProp (bounds, num_nolakec, filter_nolakec, & endif endif - if (use_snow_thk) then ! chose which snow thermal conductivity to use + if (use_T_rho_dependent_snowthk) then ! chose which snow thermal conductivity to use if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) !write(iulog,*)"CAW bw(c,j)",bw(c,j) diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 0663bace9fc3..c00db6dd7399 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -285,7 +285,7 @@ subroutine control_init( ) use_nofire, use_lch4, use_vertsoilc, use_extralakelayers, & use_vichydro, use_century_decomp, use_cn, use_crop, use_snicar_frc, & use_snicar_ad, use_firn_percolation_and_compaction, use_extrasnowlayers,& - use_snow_thk, use_vancouver, use_mexicocity, use_noio + use_T_rho_dependent_snowthk, use_vancouver, use_mexicocity, use_noio ! cpl_bypass variables namelist /elm_inparm/ metdata_type, metdata_bypass, metdata_biases, & @@ -726,11 +726,15 @@ subroutine control_spmd() call mpi_bcast (use_vertsoilc, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_extralakelayers, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_extrasnowlayers, 1, MPI_LOGICAL, 0, mpicom, ier) +<<<<<<< HEAD <<<<<<< HEAD call mpi_bcast (use_firn_percolation_and_compaction, 1, MPI_LOGICAL, 0, mpicom, ier) ======= call mpi_bcast (use_snow_thk, 1, MPI_LOGICAL, 0, mpicom, ier) >>>>>>> aa0f583905 (Added new snow Thermal Conductivity in SoilTemperatureMod.F90) +======= + call mpi_bcast (use_T_rho_dependent_snowthk, 1, MPI_LOGICAL, 0, mpicom, ier) +>>>>>>> 7d87f8c1ac (changed flag name for new snow thk flag to be more descriptive) call mpi_bcast (use_vichydro, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_century_decomp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_cn, 1, MPI_LOGICAL, 0, mpicom, ier) @@ -1034,11 +1038,15 @@ subroutine control_print () write(iulog,*) ' use_lake_wat_storage = ', use_lake_wat_storage write(iulog,*) ' use_extralakelayers = ', use_extralakelayers write(iulog,*) ' use_extrasnowlayers = ', use_extrasnowlayers +<<<<<<< HEAD <<<<<<< HEAD write(iulog,*) ' use_firn_percolation_and_compaction = ', use_firn_percolation_and_compaction ======= write(iulog,*) ' use_snow_thk = ', use_snow_thk >>>>>>> aa0f583905 (Added new snow Thermal Conductivity in SoilTemperatureMod.F90) +======= + write(iulog,*) ' use_T_rho_dependent_snowthk = ', use_T_rho_dependent_snowthk +>>>>>>> 7d87f8c1ac (changed flag name for new snow thk flag to be more descriptive) write(iulog,*) ' use_vichydro = ', use_vichydro write(iulog,*) ' use_century_decomp = ', use_century_decomp write(iulog,*) ' use_cn = ', use_cn diff --git a/components/elm/src/main/elm_varctl.F90 b/components/elm/src/main/elm_varctl.F90 index cbba48c84cba..fb9ca0dbd24e 100644 --- a/components/elm/src/main/elm_varctl.F90 +++ b/components/elm/src/main/elm_varctl.F90 @@ -376,11 +376,11 @@ module elm_varctl logical, public :: use_snicar_ad = .false. logical, public :: use_extrasnowlayers = .false. logical, public :: use_firn_percolation_and_compaction = .false. - logical, public :: use_snow_thk = .false. logical, public :: use_vancouver = .false. logical, public :: use_mexicocity = .false. logical, public :: use_noio = .false. logical, public :: use_var_soil_thick = .false. + logical, public :: use_T_rho_dependent_snowthk = .false. logical, public :: use_atm_downscaling_to_topunit = .false. character(len = SHR_KIND_CS), public :: precip_downscaling_method = 'ERMM' ! Precip downscaling method values can be ERMM or FNM logical, public :: use_lake_wat_storage = .false. From 7af3988119f772deb04acaf5de5817ff93757fa4 Mon Sep 17 00:00:00 2001 From: Chloe Date: Fri, 19 Jul 2024 12:32:33 -0700 Subject: [PATCH 502/904] clean up master & feature branch merge --- components/elm/src/main/controlMod.F90 | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index c00db6dd7399..1d8f48d7cc7d 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -726,15 +726,8 @@ subroutine control_spmd() call mpi_bcast (use_vertsoilc, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_extralakelayers, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_extrasnowlayers, 1, MPI_LOGICAL, 0, mpicom, ier) -<<<<<<< HEAD -<<<<<<< HEAD call mpi_bcast (use_firn_percolation_and_compaction, 1, MPI_LOGICAL, 0, mpicom, ier) -======= - call mpi_bcast (use_snow_thk, 1, MPI_LOGICAL, 0, mpicom, ier) ->>>>>>> aa0f583905 (Added new snow Thermal Conductivity in SoilTemperatureMod.F90) -======= call mpi_bcast (use_T_rho_dependent_snowthk, 1, MPI_LOGICAL, 0, mpicom, ier) ->>>>>>> 7d87f8c1ac (changed flag name for new snow thk flag to be more descriptive) call mpi_bcast (use_vichydro, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_century_decomp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_cn, 1, MPI_LOGICAL, 0, mpicom, ier) @@ -1038,15 +1031,8 @@ subroutine control_print () write(iulog,*) ' use_lake_wat_storage = ', use_lake_wat_storage write(iulog,*) ' use_extralakelayers = ', use_extralakelayers write(iulog,*) ' use_extrasnowlayers = ', use_extrasnowlayers -<<<<<<< HEAD -<<<<<<< HEAD write(iulog,*) ' use_firn_percolation_and_compaction = ', use_firn_percolation_and_compaction -======= - write(iulog,*) ' use_snow_thk = ', use_snow_thk ->>>>>>> aa0f583905 (Added new snow Thermal Conductivity in SoilTemperatureMod.F90) -======= write(iulog,*) ' use_T_rho_dependent_snowthk = ', use_T_rho_dependent_snowthk ->>>>>>> 7d87f8c1ac (changed flag name for new snow thk flag to be more descriptive) write(iulog,*) ' use_vichydro = ', use_vichydro write(iulog,*) ' use_century_decomp = ', use_century_decomp write(iulog,*) ' use_cn = ', use_cn From c2219551fdade1192ca6992b8c9ea27bca2ebbcc Mon Sep 17 00:00:00 2001 From: Chloe Date: Wed, 24 Jul 2024 12:07:11 -0700 Subject: [PATCH 503/904] removed testing write statements --- components/elm/src/biogeophys/SoilTemperatureMod.F90 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/elm/src/biogeophys/SoilTemperatureMod.F90 b/components/elm/src/biogeophys/SoilTemperatureMod.F90 index 3011538cedb6..9bda1ab7918e 100644 --- a/components/elm/src/biogeophys/SoilTemperatureMod.F90 +++ b/components/elm/src/biogeophys/SoilTemperatureMod.F90 @@ -952,7 +952,6 @@ subroutine SoilThermProp (bounds, num_nolakec, filter_nolakec, & if (use_T_rho_dependent_snowthk) then ! chose which snow thermal conductivity to use if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) - !write(iulog,*)"CAW bw(c,j)",bw(c,j) do i = 1, 5 if (i == 1) then @@ -981,10 +980,6 @@ subroutine SoilThermProp (bounds, num_nolakec, filter_nolakec, & thk(c,j) = k_snw_vals(size(k_snw_tmps)) end if - ! write(iulog,*)"CAW snow layer:",j - ! write(iulog,*)"CAW snow temp:",t_soisno(c,j) - ! write(iulog,*)"CAW snow density:",bw(c,j) - ! write(iulog,*)"CAW snow thk:",thk(c,j) end if From a4e9c067a4edebadbd31394e2b27e3c01d7cd39b Mon Sep 17 00:00:00 2001 From: Chloe Date: Fri, 16 Aug 2024 14:17:53 -0700 Subject: [PATCH 504/904] minor stylistic / precision mods --- .../elm/src/biogeophys/SoilTemperatureMod.F90 | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/components/elm/src/biogeophys/SoilTemperatureMod.F90 b/components/elm/src/biogeophys/SoilTemperatureMod.F90 index 9bda1ab7918e..524ee65d1205 100644 --- a/components/elm/src/biogeophys/SoilTemperatureMod.F90 +++ b/components/elm/src/biogeophys/SoilTemperatureMod.F90 @@ -860,7 +860,13 @@ subroutine SoilThermProp (bounds, num_nolakec, filter_nolakec, & real(r8), parameter :: rho_ice = 917._r8 real(r8) :: k_snw_vals(5) real(r8) :: k_snw_tmps(5) - data k_snw_tmps(:) /223.0, 248.0, 263.0, 268.0, 273.0/ + real(r8) :: k_snw_coe1(5) + real(r8) :: k_snw_coe2(5) + real(r8) :: k_snw_coe3(5) + data k_snw_tmps(:) /223.0_r8, 248.0_r8, 263.0_r8, 268.0_r8, 273.0_r8/ + data k_snw_coe1(:) /2.564_r8,2.172_r8,1.985_r8,1.883_r8,1.776_r8/ + data k_snw_coe2(:) /-0.059_r8, 0.015_r8, 0.073_r8, 0.107_r8, 0.147_r8/ + data k_snw_coe3(:) /0.0205_r8, 0.0252_r8, 0.0336_r8, 0.0386_r8, 0.0455_r8/ !----------------------------------------------------------------------- event = 'SoilThermProp' call t_start_lnd( event ) @@ -949,27 +955,17 @@ subroutine SoilThermProp (bounds, num_nolakec, filter_nolakec, & endif endif - if (use_T_rho_dependent_snowthk) then ! chose which snow thermal conductivity to use + if (use_T_rho_dependent_snowthk) then ! choose which snow thermal conductivity to use if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then - bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) + bw(c,j) = (h2osoi_ice(c,j) + h2osoi_liq(c,j)) / (frac_sno(c) * dz(c,j)) do i = 1, 5 - if (i == 1) then - k_snw_vals(i) = 2.564 * (bw(c,j)/ rho_ice)**2 - 0.059 * (bw(c,j)/ rho_ice) + 0.0205 - else if (i == 2) then - k_snw_vals(i) = 2.172 * (bw(c,j)/ rho_ice)**2 + 0.015 * (bw(c,j)/ rho_ice) + 0.0252 - else if (i == 3) then - k_snw_vals(i) = 1.985 * (bw(c,j)/ rho_ice)**2 + 0.073 * (bw(c,j)/ rho_ice) + 0.0336 - else if (i == 4) then - k_snw_vals(i) = 1.883 * (bw(c,j)/ rho_ice)**2 + 0.107 * (bw(c,j)/ rho_ice) + 0.0386 - else if (i == 5) then - k_snw_vals(i) = 1.776 * (bw(c,j)/ rho_ice)**2 + 0.147 * (bw(c,j)/ rho_ice) + 0.0455 - end if + k_snw_vals(i) = k_snw_coe1(i) * (bw(c,j) / rho_ice)**2 - k_snw_coe2(i) * (bw(c,j) / rho_ice) + k_snw_coe3(i) end do do i = 1, size(k_snw_tmps) - 1 if (k_snw_tmps(i) <= t_soisno(c,j) .and. t_soisno(c,j) <= k_snw_tmps(i + 1)) then - thk(c,j) = k_snw_vals(i) + (t_soisno(c,j) - k_snw_tmps(i))*(k_snw_vals(i + 1)-k_snw_vals(i))/(k_snw_tmps(i + 1)-k_snw_tmps(i)) + thk(c,j) = k_snw_vals(i) + (t_soisno(c,j) - k_snw_tmps(i)) * (k_snw_vals(i + 1)-k_snw_vals(i)) / (k_snw_tmps(i + 1) - k_snw_tmps(i)) end if end do From f35d4d75f6bc958667a524fda3587cb08462adbe Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 16 Aug 2024 19:39:58 -0400 Subject: [PATCH 505/904] disable gh testing for now --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 2 +- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 1 + .github/workflows/eamxx_default_files.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 366fef018f20..4cb89ed3cb9d 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -21,7 +21,7 @@ on: jobs: ci: - if: ${{ github.event.repository.name == 'e3sm' }} + if: false runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index 595518c326d7..2553c63cdb5f 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -14,6 +14,7 @@ on: jobs: ci-w: + if: false runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index e0a8e19f9c01..7f8f2a177515 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -11,6 +11,7 @@ on: jobs: scream-defaults: + if: false runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} From 4fa577ebced271a90ee7c29cb308eada86426d6f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 19:12:45 -0700 Subject: [PATCH 506/904] Removes debug print statements --- ...eamxx_mam_constituent_fluxes_interface.cpp | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index c74dbd7e4fa0..c3a33687dad6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -1,4 +1,3 @@ -#include #include #include namespace scream { @@ -282,30 +281,10 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - auto start = std::chrono::steady_clock::now(); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, constituent_fluxes_, // output wet_aero_); - auto stop = std::chrono::steady_clock::now(); - auto duration = (stop - start); - - // To get the value of duration use the count() - // member function on the duration object - printf("TIME:%e\n", - std::chrono::duration(duration).count()); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - } // run_impl ends // ============================================================================= From 796403f204752d74a724c6c8ffd2003d66ac9b06 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 17 Aug 2024 17:53:48 -0400 Subject: [PATCH 507/904] fix errors and warning in rrtmgp-k --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 22 +++++++++++++++---- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 6 ++--- .../rrtmgp/scream_rrtmgp_interface.hpp | 4 ++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 62de7f8416cd..e89e02fe301b 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -1171,10 +1171,24 @@ void RRTMGPRadiation::run_impl (const double dt) { aero_tau_lw_k(i,k,b) = d_aero_tau_lw(icol,b,k); }); } else { - Kokkos::deep_copy(aero_tau_sw_k, 0); - Kokkos::deep_copy(aero_ssa_sw_k, 0); - Kokkos::deep_copy(aero_g_sw_k , 0); - Kokkos::deep_copy(aero_tau_lw_k, 0); + // cuda complains (in warning only) about these being not allowed... + // Kokkos::deep_copy(aero_tau_sw_k, 0); + // Kokkos::deep_copy(aero_ssa_sw_k, 0); + // Kokkos::deep_copy(aero_g_sw_k , 0); + // Kokkos::deep_copy(aero_tau_lw_k, 0); + // So, do the manual labor instead: + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_sw_k(i,k,b) = 0.0; + aero_ssa_sw_k(i,k,b) = 0.0; + aero_g_sw_k (i,k,b) = 0.0; + }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlwbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_lw_k(i,k,b) = 0.0; + }); } #endif }); diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 8c236e64e811..aa6326c81811 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -112,7 +112,7 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template ::type* = nullptr> +template ::type* dummy = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; @@ -137,7 +137,7 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val return pass; } -template ::type* = nullptr> +template ::type* dummy = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; @@ -166,7 +166,7 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val return pass; } -template ::type* = nullptr> +template ::type* dummy = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 0bceb1282e5f..216722b3766b 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -1190,14 +1190,14 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template::type* = nullptr> +template::type* dummy = nullptr> static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { Kokkos::parallel_for(arr_out.size(), KOKKOS_LAMBDA(int i) { arr_out(i) = std::min(std::max(arr_in(i), lower), upper); }); } -template::type* = nullptr> +template::type* dummy = nullptr> static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { Kokkos::parallel_for(MDRP::template get<2>({arr_out.extent(0), arr_out.extent(1)}), KOKKOS_LAMBDA(int i, int j) { arr_out(i, j) = std::min(std::max(arr_in(i, j), lower), upper); From 87ea43c4f5cefdf12bd1ee869c3508230db1f8b1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 17 Aug 2024 15:20:01 -0700 Subject: [PATCH 508/904] Edits and adds some comments to provide clarity and TODOs --- .../mam/eamxx_mam_constituent_fluxes_functions.hpp | 3 +++ .../mam/eamxx_mam_constituent_fluxes_interface.cpp | 8 ++++---- .../mam/eamxx_mam_constituent_fluxes_interface.hpp | 2 +- ...mxx_mam_srf_and_online_emissions_process_interface.cpp | 4 +++- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 2 ++ 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index e7ffb6376c35..12c9c172f17d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -33,6 +33,9 @@ void update_gas_aerosols_using_constituents( // Create a policy to loop over columns annd number of constituents // to update + // FIXME: TODO:We don't need a team for "nconstituents", so we can make the + // kookos_for + // simple by using just ncols const auto policy = ekat::ExeSpaceUtils:: get_default_team_policy(ncol, nconstituents); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index c3a33687dad6..96d26ca22f9e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -267,13 +267,13 @@ void MAMConstituentFluxes::run_impl(const double dt) { // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". + // mixing ratios), we are *not* doing any wet to dry conversions in the + // "preprocess" for this process. We are simply updating the MAM4xx + // tracers using the "constituent fluxes". // We are converting wet atm to dry atm. Since we do not use or update // any of the water constituents (qc, qv, qi etc.), we should be okay // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. + // are built following HAERO data structures. // ------------------------------------------------------------------- // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 7e77fccf2afc..c68065f21788 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -27,7 +27,7 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { mam_coupling::DryAtmosphere dry_atm_; // aerosol state variables - mam_coupling::AerosolState wet_aero_; //, dry_aero_; + mam_coupling::AerosolState wet_aero_; // buffer for sotring temporary variables mam_coupling::Buffer buffer_; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f2a68ee6b5a7..850a82d0896d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -42,6 +42,8 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file auto srf_map_file = m_params.get("srf_remap_file", ""); + // FIXME: We can extract the following info about each species + // in a separate hpp file //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- @@ -221,7 +223,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - // Zero output + // Zero-out output Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 2d29075e41a4..b8ac7de768db 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -28,6 +28,8 @@ srfEmissFunctions::create_horiz_remapper( const int ncols_model = model_grid->get_num_global_dofs(); std::shared_ptr remapper; + // if the file's grid is same as model's native grid, we identity remapper + // (i.e., no interpolation) if(ncols_data == ncols_model) { remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); From b3ffc69e30889dac9a9344617c9b9f6072ddca00 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 19 Aug 2024 10:49:08 -0600 Subject: [PATCH 509/904] Limit threads available to compute-sanitizer racecheck --- components/eamxx/scripts/test_all_scream.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index 2d1ee9378ff8..f5350954b1c3 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -2,7 +2,7 @@ SharedArea, safe_copy from git_utils import get_current_head, get_current_commit -from test_factory import create_tests, COV +from test_factory import create_tests, COV, CSR from machines_specs import get_mach_compilation_resources, get_mach_testing_resources, \ get_mach_baseline_root_dir, setup_mach_env, is_cuda_machine, \ @@ -386,6 +386,15 @@ def generate_cmake_config(self, test, for_ctest=False): if "SCREAM_DYNAMICS_DYCORE" not in custom_opts_keys: result += " -DSCREAM_DYNAMICS_DYCORE=HOMME" + # For the compute-sanitizer tool 'racecheck', if no option --racecheck-num-workers + # is provided, it will attempt to use all threads available on node. This can cause + # issues if other test cases are being run in parallel. If the option was not specified, + # limit the number of threads availible to racecheck to the number of compile resources. + if self._parallel and isinstance(test, CSR) and not '--racecheck-num-workers' in result: + new_option = ' --racecheck-num-workers=' + str(test.compile_res_count) + index = result.index('--tool=racecheck') + len('--tool=racecheck') + result = result[:index] + new_option + result[index:] + return result ############################################################################### From 4375b3144492349002c5294e0b863a4732144b20 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 19 Aug 2024 10:50:16 -0600 Subject: [PATCH 510/904] Add single quotes to CS options If we ever need to add more options, not having these quotes might lead to hard to find errors --- components/eamxx/scripts/test_factory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py index 0b3f0c738c88..f85461ad23b4 100644 --- a/components/eamxx/scripts/test_factory.py +++ b/components/eamxx/scripts/test_factory.py @@ -176,7 +176,7 @@ def __init__(self, _): "debug with compute sanitizer memcheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=memcheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -210,7 +210,7 @@ def __init__(self, _): "debug with compute sanitizer initcheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=initcheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=initcheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -227,7 +227,7 @@ def __init__(self, _): "debug with compute sanitizer synccheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=synccheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=synccheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" From 8fb6e06d920aacefbfbaed525c3f0bb737523430 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 15 Aug 2024 15:53:31 -0600 Subject: [PATCH 511/904] Use C++ in shoc property tests --- .../shoc/tests/shoc_assumed_pdf_tests.cpp | 42 ++++++++++++++----- .../shoc/tests/shoc_brunt_length_tests.cpp | 7 +++- .../shoc/tests/shoc_check_length_tests.cpp | 7 +++- .../shoc/tests/shoc_check_tke_tests.cpp | 7 +++- .../shoc/tests/shoc_clip_third_moms_tests.cpp | 7 +++- .../tests/shoc_compute_diag_third_tests.cpp | 11 ++++- .../shoc_compute_shoc_temperature_tests.cpp | 18 +++++--- .../tests/shoc_compute_shoc_vapor_tests.cpp | 6 ++- .../shoc/tests/shoc_diag_obklen_tests.cpp | 18 ++++++-- .../tests/shoc_diag_second_mom_srf_test.cpp | 4 +- .../shoc_diag_second_mom_ubycond_test.cpp | 5 ++- ...shoc_diag_second_moments_lbycond_tests.cpp | 7 +++- .../tests/shoc_diag_second_moments_tests.cpp | 9 +++- .../shoc_diag_second_shoc_moments_tests.cpp | 9 +++- .../shoc/tests/shoc_diag_third_tests.cpp | 20 +++++++-- .../tests/shoc_eddy_diffusivities_tests.cpp | 21 +++++++--- .../shoc/tests/shoc_energy_fixer_tests.cpp | 22 ++++++++-- .../shoc/tests/shoc_energy_integral_tests.cpp | 9 +++- .../tests/shoc_energy_update_dse_tests.cpp | 8 +++- .../physics/shoc/tests/shoc_grid_tests.cpp | 6 ++- .../shoc/tests/shoc_impli_comp_tmpi_tests.cpp | 6 ++- .../tests/shoc_impli_dp_inverse_tests.cpp | 6 ++- .../shoc/tests/shoc_l_inf_length_tests.cpp | 7 +++- .../physics/shoc/tests/shoc_length_tests.cpp | 18 ++++++-- .../shoc/tests/shoc_linear_interp_tests.cpp | 16 +++++-- .../physics/shoc/tests/shoc_main_tests.cpp | 15 ++++++- .../shoc/tests/shoc_mix_length_tests.cpp | 10 ++++- .../tests/shoc_pblintd_check_pblh_tests.cpp | 6 ++- .../tests/shoc_pblintd_cldcheck_tests.cpp | 5 ++- .../shoc/tests/shoc_pblintd_height_tests.cpp | 21 +++++++--- .../shoc/tests/shoc_pblintd_init_pot_test.cpp | 8 ++-- .../tests/shoc_pblintd_surf_temp_tests.cpp | 7 +++- .../physics/shoc/tests/shoc_pblintd_tests.cpp | 8 +++- .../shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp | 12 ++++-- .../shoc/tests/shoc_tke_column_stab_tests.cpp | 12 ++++-- .../tests/shoc_tke_isotropic_ts_tests.cpp | 12 ++++-- .../shoc/tests/shoc_tke_shr_prod_tests.cpp | 12 ++++-- .../src/physics/shoc/tests/shoc_tke_tests.cpp | 16 +++++-- ...shoc_update_prognostics_implicit_tests.cpp | 14 +++++-- .../shoc/tests/shoc_varorcovar_tests.cpp | 30 ++++++++++--- .../shoc/tests/shoc_vertflux_tests.cpp | 7 +++- 41 files changed, 362 insertions(+), 129 deletions(-) diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index 3f1ab03f1a03..3672101c7dd5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -102,8 +102,6 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } // Check that the inputs make sense - - // Load input data for(Int s = 0; s < shcol; ++s) { for(Int n = 0; n < nlev; ++n) { const auto offset = n + s * nlev; @@ -142,8 +140,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { // For this test we want exactly two columns REQUIRE(SDS.shcol == 2); - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Verify the result // Make sure cloud fraction is either 1 or 0 and all @@ -182,8 +186,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Verify the result // Make sure cloud fraction is either 1 or 0 and all @@ -240,8 +250,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Check the result @@ -324,8 +340,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp index 4c9ff408e675..3018d0e5aa6a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp @@ -90,8 +90,11 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { } } - // Call the fortran implementation - compute_brunt_shoc_length(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + compute_brunt_shoc_length_f(SDS.nlev,SDS.nlevi,SDS.shcol,SDS.dz_zt,SDS.thv,SDS.thv_zi,SDS.brunt); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp index 1069b51d408b..bd023e2ed794 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp @@ -75,8 +75,11 @@ struct UnitWrap::UnitTest::TestCheckShocLength { REQUIRE(SDS.host_dy[s] > 0); } - // Call the fortran implementation - check_length_scale_shoc_length(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + check_length_scale_shoc_length_f(SDS.nlev,SDS.shcol,SDS.host_dx,SDS.host_dy,SDS.shoc_mix); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp index ca22ffe3b2ef..75581e2283ce 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp @@ -52,8 +52,11 @@ struct UnitWrap::UnitTest::TestShocCheckTke { // Check some input REQUIRE((SDS.shcol > 0 && SDS.nlev > 0)); - // call the fortran implementation - check_tke(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + check_tke_f(SDS.nlev, SDS.shcol, SDS.tke); + SDS.transpose(); // Check the result against the input values for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp index ee8eb4df9523..28bef1aae259 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp @@ -82,8 +82,11 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { REQUIRE(w3_large == true); } - // Call the fortran implementation - clipping_diag_third_shoc_moments(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + clipping_diag_third_shoc_moments_f(SDS.nlevi,SDS.shcol,SDS.w_sec_zi,SDS.w3); + SDS.transpose(); // Check the result // For large values of w3, verify that the result has been reduced diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp index 0e58b658d475..d63bf34bb50e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp @@ -145,8 +145,15 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { } } - // Call the fortran implementation - compute_diag_third_shoc_moment(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_diag_third_shoc_moment_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.tke,SDS.dz_zt, + SDS.dz_zi,SDS.isotropy_zi, + SDS.brunt_zi,SDS.w_sec_zi,SDS.thetal_zi, + SDS.w3); + SDS.transpose(); // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp index ae359978cf32..27452cdcf703 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp @@ -67,8 +67,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature is equal to thetal for(Int s = 0; s < shcol; ++s) { @@ -113,8 +115,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature is greather than thetal for(Int s = 0; s < shcol; ++s) { @@ -172,8 +176,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature be less than thetal for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp index 004f072405ba..afd184e18231 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp @@ -62,8 +62,10 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { } } - // Call the fortran implementation - compute_shoc_vapor(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_vapor_f(SDS.shcol, SDS.nlev, SDS.qw, SDS.ql, SDS.qv); + SDS.transpose(); // go back to C layout // Verify the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp index 3662cc2741a0..930faf31d6b2 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp @@ -83,8 +83,13 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { REQUIRE( (SDS.qv_sfc[s] > 0 && SDS.qv_sfc[s] < 0.1) ); } - // Call the fortran implementation - shoc_diag_obklen(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, + SDS.obklen); + SDS.transpose(); // Check the result @@ -148,8 +153,13 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { SDS.wthl_sfc[s]+SDS.wqw_sfc[s]); } - // Call the fortran implementation - shoc_diag_obklen(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, + SDS.obklen); + SDS.transpose(); // Verify that DIMENSIONLESS obukhov length decreases as columns // increases due to the increasing surface fluxes diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp index 507103e59a1b..25f56139d776 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp @@ -51,8 +51,8 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { SDS.vw_sfc[s] = vw_sfc[s]; } - // Call the fortran implementation - diag_second_moments_srf(SDS); + // Call the C++ implementation + shoc_diag_second_moments_srf_f(SDS.shcol, SDS.wthl_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.ustar2, SDS.wstar); // Verify the output for (Int s = 0; s < shcol; ++s){ diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp index 68fccb11bfcb..d4dbd3e7b71f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp @@ -42,8 +42,9 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { REQUIRE(SDS.shcol == shcol); REQUIRE(shcol > 0); - // Call the fortran implementation - diag_second_moments_ubycond(SDS); + // Call the C++ implementation + shoc_diag_second_moments_ubycond_f(SDS.shcol, SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, + SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec); // Verify the result // all output should be zero. diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp index bd2a3ac60ee9..1f1f3df8e5cc 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp @@ -75,8 +75,11 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { REQUIRE(SDS.ustar2[s] >= 0); } - // Call the fortran implementation - diag_second_moments_lbycond(SDS); + // Call the C++ implementation + diag_second_moments_lbycond_f(SDS.shcol, SDS.wthl_sfc, SDS.wqw_sfc, SDS.uw_sfc, + SDS.vw_sfc, SDS.ustar2, SDS.wstar, SDS.wthl_sec, + SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, + SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec); // Verify output is as expected for (Int s = 0; s < shcol; ++s){ diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp index 1ff4df42d168..f83bfa313f9c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp @@ -162,8 +162,13 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { } } - // Call the fortran implementation - diag_second_moments(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + diag_second_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, + SDS.tke, SDS.isotropy, SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, SDS.uw_sec, + SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); + SDS.transpose(); // go back to C layout // Verify output makes sense for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp index 86084fef344a..0385335c808b 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp @@ -160,8 +160,13 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { } } - // Call the fortran implementation - diag_second_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + diag_second_shoc_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, SDS.tke, SDS.isotropy, + SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.uw_sfc, SDS.vw_sfc, SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, + SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); + SDS.transpose(); // go back to C layout // Verify output makes sense for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp index e6430fa00d7f..82b4d88c0f70 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp @@ -142,8 +142,14 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } } - // Call the fortran implementation - diag_third_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, + SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, + SDS.w3); + SDS.transpose(); // Check to make sure there is at least one // positive w3 value for convective boundary layer @@ -186,8 +192,14 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } } - // Call the fortran implementation - diag_third_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, + SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, + SDS.w3); + SDS.transpose(); // Verify that new result is greater or equal in magnitude // that the result from test one diff --git a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp index 4cb8d7a99c19..0a96e5adc629 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp @@ -97,8 +97,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the answers in the columns are different for(Int s = 0; s < shcol-1; ++s) { @@ -164,8 +167,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the answers in the columns are larger // when the length scale and shear term are larger @@ -234,8 +240,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the diffusivities are smaller // in the columns where isotropy and tke are smaller diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp index 8ebf2a69c88e..dfb37104f018 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp @@ -157,8 +157,15 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } } - // Call the fortran implementation - shoc_energy_fixer(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, + SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, + SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, + SDS.host_dse); + SDS.transpose(); // Check test // Verify that the dry static energy has not changed if surface @@ -231,8 +238,15 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } } - // Call the fortran implementation - shoc_energy_fixer(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, + SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, + SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, + SDS.host_dse); + SDS.transpose(); // Verify the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp index 705d5ad02f25..8142da9371da 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp @@ -107,8 +107,13 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { } } - // Call the fortran implementation - shoc_energy_integrals(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_integrals_f(SDS.shcol, SDS.nlev, SDS.host_dse, SDS.pdel, + SDS.rtm, SDS.rcm, SDS.u_wind, SDS.v_wind, + SDS.se_int, SDS.ke_int, SDS.wv_int, SDS.wl_int); + SDS.transpose(); // Check test for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp index 74d5094076b0..a4e01eadb852 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp @@ -108,8 +108,12 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { } } - // Call the fortran implementation - update_host_dse(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + update_host_dse_f(SDS.shcol,SDS.nlev,SDS.thlm,SDS.shoc_ql,SDS.inv_exner,SDS.zt_grid, + SDS.phis,SDS.host_dse); + SDS.transpose(); // Check test for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp index d14fdf199175..6a79eeca02ee 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp @@ -79,8 +79,10 @@ struct UnitWrap::UnitTest::TestShocGrid { } } - // Call the fortran implementation - shoc_grid(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_grid_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zt_grid, SDS.zi_grid, SDS.pdel, SDS.dz_zt, SDS.dz_zi, SDS.rho_zt); + SDS.transpose(); // go back to C layout // First check that dz is correct for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp index 6b641b57d61d..53280dab17a6 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp @@ -86,8 +86,10 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { } } - // Call the fortran implementation - compute_tmpi(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_tmpi_f(SDS.nlevi, SDS.shcol, SDS.dtime, SDS.rho_zi, SDS.dz_zi, SDS.tmpi); + SDS.transpose(); // go back to C layout // Verify result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp index e2fabf24cb36..aad9c0266869 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp @@ -73,8 +73,10 @@ struct UnitWrap::UnitTest::TestImpDpInverse { } } - // Call the fortran implementation - dp_inverse(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + dp_inverse_f(SDS.nlev, SDS.shcol, SDS.rho_zt, SDS.dz_zt, SDS.rdp_zt); + SDS.transpose(); // go back to C layout // Verify result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp index dee0e30beda4..84faaebe4b91 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp @@ -88,8 +88,11 @@ struct UnitWrap::UnitTest::TestLInfShocLength { } } - // Call the fortran implementation - compute_l_inf_shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_l_inf_shoc_length_f(SDS.nlev,SDS.shcol,SDS.zt_grid,SDS.dz_zt,SDS.tke,SDS.l_inf); + SDS.transpose(); // Check the results // Make sure result is bounded correctly diff --git a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp index 873feffeb455..0f890db9ce11 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp @@ -120,8 +120,13 @@ struct UnitWrap::UnitTest::TestShocLength { } } - // Call the Fortran implementation - shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, + SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, + SDS.thv,SDS.brunt,SDS.shoc_mix); + SDS.transpose(); // Verify output for(Int s = 0; s < shcol; ++s) { @@ -169,8 +174,13 @@ struct UnitWrap::UnitTest::TestShocLength { SDS.host_dy[s] = host_dy_small; } - // call fortran implentation - shoc_length(SDS); + // call C++ implentation + SDS.transpose(); + // expects data in fortran layout + shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, + SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, + SDS.thv,SDS.brunt,SDS.shoc_mix); + SDS.transpose(); // Verify output for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp index 2c55de1433df..0a9c666688b5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp @@ -99,8 +99,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - // Call the fortran implementation - linear_interp(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + linear_interp_f(SDS.x1, SDS.x2, SDS.y1, SDS.y2, SDS.km1, SDS.km2, SDS.ncol, SDS.minthresh); + SDS.transpose(); // go back to C layout // First check that all output temperatures are greater than zero @@ -191,7 +193,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - linear_interp(SDS2); + // Call the C++ implementation + SDS2.transpose(); // _f expects data in fortran layout + linear_interp_f(SDS2.x1, SDS2.x2, SDS2.y1, SDS2.y2, SDS2.km1, SDS2.km2, SDS2.ncol, SDS2.minthresh); + SDS2.transpose(); // go back to C layout // Check the result, make sure output is bounded correctly @@ -275,7 +280,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - linear_interp(d); + // Call the C++ implementation + d.transpose(); // _f expects data in fortran layout + linear_interp_f(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); + d.transpose(); // go back to C layout // The combination of single-precision and randomness generating points // close together can result in larger error margins. diff --git a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp index e699c3283284..5c3eb8f9860e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp @@ -258,8 +258,19 @@ struct UnitWrap::UnitTest::TestShocMain { } - // Call the fortran implementation - shoc_main(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + const int npbl = shoc_init_f(SDS.nlev, SDS.pref_mid, SDS.nbot_shoc, SDS.ntop_shoc); + + shoc_main_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, npbl, SDS.host_dx, SDS.host_dy, + SDS.thv, SDS.zt_grid, SDS.zi_grid, SDS.pres, SDS.presi, SDS.pdel, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.wtracer_sfc, SDS.num_qtracers, + SDS.w_field, SDS.inv_exner, SDS.phis, SDS.host_dse, SDS.tke, SDS.thetal, SDS.qw, + SDS.u_wind, SDS.v_wind, SDS.qtracers, SDS.wthv_sec, SDS.tkh, SDS.tk, SDS.shoc_ql, + SDS.shoc_cldfrac, SDS.pblh, SDS.shoc_mix, SDS.isotropy, SDS.w_sec, SDS.thl_sec, + SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.wtke_sec, SDS.uw_sec, + SDS.vw_sec, SDS.w3, SDS.wqls_sec, SDS.brunt, SDS.shoc_ql2); + SDS.transpose(); // go back to C layout // Make sure output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp index 40303c6d0487..4d6b5dfdce1c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp @@ -90,8 +90,14 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { } } - // Call the fortran implementation - compute_shoc_mix_shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_shoc_mix_shoc_length_f(SDS.nlev, SDS.shcol, + SDS.tke, SDS.brunt, + SDS.zt_grid, + SDS.l_inf, SDS.shoc_mix); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp index 7ee84767a938..2d5f01c8ff27 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp @@ -67,8 +67,10 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { } } - // Call the fortran implementation - pblintd_check_pblh(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_check_pblh_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.nlev, SDS.z, SDS.ustar, SDS.check, SDS.pblh); + SDS.transpose(); // go back to C layout // Check the result // Check that PBL height is greater than zero. This is an diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp index 41b61778630c..df65050acacc 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp @@ -79,8 +79,9 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { } } - // Call the fortran implementation - pblintd_cldcheck(SDS); + // Call the C++ implementation + SDS.transpose(); + shoc_pblintd_cldcheck_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zi, SDS.cldn, SDS.pblh); // Check the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp index ca30fc4db5b0..9c33c7de68be 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp @@ -85,8 +85,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { REQUIRE(SDS.ustar[s+1] > SDS.ustar[s]); } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { @@ -119,8 +122,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { SDS.thv[offset] = SDS.thv[offset-1] + 2; } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { @@ -158,8 +164,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check that PBLH is zero (not modified) everywhere for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp index 910f71836ea7..b01c39eed95a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp @@ -76,8 +76,8 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } } - // call the fortran implementation - pblintd_init_pot(SDS); + // call the C++ implementation + shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); // Check the result. // Verify that virtual potential temperature is idential @@ -125,8 +125,8 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } } - // Call the fortran implementation - pblintd_init_pot(SDS); + // Call the C++ implementation + shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); // Check test // Verify that column with condensate loading diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp index 55e11953db02..ccf3d110fc8d 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp @@ -83,8 +83,11 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { } } - // Call the fortran implementation - pblintd_surf_temp(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_surf_temp_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.z, SDS.ustar, SDS.obklen, + SDS.kbfs, SDS.thv, SDS.tlv, SDS.pblh, SDS.check, SDS.rino); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp index 7e27f8cce296..b5cd117228bf 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp @@ -121,8 +121,12 @@ struct UnitWrap::UnitTest::TestPblintd { } } - // Call the fortran implementation - pblintd(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.npbl, SDS.z, SDS.zi, + SDS.thl, SDS.ql, SDS.q, SDS.u, SDS.v, SDS.ustar, SDS.obklen, + SDS.kbfs, SDS.cldn, SDS.pblh); + SDS.transpose(); // go back to C layout // Make sure PBL height is reasonable // Should be larger than second lowest zi level and lower diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp index 1434cb27f4ec..ea2c95470d89 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp @@ -94,8 +94,10 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } } - // Call the fortran implementation - adv_sgs_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); + SDS.transpose(); // go back to C layout // Check to make sure that there has been // TKE growth @@ -159,8 +161,10 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } } - // Call the fortran implementation - adv_sgs_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); + SDS.transpose(); // go back to C layout // Check to make sure that the column with // the smallest length scale has larger diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp index 7cbb00255a76..eb68bbc11df1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp @@ -75,8 +75,10 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } } - // Call the fortran implementation - integ_column_stability(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); + SDS.transpose(); // go back to C layout // Check test // Verify that output is zero @@ -107,8 +109,10 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } } - // Call the fortran implementation - integ_column_stability(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); + SDS.transpose(); // go back to C layout // Check test // Verify that output is negative diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp index 0baeb12b029a..c90128857cbb 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp @@ -81,8 +81,10 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } } - // Call the fortran implementation - isotropic_ts(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -145,8 +147,10 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } } - // Call the fortran implementation - isotropic_ts(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp index 8b088852af9c..0672ce3b3ae6 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp @@ -93,8 +93,10 @@ struct UnitWrap::UnitTest::TestShocShearProd { } } - // Call the fortran implementation - compute_shr_prod(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); + SDS.transpose(); // go back to C layout // Check test for(Int s = 0; s < shcol; ++s) { @@ -142,8 +144,10 @@ struct UnitWrap::UnitTest::TestShocShearProd { } } - // Call the fortran implementation - compute_shr_prod(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); + SDS.transpose(); // go back to C layout // Check test // Verify that shear term is zero everywhere diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp index bc975c9b59c1..f09ecc79c7d3 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp @@ -153,8 +153,12 @@ struct UnitWrap::UnitTest::TestShocTke { } } - // Call the fortran implementation - shoc_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, + SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, + SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check test // Make sure that TKE has increased everwhere relative @@ -226,8 +230,12 @@ struct UnitWrap::UnitTest::TestShocTke { } } - // Call the fortran implementation - shoc_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, + SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, + SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp index 45268023faef..83cfb8d5707e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp @@ -258,10 +258,18 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { } } - // Call the fortran implementation - update_prognostics_implicit(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + update_prognostics_implicit_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.num_tracer, SDS.dtime, + SDS.dz_zt, SDS.dz_zi, SDS.rho_zt, SDS.zt_grid, SDS.zi_grid, + SDS.tk, SDS.tkh, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.wtracer_sfc, SDS.thetal, SDS.qw, SDS.tracer, SDS.tke, SDS.u_wind, SDS.v_wind); + SDS.transpose(); // go back to C layout + // Call linear interp to get rho value at surface for checking - linear_interp(SDSL); + SDSL.transpose(); // _f expects data in fortran layout + linear_interp_f(SDSL.x1, SDSL.x2, SDSL.y1, SDSL.y2, SDSL.km1, SDSL.km2, SDSL.ncol, SDSL.minthresh); + SDSL.transpose(); // go back to C layout // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp index 74990b2c43f2..4a743009f244 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp @@ -115,8 +115,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for variance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for variance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -172,8 +178,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for covariance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for covariance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -250,8 +262,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for variance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for variance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp index e8b1b5a8cdcd..7ac7c7c53160 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp @@ -95,8 +95,11 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { } } - // Call the fortran implementation - calc_shoc_vertflux(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + calc_shoc_vertflux_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.tkh_zi, SDS.dz_zi, SDS.invar, SDS.vertflux); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { From 7c473e334e68aef5d142c55e245109f624a6aaaa Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Mon, 19 Aug 2024 22:26:49 +0000 Subject: [PATCH 512/904] Update oneapi to latest default --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 67611a0de225..446262a3a9b2 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3416,7 +3416,7 @@ cmake - oneapi/eng-compiler/2023.05.15.007 + oneapi/eng-compiler/2024.04.15.002 spack cmake From a4b7c3b9a75acdef005b3bbfc6dd671c7b2c2248 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Mon, 19 Aug 2024 22:29:07 +0000 Subject: [PATCH 513/904] Run BGC couppled cases on at least 2 nodes --- cime_config/allactive/config_pesall.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index e99230e7cb04..78ba011d9b78 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -1472,6 +1472,21 @@ + + + sunspot: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 2 nodes pure-MPI + + -2 + -2 + -2 + -2 + -2 + -2 + -2 + -2 + + + From e3e52e2b833f57d74231107cfcc3efc4c940b393 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Mon, 19 Aug 2024 22:34:11 +0000 Subject: [PATCH 514/904] Remove -check flags from debug builds They lead to link errors in currently available oneapi versions. --- cime_config/machines/cmake_macros/oneapi-ifx.cmake | 2 +- cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/oneapi-ifx.cmake b/cime_config/machines/cmake_macros/oneapi-ifx.cmake index e98a65d32a67..9ab0cdda7d51 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifx.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifx.cmake @@ -7,7 +7,7 @@ endif() string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0") string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake index 9d08ca6c6302..d7dfae002192 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake @@ -7,7 +7,7 @@ endif() string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0") string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99") From c6e75e3fa6b3f9c1c1814ca7d368f4111a626949 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Mon, 19 Aug 2024 22:50:35 +0000 Subject: [PATCH 515/904] Cleanup unused mpilib refs --- cime_config/machines/config_machines.xml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 446262a3a9b2..c94a2bbcf7d2 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3365,7 +3365,7 @@ uan-.* LINUX oneapi-ifx,oneapi-ifxgpu,gnu - mpich,impi,openmpi + mpich CSC249ADSE15_CNDA /gila/CSC249ADSE15_CNDA/performance_archive .* @@ -3398,13 +3398,6 @@ $ENV{GPU_TILE_COMPACT} - - mpirun - - --tag-output -n {{ total_tasks }} - --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread - - /soft/packaging/lmod/lmod/init/sh /soft/packaging/lmod/lmod/init/csh @@ -3434,12 +3427,6 @@ 1 - - 10 - omp - spread - unit - level_zero:gpu NO_GPU From eae53e54007c2be42ef3ee9f0d809e1ff88c53d9 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 20 Aug 2024 03:21:06 +0000 Subject: [PATCH 516/904] Run ne4-cases at 96x1 MPIxOMP --- cime_config/allactive/config_pesall.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 78ba011d9b78..84c0e24d8c9b 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -1759,6 +1759,21 @@ + + + allactive+sunspot: default, 96 mpi x 1 omp @ root 0 + + 96 + 96 + 96 + 96 + 96 + 96 + 96 + 96 + + + From ffec269d010a34f04b3ff132981dbeb0bcc24ac3 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 20 Aug 2024 03:22:36 +0000 Subject: [PATCH 517/904] Add initial sunspot machine file for eamxx --- components/eamxx/cmake/machine-files/sunspot.cmake | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 components/eamxx/cmake/machine-files/sunspot.cmake diff --git a/components/eamxx/cmake/machine-files/sunspot.cmake b/components/eamxx/cmake/machine-files/sunspot.cmake new file mode 100644 index 000000000000..09ff6d4e6c48 --- /dev/null +++ b/components/eamxx/cmake/machine-files/sunspot.cmake @@ -0,0 +1,10 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) + +set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE) +set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE) +set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "") +set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "") From 04258da74cb7eb2845cccf6ea3a54c71109b378c Mon Sep 17 00:00:00 2001 From: Chloe Date: Tue, 20 Aug 2024 09:09:15 -0700 Subject: [PATCH 518/904] minor stylistic change --- components/elm/src/biogeophys/SoilTemperatureMod.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/elm/src/biogeophys/SoilTemperatureMod.F90 b/components/elm/src/biogeophys/SoilTemperatureMod.F90 index 524ee65d1205..d4a1074bf4a7 100644 --- a/components/elm/src/biogeophys/SoilTemperatureMod.F90 +++ b/components/elm/src/biogeophys/SoilTemperatureMod.F90 @@ -864,7 +864,7 @@ subroutine SoilThermProp (bounds, num_nolakec, filter_nolakec, & real(r8) :: k_snw_coe2(5) real(r8) :: k_snw_coe3(5) data k_snw_tmps(:) /223.0_r8, 248.0_r8, 263.0_r8, 268.0_r8, 273.0_r8/ - data k_snw_coe1(:) /2.564_r8,2.172_r8,1.985_r8,1.883_r8,1.776_r8/ + data k_snw_coe1(:) /2.564_r8, 2.172_r8, 1.985_r8, 1.883_r8, 1.776_r8/ data k_snw_coe2(:) /-0.059_r8, 0.015_r8, 0.073_r8, 0.107_r8, 0.147_r8/ data k_snw_coe3(:) /0.0205_r8, 0.0252_r8, 0.0336_r8, 0.0386_r8, 0.0455_r8/ !----------------------------------------------------------------------- @@ -982,9 +982,9 @@ subroutine SoilThermProp (bounds, num_nolakec, filter_nolakec, & else ! Thermal conductivity of snow, which from Jordan (1991) pp. 18 ! Only examine levels from snl(c)+1 -> 0 where snl(c) < 1 - if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then - bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) - thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) + if (snl(c) + 1 < 1 .AND. (j >= snl(c) + 1) .AND. (j <= 0)) then + bw(c,j) = (h2osoi_ice(c,j) + h2osoi_liq(c,j)) / (frac_sno(c) * dz(c,j)) + thk(c,j) = tkair + (7.75e-5_r8 * bw(c,j) + 1.105e-6_r8 * bw(c,j) * bw(c,j)) * (tkice - tkair) end if endif end do From 7c182a67279b6499b34ba9d2d4aee6953b221c1e Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 27 Jun 2024 17:26:26 -0500 Subject: [PATCH 519/904] data land for moab driver use existing lnd_domain attribute in infodata we do not differentiate on coupler side if data land or not still need to send the proper land domain file --- .../data_comps/dlnd/src/dlnd_comp_mod.F90 | 181 +++++++++++++++++- .../data_comps/dlnd/src/lnd_comp_mct.F90 | 40 +++- 2 files changed, 216 insertions(+), 5 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index bf723259be48..7768b2dfe815 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -29,6 +29,10 @@ module dlnd_comp_mod use dlnd_shr_mod , only: domain_fracname ! namelist input use dlnd_shr_mod , only: nullstr +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! id of moab lnd app + use iso_c_binding +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -100,6 +104,12 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & scmMode, scmlat, scmlon) ! !DESCRIPTION: initialize dlnd model +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & + iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh +#endif implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -135,6 +145,18 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & character(nec_len) :: nec_str ! elevation class, as character string character(*), parameter :: domain_fracname_unset = 'null' +#ifdef HAVE_MOAB + character*400 tagname + real(R8) latv, lonv + integer iv, tagindex, ilat, ilon !, arrsize, nfields ! ierr is already defined as integer above + real(R8), allocatable, target :: data(:) + integer(IN), pointer :: idata(:) ! temporary + real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif + !--- formats --- character(*), parameter :: F00 = "('(dlnd_comp_init) ',8a)" character(*), parameter :: F0L = "('(dlnd_comp_init) ',a, l2)" @@ -256,6 +278,119 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & call t_stopf('dlnd_initmctdom') +#ifdef HAVE_MOAB + ilat = mct_aVect_indexRA(ggrid%data,'lat') + ilon = mct_aVect_indexRA(ggrid%data,'lon') + allocate(moab_vert_coords(lsize*3)) + do iv = 1, lsize + lonv = ggrid%data%rAttr(ilon, iv) * SHR_CONST_PI/180. + latv = ggrid%data%rAttr(ilat, iv) * SHR_CONST_PI/180. + moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*iv )=SIN(latv) + enddo + + ! create the vertices with coordinates from MCT domain + ierr = iMOAB_CreateVertices(mlnid, lsize*3, 3, moab_vert_coords) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to create MOAB vertices in data lnd model') + + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, & + 0, & ! dense, integer + 1, & ! number of components + tagindex ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ') + + ! get list of global IDs for Dofs + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsize, & + 0, & ! vertex type + idata) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mlnid, lsize, idata ); + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to resolve shared entities') + + deallocate(moab_vert_coords) + deallocate(idata) + + ierr = iMOAB_UpdateMeshInfo( mlnid ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to update mesh info ') + + allocate(data(lsize)) + ierr = iMOAB_DefineTagStorage( mlnid, "area:aream:frac:mask"//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create tag: area:aream:frac:mask' ) + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:) + tagname='area'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to get area tag ') + + ! set the same data for aream (model area) as area + ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:) + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set aream tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'mask'),:) + tagname='mask'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set mask tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'frac'),:) + tagname='frac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set frac tag ') + + deallocate(data) + + ! define tags + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_x2l_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_x2l_fields tags ') + + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_l2x_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_l2x_fields tags ') +#ifdef MOABDEBUG + ! debug test + outfile = 'LndDataMesh.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ! write out the mesh file to disk + ierr = iMOAB_WriteMesh(mlnid, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' ERROR in writing data mesh lnd ') + endif +#endif +#endif !---------------------------------------------------------------------------- ! Initialize MCT attribute vectors !---------------------------------------------------------------------------- @@ -339,8 +474,15 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & inst_suffix, logunit, case_name) ! !DESCRIPTION: run method for dlnd model - implicit none +#ifdef HAVE_MOAB +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif + use seq_flds_mod , only: seq_flds_l2x_fields + use seq_flds_mod , only: moab_set_tag_from_av +#endif + implicit none ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) , intent(in) :: EClock type(mct_aVect) , intent(inout) :: x2l @@ -366,6 +508,17 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & integer(IN) :: nu ! unit number logical :: write_restart ! restart now character(len=18) :: date_str +#ifdef HAVE_MOAB + real(R8), allocatable, target :: datam(:) + type(mct_list) :: temp_list + integer :: size_list, index_list, lsize + type(mct_string) :: mctOStr ! + character*400 tagname, mct_field +#ifdef MOABDEBUG + integer :: cur_dlnd_stepno, ierr + character*100 outfile, wopts, lnum +#endif +#endif character(*), parameter :: F00 = "('(dlnd_comp_run) ',8a)" character(*), parameter :: F04 = "('(dlnd_comp_run) ',2a,2i8,'s')" @@ -464,6 +617,32 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & call t_stopf('DLND_RUN') +#ifdef HAVE_MOAB + lsize = mct_avect_lsize(l2x) ! is it the same as mct_avect_lsize(avstrm) ? + allocate(datam(lsize)) ! + call mct_list_init(temp_list ,seq_flds_l2x_fields) + size_list=mct_list_nitem (temp_list) + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call moab_set_tag_from_av(tagname, l2x, index_list, mlnid, datam, lsize) ! loop over all a2x fields, not just a few + enddo + call mct_list_clean(temp_list) + deallocate(datam) ! maybe we should keep it around, deallocate at the final only? + +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_dlnd_stepno ) + write(lnum,"(I0.2)")cur_dlnd_stepno + outfile = 'dlnd_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mlnid, outfile, wopts) + if (ierr > 0 ) then + write(logunit,*) 'Failed to write data lnd component state ' + endif +#endif +#endif + end subroutine dlnd_comp_run !=============================================================================== diff --git a/components/data_comps/dlnd/src/lnd_comp_mct.F90 b/components/data_comps/dlnd/src/lnd_comp_mct.F90 index f5193ca84580..8361a8522d8a 100644 --- a/components/data_comps/dlnd/src/lnd_comp_mct.F90 +++ b/components/data_comps/dlnd/src/lnd_comp_mct.F90 @@ -16,7 +16,11 @@ module lnd_comp_mct use dlnd_comp_mod , only: dlnd_comp_init, dlnd_comp_run, dlnd_comp_final use dlnd_shr_mod , only: dlnd_shr_read_namelists use seq_flds_mod , only: seq_flds_x2l_fields, seq_flds_l2x_fields - +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! iMOAB app id for lnd + use iso_c_binding + use iMOAB , only: iMOAB_RegisterApplication +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -52,7 +56,9 @@ module lnd_comp_mct !=============================================================================== subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) - +#ifdef HAVE_MOAB + use shr_stream_mod, only: shr_stream_getDomainInfo, shr_stream_getFile +#endif ! !DESCRIPTION: initialize dlnd model implicit none @@ -78,6 +84,16 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) real(R8) :: scmLon = shr_const_SPVAL ! single column lon character(*), parameter :: subName = "(lnd_init_mct) " !------------------------------------------------------------------------------- +#ifdef HAVE_MOAB + character(CL) :: filePath ! generic file path + character(CL) :: fileName ! generic file name + character(CS) :: timeName ! domain file: time variable name + character(CS) :: lonName ! domain file: lon variable name + character(CS) :: latName ! domain file: lat variable name + character(CS) :: hgtName ! domain file: hgt variable name + character(CS) :: maskName ! domain file: mask variable name + character(CS) :: areaName ! domain file: area variable name +#endif ! Set cdata pointers call seq_cdata_setptrs(cdata, & @@ -146,13 +162,29 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) !---------------------------------------------------------------------------- ! Initialize dlnd !---------------------------------------------------------------------------- - +#ifdef HAVE_MOAB + ierr = iMOAB_RegisterApplication(trim("DLND")//C_NULL_CHAR, mpicom, compid, mlnid) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data lnd comp' + call shr_sys_abort(subname//' ERROR in registering data lnd comp') + endif +#endif call dlnd_comp_init(Eclock, x2l, l2x, & seq_flds_x2l_fields, seq_flds_l2x_fields, & SDLND, gsmap, ggrid, mpicom, compid, my_task, master_task, & inst_suffix, inst_name, logunit, read_restart, & scmMode, scmlat, scmlon) - +#ifdef HAVE_MOAB + if (my_task == master_task) then + call shr_stream_getDomainInfo(SDLND%stream(1), filePath,fileName,timeName,lonName, & + latName,hgtName,maskName,areaName) + call shr_stream_getFile(filePath,fileName) + ! send path of data lnd domain to MOAB coupler. + call seq_infodata_PutData( infodata, lnd_domain=fileName) ! we use the same one for regular case + ! in regular case, it is copied from fatmlndfrc ; so we don't know if it is data land or not + write(logunit,*), ' filename: ', filename + endif +#endif !---------------------------------------------------------------------------- ! Fill infodata that needs to be returned from dlnd !---------------------------------------------------------------------------- From 2ad116ba0b787f1c9ab391eb3d95025a27cb19db Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 28 Jun 2024 09:12:11 -0500 Subject: [PATCH 520/904] domain file or land data case domain file is actually set, in shr_strdata_type%domainFile all the other data models use the first stream file for domain information land is the one that actually has that member set in the SDLND structure it is easier to get it from there, then from stream the domain file in regular land case is defined by fatmlndfrac file --- .../data_comps/dlnd/src/lnd_comp_mct.F90 | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/components/data_comps/dlnd/src/lnd_comp_mct.F90 b/components/data_comps/dlnd/src/lnd_comp_mct.F90 index 8361a8522d8a..b699ec217f00 100644 --- a/components/data_comps/dlnd/src/lnd_comp_mct.F90 +++ b/components/data_comps/dlnd/src/lnd_comp_mct.F90 @@ -84,16 +84,6 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) real(R8) :: scmLon = shr_const_SPVAL ! single column lon character(*), parameter :: subName = "(lnd_init_mct) " !------------------------------------------------------------------------------- -#ifdef HAVE_MOAB - character(CL) :: filePath ! generic file path - character(CL) :: fileName ! generic file name - character(CS) :: timeName ! domain file: time variable name - character(CS) :: lonName ! domain file: lon variable name - character(CS) :: latName ! domain file: lat variable name - character(CS) :: hgtName ! domain file: hgt variable name - character(CS) :: maskName ! domain file: mask variable name - character(CS) :: areaName ! domain file: area variable name -#endif ! Set cdata pointers call seq_cdata_setptrs(cdata, & @@ -176,13 +166,9 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) scmMode, scmlat, scmlon) #ifdef HAVE_MOAB if (my_task == master_task) then - call shr_stream_getDomainInfo(SDLND%stream(1), filePath,fileName,timeName,lonName, & - latName,hgtName,maskName,areaName) - call shr_stream_getFile(filePath,fileName) - ! send path of data lnd domain to MOAB coupler. - call seq_infodata_PutData( infodata, lnd_domain=fileName) ! we use the same one for regular case + call seq_infodata_PutData( infodata, lnd_domain=SDLND%domainFile) ! we use the same one for regular case ! in regular case, it is copied from fatmlndfrc ; so we don't know if it is data land or not - write(logunit,*), ' filename: ', filename + write(logunit,*), ' use this land domain file: ', SDLND%domainFile endif #endif !---------------------------------------------------------------------------- From 9f0a28c35186b7f68daf7cf79b1643258ac73485 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 15 Jul 2024 10:33:50 -0500 Subject: [PATCH 521/904] apply weights is not used anymore in fractions iMOAB_ApplyScalarProjectionWeights is not used anymore all maps are done now with seq_map method, and there is only one instance of iMOAB_ApplyScalarProjectionWeights in E3SM using moab driver --- driver-moab/main/seq_frac_mct.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/seq_frac_mct.F90 b/driver-moab/main/seq_frac_mct.F90 index a89fc0b9aa4f..031845a102ef 100644 --- a/driver-moab/main/seq_frac_mct.F90 +++ b/driver-moab/main/seq_frac_mct.F90 @@ -180,7 +180,7 @@ module seq_frac_mct use iMOAB, only : iMOAB_DefineTagStorage, iMOAB_SetDoubleTagStorage, & iMOAB_GetMeshInfo, iMOAB_SetDoubleTagStorageWithGid, iMOAB_WriteMesh, & - iMOAB_ApplyScalarProjectionWeights, iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & + iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & iMOAB_FreeSenderBuffers, iMOAB_GetDoubleTagStorage #ifdef MOABDEBUG use seq_comm_mct, only : num_moab_exports From f8a137341ecf1e2f653389c1130ca216a896c81d Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 20 Aug 2024 10:10:44 -0600 Subject: [PATCH 522/904] Use eps to compare shoc_ql2==0 --- .../eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index 3672101c7dd5..d53bf4935dbd 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -160,9 +160,8 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] == 0); - REQUIRE(SDS.shoc_ql2[offset] == 0); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); REQUIRE(SDS.shoc_ql[offset] >= 0); - } } @@ -207,7 +206,7 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] != 0); REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); - REQUIRE(SDS.shoc_ql2[offset] == 0); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); REQUIRE(SDS.shoc_ql[offset] >= 0); REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); } From a39296ce127815b05ef80253d1a986e75fc75dae Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 20 Aug 2024 16:46:20 -0400 Subject: [PATCH 523/904] re-enable MMF2 test --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 059c280e2bb4..0248077934d6 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -370,7 +370,7 @@ "ERP_Ln9.ne4pg2_oQU480.WCYCL20TRNS-MMF1.allactive-mmf_fixed_subcycle", "ERS_Ln9.ne4pg2_ne4pg2.FRCE-MMF1.eam-cosp_nhtfrq9", "SMS_Ln5.ne4_ne4.FSCM-ARM97-MMF1", - # "SMS_Ln3.ne4pg2_ne4pg2.F2010-MMF2", - temporarily disabled due to ongoing dev issues + "SMS_Ln3.ne4pg2_oQU480.F2010-MMF2", ) }, From 2aaccf32f6909456f5ce4fdc0e3ebcd89f81b3f7 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 20 Aug 2024 14:47:07 -0600 Subject: [PATCH 524/904] add comment about non-bfb with 0 --- .../eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index d53bf4935dbd..e947946fa203 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -160,7 +160,7 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] == 0); - REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 REQUIRE(SDS.shoc_ql[offset] >= 0); } } @@ -206,7 +206,7 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] != 0); REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); - REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 REQUIRE(SDS.shoc_ql[offset] >= 0); REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); } From fd620a5c8779956548980e335498ffef16552581 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 21 Aug 2024 09:37:50 -0400 Subject: [PATCH 525/904] gen custom_pelayout, better case loc, and default proj --- run_e3sm.template.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 413f2bf75286..86e360157598 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -15,7 +15,7 @@ main() { # Machine and project readonly MACHINE=pm-cpu # BEFORE RUNNING: CHANGE this to your project -readonly PROJECT="e3sm" +readonly PROJECT="$(sacctmgr show user $USER format=DefaultAccount | tail -n1 | tr -d ' ')" # Simulation readonly COMPSET="WCYCL1850" @@ -43,7 +43,7 @@ readonly GET_REFCASE=TRUE #readonly RUN_REFDATE="" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' # Set paths -readonly CASE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}" +readonly CASE_ROOT="${PSCRATCH}/E3SMv3/${CASE_NAME}" readonly CODE_ROOT="${HOME}/E3SMv3/code/${CHECKOUT}" # Sub-directories @@ -417,6 +417,12 @@ then # Number of cores per node (machine specific) if [ "${MACHINE}" == "pm-cpu" ]; then ncore=128 + elif [ "${MACHINE}" == "chrysalis" ]; then + ncore=64 + elif [ "${MACHINE}" == "compy" ]; then + ncore=40 + elif [ "${MACHINE}" == "anvil" ]; then + ncore=36 else echo 'ERROR: MACHINE = '${MACHINE}' is not supported for custom PE layout.' exit 400 From ec9833930b90653215a118f7295bf8bba4998632 Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Thu, 2 Mar 2023 07:33:31 -0800 Subject: [PATCH 526/904] Add F2010-SCREAMv1 test to e3sm_gpucxx suite --- cime_config/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/tests.py b/cime_config/tests.py index 059c280e2bb4..e0f2967fdadd 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -762,6 +762,7 @@ "e3sm_gpucxx" : { "tests" : ( "SMS_Ln9.ne4pg2_ne4pg2.F2010-MMF1", + "ERP_Ln9.ne4pg2_ne4pg2.F2010-SCREAMv1", ) }, From b326f278047eacc245c4d92e87d836ceee68f4af Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Wed, 21 Aug 2024 09:40:48 -0700 Subject: [PATCH 527/904] Revert to external cosp.F90 to fix build error for EAMxx --- components/eamxx/src/physics/cosp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/cosp/CMakeLists.txt b/components/eamxx/src/physics/cosp/CMakeLists.txt index e6ba4c363794..cd5c966a501a 100644 --- a/components/eamxx/src/physics/cosp/CMakeLists.txt +++ b/components/eamxx/src/physics/cosp/CMakeLists.txt @@ -17,7 +17,7 @@ set(EXTERNAL_SRC ${EAM_COSP_DIR}/external/src/cosp_constants.F90 ${EAM_COSP_DIR}/external/src/simulator/cosp_cloudsat_interface.F90 ${EAM_COSP_DIR}/external/src/cosp_config.F90 - ${EAM_COSP_DIR}/local/cosp.F90 + ${EAM_COSP_DIR}/external/src/cosp.F90 ${EAM_COSP_DIR}/external/src/cosp_stats.F90 ${EAM_COSP_DIR}/external/src/simulator/quickbeam/quickbeam.F90 ${EAM_COSP_DIR}/external/src/simulator/parasol/parasol.F90 From 118865ff3a59f240efb4527dae5b6467449cf991 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 21 Aug 2024 11:19:28 -0600 Subject: [PATCH 528/904] Fix hist regex for scream The file names in the output testmods were producing files that were getting skipped by CIME due to not matching the regex in config_archive.xml. --- cime_config/config_archive.xml | 2 +- .../testdefs/testmods_dirs/scream/output/diags/shell_commands | 2 +- .../testdefs/testmods_dirs/scream/output/phys/shell_commands | 2 +- .../testmods_dirs/scream/output/phys_dyn/shell_commands | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cime_config/config_archive.xml b/cime_config/config_archive.xml index 84de6e0f171a..dee8d1992981 100644 --- a/cime_config/config_archive.xml +++ b/cime_config/config_archive.xml @@ -29,7 +29,7 @@ r\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* rhist\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* - hi\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d*\.\d{4}-\d{2}-\d{2}-\d{5}\.nc$ + .*\.h\..*\.nc$ diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands index 904f46b580f2..54bbf0aba648 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands @@ -29,7 +29,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.diags.hi \ + --prefix ${CASE}.scream.diags.h \ --grid "Physics ${PGTYPE}" \ --fields ${FIELDS} diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands index 116cdf111b43..e327f4d03365 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands @@ -26,7 +26,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.phys.hi \ + --prefix ${CASE}.scream.phys.h \ --grid "Physics ${PGTYPE}" \ --fields ${FIELDS} diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands index 0952b4afe3f9..c20a77d30d5c 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands @@ -26,7 +26,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.phys_dyn.hi \ + --prefix ${CASE}.scream.phys_dyn.h \ --grid Dynamics \ --io-grid 'Physics GLL' \ --fields ${FIELDS} From 49e1cb2ebda26e15c33ab62245606db132211576 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Tue, 13 Aug 2024 12:30:05 -0500 Subject: [PATCH 529/904] Add additional fields to history files --- .../mpas-albany-landice/cime_config/buildnml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 4d927c4370fd..e345853f84a5 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -229,25 +229,31 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') -# lines.append(' ') -# lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append('') lines.append('') lines.append('') @@ -263,22 +269,16 @@ def buildnml(case, caseroot, compname): lines.append(' filename_interval="0001-00-00_00:00:00"') lines.append(' clobber_mode="truncate"') lines.append(' packages="globalStatsAMPKG"') - lines.append(' output_interval="0000_01:00:00">') + lines.append(' output_interval="0000_00:00:01">') lines.append('') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') + lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append('') lines.append('') lines.append(' -'none' +'temperature' .true. 'file' 0.0 From 36147b9482bf6ca18009f368f02c083c2a63f39d Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 14 Aug 2024 16:03:06 -0500 Subject: [PATCH 531/904] Adjust MALI buildnml to look for mesh-specific albany_input.yaml files Also create the corresponding albany_input.*.yaml files --- .../namelist_files/albany_input.ais20km.yaml | 231 ++++++++++++++++ .../albany_input.ais_4to20km.yaml | 246 ++++++++++++++++++ .../albany_input.ais_8to30km.yaml | 246 ++++++++++++++++++ .../albany_input.aisgis20km.yaml | 231 ++++++++++++++++ .../albany_input.gis_1to10km_r01.yaml | 231 ++++++++++++++++ ...yaml => albany_input.gis_1to10km_r02.yaml} | 118 +++++---- .../albany_input.gis_20km_r01.yaml | 231 ++++++++++++++++ .../mpas-albany-landice/cime_config/buildnml | 6 +- 8 files changed, 1479 insertions(+), 61 deletions(-) create mode 100644 components/mpas-albany-landice/bld/namelist_files/albany_input.ais20km.yaml create mode 100644 components/mpas-albany-landice/bld/namelist_files/albany_input.ais_4to20km.yaml create mode 100644 components/mpas-albany-landice/bld/namelist_files/albany_input.ais_8to30km.yaml create mode 100644 components/mpas-albany-landice/bld/namelist_files/albany_input.aisgis20km.yaml create mode 100644 components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r01.yaml rename components/mpas-albany-landice/bld/namelist_files/{albany_input.yaml => albany_input.gis_1to10km_r02.yaml} (81%) create mode 100644 components/mpas-albany-landice/bld/namelist_files/albany_input.gis_20km_r01.yaml diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.ais20km.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.ais20km.yaml new file mode 100644 index 000000000000..589e163bba7f --- /dev/null +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.ais20km.yaml @@ -0,0 +1,231 @@ +%YAML 1.1 +--- +ANONYMOUS: + +# Discretization Description + Discretization: + #Exodus Output File Name: albany_output.exo + + Piro: +# Nonlinear Solver Information + NOX: + Nonlinear Solver: Line Search Based + Line Search: + Full Step: + Full Step: 1.0e+00 + Method: Backtrack + Solver Options: + Status Test Check Type: Minimal + Status Tests: + Test Type: Combo + Combo Type: OR + Number of Tests: 2 + Test 0: + Test Type: Combo + Combo Type: AND + Number of Tests: 2 + Test 0: + Test Type: NormF + Norm Type: Two Norm + Scale Type: Unscaled + Tolerance: 1.0e-03 + Test 1: + Test Type: RelativeNormF + Norm Type: Two Norm + Tolerance: 0.9999 + Test 1: + Test Type: MaxIters + Maximum Iterations: 100 + Printing: + Output Precision: 3 + Output Processor: 0 + Output Information: + Error: true + Warning: true + Outer Iteration: true + Parameters: false + Details: false + Linear Solver Details: false + Stepper Iteration: true + Stepper Details: true + Stepper Parameters: true + + Direction: + Method: Newton + Newton: + Forcing Term Method: Type 2 + Rescue Bad Newton Solve: true + Linear Solver: + Write Linear System: false + Tolerance: 1.0e-8 + + Stratimikos Linear Solver: + Stratimikos: + +# Linear Solver Information + Linear Solver Type: Belos + Linear Solver Types: + Belos: + Solver Type: Block GMRES + Solver Types: + Block GMRES: + Output Frequency: 20 + Output Style: 1 + Verbosity: 33 + Maximum Iterations: 200 + Block Size: 1 + Num Blocks: 200 + Flexible Gmres: false + VerboseObject: + Output File: none + Verbosity Level: low + +# Preconditioner Information + Preconditioner Type: MueLu + Preconditioner Types: + + Ifpack2: + Overlap: 1 + Prec Type: ILUT + + MueLu: + Matrix: + PDE equations: 2 + Factories: + myLineDetectionFact: + factory: LineDetectionFactory + 'linedetection: orientation': coordinates + mySemiCoarsenPFact1: + factory: SemiCoarsenPFactory + 'semicoarsen: coarsen rate': 14 + UncoupledAggregationFact2: + factory: UncoupledAggregationFactory + 'aggregation: ordering': graph + 'aggregation: max selected neighbors': 0 + 'aggregation: min agg size': 3 + 'aggregation: phase3 avoid singletons': true + MyCoarseMap2: + factory: CoarseMapFactory + Aggregates: UncoupledAggregationFact2 + myTentativePFact2: + 'tentative: calculate qr': true + factory: TentativePFactory + Aggregates: UncoupledAggregationFact2 + CoarseMap: MyCoarseMap2 + mySaPFact2: + 'sa: eigenvalue estimate num iterations': 10 + 'sa: damping factor': 1.33333e+00 + factory: SaPFactory + P: myTentativePFact2 + myTransferCoordinatesFact: + factory: CoordinatesTransferFactory + CoarseMap: MyCoarseMap2 + Aggregates: UncoupledAggregationFact2 + myTogglePFact: + factory: TogglePFactory + 'semicoarsen: number of levels': 2 + TransferFactories: + P1: mySemiCoarsenPFact1 + P2: mySaPFact2 + Ptent1: mySemiCoarsenPFact1 + Ptent2: myTentativePFact2 + Nullspace1: mySemiCoarsenPFact1 + Nullspace2: myTentativePFact2 + myRestrictorFact: + factory: TransPFactory + P: myTogglePFact + myToggleTransferCoordinatesFact: + factory: ToggleCoordinatesTransferFactory + Chosen P: myTogglePFact + TransferFactories: + Coordinates1: mySemiCoarsenPFact1 + Coordinates2: myTransferCoordinatesFact + myRAPFact: + factory: RAPFactory + P: myTogglePFact + R: myRestrictorFact + TransferFactories: + For Coordinates: myToggleTransferCoordinatesFact + myRepartitionHeuristicFact: + factory: RepartitionHeuristicFactory + A: myRAPFact + 'repartition: min rows per proc': 3000 + 'repartition: max imbalance': 1.327e+00 + 'repartition: start level': 1 + myZoltanInterface: + factory: ZoltanInterface + A: myRAPFact + Coordinates: myToggleTransferCoordinatesFact + number of partitions: myRepartitionHeuristicFact + myRepartitionFact: + factory: RepartitionFactory + A: myRAPFact + Partition: myZoltanInterface + 'repartition: remap parts': true + number of partitions: myRepartitionHeuristicFact + myRebalanceProlongatorFact: + factory: RebalanceTransferFactory + type: Interpolation + P: myTogglePFact + Coordinates: myToggleTransferCoordinatesFact + Nullspace: myTogglePFact + myRebalanceRestrictionFact: + factory: RebalanceTransferFactory + type: Restriction + R: myRestrictorFact + myRebalanceAFact: + factory: RebalanceAcFactory + A: myRAPFact + TransferFactories: { } + mySmoother1: + factory: TrilinosSmoother + type: LINESMOOTHING_BANDEDRELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother3: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother4: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': pre + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 4 + 'relaxation: damping factor': 1.0 + Hierarchy: + max levels: 7 + 'coarse: max size': 2000 + verbosity: None + Finest: + Smoother: mySmoother1 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + All: + startLevel: 1 + Smoother: mySmoother4 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.ais_4to20km.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.ais_4to20km.yaml new file mode 100644 index 000000000000..c3bc0cf8bd56 --- /dev/null +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.ais_4to20km.yaml @@ -0,0 +1,246 @@ +%YAML 1.1 +--- +ANONYMOUS: + Problem: + Basal Cubature Degree: 4 + LandIce Field Norm: + sliding_velocity_basalside: + Regularization Type: Given Value + Regularization Value: 1.0e-4 + LandIce BCs: + BC 0: + Basal Friction Coefficient: + Type: Power Law + Power Exponent: 0.2 + Mu Type: Field + Effective Pressure Type: Constant + Effective Pressure: 1.0 + # Zero Effective Pressure On Floating Ice At Nodes: true + Zero Beta On Floating Ice: true +# Discretization Description + Discretization: + #Exodus Output File Name: albany_output.exo + + Piro: +# Nonlinear Solver Information + NOX: + Nonlinear Solver: Line Search Based + Line Search: + Full Step: + Full Step: 1.0e+00 + Method: Backtrack + Solver Options: + Status Test Check Type: Minimal + Status Tests: + Test Type: Combo + Combo Type: OR + Number of Tests: 2 + Test 0: + Test Type: Combo + Combo Type: AND + Number of Tests: 2 + Test 0: + Test Type: NormF + Norm Type: Two Norm + Scale Type: Unscaled + Tolerance: 1.0e-03 + Test 1: + Test Type: RelativeNormF + Norm Type: Two Norm + Tolerance: 0.9999 + Test 1: + Test Type: MaxIters + Maximum Iterations: 100 + Printing: + Output Precision: 3 + Output Processor: 0 + Output Information: + Error: true + Warning: true + Outer Iteration: true + Parameters: false + Details: false + Linear Solver Details: false + Stepper Iteration: true + Stepper Details: true + Stepper Parameters: true + + Direction: + Method: Newton + Newton: + Forcing Term Method: Type 2 + Rescue Bad Newton Solve: true + Linear Solver: + Write Linear System: false + Tolerance: 1.0e-8 + + Stratimikos Linear Solver: + Stratimikos: + +# Linear Solver Information + Linear Solver Type: Belos + Linear Solver Types: + Belos: + Solver Type: Block GMRES + Solver Types: + Block GMRES: + Output Frequency: 20 + Output Style: 1 + Verbosity: 33 + Maximum Iterations: 200 + Block Size: 1 + Num Blocks: 200 + Flexible Gmres: false + VerboseObject: + Output File: none + Verbosity Level: low + +# Preconditioner Information + Preconditioner Type: MueLu + Preconditioner Types: + + Ifpack2: + Overlap: 1 + Prec Type: ILUT + + MueLu: + Matrix: + PDE equations: 2 + Factories: + myLineDetectionFact: + factory: LineDetectionFactory + 'linedetection: orientation': coordinates + mySemiCoarsenPFact1: + factory: SemiCoarsenPFactory + 'semicoarsen: coarsen rate': 14 + UncoupledAggregationFact2: + factory: UncoupledAggregationFactory + 'aggregation: ordering': graph + 'aggregation: max selected neighbors': 0 + 'aggregation: min agg size': 3 + 'aggregation: phase3 avoid singletons': true + MyCoarseMap2: + factory: CoarseMapFactory + Aggregates: UncoupledAggregationFact2 + myTentativePFact2: + 'tentative: calculate qr': true + factory: TentativePFactory + Aggregates: UncoupledAggregationFact2 + CoarseMap: MyCoarseMap2 + mySaPFact2: + 'sa: eigenvalue estimate num iterations': 10 + 'sa: damping factor': 1.33333e+00 + factory: SaPFactory + P: myTentativePFact2 + myTransferCoordinatesFact: + factory: CoordinatesTransferFactory + CoarseMap: MyCoarseMap2 + Aggregates: UncoupledAggregationFact2 + myTogglePFact: + factory: TogglePFactory + 'semicoarsen: number of levels': 2 + TransferFactories: + P1: mySemiCoarsenPFact1 + P2: mySaPFact2 + Ptent1: mySemiCoarsenPFact1 + Ptent2: myTentativePFact2 + Nullspace1: mySemiCoarsenPFact1 + Nullspace2: myTentativePFact2 + myRestrictorFact: + factory: TransPFactory + P: myTogglePFact + myToggleTransferCoordinatesFact: + factory: ToggleCoordinatesTransferFactory + Chosen P: myTogglePFact + TransferFactories: + Coordinates1: mySemiCoarsenPFact1 + Coordinates2: myTransferCoordinatesFact + myRAPFact: + factory: RAPFactory + P: myTogglePFact + R: myRestrictorFact + TransferFactories: + For Coordinates: myToggleTransferCoordinatesFact + myRepartitionHeuristicFact: + factory: RepartitionHeuristicFactory + A: myRAPFact + 'repartition: min rows per proc': 3000 + 'repartition: max imbalance': 1.327e+00 + 'repartition: start level': 1 + myZoltanInterface: + factory: ZoltanInterface + A: myRAPFact + Coordinates: myToggleTransferCoordinatesFact + number of partitions: myRepartitionHeuristicFact + myRepartitionFact: + factory: RepartitionFactory + A: myRAPFact + Partition: myZoltanInterface + 'repartition: remap parts': true + number of partitions: myRepartitionHeuristicFact + myRebalanceProlongatorFact: + factory: RebalanceTransferFactory + type: Interpolation + P: myTogglePFact + Coordinates: myToggleTransferCoordinatesFact + Nullspace: myTogglePFact + myRebalanceRestrictionFact: + factory: RebalanceTransferFactory + type: Restriction + R: myRestrictorFact + myRebalanceAFact: + factory: RebalanceAcFactory + A: myRAPFact + TransferFactories: { } + mySmoother1: + factory: TrilinosSmoother + type: LINESMOOTHING_BANDEDRELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother3: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother4: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': pre + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 4 + 'relaxation: damping factor': 1.0 + Hierarchy: + max levels: 7 + 'coarse: max size': 2000 + verbosity: None + Finest: + Smoother: mySmoother1 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + All: + startLevel: 1 + Smoother: mySmoother4 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.ais_8to30km.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.ais_8to30km.yaml new file mode 100644 index 000000000000..c3bc0cf8bd56 --- /dev/null +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.ais_8to30km.yaml @@ -0,0 +1,246 @@ +%YAML 1.1 +--- +ANONYMOUS: + Problem: + Basal Cubature Degree: 4 + LandIce Field Norm: + sliding_velocity_basalside: + Regularization Type: Given Value + Regularization Value: 1.0e-4 + LandIce BCs: + BC 0: + Basal Friction Coefficient: + Type: Power Law + Power Exponent: 0.2 + Mu Type: Field + Effective Pressure Type: Constant + Effective Pressure: 1.0 + # Zero Effective Pressure On Floating Ice At Nodes: true + Zero Beta On Floating Ice: true +# Discretization Description + Discretization: + #Exodus Output File Name: albany_output.exo + + Piro: +# Nonlinear Solver Information + NOX: + Nonlinear Solver: Line Search Based + Line Search: + Full Step: + Full Step: 1.0e+00 + Method: Backtrack + Solver Options: + Status Test Check Type: Minimal + Status Tests: + Test Type: Combo + Combo Type: OR + Number of Tests: 2 + Test 0: + Test Type: Combo + Combo Type: AND + Number of Tests: 2 + Test 0: + Test Type: NormF + Norm Type: Two Norm + Scale Type: Unscaled + Tolerance: 1.0e-03 + Test 1: + Test Type: RelativeNormF + Norm Type: Two Norm + Tolerance: 0.9999 + Test 1: + Test Type: MaxIters + Maximum Iterations: 100 + Printing: + Output Precision: 3 + Output Processor: 0 + Output Information: + Error: true + Warning: true + Outer Iteration: true + Parameters: false + Details: false + Linear Solver Details: false + Stepper Iteration: true + Stepper Details: true + Stepper Parameters: true + + Direction: + Method: Newton + Newton: + Forcing Term Method: Type 2 + Rescue Bad Newton Solve: true + Linear Solver: + Write Linear System: false + Tolerance: 1.0e-8 + + Stratimikos Linear Solver: + Stratimikos: + +# Linear Solver Information + Linear Solver Type: Belos + Linear Solver Types: + Belos: + Solver Type: Block GMRES + Solver Types: + Block GMRES: + Output Frequency: 20 + Output Style: 1 + Verbosity: 33 + Maximum Iterations: 200 + Block Size: 1 + Num Blocks: 200 + Flexible Gmres: false + VerboseObject: + Output File: none + Verbosity Level: low + +# Preconditioner Information + Preconditioner Type: MueLu + Preconditioner Types: + + Ifpack2: + Overlap: 1 + Prec Type: ILUT + + MueLu: + Matrix: + PDE equations: 2 + Factories: + myLineDetectionFact: + factory: LineDetectionFactory + 'linedetection: orientation': coordinates + mySemiCoarsenPFact1: + factory: SemiCoarsenPFactory + 'semicoarsen: coarsen rate': 14 + UncoupledAggregationFact2: + factory: UncoupledAggregationFactory + 'aggregation: ordering': graph + 'aggregation: max selected neighbors': 0 + 'aggregation: min agg size': 3 + 'aggregation: phase3 avoid singletons': true + MyCoarseMap2: + factory: CoarseMapFactory + Aggregates: UncoupledAggregationFact2 + myTentativePFact2: + 'tentative: calculate qr': true + factory: TentativePFactory + Aggregates: UncoupledAggregationFact2 + CoarseMap: MyCoarseMap2 + mySaPFact2: + 'sa: eigenvalue estimate num iterations': 10 + 'sa: damping factor': 1.33333e+00 + factory: SaPFactory + P: myTentativePFact2 + myTransferCoordinatesFact: + factory: CoordinatesTransferFactory + CoarseMap: MyCoarseMap2 + Aggregates: UncoupledAggregationFact2 + myTogglePFact: + factory: TogglePFactory + 'semicoarsen: number of levels': 2 + TransferFactories: + P1: mySemiCoarsenPFact1 + P2: mySaPFact2 + Ptent1: mySemiCoarsenPFact1 + Ptent2: myTentativePFact2 + Nullspace1: mySemiCoarsenPFact1 + Nullspace2: myTentativePFact2 + myRestrictorFact: + factory: TransPFactory + P: myTogglePFact + myToggleTransferCoordinatesFact: + factory: ToggleCoordinatesTransferFactory + Chosen P: myTogglePFact + TransferFactories: + Coordinates1: mySemiCoarsenPFact1 + Coordinates2: myTransferCoordinatesFact + myRAPFact: + factory: RAPFactory + P: myTogglePFact + R: myRestrictorFact + TransferFactories: + For Coordinates: myToggleTransferCoordinatesFact + myRepartitionHeuristicFact: + factory: RepartitionHeuristicFactory + A: myRAPFact + 'repartition: min rows per proc': 3000 + 'repartition: max imbalance': 1.327e+00 + 'repartition: start level': 1 + myZoltanInterface: + factory: ZoltanInterface + A: myRAPFact + Coordinates: myToggleTransferCoordinatesFact + number of partitions: myRepartitionHeuristicFact + myRepartitionFact: + factory: RepartitionFactory + A: myRAPFact + Partition: myZoltanInterface + 'repartition: remap parts': true + number of partitions: myRepartitionHeuristicFact + myRebalanceProlongatorFact: + factory: RebalanceTransferFactory + type: Interpolation + P: myTogglePFact + Coordinates: myToggleTransferCoordinatesFact + Nullspace: myTogglePFact + myRebalanceRestrictionFact: + factory: RebalanceTransferFactory + type: Restriction + R: myRestrictorFact + myRebalanceAFact: + factory: RebalanceAcFactory + A: myRAPFact + TransferFactories: { } + mySmoother1: + factory: TrilinosSmoother + type: LINESMOOTHING_BANDEDRELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother3: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother4: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': pre + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 4 + 'relaxation: damping factor': 1.0 + Hierarchy: + max levels: 7 + 'coarse: max size': 2000 + verbosity: None + Finest: + Smoother: mySmoother1 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + All: + startLevel: 1 + Smoother: mySmoother4 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.aisgis20km.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.aisgis20km.yaml new file mode 100644 index 000000000000..589e163bba7f --- /dev/null +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.aisgis20km.yaml @@ -0,0 +1,231 @@ +%YAML 1.1 +--- +ANONYMOUS: + +# Discretization Description + Discretization: + #Exodus Output File Name: albany_output.exo + + Piro: +# Nonlinear Solver Information + NOX: + Nonlinear Solver: Line Search Based + Line Search: + Full Step: + Full Step: 1.0e+00 + Method: Backtrack + Solver Options: + Status Test Check Type: Minimal + Status Tests: + Test Type: Combo + Combo Type: OR + Number of Tests: 2 + Test 0: + Test Type: Combo + Combo Type: AND + Number of Tests: 2 + Test 0: + Test Type: NormF + Norm Type: Two Norm + Scale Type: Unscaled + Tolerance: 1.0e-03 + Test 1: + Test Type: RelativeNormF + Norm Type: Two Norm + Tolerance: 0.9999 + Test 1: + Test Type: MaxIters + Maximum Iterations: 100 + Printing: + Output Precision: 3 + Output Processor: 0 + Output Information: + Error: true + Warning: true + Outer Iteration: true + Parameters: false + Details: false + Linear Solver Details: false + Stepper Iteration: true + Stepper Details: true + Stepper Parameters: true + + Direction: + Method: Newton + Newton: + Forcing Term Method: Type 2 + Rescue Bad Newton Solve: true + Linear Solver: + Write Linear System: false + Tolerance: 1.0e-8 + + Stratimikos Linear Solver: + Stratimikos: + +# Linear Solver Information + Linear Solver Type: Belos + Linear Solver Types: + Belos: + Solver Type: Block GMRES + Solver Types: + Block GMRES: + Output Frequency: 20 + Output Style: 1 + Verbosity: 33 + Maximum Iterations: 200 + Block Size: 1 + Num Blocks: 200 + Flexible Gmres: false + VerboseObject: + Output File: none + Verbosity Level: low + +# Preconditioner Information + Preconditioner Type: MueLu + Preconditioner Types: + + Ifpack2: + Overlap: 1 + Prec Type: ILUT + + MueLu: + Matrix: + PDE equations: 2 + Factories: + myLineDetectionFact: + factory: LineDetectionFactory + 'linedetection: orientation': coordinates + mySemiCoarsenPFact1: + factory: SemiCoarsenPFactory + 'semicoarsen: coarsen rate': 14 + UncoupledAggregationFact2: + factory: UncoupledAggregationFactory + 'aggregation: ordering': graph + 'aggregation: max selected neighbors': 0 + 'aggregation: min agg size': 3 + 'aggregation: phase3 avoid singletons': true + MyCoarseMap2: + factory: CoarseMapFactory + Aggregates: UncoupledAggregationFact2 + myTentativePFact2: + 'tentative: calculate qr': true + factory: TentativePFactory + Aggregates: UncoupledAggregationFact2 + CoarseMap: MyCoarseMap2 + mySaPFact2: + 'sa: eigenvalue estimate num iterations': 10 + 'sa: damping factor': 1.33333e+00 + factory: SaPFactory + P: myTentativePFact2 + myTransferCoordinatesFact: + factory: CoordinatesTransferFactory + CoarseMap: MyCoarseMap2 + Aggregates: UncoupledAggregationFact2 + myTogglePFact: + factory: TogglePFactory + 'semicoarsen: number of levels': 2 + TransferFactories: + P1: mySemiCoarsenPFact1 + P2: mySaPFact2 + Ptent1: mySemiCoarsenPFact1 + Ptent2: myTentativePFact2 + Nullspace1: mySemiCoarsenPFact1 + Nullspace2: myTentativePFact2 + myRestrictorFact: + factory: TransPFactory + P: myTogglePFact + myToggleTransferCoordinatesFact: + factory: ToggleCoordinatesTransferFactory + Chosen P: myTogglePFact + TransferFactories: + Coordinates1: mySemiCoarsenPFact1 + Coordinates2: myTransferCoordinatesFact + myRAPFact: + factory: RAPFactory + P: myTogglePFact + R: myRestrictorFact + TransferFactories: + For Coordinates: myToggleTransferCoordinatesFact + myRepartitionHeuristicFact: + factory: RepartitionHeuristicFactory + A: myRAPFact + 'repartition: min rows per proc': 3000 + 'repartition: max imbalance': 1.327e+00 + 'repartition: start level': 1 + myZoltanInterface: + factory: ZoltanInterface + A: myRAPFact + Coordinates: myToggleTransferCoordinatesFact + number of partitions: myRepartitionHeuristicFact + myRepartitionFact: + factory: RepartitionFactory + A: myRAPFact + Partition: myZoltanInterface + 'repartition: remap parts': true + number of partitions: myRepartitionHeuristicFact + myRebalanceProlongatorFact: + factory: RebalanceTransferFactory + type: Interpolation + P: myTogglePFact + Coordinates: myToggleTransferCoordinatesFact + Nullspace: myTogglePFact + myRebalanceRestrictionFact: + factory: RebalanceTransferFactory + type: Restriction + R: myRestrictorFact + myRebalanceAFact: + factory: RebalanceAcFactory + A: myRAPFact + TransferFactories: { } + mySmoother1: + factory: TrilinosSmoother + type: LINESMOOTHING_BANDEDRELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother3: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother4: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': pre + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 4 + 'relaxation: damping factor': 1.0 + Hierarchy: + max levels: 7 + 'coarse: max size': 2000 + verbosity: None + Finest: + Smoother: mySmoother1 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + All: + startLevel: 1 + Smoother: mySmoother4 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r01.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r01.yaml new file mode 100644 index 000000000000..589e163bba7f --- /dev/null +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r01.yaml @@ -0,0 +1,231 @@ +%YAML 1.1 +--- +ANONYMOUS: + +# Discretization Description + Discretization: + #Exodus Output File Name: albany_output.exo + + Piro: +# Nonlinear Solver Information + NOX: + Nonlinear Solver: Line Search Based + Line Search: + Full Step: + Full Step: 1.0e+00 + Method: Backtrack + Solver Options: + Status Test Check Type: Minimal + Status Tests: + Test Type: Combo + Combo Type: OR + Number of Tests: 2 + Test 0: + Test Type: Combo + Combo Type: AND + Number of Tests: 2 + Test 0: + Test Type: NormF + Norm Type: Two Norm + Scale Type: Unscaled + Tolerance: 1.0e-03 + Test 1: + Test Type: RelativeNormF + Norm Type: Two Norm + Tolerance: 0.9999 + Test 1: + Test Type: MaxIters + Maximum Iterations: 100 + Printing: + Output Precision: 3 + Output Processor: 0 + Output Information: + Error: true + Warning: true + Outer Iteration: true + Parameters: false + Details: false + Linear Solver Details: false + Stepper Iteration: true + Stepper Details: true + Stepper Parameters: true + + Direction: + Method: Newton + Newton: + Forcing Term Method: Type 2 + Rescue Bad Newton Solve: true + Linear Solver: + Write Linear System: false + Tolerance: 1.0e-8 + + Stratimikos Linear Solver: + Stratimikos: + +# Linear Solver Information + Linear Solver Type: Belos + Linear Solver Types: + Belos: + Solver Type: Block GMRES + Solver Types: + Block GMRES: + Output Frequency: 20 + Output Style: 1 + Verbosity: 33 + Maximum Iterations: 200 + Block Size: 1 + Num Blocks: 200 + Flexible Gmres: false + VerboseObject: + Output File: none + Verbosity Level: low + +# Preconditioner Information + Preconditioner Type: MueLu + Preconditioner Types: + + Ifpack2: + Overlap: 1 + Prec Type: ILUT + + MueLu: + Matrix: + PDE equations: 2 + Factories: + myLineDetectionFact: + factory: LineDetectionFactory + 'linedetection: orientation': coordinates + mySemiCoarsenPFact1: + factory: SemiCoarsenPFactory + 'semicoarsen: coarsen rate': 14 + UncoupledAggregationFact2: + factory: UncoupledAggregationFactory + 'aggregation: ordering': graph + 'aggregation: max selected neighbors': 0 + 'aggregation: min agg size': 3 + 'aggregation: phase3 avoid singletons': true + MyCoarseMap2: + factory: CoarseMapFactory + Aggregates: UncoupledAggregationFact2 + myTentativePFact2: + 'tentative: calculate qr': true + factory: TentativePFactory + Aggregates: UncoupledAggregationFact2 + CoarseMap: MyCoarseMap2 + mySaPFact2: + 'sa: eigenvalue estimate num iterations': 10 + 'sa: damping factor': 1.33333e+00 + factory: SaPFactory + P: myTentativePFact2 + myTransferCoordinatesFact: + factory: CoordinatesTransferFactory + CoarseMap: MyCoarseMap2 + Aggregates: UncoupledAggregationFact2 + myTogglePFact: + factory: TogglePFactory + 'semicoarsen: number of levels': 2 + TransferFactories: + P1: mySemiCoarsenPFact1 + P2: mySaPFact2 + Ptent1: mySemiCoarsenPFact1 + Ptent2: myTentativePFact2 + Nullspace1: mySemiCoarsenPFact1 + Nullspace2: myTentativePFact2 + myRestrictorFact: + factory: TransPFactory + P: myTogglePFact + myToggleTransferCoordinatesFact: + factory: ToggleCoordinatesTransferFactory + Chosen P: myTogglePFact + TransferFactories: + Coordinates1: mySemiCoarsenPFact1 + Coordinates2: myTransferCoordinatesFact + myRAPFact: + factory: RAPFactory + P: myTogglePFact + R: myRestrictorFact + TransferFactories: + For Coordinates: myToggleTransferCoordinatesFact + myRepartitionHeuristicFact: + factory: RepartitionHeuristicFactory + A: myRAPFact + 'repartition: min rows per proc': 3000 + 'repartition: max imbalance': 1.327e+00 + 'repartition: start level': 1 + myZoltanInterface: + factory: ZoltanInterface + A: myRAPFact + Coordinates: myToggleTransferCoordinatesFact + number of partitions: myRepartitionHeuristicFact + myRepartitionFact: + factory: RepartitionFactory + A: myRAPFact + Partition: myZoltanInterface + 'repartition: remap parts': true + number of partitions: myRepartitionHeuristicFact + myRebalanceProlongatorFact: + factory: RebalanceTransferFactory + type: Interpolation + P: myTogglePFact + Coordinates: myToggleTransferCoordinatesFact + Nullspace: myTogglePFact + myRebalanceRestrictionFact: + factory: RebalanceTransferFactory + type: Restriction + R: myRestrictorFact + myRebalanceAFact: + factory: RebalanceAcFactory + A: myRAPFact + TransferFactories: { } + mySmoother1: + factory: TrilinosSmoother + type: LINESMOOTHING_BANDEDRELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother3: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother4: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': pre + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 4 + 'relaxation: damping factor': 1.0 + Hierarchy: + max levels: 7 + 'coarse: max size': 2000 + verbosity: None + Finest: + Smoother: mySmoother1 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + All: + startLevel: 1 + Smoother: mySmoother4 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r02.yaml similarity index 81% rename from components/mpas-albany-landice/bld/namelist_files/albany_input.yaml rename to components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r02.yaml index 8b770a290ae4..2b70fd9b30fa 100644 --- a/components/mpas-albany-landice/bld/namelist_files/albany_input.yaml +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r02.yaml @@ -12,35 +12,34 @@ ANONYMOUS: Basal Friction Coefficient: Type: Power Law Power Exponent: 1.0 - Mu Type: Field + Mu Type: Field Effective Pressure Type: Hydrostatic Computed At Nodes Zero Effective Pressure On Floating Ice At Nodes: true Zero Beta On Floating Ice: false Use Pressurized Bed Above Sea Level: false - # Discretization Description Discretization: #Exodus Output File Name: albany_output.exo - - Piro: + + Piro: # Nonlinear Solver Information - NOX: + NOX: Nonlinear Solver: Line Search Based - Line Search: - Full Step: + Line Search: + Full Step: Full Step: 1.0e+00 Method: Backtrack - Solver Options: + Solver Options: Status Test Check Type: Minimal - Status Tests: + Status Tests: Test Type: Combo Combo Type: OR Number of Tests: 2 - Test 0: + Test 0: Test Type: Combo Combo Type: AND Number of Tests: 2 - Test 0: + Test 0: Test Type: NormF Norm Type: Two Norm Scale Type: Unscaled @@ -49,13 +48,13 @@ ANONYMOUS: Test Type: RelativeNormF Norm Type: Two Norm Tolerance: 0.9999 - Test 1: + Test 1: Test Type: MaxIters Maximum Iterations: 100 - Printing: + Printing: Output Precision: 3 Output Processor: 0 - Output Information: + Output Information: Error: true Warning: true Outer Iteration: true @@ -66,21 +65,21 @@ ANONYMOUS: Stepper Details: true Stepper Parameters: true - Direction: + Direction: Method: Newton - Newton: - Forcing Term Method: Type 2 + Newton: + Forcing Term Method: Type 2 Rescue Bad Newton Solve: true Linear Solver: Write Linear System: false Tolerance: 1.0e-8 - - Stratimikos Linear Solver: + + Stratimikos Linear Solver: Stratimikos: - + # Linear Solver Information Linear Solver Type: Belos - Linear Solver Types: + Linear Solver Types: Belos: Solver Type: Block GMRES Solver Types: @@ -95,133 +94,133 @@ ANONYMOUS: VerboseObject: Output File: none Verbosity Level: low - + # Preconditioner Information Preconditioner Type: MueLu - Preconditioner Types: - - Ifpack2: + Preconditioner Types: + + Ifpack2: Overlap: 1 - Prec Type: ILUT - - MueLu: - Matrix: + Prec Type: ILUT + + MueLu: + Matrix: PDE equations: 2 - Factories: - myLineDetectionFact: + Factories: + myLineDetectionFact: factory: LineDetectionFactory 'linedetection: orientation': coordinates - mySemiCoarsenPFact1: + mySemiCoarsenPFact1: factory: SemiCoarsenPFactory 'semicoarsen: coarsen rate': 14 - UncoupledAggregationFact2: + UncoupledAggregationFact2: factory: UncoupledAggregationFactory 'aggregation: ordering': graph 'aggregation: max selected neighbors': 0 'aggregation: min agg size': 3 'aggregation: phase3 avoid singletons': true - MyCoarseMap2: + MyCoarseMap2: factory: CoarseMapFactory Aggregates: UncoupledAggregationFact2 - myTentativePFact2: + myTentativePFact2: 'tentative: calculate qr': true factory: TentativePFactory Aggregates: UncoupledAggregationFact2 CoarseMap: MyCoarseMap2 - mySaPFact2: + mySaPFact2: 'sa: eigenvalue estimate num iterations': 10 'sa: damping factor': 1.33333e+00 factory: SaPFactory P: myTentativePFact2 - myTransferCoordinatesFact: + myTransferCoordinatesFact: factory: CoordinatesTransferFactory CoarseMap: MyCoarseMap2 Aggregates: UncoupledAggregationFact2 - myTogglePFact: + myTogglePFact: factory: TogglePFactory 'semicoarsen: number of levels': 2 - TransferFactories: + TransferFactories: P1: mySemiCoarsenPFact1 P2: mySaPFact2 Ptent1: mySemiCoarsenPFact1 Ptent2: myTentativePFact2 Nullspace1: mySemiCoarsenPFact1 Nullspace2: myTentativePFact2 - myRestrictorFact: + myRestrictorFact: factory: TransPFactory P: myTogglePFact - myToggleTransferCoordinatesFact: + myToggleTransferCoordinatesFact: factory: ToggleCoordinatesTransferFactory Chosen P: myTogglePFact - TransferFactories: + TransferFactories: Coordinates1: mySemiCoarsenPFact1 Coordinates2: myTransferCoordinatesFact - myRAPFact: + myRAPFact: factory: RAPFactory P: myTogglePFact R: myRestrictorFact - TransferFactories: + TransferFactories: For Coordinates: myToggleTransferCoordinatesFact - myRepartitionHeuristicFact: + myRepartitionHeuristicFact: factory: RepartitionHeuristicFactory A: myRAPFact 'repartition: min rows per proc': 3000 'repartition: max imbalance': 1.327e+00 'repartition: start level': 1 - myZoltanInterface: + myZoltanInterface: factory: ZoltanInterface A: myRAPFact Coordinates: myToggleTransferCoordinatesFact number of partitions: myRepartitionHeuristicFact - myRepartitionFact: + myRepartitionFact: factory: RepartitionFactory A: myRAPFact Partition: myZoltanInterface 'repartition: remap parts': true number of partitions: myRepartitionHeuristicFact - myRebalanceProlongatorFact: + myRebalanceProlongatorFact: factory: RebalanceTransferFactory type: Interpolation P: myTogglePFact Coordinates: myToggleTransferCoordinatesFact Nullspace: myTogglePFact - myRebalanceRestrictionFact: + myRebalanceRestrictionFact: factory: RebalanceTransferFactory type: Restriction R: myRestrictorFact - myRebalanceAFact: + myRebalanceAFact: factory: RebalanceAcFactory A: myRAPFact TransferFactories: { } - mySmoother1: + mySmoother1: factory: TrilinosSmoother type: LINESMOOTHING_BANDEDRELAXATION 'smoother: pre or post': both - ParameterList: + ParameterList: 'relaxation: type': Gauss-Seidel 'relaxation: sweeps': 1 'relaxation: damping factor': 1.0 - mySmoother3: + mySmoother3: factory: TrilinosSmoother type: RELAXATION 'smoother: pre or post': both - ParameterList: + ParameterList: 'relaxation: type': Gauss-Seidel 'relaxation: sweeps': 1 'relaxation: damping factor': 1.0 - mySmoother4: + mySmoother4: factory: TrilinosSmoother type: RELAXATION 'smoother: pre or post': pre - ParameterList: + ParameterList: 'relaxation: type': Gauss-Seidel 'relaxation: sweeps': 4 'relaxation: damping factor': 1.0 - Hierarchy: + Hierarchy: max levels: 7 'coarse: max size': 2000 verbosity: None - Finest: + Finest: Smoother: mySmoother1 CoarseSolver: mySmoother4 P: myRebalanceProlongatorFact @@ -232,7 +231,7 @@ ANONYMOUS: A: myRebalanceAFact Coordinates: myRebalanceProlongatorFact Importer: myRepartitionFact - All: + All: startLevel: 1 Smoother: mySmoother4 CoarseSolver: mySmoother4 @@ -244,4 +243,3 @@ ANONYMOUS: A: myRebalanceAFact Coordinates: myRebalanceProlongatorFact Importer: myRepartitionFact - diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_20km_r01.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_20km_r01.yaml new file mode 100644 index 000000000000..589e163bba7f --- /dev/null +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_20km_r01.yaml @@ -0,0 +1,231 @@ +%YAML 1.1 +--- +ANONYMOUS: + +# Discretization Description + Discretization: + #Exodus Output File Name: albany_output.exo + + Piro: +# Nonlinear Solver Information + NOX: + Nonlinear Solver: Line Search Based + Line Search: + Full Step: + Full Step: 1.0e+00 + Method: Backtrack + Solver Options: + Status Test Check Type: Minimal + Status Tests: + Test Type: Combo + Combo Type: OR + Number of Tests: 2 + Test 0: + Test Type: Combo + Combo Type: AND + Number of Tests: 2 + Test 0: + Test Type: NormF + Norm Type: Two Norm + Scale Type: Unscaled + Tolerance: 1.0e-03 + Test 1: + Test Type: RelativeNormF + Norm Type: Two Norm + Tolerance: 0.9999 + Test 1: + Test Type: MaxIters + Maximum Iterations: 100 + Printing: + Output Precision: 3 + Output Processor: 0 + Output Information: + Error: true + Warning: true + Outer Iteration: true + Parameters: false + Details: false + Linear Solver Details: false + Stepper Iteration: true + Stepper Details: true + Stepper Parameters: true + + Direction: + Method: Newton + Newton: + Forcing Term Method: Type 2 + Rescue Bad Newton Solve: true + Linear Solver: + Write Linear System: false + Tolerance: 1.0e-8 + + Stratimikos Linear Solver: + Stratimikos: + +# Linear Solver Information + Linear Solver Type: Belos + Linear Solver Types: + Belos: + Solver Type: Block GMRES + Solver Types: + Block GMRES: + Output Frequency: 20 + Output Style: 1 + Verbosity: 33 + Maximum Iterations: 200 + Block Size: 1 + Num Blocks: 200 + Flexible Gmres: false + VerboseObject: + Output File: none + Verbosity Level: low + +# Preconditioner Information + Preconditioner Type: MueLu + Preconditioner Types: + + Ifpack2: + Overlap: 1 + Prec Type: ILUT + + MueLu: + Matrix: + PDE equations: 2 + Factories: + myLineDetectionFact: + factory: LineDetectionFactory + 'linedetection: orientation': coordinates + mySemiCoarsenPFact1: + factory: SemiCoarsenPFactory + 'semicoarsen: coarsen rate': 14 + UncoupledAggregationFact2: + factory: UncoupledAggregationFactory + 'aggregation: ordering': graph + 'aggregation: max selected neighbors': 0 + 'aggregation: min agg size': 3 + 'aggregation: phase3 avoid singletons': true + MyCoarseMap2: + factory: CoarseMapFactory + Aggregates: UncoupledAggregationFact2 + myTentativePFact2: + 'tentative: calculate qr': true + factory: TentativePFactory + Aggregates: UncoupledAggregationFact2 + CoarseMap: MyCoarseMap2 + mySaPFact2: + 'sa: eigenvalue estimate num iterations': 10 + 'sa: damping factor': 1.33333e+00 + factory: SaPFactory + P: myTentativePFact2 + myTransferCoordinatesFact: + factory: CoordinatesTransferFactory + CoarseMap: MyCoarseMap2 + Aggregates: UncoupledAggregationFact2 + myTogglePFact: + factory: TogglePFactory + 'semicoarsen: number of levels': 2 + TransferFactories: + P1: mySemiCoarsenPFact1 + P2: mySaPFact2 + Ptent1: mySemiCoarsenPFact1 + Ptent2: myTentativePFact2 + Nullspace1: mySemiCoarsenPFact1 + Nullspace2: myTentativePFact2 + myRestrictorFact: + factory: TransPFactory + P: myTogglePFact + myToggleTransferCoordinatesFact: + factory: ToggleCoordinatesTransferFactory + Chosen P: myTogglePFact + TransferFactories: + Coordinates1: mySemiCoarsenPFact1 + Coordinates2: myTransferCoordinatesFact + myRAPFact: + factory: RAPFactory + P: myTogglePFact + R: myRestrictorFact + TransferFactories: + For Coordinates: myToggleTransferCoordinatesFact + myRepartitionHeuristicFact: + factory: RepartitionHeuristicFactory + A: myRAPFact + 'repartition: min rows per proc': 3000 + 'repartition: max imbalance': 1.327e+00 + 'repartition: start level': 1 + myZoltanInterface: + factory: ZoltanInterface + A: myRAPFact + Coordinates: myToggleTransferCoordinatesFact + number of partitions: myRepartitionHeuristicFact + myRepartitionFact: + factory: RepartitionFactory + A: myRAPFact + Partition: myZoltanInterface + 'repartition: remap parts': true + number of partitions: myRepartitionHeuristicFact + myRebalanceProlongatorFact: + factory: RebalanceTransferFactory + type: Interpolation + P: myTogglePFact + Coordinates: myToggleTransferCoordinatesFact + Nullspace: myTogglePFact + myRebalanceRestrictionFact: + factory: RebalanceTransferFactory + type: Restriction + R: myRestrictorFact + myRebalanceAFact: + factory: RebalanceAcFactory + A: myRAPFact + TransferFactories: { } + mySmoother1: + factory: TrilinosSmoother + type: LINESMOOTHING_BANDEDRELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother3: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': both + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 1 + 'relaxation: damping factor': 1.0 + mySmoother4: + factory: TrilinosSmoother + type: RELAXATION + 'smoother: pre or post': pre + ParameterList: + 'relaxation: type': Gauss-Seidel + 'relaxation: sweeps': 4 + 'relaxation: damping factor': 1.0 + Hierarchy: + max levels: 7 + 'coarse: max size': 2000 + verbosity: None + Finest: + Smoother: mySmoother1 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + All: + startLevel: 1 + Smoother: mySmoother4 + CoarseSolver: mySmoother4 + P: myRebalanceProlongatorFact + Nullspace: myRebalanceProlongatorFact + CoarseNumZLayers: myLineDetectionFact + LineDetection_Layers: myLineDetectionFact + LineDetection_VertLineIds: myLineDetectionFact + A: myRebalanceAFact + Coordinates: myRebalanceProlongatorFact + Importer: myRepartitionFact + diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index e345853f84a5..a42011e54073 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -300,7 +300,11 @@ def buildnml(case, caseroot, compname): if os.path.exists("{}/SourceMods/src.mali/{}".format(caseroot, albany_input_name)): safe_copy("{}/SourceMods/src.mali/{}".format(caseroot, albany_input_name), os.path.join(rundir, albany_input_name)) else: - safe_copy("{}/components/mpas-albany-landice/bld/namelist_files/{}".format(srcroot, albany_input_name), os.path.join(rundir, albany_input_name)) + # Each supported grid should have a corresponding albany_input.yaml file in + # components/mpas-albany-landice/bld/namelist_files/ + # Naming convention is albany_input.GRID_PREFIX.yaml + albany_input_name_grid = f"albany_input.{grid_prefix}.yaml" + safe_copy(f"{srcroot}/components/mpas-albany-landice/bld/namelist_files/{albany_input_name_grid}", os.path.join(rundir, albany_input_name)) ############################################################################### def _main_func(): From 57c057e5e396ee893d1ef98edfa3275cd56ed2cf Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 14 Aug 2024 16:28:32 -0500 Subject: [PATCH 532/904] Update glcshelf test to use ais8to30 mesh This was not possible when the mesh was added due to issues with the treatment of temperature in MALI. Now that those issues have been resolved, this test can be updated to the newer mesh. --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 059c280e2bb4..183004df2dd2 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -302,7 +302,7 @@ "SMS.ne30_f19_g16_rx1.A", "ERS_Ld5.T62_oQU120.CMPASO-NYF", "ERS.f09_g16_g.MALISIA", - "ERS_Ld5.TL319_oQU240wLI_ais20.MPAS_LISIO_JRA1p5.mpaso-ocn_glcshelf", + "ERS_Ld5.TL319_oQU240wLI_ais8to30.MPAS_LISIO_JRA1p5.mpaso-ocn_glcshelf", "SMS_P12x2.ne4pg2_oQU480.WCYCL1850NS.allactive-mach_mods", "ERS_Ln9.ne4pg2_ne4pg2.F2010-MMF1.eam-mmf_crmout", ) From bd084b04a0980bf9a3e1614997f7c32d8cecd75e Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Wed, 21 Aug 2024 15:27:19 -0500 Subject: [PATCH 533/904] Add and move a test to e3sm_integration. ERS.ne30pg2_r05_IcoswISC30E3r5.GPMPAS-JRA.mosart-rof_ocn_2way is moved from e3sm_developer to e3sm_integration because it is to large and was taking to much time/memory. To do that, it was removed from land_developer by removing the inheritance of e3sm_mosart_exonoshare Also add a moab-coupler test to e3sm-integration. --- cime_config/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 059c280e2bb4..c6be199ea73a 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -80,7 +80,7 @@ "e3sm_land_developer" : { "share" : True, "time" : "0:45:00", - "inherit" : ("e3sm_mosart_developer", "e3sm_mosart_exenoshare", "e3sm_land_exeshare", "e3sm_land_exenoshare", "e3sm_land_debug", "fates_elm_developer"), + "inherit" : ("e3sm_mosart_developer", "e3sm_land_exeshare", "e3sm_land_exenoshare", "e3sm_land_debug", "fates_elm_developer"), "tests" : ( "ERS.f19_f19.I1850ELMCN", "ERS.f19_f19.I20TRELMCN", @@ -321,6 +321,7 @@ "time" : "03:00:00", "tests" : ( "ERS.ne4pg2_oQU480.WCYCL1850NS", + "ERS_Vmoab.ne4pg2_oQU480.WCYCL1850NS", "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.allactive-wcprod", "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCLSSP370.allactive-wcprodssp", "ERS_Ld3.ne4pg2_oQU480.F2010", @@ -337,6 +338,7 @@ "ERS_Ld3.ne30pg2_r05_IcoswISC30E3r5.WCYCL1850.allactive-nlmaps", "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.CRYO1850-DISMF", "ERS.hcru_hcru.I20TRGSWCNPRDCTCBC.elm-erosion", + "ERS.ne30pg2_r05_IcoswISC30E3r5.GPMPAS-JRA.mosart-rof_ocn_2way", ) }, From f6430aa5730b6baa6ad766fdf6fe75f3ee39bb8a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 10 Jul 2024 15:04:00 -0500 Subject: [PATCH 534/904] instance MOAB meshes on eamxx side need to change semoab_mod to accept pgN as a parameter --- .../homme/interface/phys_grid_mod.F90 | 51 +++++++++++++++++++ components/homme/src/share/semoab_mod.F90 | 5 +- .../homme/src/theta-l_kokkos/CMakeLists.txt | 1 + 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 index 4d928c800b03..5eaab1204e65 100644 --- a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 @@ -542,6 +542,14 @@ subroutine phys_grid_init (pgN) use gllfvremap_mod, only: gfr_init use homme_context_mod, only: elem, par use dimensions_mod, only: nelem, nelemd +#ifdef HAVE_MOAB + use seq_comm_mct, only: MHID, MHFID ! id of homme moab coarse and fine applications + use seq_comm_mct, only: ATMID + use seq_comm_mct, only: mhpgid ! id of pgx moab application + use semoab_mod, only: create_moab_meshes + use iMOAB, only : iMOAB_RegisterApplication + use iso_c_binding +#endif ! ! Input(s) ! @@ -554,6 +562,10 @@ subroutine phys_grid_init (pgN) character(2) :: str type(pg_specs_t), pointer :: pg +#ifdef HAVE_MOAB + integer :: ATM_ID1 + character*32 appname +#endif pg => pg_specs(pgN) if (pg%inited) then @@ -605,6 +617,45 @@ subroutine phys_grid_init (pgN) call compute_global_dofs (pg) call compute_global_coords (pg) call compute_global_area (pg) +#ifdef HAVE_MOAB + if (pgN > 0) then + appname="HM_COARSE"//C_NULL_CHAR + ATM_ID1 = 120 ! + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, MHID) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHID=", MHID + write(iulog,*) " " + endif + appname="HM_FINE"//C_NULL_CHAR + ATM_ID1 = 119 ! this number should not conflict with other components IDs; how do we know? + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, MHFID) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app for fine mesh') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHFID=", MHFID + write(iulog,*) " " + endif + appname="HM_PGX"//C_NULL_CHAR + ATM_ID1 = ATMID(1) ! this number should not conflict with other components IDs; how do we know? + ! + ! in this case, we reuse the main atm id, mhid will not be used for intersection anymore + ! still, need to be careful + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, mhpgid) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app for fine mesh') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHPGID=", mhpgid + write(iulog,*) " " + endif + + call create_moab_meshes(par, elem, pgN) + endif +#endif end subroutine phys_grid_init diff --git a/components/homme/src/share/semoab_mod.F90 b/components/homme/src/share/semoab_mod.F90 index 2d8bb099ade3..4841cb84b543 100644 --- a/components/homme/src/share/semoab_mod.F90 +++ b/components/homme/src/share/semoab_mod.F90 @@ -23,7 +23,7 @@ module semoab_mod use seq_comm_mct, only: MHPGID ! app id on moab side, for PGx style mesh, uniform from se use seq_comm_mct, only: atm_pg_active ! turn it on when PG style mesh active - use dyn_grid, only: fv_nphys ! phys grid mesh will be replicated too + !use dyn_grid, only: fv_nphys ! phys grid mesh will be replicated too use gllfvremap_mod, only: gfr_f_get_corner_latlon use control_mod, only : west, east, south, north ! 1, 2, 3, 4 @@ -76,7 +76,7 @@ integer function search_in(intarr, leng, value) end function search_in - subroutine create_moab_meshes(par, elem) + subroutine create_moab_meshes(par, elem, fv_nphys) use ISO_C_BINDING use iMOAB, only: iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & @@ -86,6 +86,7 @@ subroutine create_moab_meshes(par, elem) type (element_t), intent(inout) :: elem(:) type (parallel_t) , intent(in) :: par + integer , intent(in) :: fv_nphys integer ierr, i, j, ie, iv, block_ID, k, numvals integer icol, irow, je, linx ! local indices in fine el connect diff --git a/components/homme/src/theta-l_kokkos/CMakeLists.txt b/components/homme/src/theta-l_kokkos/CMakeLists.txt index 739866960e36..a585090ea76a 100644 --- a/components/homme/src/theta-l_kokkos/CMakeLists.txt +++ b/components/homme/src/theta-l_kokkos/CMakeLists.txt @@ -106,6 +106,7 @@ MACRO(THETAL_KOKKOS_SETUP) ${SRC_SHARE_DIR}/planar_mod.F90 ${SRC_SHARE_DIR}/geometry_mod.F90 ${SRC_SHARE_DIR}/planar_mesh_mod.F90 + ${SRC_SHARE_DIR}/semoab_mod.F90 ) IF (NOT HOMME_ENABLE_COMPOSE) From 7e04e52e4dba370899396e1bf239eb4a5e22905d Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 10 Jul 2024 23:04:52 -0500 Subject: [PATCH 535/904] phys atm model --- .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 171 +++++++++++++++++- 1 file changed, 170 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index dc57d828bbdb..b562dbce99db 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -17,6 +17,15 @@ module atm_comp_mct ! MPI use mpi +#ifdef HAVE_MOAB + use seq_comm_mct , only: mphaid ! atm physics grid id in MOAB, on atm pes + use iso_c_binding + use seq_comm_mct, only : num_moab_exports +#ifdef MOABCOMP + use seq_comm_mct, only: seq_comm_compare_mb_mct +#endif +#endif + implicit none save private @@ -107,7 +116,6 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) character(len=256) :: caseid, username, hostname character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c logical (kind=c_bool) :: restarted_run - !------------------------------------------------------------------------------- ! Grab some data from the cdata structure (coming from the coupler) @@ -178,6 +186,10 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Init MCT domain structure call atm_domain_mct (lsize, gsMap_atm, dom_atm) +#ifdef HAVE_MOAB + call moab_atm_phys_scream() +#endif + ! Init import/export mct attribute vectors call mct_aVect_init(x2a, rList=seq_flds_x2a_fields, lsize=lsize) call mct_aVect_init(a2x, rList=seq_flds_a2x_fields, lsize=lsize) @@ -356,4 +368,161 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) call mct_gGrid_importRAttr(dom_atm,"aream",data1,lsize) end subroutine atm_domain_mct +#ifdef HAVE_MOAB + subroutine moab_atm_phys_scream() + + use iMOAB, only : iMOAB_RegisterApplication, iMOAB_CreateVertices, iMOAB_WriteMesh, & + iMOAB_DefineTagStorage, iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_UpdateMeshInfo + use iso_c_binding, only: c_int, c_loc + use scream_f2c_mod, only: scream_get_num_global_cols, scream_get_num_local_cols, & + scream_get_local_cols_gids + use scream_f2c_mod, only: scream_get_cols_latlon, scream_get_cols_area + use seq_flds_mod, only: seq_flds_dom_fields + use shr_kind_mod , only: r8 => shr_kind_r8, cl=>shr_kind_cl, cxx=>shr_kind_cxx + use shr_const_mod, only: SHR_CONST_PI + use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields + + ! Local variables + ! + integer(kind=c_int) :: num_local_cols, num_global_cols + character*100 outfile, wopts + character(CXX) :: tagname ! will store all seq_flds_a2x_fields + character*32 appname + integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) + !------------------------------------------------------------------- + ! + ! Local Variables + ! + integer :: ierr,mpi_ierr ! error codes + integer :: i ! for loops along dofs + integer :: tagtype, tagindex, numco, ent_type + real(r8), pointer :: data1(:) + real(r8), pointer :: data2(:) ! temporary + integer(kind=c_int), allocatable, target :: col_gids(:) + real(r8), allocatable, target :: moab_vert_coords(:) + real(r8) :: latv, lonv + !------------------------------------------------------------------- + + appname="ATM_PHYS_SCREAM"//C_NULL_CHAR + ATM_PHYS = 200 + ATM_ID ! + ierr = iMOAB_RegisterApplication(appname, mpicom_atm, ATM_PHYS, mphaid) + if (ierr > 0 ) then + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + if (my_task == master_task) then + print *, " moab_atm_phys_scream:: register MOAB app:", trim(appname), " mphaid=", mphaid + endif + ! find global ids + num_local_cols = scream_get_num_local_cols() + num_global_cols = scream_get_num_global_cols() + + allocate(col_gids(num_local_cols)) + + call scream_get_local_cols_gids(c_loc(col_gids)) + + ! Fill in correct values for domain components + allocate(moab_vert_coords(num_local_cols*3)) + allocate( data1(num_local_cols), data2(num_local_cols) ) + call scream_get_cols_latlon(c_loc(data1),c_loc(data2)) + do i=1,num_local_cols + latv = data1(i) * SHR_CONST_PI/180 + lonv = data2(i) * SHR_CONST_PI/180 + moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*i )=SIN(latv) + enddo + ierr = iMOAB_CreateVertices(mphaid, num_local_cols*3, 3, moab_vert_coords) + if (ierr > 0 ) then + print *, "Error: fail to create MOAB vertices in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + tagtype = 0 ! dense, integer + numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define GLOBAL_ID tag in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ent_type = 0 ! vertex type + ierr = iMOAB_SetIntTagStorage(mphaid, tagname, num_local_cols , ent_type, col_gids) + if (ierr > 0 ) then + print *, "Error: fail to set GLOBAL_ID tag in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ierr = iMOAB_ResolveSharedEntities( mphaid, num_local_cols, col_gids) + if (ierr > 0 ) then + print *, "Error: fail to resolve shared ents in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + tagname=trim(seq_flds_dom_fields)//C_NULL_CHAR ! mask is double too lat:lon:hgt:area:aream:mask:frac + tagtype = 1 ! dense, double + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, 'Error: fail to create tags from seq_flds_dom_fields ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + call scream_get_cols_area(c_loc(data1)) + tagname = 'area'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set area ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + ! Mask and frac are both exactly 1 + data1 = 1.0 + tagname = 'mask'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set mask ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + tagname = 'frac'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set frac ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + !call mct_gGrid_importRAttr(dom_atm,"mask",data1,lsize) + !call mct_gGrid_importRAttr(dom_atm,"frac",data1,lsize) + + ! Aream is computed by mct, so give invalid initial value + data1 = -9999.0_R8 +#ifdef MOABDEBUG + outfile = 'AtmPhys.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) + if (ierr > 0 ) then + print *, "Error: fail to write PhysAtm mesh " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif +#endif + ! define fields seq_flds_a2x_fields + tagtype = 1 ! dense, double + numco = 1 ! one value per vertex / entity + tagname = trim(seq_flds_a2x_fields)//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define seq_flds_a2x_fields " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ! make sure this is defined too; it could have the same fields, but in different order, or really different + ! fields; need to make sure we have them + tagname = trim(seq_flds_x2a_fields)//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define seq_flds_x2a_fields " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + deallocate(col_gids) + deallocate(data1) + deallocate(data2) + end subroutine moab_atm_phys_scream +#endif + end module atm_comp_mct From b745f6b53ab1029503559dc218491620114e6042 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 11 Jul 2024 14:28:25 -0500 Subject: [PATCH 536/904] update mesh info is needed also, set aream to -9999.0, like mct it will be reset later in areacor --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index b562dbce99db..0cd903837197 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -457,6 +457,11 @@ subroutine moab_atm_phys_scream() print *, "Error: fail to resolve shared ents in phys atm model" call mpi_abort(mpicom_atm,ierr,mpi_ierr) endif + ierr = iMOAB_UpdateMeshInfo(mphaid) + if (ierr > 0 ) then + print *, "Error: fail to update mesh info in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif tagname=trim(seq_flds_dom_fields)//C_NULL_CHAR ! mask is double too lat:lon:hgt:area:aream:mask:frac tagtype = 1 ! dense, double ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) @@ -493,6 +498,13 @@ subroutine moab_atm_phys_scream() ! Aream is computed by mct, so give invalid initial value data1 = -9999.0_R8 + tagname = 'aream'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set aream ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + #ifdef MOABDEBUG outfile = 'AtmPhys.h5m'//C_NULL_CHAR wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR From b0446efa38313351e009d5b2c41f1fb3fd2c5ead Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 27 Jul 2024 15:01:45 -0500 Subject: [PATCH 537/904] initial fields to and from moab data structures modify the SCDataManager to add moab 2d fields duplicate just 2d fields, for devive and host, for import and export allocate a2x_am and x2a_am, and use extensively #ifdef HAVE_MOAB --- .../atmosphere_surface_coupling_exporter.cpp | 31 +++++ .../atmosphere_surface_coupling_exporter.hpp | 7 ++ .../atmosphere_surface_coupling_importer.cpp | 34 ++++++ .../atmosphere_surface_coupling_importer.hpp | 20 ++++ .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 111 ++++++++++++++++-- .../mct_coupling/scream_cxx_f90_interface.cpp | 18 ++- .../eamxx/src/mct_coupling/scream_f2c_mod.F90 | 15 ++- .../src/share/atm_process/SCDataManager.hpp | 15 ++- 8 files changed, 238 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp index 7a0253ec908a..9044df7e76cc 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp @@ -137,6 +137,13 @@ void SurfaceCouplingExporter::setup_surface_coupling_data(const SCDataManager &s m_num_cols, m_num_cpl_exports); m_cpl_exports_view_d = Kokkos::create_mirror_view(DefaultDevice(), m_cpl_exports_view_h); +#ifdef HAVE_MOAB + // The export data is of size num_cpl_exports,ncols. All other data is of size num_scream_exports + m_moab_cpl_exports_view_h = decltype(m_moab_cpl_exports_view_h) (sc_data_manager.get_field_data_moab_ptr(), + m_num_cpl_exports, m_num_cols); + m_moab_cpl_exports_view_d = Kokkos::create_mirror_view(DefaultDevice(), m_moab_cpl_exports_view_h); +#endif + m_export_field_names = new name_t[m_num_scream_exports]; std::memcpy(m_export_field_names, sc_data_manager.get_field_name_ptr(), m_num_scream_exports*32*sizeof(char)); @@ -536,6 +543,12 @@ void SurfaceCouplingExporter::do_export_to_cpl(const bool called_during_initiali // Any field not exported by scream, or not exported // during initialization, is set to 0.0 Kokkos::deep_copy(m_cpl_exports_view_d, 0.0); +#ifdef HAVE_MOAB + // Any field not exported by scream, or not exported + // during initialization, is set to 0.0 + Kokkos::deep_copy(m_moab_cpl_exports_view_d, 0.0); + const auto moab_cpl_exports_view_d = m_moab_cpl_exports_view_d; +#endif const auto cpl_exports_view_d = m_cpl_exports_view_d; const int num_exports = m_num_scream_exports; const int num_cols = m_num_cols; @@ -554,9 +567,27 @@ void SurfaceCouplingExporter::do_export_to_cpl(const bool called_during_initiali cpl_exports_view_d(icol,info.cpl_indx) = info.constant_multiple*info.data[offset]; } }); +#ifdef HAVE_MOAB + Kokkos::parallel_for(export_policy, KOKKOS_LAMBDA(const int& i) { + const int ifield = i / num_cols; + const int icol = i % num_cols; + const auto& info = col_info(ifield); + const auto offset = icol*info.col_stride + info.col_offset; + // if this is during initialization, check whether or not the field should be exported + bool do_export = (not called_during_initialization || info.transfer_during_initialization); + if (do_export) { + moab_cpl_exports_view_d(info.cpl_indx, icol) = info.constant_multiple*info.data[offset]; + } + }); +#endif // Deep copy fields from device to cpl host array Kokkos::deep_copy(m_cpl_exports_view_h,m_cpl_exports_view_d); +#ifdef HAVE_MOAB + // Deep copy fields from device to cpl host array + Kokkos::deep_copy(m_moab_cpl_exports_view_h,m_moab_cpl_exports_view_d); +#endif + } // ========================================================================================= void SurfaceCouplingExporter::finalize_impl() diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp index d8ccf5862f3c..ca7772fdf1be 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp @@ -141,6 +141,13 @@ class SurfaceCouplingExporter : public AtmosphereProcess view_2d m_cpl_exports_view_d; uview_2d m_cpl_exports_view_h; +#ifdef HAVE_MOAB + // Views storing a 2d array with dims (num_fields, num_cols) for moab cpl export data. + // The field cols strides faster, since that's what moab does (so we can "view" the + // pointer to the whole a2x_am(:,:) array from Fortran) + view_2d m_moab_cpl_exports_view_d; + uview_2d m_moab_cpl_exports_view_h; +#endif // Array storing the field names for exports name_t* m_export_field_names; std::vector m_export_field_names_vector; diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 7573993a3189..7686319ad3d8 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -74,6 +74,13 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptrget_params().get("iop_srf_prop")) { // Overwrite imports with data from IOP file @@ -232,6 +263,9 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in const auto& info_d = col_info_d(ifield); const auto offset = icol*info_d.col_stride + info_d.col_offset; info_d.data[offset] = col_val; +#ifdef HAVE_MOAB + // TODO +#endif }); } } diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp index 3a34a8b2951e..2bb0aa3ad4db 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp @@ -55,12 +55,24 @@ class SurfaceCouplingImporter : public AtmosphereProcess // which are not needed during initialization. void do_import(const bool called_during_initialization=false); +#ifdef HAVE_MOAB + // Function which performes the import from moab coupler to scream fields. + // If calling in initialize_impl(), set + // called_during_initialization=true to avoid importing to fields + // which are not needed during initialization. + void do_import_from_moab(const bool called_during_initialization=false); +#endif + // Take and store data from SCDataManager void setup_surface_coupling_data(const SCDataManager &sc_data_manager); // Overwrite imports for IOP cases with IOP file surface data void overwrite_iop_imports (const bool called_during_initialization); +#ifdef HAVE_MOAB + // Overwrite imports for IOP cases with IOP file surface data + void overwrite_iop_imports_from_moab (const bool called_during_initialization); +#endif protected: // The three main overrides for the subcomponent @@ -83,6 +95,14 @@ class SurfaceCouplingImporter : public AtmosphereProcess view_2d m_cpl_imports_view_d; uview_2d m_cpl_imports_view_h; +#ifdef HAVE_MOAB + // Views storing a 2d array with dims (num_fields,num_cols) for import data. + // The colums index strides faster, since that's what moab does (so we can "view" the + // pointer to the whole x2a_am(:,:) array from Fortran) + view_2d m_moab_cpl_imports_view_d; + uview_2d m_moab_cpl_imports_view_h; +#endif + // Array storing the field names for imports name_t* m_import_field_names; diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 0cd903837197..54f0338f0c66 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -20,7 +20,6 @@ module atm_comp_mct #ifdef HAVE_MOAB use seq_comm_mct , only: mphaid ! atm physics grid id in MOAB, on atm pes use iso_c_binding - use seq_comm_mct, only : num_moab_exports #ifdef MOABCOMP use seq_comm_mct, only: seq_comm_compare_mb_mct #endif @@ -54,6 +53,17 @@ module atm_comp_mct integer :: atm_log_unit + +#ifdef HAVE_MOAB + ! to store all fields to be set in moab + integer , private :: mblsize, totalmbls, nsend, totalmbls_r, nrecv + real(r8) , allocatable, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes + real(r8) , allocatable, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes +#ifdef MOABCOMP + integer :: mpicom_atm_moab ! used just for mpi-reducing the difference between moab tags and mct avs + integer :: rank2 +#endif +#endif !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -89,7 +99,13 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use seq_comm_mct, only: seq_comm_inst, seq_comm_name, seq_comm_suffix use shr_file_mod, only: shr_file_getunit, shr_file_setIO use shr_sys_mod, only: shr_sys_abort - +#ifdef HAVE_MOAB + use shr_kind_mod , only: cxx=>shr_kind_cxx + use iMOAB, only: iMOAB_SetDoubleTagStorage +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#endif ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) , intent(inout) :: EClock type(seq_cdata) , intent(inout) :: cdata @@ -109,7 +125,13 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) type(c_ptr) :: x2a_ptr, a2x_ptr character(len=256) :: atm_log_fname ! name of ATM log file character(CL) :: calendar ! calendar string - +#ifdef HAVE_MOAB + integer :: ent_type + character(CXX) :: tagname ! will store all seq_flds_a2x_fields +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif ! TODO: read this from the namelist? character(len=256) :: yaml_fname = "./data/scream_input.yaml" character(kind=c_char,len=256), target :: yaml_fname_c, atm_log_fname_c @@ -207,12 +229,33 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Init surface coupling stuff in the AD call scream_set_cpl_indices (x2a, a2x) +#ifdef HAVE_MOAB + mblsize = lsize + nsend = mct_avect_nRattr(a2x) + totalmbls = mblsize * nsend ! size of the double array + allocate (a2x_am(mblsize, nsend) ) + a2x_am = 0. ! initialize everything with 0 + + nrecv = mct_avect_nRattr(x2a) + totalmbls_r = mblsize * nrecv ! size of the double array used to receive + allocate (x2a_am(mblsize, nrecv) ) ! these will be received by moab tags, then used to set cam in surf data + x2a_am = 0. ! initialize everything with 0 +#endif + call scream_setup_surface_coupling (c_loc(import_field_names), c_loc(import_cpl_indices), & - c_loc(x2a%rAttr), c_loc(import_vector_components), & + c_loc(x2a%rAttr), & +#ifdef HAVE_MOAB + c_loc(x2a_am), & +#endif + c_loc(import_vector_components), & c_loc(import_constant_multiple), c_loc(do_import_during_init), & num_cpl_imports, num_scream_imports, import_field_size, & c_loc(export_field_names), c_loc(export_cpl_indices), & - c_loc(a2x%rAttr), c_loc(export_vector_components), & + c_loc(a2x%rAttr), & +#ifdef HAVE_MOAB + c_loc(a2x_am), & +#endif + c_loc(export_vector_components), & c_loc(export_constant_multiple), c_loc(do_export_during_init), & num_cpl_exports, num_scream_exports, export_field_size) @@ -220,6 +263,24 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(trim(username),username_c) call string_f2c(trim(hostname),hostname_c) call scream_init_atm (caseid_c,hostname_c,username_c) +#ifdef HAVE_MOAB + ! data should be set now inside moab from import and export fields + ! do we import and export or just export at init stage ? + tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, totalmbls , ent_type, a2x_am ) + if (ierr /= 0) then + print *,'moab error in setting data in a2x_am ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + end if +#ifdef MOABDEBUG + outfile = 'AtmPhys_init.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) + if (ierr > 0 ) & + call mpi_abort(mpicom_atm,ierr,mpi_ierr) +#endif +#endif end subroutine atm_init_mct @@ -227,6 +288,14 @@ end subroutine atm_init_mct subroutine atm_run_mct(EClock, cdata, x2a, a2x) use iso_c_binding, only: c_double use scream_f2c_mod, only: scream_run +#ifdef HAVE_MOAB + use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields + use shr_kind_mod , only: cxx=>shr_kind_cxx + use iMOAB, only: iMOAB_SetDoubleTagStorage, iMOAB_GetDoubleTagStorage +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#endif ! !INPUT/OUTPUT PARAMETERS: @@ -241,7 +310,14 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) type(mct_gGrid) , pointer :: ggrid real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream - +#ifdef HAVE_MOAB + integer :: ent_type + character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields +#ifdef MOABDEBUG + integer :: cur_atm_stepno, ierr + character*100 :: outfile, wopts, lnum +#endif +#endif !------------------------------------------------------------------------------- call seq_cdata_setptrs(cdata, & @@ -251,10 +327,29 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) ! Get time step info call seq_timemgr_EClockGetData (EClock, next_cday=nextsw_cday, dtime=dt_scream) - +#ifdef HAVE_MOAB + ! import data from moab data structures + tagname=trim(seq_flds_x2a_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_GetDoubleTagStorage ( mphaid, tagname, totalmbls_r , ent_type, x2a_am ) + +#endif ! Run scream call scream_run( dt_scream ) +#ifdef HAVE_MOAB + ! export data to moab data structures + tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, totalmbls , ent_type, a2x_am ) +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_atm_stepno ) + write(lnum,"(I0.2)")cur_atm_stepno + outfile = 'AtmPhys_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) +#endif +#endif ! Set time of next radiadtion computation call seq_infodata_PutData(infodata, nextsw_cday=nextsw_cday) @@ -386,7 +481,9 @@ subroutine moab_atm_phys_scream() ! Local variables ! integer(kind=c_int) :: num_local_cols, num_global_cols +#ifdef MOABDEBUG character*100 outfile, wopts +#endif character(CXX) :: tagname ! will store all seq_flds_a2x_fields character*32 appname integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index e9129e6fcea3..4b630614c941 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -165,11 +165,19 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, } void scream_setup_surface_coupling (const char*& import_field_names, int*& import_cpl_indices, - double*& x2a_ptr, int*& import_vector_components, + double*& x2a_ptr, +#ifdef HAVE_MOAB + double*& x2a_moab_ptr, +#endif + int*& import_vector_components, double*& import_constant_multiple, bool*& do_import_during_init, const int& num_cpl_imports, const int& num_scream_imports, const int& import_field_size, char*& export_field_names, int*& export_cpl_indices, - double*& a2x_ptr, int*& export_vector_components, + double*& a2x_ptr, +#ifdef HAVE_MOAB + double*& a2x_moab_ptr, +#endif + int*& export_vector_components, double*& export_constant_multiple, bool*& do_export_during_init, const int& num_cpl_exports, const int& num_scream_exports, const int& export_field_size) { @@ -197,10 +205,16 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Import, num_cpl_imports, num_scream_imports, import_field_size, x2a_ptr, +#ifdef HAVE_MOAB + x2a_moab_ptr, +#endif names_in[0], import_cpl_indices, import_vector_components, import_constant_multiple, do_import_during_init); ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Export, num_cpl_exports, num_scream_exports, export_field_size, a2x_ptr, +#ifdef HAVE_MOAB + x2a_moab_ptr, +#endif names_out[0], export_cpl_indices, export_vector_components, export_constant_multiple, do_export_during_init); }); diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index f7a0bc9c3794..5e09def98df0 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -47,11 +47,19 @@ end subroutine scream_get_cols_area ! responsible to handle import/export operation from/into the component ! coupler surface fluxes/state structures subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices, & - x2a_ptr, import_vector_components, & + x2a_ptr, & +#ifdef HAVE_MOAB + x2a_moab_ptr, & +#endif + import_vector_components, & import_constant_multiple, do_import_during_init, & num_cpl_imports, num_scream_imports, import_field_size, & export_field_names, export_cpl_indices, & - a2x_ptr, export_vector_components, & + a2x_ptr, & +#ifdef HAVE_MOAB + a2x_moab_ptr, & +#endif + export_vector_components, & export_constant_multiple, do_export_during_init, & num_cpl_exports, num_scream_exports, export_field_size) bind(c) use iso_c_binding, only: c_ptr, c_int @@ -64,6 +72,9 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices type(c_ptr), intent(in) :: export_field_names, export_cpl_indices, & a2x_ptr, export_vector_components, & export_constant_multiple, do_export_during_init +#ifdef HAVE_MOAB + type(c_ptr), intent(in) :: x2a_moab_ptr, a2x_moab_ptr +#endif integer(kind=c_int), intent(in) :: num_cpl_imports, num_scream_imports, & num_cpl_exports, num_scream_exports integer(kind=c_int), intent(in) :: import_field_size, export_field_size diff --git a/components/eamxx/src/share/atm_process/SCDataManager.hpp b/components/eamxx/src/share/atm_process/SCDataManager.hpp index 02af15423e62..51384750d435 100644 --- a/components/eamxx/src/share/atm_process/SCDataManager.hpp +++ b/components/eamxx/src/share/atm_process/SCDataManager.hpp @@ -22,7 +22,11 @@ struct SCDataManager { ~SCDataManager() = default; void setup_internals (const int num_cpl_fields, const int num_scream_fields, const int field_size, - Real* field_data_ptr, char* field_names, int* field_cpl_indices_ptr, + Real* field_data_ptr, +#ifdef HAVE_MOAB + Real* field_data_moab_ptr, +#endif + char* field_names, int* field_cpl_indices_ptr, int* field_vector_components_ptr, Real* field_constant_multiple_ptr, bool* transfer_during_init_ptr) { @@ -38,6 +42,9 @@ struct SCDataManager { EKAT_ASSERT_MSG(field_constant_multiple_ptr !=nullptr, "Error! Ptr for constant multiple is null."); EKAT_ASSERT_MSG(transfer_during_init_ptr !=nullptr, "Error! Ptr for initial transfer boolean is null."); m_field_data = decltype(m_field_data) (field_data_ptr, m_field_size, m_num_cpl_fields); +#ifdef HAVE_MOAB + m_field_data_moab = decltype(m_field_data_moab) (field_data_moab_ptr, m_num_cpl_fields, m_field_size); +#endif m_field_cpl_indices = decltype(m_field_cpl_indices) (field_cpl_indices_ptr, m_num_scream_fields); m_field_vector_components = decltype(m_field_vector_components) (field_vector_components_ptr, m_num_scream_fields); m_field_constant_multiple = decltype(m_field_constant_multiple) (field_constant_multiple_ptr, m_num_scream_fields); @@ -65,7 +72,11 @@ struct SCDataManager { Real* get_field_data_ptr () const { return m_field_data.data(); } - +#ifdef HAVE_MOAB + Real* get_field_data_moab_ptr () const { + return m_field_data_moab.data(); + } +#endif Real get_field_data_view_entry(const int i, const int f) { return m_field_data(i, f); } From d881e0486f082e89f8eb1392c2e0df7ea7cdd71c Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 27 Jul 2024 15:49:37 -0500 Subject: [PATCH 538/904] setup_internals moab changes --- components/eamxx/src/control/atmosphere_driver.cpp | 6 ++++++ components/eamxx/src/control/atmosphere_driver.hpp | 3 +++ components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 5 +++-- components/eamxx/src/share/atm_process/SCDataManager.hpp | 3 +++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index c3da1bacd3f0..211b75368981 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -300,6 +300,9 @@ void AtmosphereDriver::create_grids() void AtmosphereDriver::setup_surface_coupling_data_manager(SurfaceCouplingTransferType transfer_type, const int num_cpl_fields, const int num_scream_fields, const int field_size, Real* data_ptr, +#ifdef HAVE_MOAB + Real* data_ptr_moab, +#endif char* names_ptr, int* cpl_indices_ptr, int* vec_comps_ptr, Real* constant_multiple_ptr, bool* do_transfer_during_init_ptr) { @@ -318,6 +321,9 @@ void AtmosphereDriver::setup_surface_coupling_data_manager(SurfaceCouplingTransf } else EKAT_ERROR_MSG("Error! Unexpected SurfaceCouplingTransferType."); sc_data_mgr->setup_internals(num_cpl_fields, num_scream_fields, field_size, data_ptr, +#ifdef HAVE_MOAB + data_ptr_moab, +#endif names_ptr, cpl_indices_ptr, vec_comps_ptr, constant_multiple_ptr, do_transfer_during_init_ptr); } diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index d43d233f4e97..7e96225210db 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -87,6 +87,9 @@ class AtmosphereDriver void setup_surface_coupling_data_manager(SurfaceCouplingTransferType transfer_type, const int num_cpl_fields, const int num_scream_fields, const int field_size, Real* data_ptr, +#ifdef HAVE_MOAB + Real* data_ptr_moab, +#endif char* names_ptr, int* cpl_indices_ptr, int* vec_comps_ptr, Real* constant_multiple_ptr, bool* do_transfer_during_init_ptr); diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 54f0338f0c66..e9f3397f827b 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -57,8 +57,8 @@ module atm_comp_mct #ifdef HAVE_MOAB ! to store all fields to be set in moab integer , private :: mblsize, totalmbls, nsend, totalmbls_r, nrecv - real(r8) , allocatable, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes - real(r8) , allocatable, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes + real(r8) , allocatable, target, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes + real(r8) , allocatable, target, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes #ifdef MOABCOMP integer :: mpicom_atm_moab ! used just for mpi-reducing the difference between moab tags and mct avs integer :: rank2 @@ -101,6 +101,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use shr_sys_mod, only: shr_sys_abort #ifdef HAVE_MOAB use shr_kind_mod , only: cxx=>shr_kind_cxx + use mct_mod, only: mct_avect_nRattr use iMOAB, only: iMOAB_SetDoubleTagStorage #ifdef MOABDEBUG use iMOAB, only: iMOAB_WriteMesh diff --git a/components/eamxx/src/share/atm_process/SCDataManager.hpp b/components/eamxx/src/share/atm_process/SCDataManager.hpp index 51384750d435..2dcfd3c5016c 100644 --- a/components/eamxx/src/share/atm_process/SCDataManager.hpp +++ b/components/eamxx/src/share/atm_process/SCDataManager.hpp @@ -116,6 +116,9 @@ struct SCDataManager { int m_num_scream_fields; view_2d m_field_data; +#ifdef HAVE_MOAB + view_2d m_field_data_moab; +#endif name_t* m_field_names; view_1d m_field_cpl_indices; view_1d m_field_vector_components; From 9e87950a0604fb652b492c16de0f006592c52b7a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 27 Jul 2024 16:29:12 -0500 Subject: [PATCH 539/904] type, x2a instead of a2x --- components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 4b630614c941..52e0d692e0a5 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -213,7 +213,7 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Export, num_cpl_exports, num_scream_exports, export_field_size, a2x_ptr, #ifdef HAVE_MOAB - x2a_moab_ptr, + a2x_moab_ptr, #endif names_out[0], export_cpl_indices, export_vector_components, export_constant_multiple, do_export_during_init); From 8f0e75a01f65e75fdc0544fb3b030a1b85e79c1b Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 27 Jul 2024 21:48:55 -0500 Subject: [PATCH 540/904] remove unused moab import functions --- .../control/atmosphere_surface_coupling_importer.hpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp index 2bb0aa3ad4db..6cf04c4166d2 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp @@ -55,24 +55,12 @@ class SurfaceCouplingImporter : public AtmosphereProcess // which are not needed during initialization. void do_import(const bool called_during_initialization=false); -#ifdef HAVE_MOAB - // Function which performes the import from moab coupler to scream fields. - // If calling in initialize_impl(), set - // called_during_initialization=true to avoid importing to fields - // which are not needed during initialization. - void do_import_from_moab(const bool called_during_initialization=false); -#endif - // Take and store data from SCDataManager void setup_surface_coupling_data(const SCDataManager &sc_data_manager); // Overwrite imports for IOP cases with IOP file surface data void overwrite_iop_imports (const bool called_during_initialization); -#ifdef HAVE_MOAB - // Overwrite imports for IOP cases with IOP file surface data - void overwrite_iop_imports_from_moab (const bool called_during_initialization); -#endif protected: // The three main overrides for the subcomponent From ff21995779dc890737461850a1884d3caf01473f Mon Sep 17 00:00:00 2001 From: Vijay Mahadevan Date: Tue, 30 Jul 2024 14:12:19 -0500 Subject: [PATCH 541/904] Fix a build error when MOABDEBUG is undefined --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index e9f3397f827b..0a0a53007bde 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -312,10 +312,10 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream #ifdef HAVE_MOAB - integer :: ent_type + integer :: ent_type, ierr character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields #ifdef MOABDEBUG - integer :: cur_atm_stepno, ierr + integer :: cur_atm_stepno character*100 :: outfile, wopts, lnum #endif #endif From 07dc9f04134517fee735fbee158bf1862f26b75e Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 13 Aug 2024 16:49:40 -0500 Subject: [PATCH 542/904] add comparison logic (MOABCOMP) for atm imported fields --- .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 0a0a53007bde..96b243c7aabb 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -211,6 +211,10 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) #ifdef HAVE_MOAB call moab_atm_phys_scream() +#ifdef MOABCOMP + mpicom_atm_moab = mpicom_atm ! just to store it to be used later in MOABCOMP + rank2 = my_task ! again, just to use it for MOABCOMP output +#endif #endif ! Init import/export mct attribute vectors @@ -296,6 +300,16 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) #ifdef MOABDEBUG use iMOAB, only: iMOAB_WriteMesh #endif +#ifdef MOABCOMP + use mct_mod, only: mct_list, mct_string, mct_list_nitem, mct_string_toChar + use seq_comm_mct, only : num_moab_exports + + real(r8) :: difference + type(mct_list) :: temp_list + integer :: size_list, index_list, ent_type + type(mct_string) :: mctOStr ! + character(CXX) :: mct_field, modelStr +#endif #endif ! !INPUT/OUTPUT PARAMETERS: @@ -312,7 +326,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream #ifdef HAVE_MOAB - integer :: ent_type, ierr + integer :: ierr character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields #ifdef MOABDEBUG integer :: cur_atm_stepno @@ -329,6 +343,24 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) ! Get time step info call seq_timemgr_EClockGetData (EClock, next_cday=nextsw_cday, dtime=dt_scream) #ifdef HAVE_MOAB + +#ifdef MOABCOMP + ! loop over all fields in seq_flds_x2a_fields + call mct_list_init(temp_list ,seq_flds_x2a_fields) + size_list=mct_list_nitem (temp_list) + ent_type = 0 ! entity type is vertex for phys atm + if (rank2 .eq. 0) print *, num_moab_exports, trim(seq_flds_x2a_fields) + modelStr ='atm run' + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call seq_comm_compare_mb_mct(modelStr, mpicom_atm_moab, x2a, mct_field, mphaid, tagname, ent_type, difference) + enddo + call mct_list_clean(temp_list) +#endif + + ! import data from moab data structures tagname=trim(seq_flds_x2a_fields)//C_NULL_CHAR ent_type = 0 ! vertices, point cloud From 00214222b49a8d84f51a7b977c665af935944d39 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 13 Aug 2024 17:12:13 -0500 Subject: [PATCH 543/904] compile error --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 96b243c7aabb..b5513afb4cca 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -301,7 +301,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) use iMOAB, only: iMOAB_WriteMesh #endif #ifdef MOABCOMP - use mct_mod, only: mct_list, mct_string, mct_list_nitem, mct_string_toChar + use mct_mod use seq_comm_mct, only : num_moab_exports real(r8) :: difference From c1f6113b1a441b6e01aeca44effc6e2741578e78 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 21 Aug 2024 16:37:26 -0500 Subject: [PATCH 544/904] new api in imoab to apply weights apply manually the commit 3bea55f109ccdca502d26909fbb0a66054a16c4b from the E3SM repo, not available yet in scream repo, because it is not synchronized to master E3SM yet I mean, the last sync was after we did merged 3bea55f109ccd to E3SM master --- driver-moab/main/seq_map_mod.F90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/driver-moab/main/seq_map_mod.F90 b/driver-moab/main/seq_map_mod.F90 index be47473f444c..82e67b8b195e 100644 --- a/driver-moab/main/seq_map_mod.F90 +++ b/driver-moab/main/seq_map_mod.F90 @@ -355,6 +355,7 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, real(r8), dimension(:), allocatable :: wghts real(kind=r8) , allocatable :: targtags(:,:), targtags_ini(:,:) real(kind=r8) :: factor + integer :: filter_type ! used for caas projection #endif ! ! Local Variables @@ -649,7 +650,8 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, call shr_sys_flush(logunit) endif #endif - ierr = iMOAB_ApplyScalarProjectionWeights ( mapper%intx_mbid, mapper%weight_identifier, fldlist_moab, fldlist_moab) + filter_type = 0 ! no + ierr = iMOAB_ApplyScalarProjectionWeights ( mapper%intx_mbid, filter_type, mapper%weight_identifier, fldlist_moab, fldlist_moab) if (ierr .ne. 0) then write(logunit,*) subname,' error in applying weights ' call shr_sys_abort(subname//' ERROR in applying weights') From 0b58cb0c955bafd28f906f1f1fb006153a0978a4 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Wed, 21 Aug 2024 23:18:20 +0000 Subject: [PATCH 545/904] Load pre-built kokkos module for GPU builds Also export ZES_ENABLE_SYSMAN=1 to avoid ext_intel_free_memory run-time errors. --- cime_config/machines/config_machines.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index c94a2bbcf7d2..b6ac815f58f6 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3411,6 +3411,9 @@ oneapi/eng-compiler/2024.04.15.002 + + kokkos/git.7ff87a5-omp-sycl + spack cmake gcc/10.3.0 @@ -3439,6 +3442,8 @@ /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh 131072 20 + $ENV{KOKKOS_ROOT} + 1 0 From 226ac0ff37064badf2f4237b111a93ecdf20ce74 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Wed, 21 Aug 2024 23:21:54 +0000 Subject: [PATCH 546/904] Fix a SYCL typo Also avoid a call to variadic printf in a SYCL kernel within an error diagnostic log. --- components/homme/src/share/compose/cedr_kokkos.hpp | 2 +- components/homme/src/share/compose/compose_slmm_siqk.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/components/homme/src/share/compose/cedr_kokkos.hpp b/components/homme/src/share/compose/cedr_kokkos.hpp index 42e423e2913a..758f4148a9a4 100644 --- a/components/homme/src/share/compose/cedr_kokkos.hpp +++ b/components/homme/src/share/compose/cedr_kokkos.hpp @@ -18,7 +18,7 @@ typedef Kokkos::Experimental::HIPSpace CedrGpuSpace; # endif # if defined KOKKOS_ENABLE_SYCL typedef Kokkos::Experimental::SYCL CedrGpuExeSpace; -typedef Kokkos::Experimental::SYCL> CedrGpuSpace; +typedef Kokkos::Experimental::SYCL CedrGpuSpace; # endif #endif diff --git a/components/homme/src/share/compose/compose_slmm_siqk.cpp b/components/homme/src/share/compose/compose_slmm_siqk.cpp index 628c023090cb..56564b0b8ca6 100644 --- a/components/homme/src/share/compose/compose_slmm_siqk.cpp +++ b/components/homme/src/share/compose/compose_slmm_siqk.cpp @@ -60,8 +60,10 @@ class TestSphereToRefKernel { // tol is on dx, not (a,b), so adjust slightly. if ( ! info.success || err > 1e4*tol_) { jinfo.nfails++; +#ifndef KOKKOS_ENABLE_SYCL printf("calc_sphere_to_ref ei %d i %d j %d: nits %d re %1.1e\n", ei, i, j, info.n_iterations, err); +#endif } jinfo.sum_nits += info.n_iterations; jinfo.max_nits = max(jinfo.max_nits, info.n_iterations); From d5c15c3ca7fa41b564b41b30aa79c146dafc399c Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Wed, 21 Aug 2024 23:30:34 +0000 Subject: [PATCH 547/904] Let SYCL kernels call a virtual function --- .../machines/cmake_macros/oneapi-ifxgpu_aurora.cmake | 1 + .../machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake index 47d513408c2c..16288ce4dee7 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake @@ -5,3 +5,4 @@ if (compile_threaded) endif() string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake new file mode 100644 index 000000000000..6835515164f0 --- /dev/null +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake @@ -0,0 +1,7 @@ + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core") +if (compile_threaded) + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") +endif() +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") From 251431904a6610f2a664b317c035f27ffd44a76c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 22 Aug 2024 15:02:09 -0600 Subject: [PATCH 548/904] Fix regex. Use lookahead to avoid restart files --- cime_config/config_archive.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/config_archive.xml b/cime_config/config_archive.xml index dee8d1992981..16320e8a93a4 100644 --- a/cime_config/config_archive.xml +++ b/cime_config/config_archive.xml @@ -29,7 +29,7 @@ r\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* rhist\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* - .*\.h\..*\.nc$ + .*\.h\.(?!rhist\.).*\.nc$ From c8679fd45253d4896c0f61d384c44e4c36b334b1 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 22 Aug 2024 20:08:23 -0700 Subject: [PATCH 549/904] options to build sk p3 and shoc separately --- components/eamxx/CMakeLists.txt | 6 ++-- .../eamxx/src/physics/p3/CMakeLists.txt | 6 ++-- .../src/physics/p3/impl/p3_main_impl.hpp | 2 +- .../eamxx/src/physics/p3/p3_functions.hpp | 20 +++++------ .../eamxx/src/physics/p3/tests/CMakeLists.txt | 2 +- .../eamxx/src/physics/shoc/CMakeLists.txt | 6 ++-- .../shoc/eamxx_shoc_process_interface.cpp | 10 +++--- .../shoc/eamxx_shoc_process_interface.hpp | 10 +++--- .../src/physics/shoc/impl/shoc_main_impl.hpp | 6 ++-- .../eamxx/src/physics/shoc/shoc_functions.hpp | 36 +++++++++---------- .../src/physics/shoc/shoc_functions_f90.cpp | 4 +-- .../src/physics/shoc/tests/CMakeLists.txt | 2 +- components/eamxx/src/scream_config.h.in | 4 ++- 13 files changed, 59 insertions(+), 55 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 398efa3bc2df..a60fc1e342fe 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -198,8 +198,10 @@ set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C inst set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) -set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels") -if (NOT SCREAM_SMALL_KERNELS) +set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for all components") +set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") +set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") +if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") endif() diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 1a01f607c630..fc0293cb35d0 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -74,14 +74,14 @@ set(P3_SK_SRCS ) set(P3_LIBS "p3") -if (SCREAM_SMALL_KERNELS) +if (SCREAM_P3_SMALL_KERNELS) add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) else() add_library(p3 ${P3_SRCS}) if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) - # Always build shoc_sk with SCREAM_SMALL_KERNELS on - target_compile_definitions(p3_sk PUBLIC "SCREAM_SMALL_KERNELS") + # Always build p3_sk with SCREAM_P3_SMALL_KERNELS on + target_compile_definitions(p3_sk PUBLIC "SCREAM_P3_SMALL_KERNELS") list(APPEND P3_LIBS "p3_sk") endif() endif() diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index ce5116cab2fa..c98a37d0557a 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -349,7 +349,7 @@ ::p3_main( Int nk, const physics::P3_Constants & p3constants) { -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_P3_SMALL_KERNELS return p3_main_internal(runtime_options, prognostic_state, diagnostic_inputs, diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 47bb4560c2e1..3a5218f5c001 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -420,7 +420,7 @@ struct Functions const uview_1d& nc_tend, Scalar& precip_liq_surf); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void cloud_sedimentation_disp( const uview_2d& qc_incld, const uview_2d& rho, @@ -468,7 +468,7 @@ struct Functions Scalar& precip_liq_surf, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void rain_sedimentation_disp( const uview_2d& rho, const uview_2d& inv_rho, @@ -518,7 +518,7 @@ struct Functions Scalar& precip_ice_surf, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void ice_sedimentation_disp( const uview_2d& rho, const uview_2d& inv_rho, @@ -562,7 +562,7 @@ struct Functions const uview_1d& bm, const uview_1d& th_atm); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void homogeneous_freezing_disp( const uview_2d& T_atm, const uview_2d& inv_exner, @@ -868,7 +868,7 @@ struct Functions const Int& timestepcount, const bool& force_abort, const Int& source_ind, const MemberType& team, const uview_1d& col_loc); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void check_values_disp(const uview_2d& qv, const uview_2d& temp, const Int& ktop, const Int& kbot, const Int& timestepcount, const bool& force_abort, const Int& source_ind, const uview_2d& col_loc, const Int& nj, const Int& nk); @@ -914,7 +914,7 @@ struct Functions Scalar& precip_ice_surf, view_1d_ptr_array& zero_init); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_init_disp( const Int& nj,const Int& nk_pack, const uview_2d& cld_frac_i, const uview_2d& cld_frac_l, @@ -988,7 +988,7 @@ struct Functions bool& is_hydromet_present, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part1_disp( const Int& nj, const Int& nk, @@ -1121,7 +1121,7 @@ struct Functions const Int& nk, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part2_disp( const Int& nj, const Int& nk, @@ -1246,7 +1246,7 @@ struct Functions const uview_1d& diag_eff_radius_qr, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part3_disp( const Int& nj, const Int& nk_pack, @@ -1319,7 +1319,7 @@ struct Functions Int nk, // number of vertical cells per column const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static Int p3_main_internal_disp( const P3Runtime& runtime_options, const P3PrognosticState& prognostic_state, diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 4e8901069e05..912c7cedac66 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -60,7 +60,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) LABELS "p3;physics;fail" ${FORCE_RUN_DIFF_FAILS}) - if (NOT SCREAM_SMALL_KERNELS) + if (NOT SCREAM_P3_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 891626b9d3df..4796b30abefd 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -104,14 +104,14 @@ if (HIP_BUILD) endif() set(SHOC_LIBS "shoc") -if (SCREAM_SMALL_KERNELS) +if (SCREAM_SHOC_SMALL_KERNELS) add_library(shoc ${SHOC_SRCS} ${SHOC_SK_SRCS}) else() add_library(shoc ${SHOC_SRCS}) if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(shoc_sk ${SHOC_SRCS} ${SHOC_SK_SRCS}) - # Always build shoc_sk with SCREAM_SMALL_KERNELS on - target_compile_definitions(shoc_sk PUBLIC "SCREAM_SMALL_KERNELS") + # Always build shoc_sk with SCREAM_SHOC_SMALL_KERNELS on + target_compile_definitions(shoc_sk PUBLIC "SCREAM_SHOC_SMALL_KERNELS") list(APPEND SHOC_LIBS "shoc_sk") endif() endif() diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 657f9487611c..32bcd9bde0e2 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -156,7 +156,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) using scalar_view_t = decltype(m_buffer.wpthlp_sfc); scalar_view_t* _1d_scalar_view_ptrs[Buffer::num_1d_scalar_ncol] = {&m_buffer.wpthlp_sfc, &m_buffer.wprtp_sfc, &m_buffer.upwp_sfc, &m_buffer.vpwp_sfc -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.se_b, &m_buffer.ke_b, &m_buffer.wv_b, &m_buffer.wl_b , &m_buffer.se_a, &m_buffer.ke_a, &m_buffer.wv_a, &m_buffer.wl_a , &m_buffer.kbfs, &m_buffer.ustar2, &m_buffer.wstar @@ -182,7 +182,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.z_mid, &m_buffer.rrho, &m_buffer.thv, &m_buffer.dz, &m_buffer.zt_grid, &m_buffer.wm_zt, &m_buffer.inv_exner, &m_buffer.thlm, &m_buffer.qw, &m_buffer.dse, &m_buffer.tke_copy, &m_buffer.qc_copy, &m_buffer.shoc_ql2, &m_buffer.shoc_mix, &m_buffer.isotropy, &m_buffer.w_sec, &m_buffer.wqls_sec, &m_buffer.brunt -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.tabs, &m_buffer.dz_zt #endif }; @@ -191,7 +191,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.z_int, &m_buffer.rrho_i, &m_buffer.zi_grid, &m_buffer.thl_sec, &m_buffer.qw_sec, &m_buffer.qwthl_sec, &m_buffer.wthl_sec, &m_buffer.wqw_sec, &m_buffer.wtke_sec, &m_buffer.uw_sec, &m_buffer.vw_sec, &m_buffer.w3 -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.dz_zi #endif }; @@ -353,7 +353,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) history_output.wqls_sec = m_buffer.wqls_sec; history_output.brunt = m_buffer.brunt; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS temporaries.se_b = m_buffer.se_b; temporaries.ke_b = m_buffer.ke_b; temporaries.wv_b = m_buffer.wv_b; @@ -485,7 +485,7 @@ void SHOCMacrophysics::run_impl (const double dt) // Run shoc main SHF::shoc_main(m_num_cols, m_num_levs, m_num_levs+1, m_npbl, m_nadv, m_num_tracers, dt, workspace_mgr,runtime_options,input,input_output,output,history_output -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , temporaries #endif ); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 469c31de7af5..4d76efd4558b 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -382,13 +382,13 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Structure for storing local variables initialized using the ATMBufferManager struct Buffer { -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS static constexpr int num_1d_scalar_ncol = 4; #else static constexpr int num_1d_scalar_ncol = 15; #endif static constexpr int num_1d_scalar_nlev = 1; -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS static constexpr int num_2d_vector_mid = 18; static constexpr int num_2d_vector_int = 12; #else @@ -401,7 +401,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_1d wprtp_sfc; uview_1d upwp_sfc; uview_1d vpwp_sfc; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS uview_1d se_b; uview_1d ke_b; uview_1d wv_b; @@ -448,7 +448,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_2d w3; uview_2d wqls_sec; uview_2d brunt; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS uview_2d rho_zt; uview_2d shoc_qv; uview_2d tabs; @@ -506,7 +506,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess SHF::SHOCOutput output; SHF::SHOCHistoryOutput history_output; SHF::SHOCRuntime runtime_options; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS SHF::SHOCTemporaries temporaries; #endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 11383e772454..a41868478b8f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -61,7 +61,7 @@ Int Functions::shoc_init( return host_view(0); } -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS template KOKKOS_FUNCTION void Functions::shoc_main_internal( @@ -586,7 +586,7 @@ Int Functions::shoc_main( const SHOCInputOutput& shoc_input_output, // Input/Output const SHOCOutput& shoc_output, // Output const SHOCHistoryOutput& shoc_history_output // Output (diagnostic) -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , const SHOCTemporaries& shoc_temporaries // Temporaries for small kernels #endif ) @@ -608,7 +608,7 @@ Int Functions::shoc_main( const Scalar Ckh = shoc_runtime.Ckh; const Scalar Ckm = shoc_runtime.Ckm; -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS using ExeSpace = typename KT::ExeSpace; // SHOC main loop diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index d1d547245b1f..8913ad53841b 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -203,7 +203,7 @@ struct Functions view_2d isotropy; }; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS struct SHOCTemporaries { SHOCTemporaries() = default; @@ -272,7 +272,7 @@ struct Functions const uview_1d& zt_grid, const Scalar& phis, const uview_1d& host_dse); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void update_host_dse_disp( const Int& shcol, const Int& nlev, @@ -324,7 +324,7 @@ struct Functions const MemberType& team, const Int& nlev, const uview_1d& tke); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void check_tke_disp( const Int& schol, const Int& nlev, @@ -363,7 +363,7 @@ struct Functions Scalar& ke_int, Scalar& wv_int, Scalar& wl_int); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_energy_integrals_disp( const Int& shcol, const Int& nlev, @@ -409,7 +409,7 @@ struct Functions const Workspace& workspace, const uview_1d& thl_sec, const uview_1d& qw_sec, const uview_1d& wthl_sec, const uview_1d& wqw_sec, const uview_1d& qwthl_sec, const uview_1d& uw_sec, const uview_1d& vw_sec, const uview_1d& wtke_sec, const uview_1d& w_sec); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void diag_second_shoc_moments_disp( const Int& shcol, const Int& nlev, const Int& nlevi, const Scalar& thl2tune, @@ -485,7 +485,7 @@ struct Functions Scalar& ustar, Scalar& kbfs, Scalar& obklen); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_diag_obklen_disp( const Int& shcol, const Int& nlev, @@ -522,7 +522,7 @@ struct Functions const Workspace& workspace, const uview_1d& brunt, const uview_1d& shoc_mix); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_length_disp( const Int& shcol, const Int& nlev, @@ -564,7 +564,7 @@ struct Functions const uview_1d& pint, const Workspace& workspace, const uview_1d& host_dse); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_energy_fixer_disp( const Int& shcol, const Int& nlev, @@ -597,7 +597,7 @@ struct Functions const uview_1d& qw, const uview_1d& ql, const uview_1d& qv); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void compute_shoc_vapor_disp( const Int& shcol, const Int& nlev, @@ -614,7 +614,7 @@ struct Functions const uview_1d& ql, const uview_1d& inv_exner, const uview_1d& tabs); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void compute_shoc_temperature_disp( const Int& shcol, const Int& nlev, @@ -650,7 +650,7 @@ struct Functions const uview_1d& tke, const uview_1d& u_wind, const uview_1d& v_wind); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void update_prognostics_implicit_disp( const Int& shcol, const Int& nlev, @@ -697,7 +697,7 @@ struct Functions const uview_1d& zi_grid, const Workspace& workspace, const uview_1d& w3); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void diag_third_shoc_moments_disp( const Int& shcol, const Int& nlev, @@ -754,7 +754,7 @@ struct Functions const uview_1d& wqls, const uview_1d& wthv_sec, const uview_1d& shoc_ql2); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_assumed_pdf_disp( const Int& shcol, const Int& nlev, @@ -983,7 +983,7 @@ struct Functions const Int& ntop_shoc, const view_1d& pref_mid); -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS KOKKOS_FUNCTION static void shoc_main_internal( const MemberType& team, @@ -1166,7 +1166,7 @@ struct Functions const SHOCInputOutput& shoc_input_output, // Input/Output const SHOCOutput& shoc_output, // Output const SHOCHistoryOutput& shoc_history_output // Output (diagnostic) -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , const SHOCTemporaries& shoc_temporaries // Temporaries for small kernels #endif ); @@ -1237,7 +1237,7 @@ struct Functions const uview_1d& cldn, const Workspace& workspace, Scalar& pblh); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void pblintd_disp( const Int& shcol, const Int& nlev, @@ -1269,7 +1269,7 @@ struct Functions const uview_1d& dz_zt, const uview_1d& dz_zi, const uview_1d& rho_zt); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_grid_disp( const Int& shcol, const Int& nlev, @@ -1327,7 +1327,7 @@ struct Functions const uview_1d& tk, const uview_1d& tkh, const uview_1d& isotropy); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_tke_disp( const Int& shcol, const Int& nlev, diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index b5a2a00d27c8..e7468062d3c5 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2889,7 +2889,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, const auto nlevi_packs = ekat::npack(nlevi); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS view_1d se_b ("se_b", shcol), ke_b ("ke_b", shcol), @@ -2923,7 +2923,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, const auto elapsed_microsec = SHF::shoc_main(shcol, nlev, nlevi, npbl, nadv, num_qtracers, dtime, workspace_mgr, shoc_runtime_options, shoc_input, shoc_input_output, shoc_output, shoc_history_output -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , shoc_temporaries #endif ); diff --git a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt index 87e55959c528..f61de7d76a4c 100644 --- a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt @@ -78,7 +78,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} ) - if (NOT SCREAM_SMALL_KERNELS) + if (NOT SCREAM_SHOC_SMALL_KERNELS) CreateUnitTest(shoc_sk_tests "${SHOC_TESTS_SRCS}" LIBS shoc_sk THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} diff --git a/components/eamxx/src/scream_config.h.in b/components/eamxx/src/scream_config.h.in index 77b1bf04ff0c..a7c0f911f2e3 100644 --- a/components/eamxx/src/scream_config.h.in +++ b/components/eamxx/src/scream_config.h.in @@ -45,8 +45,10 @@ // Whether or not to run RRTMGP debug checks #cmakedefine SCREAM_RRTMGP_DEBUG -// Whether monolithic kernels are on +// Whether or not small kernels are on #cmakedefine SCREAM_SMALL_KERNELS +#cmakedefine SCREAM_P3_SMALL_KERNELS +#cmakedefine SCREAM_SHOC_SMALL_KERNELS // The sha of the last commit #define EAMXX_GIT_VERSION "${EAMXX_GIT_VERSION}" From c5bf2a3b07524d385c9b7cfe9a58ab21a6500eef Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 22 Aug 2024 20:40:07 -0700 Subject: [PATCH 550/904] add tests for small kernels --- cime_config/tests.py | 1 + .../testmods_dirs/scream/small_kernels_p3/shell_commands | 2 ++ .../testmods_dirs/scream/small_kernels_shoc/shell_commands | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands diff --git a/cime_config/tests.py b/cime_config/tests.py index ca3dba503c42..c293e611077e 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -650,6 +650,7 @@ "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.small_kernels--scream-output-preset-5", ) }, diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands new file mode 100644 index 000000000000..ea0e647fdcd3 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands @@ -0,0 +1,2 @@ +./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_P3_SMALL_KERNELS On' +$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands new file mode 100644 index 000000000000..da0429652975 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands @@ -0,0 +1,2 @@ +./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SHOC_SMALL_KERNELS On' +$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b From 81f66f59170a1505f8c793fd6507d8d640075d64 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 23 Aug 2024 00:26:04 -0400 Subject: [PATCH 551/904] use kokkos norm min v instead of std stuff --- .../src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp index 70fb386055a1..5794e21456f7 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp @@ -45,7 +45,7 @@ void Functions::shoc_assumed_pdf_compute_s( std_s = ekat::sqrt(ekat::max(0, ekat::square(cthl)*thl2 + ekat::square(cqt)*qw2 - 2*cthl*sqrtthl2*cqt*sqrtqw2*r_qwthl)); - const auto std_s_not_small = std_s > std::sqrt(std::numeric_limits::min()) * 100; + const auto std_s_not_small = std_s > std::sqrt(Kokkos::Experimental::norm_min_v) * 100; s = qw1-qs*((1 + beta*qw1)/(1 + beta*qs)); if (std_s_not_small.any()) { C.set(std_s_not_small, sp(0.5)*(1 + ekat::erf(s/(sqrt2*std_s)))); From 4e70071923f864138608c8bb2ea3eb9ff6a386ac Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 23 Aug 2024 07:03:19 -0700 Subject: [PATCH 552/904] fix typo in test mod --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index c293e611077e..86ee99886eb1 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -650,7 +650,7 @@ "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", - "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.small_kernels--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", ) }, From f84cad1e7e817b248ebfe45f7270cb36db29b8f5 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 23 Aug 2024 10:15:38 -0400 Subject: [PATCH 553/904] improve the comment regarding PROJECT --- run_e3sm.template.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 86e360157598..49869f5c6f63 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -14,7 +14,8 @@ main() { # Machine and project readonly MACHINE=pm-cpu -# BEFORE RUNNING: CHANGE this to your project +# NOTE: The command below will return your default project on SLURM-based systems. +# If you are not using SLURM or need a different project, remove the command and set it directly readonly PROJECT="$(sacctmgr show user $USER format=DefaultAccount | tail -n1 | tr -d ' ')" # Simulation @@ -53,7 +54,7 @@ readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive # Define type of run # short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', # 'M_1x10_ndays', 'M2_1x10_ndays', 'M80_1x10_ndays', 'L_1x10_ndays' -# * can replace XS, M, etc with custom-XY with XY being the node count +# * can replace XS, M, etc. with custom-XY with XY being the node count # or 'production' for full simulation readonly run='XS_2x5_ndays' if [ "${run}" != "production" ]; then From 455ac56439af77dfd7450c8718ea8500b0afa3d0 Mon Sep 17 00:00:00 2001 From: Jayesh Krishna Date: Fri, 23 Aug 2024 09:23:18 -0500 Subject: [PATCH 554/904] PnetCDF support for HOMME+BFB on Anvil/Chrysalis Adding support for the PnetCDF library in HOMME for the BFB tests on Anvil and Chrysalis. These configuration files are used for standalone HOMME builds on Anvil and Chrysalis --- components/homme/cmake/machineFiles/anvil-bfb.cmake | 9 ++++++++- components/homme/cmake/machineFiles/chrysalis-bfb.cmake | 8 +++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/components/homme/cmake/machineFiles/anvil-bfb.cmake b/components/homme/cmake/machineFiles/anvil-bfb.cmake index d46774646eb1..ab9b1d3edc86 100644 --- a/components/homme/cmake/machineFiles/anvil-bfb.cmake +++ b/components/homme/cmake/machineFiles/anvil-bfb.cmake @@ -20,7 +20,14 @@ ENDIF() # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_BDW ON CACHE BOOL "") -SET (WITH_PNETCDF FALSE CACHE FILEPATH "") +EXECUTE_PROCESS(COMMAND pnetcdf-config --prefix + RESULT_VARIABLE PNCCONFIG_RESULT + OUTPUT_VARIABLE PNCCONFIG_OUTPUT + ERROR_VARIABLE PNCCONFIG_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE +) +SET (PnetCDF_PATH "${PNCCONFIG_OUTPUT}" CACHE STRING "") + # # anvil module system doesn't set environment variables, but will put # nc-config in our path. anvil seperates C and Fortran libraries, diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index ddeb7fa78111..b9f0d41a0606 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -20,7 +20,13 @@ SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") -SET (WITH_PNETCDF FALSE CACHE FILEPATH "") +EXECUTE_PROCESS(COMMAND pnetcdf-config --prefix + RESULT_VARIABLE PNCCONFIG_RESULT + OUTPUT_VARIABLE PNCCONFIG_OUTPUT + ERROR_VARIABLE PNCCONFIG_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE +) +SET (PnetCDF_PATH "${PNCCONFIG_OUTPUT}" CACHE STRING "") EXECUTE_PROCESS(COMMAND nf-config --prefix RESULT_VARIABLE NFCONFIG_RESULT From 5a60663a84c83adefcd3eab6c7613611ec3dd187 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 23 Aug 2024 08:55:47 -0600 Subject: [PATCH 555/904] fix commens and add file global attributes --- .../eam/tools/mkatmsrffile/mkatmsrffile.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/components/eam/tools/mkatmsrffile/mkatmsrffile.py b/components/eam/tools/mkatmsrffile/mkatmsrffile.py index 50999b6e945f..826544feb084 100644 --- a/components/eam/tools/mkatmsrffile/mkatmsrffile.py +++ b/components/eam/tools/mkatmsrffile/mkatmsrffile.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 #--------------------------------------------------------------------------------------------------- ''' -This is a replacement for the legacy gen_domain tool created for CESM. -Most legacy functionality is reproduced, with the notable exception of -the pole point latitude adjustment needed for the CESM FV grid. +This is a replacement for the legacy mkatmsrffile tool created for CESM. +All legacy functionality is reproduced. Created April, 2024 by Walter Hannah (LLNL) ''' @@ -22,12 +21,14 @@ #--------------------------------------------------------------------------------------------------- import datetime, os, numpy as np, xarray as xr, numba, itertools, subprocess as sp user, host = os.getenv('USER'), os.getenv('HOST') -source_code_meta = 'generate_domain_E3SM.py' +source_code_meta = 'mkatmsrffile.py' #--------------------------------------------------------------------------------------------------- class clr:END,RED,GREEN,MAGENTA,CYAN = '\033[0m','\033[31m','\033[32m','\033[35m','\033[36m' #--------------------------------------------------------------------------------------------------- usage = ''' python mkatmsrffile.py --map_file + --vegetation_file + --soil_water_file --dst_grid [--output-root ] [--date-stamp ] @@ -47,7 +48,6 @@ class clr:END,RED,GREEN,MAGENTA,CYAN = '\033[0m','\033[31m','\033[32m','\033[35m The following output file is created: atmsrf__.nc - ???? ''' from optparse import OptionParser @@ -71,11 +71,11 @@ class clr:END,RED,GREEN,MAGENTA,CYAN = '\033[0m','\033[31m','\033[32m','\033[35m parser.add_option('--output_root', dest='output_root', default='./', - help='Output path for domain files') + help='Output path for atmsrf files') parser.add_option('--date-stamp', dest='date_stamp', default=None, - help='Creation date stamp for domain files') + help='Creation date stamp for atmsrf files') (opts, args) = parser.parse_args() #--------------------------------------------------------------------------------------------------- def main(): @@ -252,6 +252,15 @@ def main(): ds_out['soilw'] = xr.DataArray(SOILW ,dims=['month','ncol']) ds_out['lon'] = ds_map.xc_b.rename({'n_b':'ncol'}) ds_out['lat'] = ds_map.yc_b.rename({'n_b':'ncol'}) + + ds.attrs['title'] = 'E3SM atmosphere surface data' + ds.attrs['source_code'] = source_code_meta + ds.attrs['hostname'] = str(host) + ds.attrs['history'] = f'created by {user}, '+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ds.attrs['input_map_file'] = opts.map_file + ds.attrs['input_vegetation_file'] = opts.vegetation_file + ds.attrs['input_soil_water_file'] = opts.soil_water_file + ds_out.to_netcdf(path=output_file,mode='w') #----------------------------------------------------------------------------- From 5d5961c4a098d413a29280b1ab355a4ebd31679f Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 24 Aug 2024 14:59:28 -0700 Subject: [PATCH 556/904] add more testmods and better comments --- cime_config/tests.py | 2 ++ components/eamxx/src/scream_config.h.in | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 86ee99886eb1..5534a84e3c21 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -651,6 +651,8 @@ "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_p3--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_shoc--scream-output-preset-5", ) }, diff --git a/components/eamxx/src/scream_config.h.in b/components/eamxx/src/scream_config.h.in index a7c0f911f2e3..2bd8c72f013b 100644 --- a/components/eamxx/src/scream_config.h.in +++ b/components/eamxx/src/scream_config.h.in @@ -45,9 +45,11 @@ // Whether or not to run RRTMGP debug checks #cmakedefine SCREAM_RRTMGP_DEBUG -// Whether or not small kernels are on +// Whether or not small kernels are used in ALL targets that support them #cmakedefine SCREAM_SMALL_KERNELS +// Whether or not small kernels are used in P3 #cmakedefine SCREAM_P3_SMALL_KERNELS +// Whether or not small kernels are used in SHOC #cmakedefine SCREAM_SHOC_SMALL_KERNELS // The sha of the last commit From b114661343405091a72fe5de722d68376a796a79 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Sun, 25 Aug 2024 23:40:48 -0500 Subject: [PATCH 557/904] Add alternative data stream control for SO2 species in ect_frc --- .../namelist_files/namelist_definition.xml | 18 + .../eam/src/chemistry/mozart/chemistry.F90 | 13 +- .../eam/src/chemistry/mozart/mo_chemini.F90 | 7 +- .../eam/src/chemistry/mozart/mo_extfrc.F90 | 31 +- .../eam/src/chemistry/mozart/tracer_data.F90 | 2884 +++++++++++++++++ 5 files changed, 2945 insertions(+), 8 deletions(-) create mode 100644 components/eam/src/chemistry/mozart/tracer_data.F90 diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index b3ecb5e9290a..8228c7d8d2e3 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -5138,6 +5138,17 @@ Format: YYYY Default: 0 + +The cycle year of the volcanic SO2 forcing (3D emissions) data. +Use this in place of ext_frc_cycle_yr if non-default, to allow different +speciciations volcanic emission and aerosols/precursors, which may be +invoked for single forcing simulation design. +if ext_frc_volc_type is 'CYCLICAL'. +Format: YYYY +Default: -1 + + Default: current model date @@ -5189,6 +5200,13 @@ Can be set to 'CYCLICAL', 'SERIAL', 'INTERP_MISSING_MONTHS', or 'FIXED'. Default: 'CYCLICAL' + +Type of time interpolation for fixed lower boundary data. See also ext_frc_volc_cycle_yr. +Can be set to 'NULL','CYCLICAL', 'SERIAL', 'INTERP_MISSING_MONTHS', or 'FIXED'. +Default: 'NULL' + + The cycle year of the fixed lower boundary data diff --git a/components/eam/src/chemistry/mozart/chemistry.F90 b/components/eam/src/chemistry/mozart/chemistry.F90 index f78ceda1dea7..3412e1b9bf92 100644 --- a/components/eam/src/chemistry/mozart/chemistry.F90 +++ b/components/eam/src/chemistry/mozart/chemistry.F90 @@ -106,6 +106,13 @@ module chemistry integer :: ext_frc_fixed_ymd = 0 integer :: ext_frc_fixed_tod = 0 + ! alternative type and cycle year for volcanic and other SO2 for controlling single forcing experiments + ! if NULL, same treatment as for the other species. + ! Possible in the future to modify further and apply to volcanic sector of SO2 alone + + character(len=24) :: ext_frc_volc_type = 'NULL' !'NULL'|'CYCLICAL'|'SERIAL'|'INTERP_MISSING_MONTHS' + integer :: ext_frc_volc_cycle_yr = -1 + real(r8) :: dms_emis_scale = 1._r8 ! fixed stratosphere @@ -515,7 +522,7 @@ subroutine chem_readnl(nlfile) srf_emis_type, srf_emis_cycle_yr, srf_emis_fixed_ymd, srf_emis_fixed_tod, srf_emis_specifier, & fstrat_file, fstrat_list, fstrat_efold_list, & ext_frc_specifier, ext_frc_type, ext_frc_cycle_yr, ext_frc_fixed_ymd, ext_frc_fixed_tod, & - dms_emis_scale + ext_frc_volc_type, ext_frc_volc_cycle_yr, dms_emis_scale namelist /chem_inparm/ chem_rad_passive @@ -689,6 +696,8 @@ subroutine chem_readnl(nlfile) call mpibcast (ext_frc_cycle_yr, 1, mpiint, 0, mpicom) call mpibcast (ext_frc_fixed_ymd, 1, mpiint, 0, mpicom) call mpibcast (ext_frc_fixed_tod, 1, mpiint, 0, mpicom) + call mpibcast (ext_frc_volc_type, len(ext_frc_volc_type), mpichar, 0, mpicom) + call mpibcast (ext_frc_volc_cycle_yr, 1, mpiint, 0, mpicom) call mpibcast (dms_emis_scale, 1, mpir8, 0, mpicom) @@ -1033,6 +1042,8 @@ subroutine chem_init(phys_state, pbuf2d, species_class) , ext_frc_cycle_yr & , ext_frc_fixed_ymd & , ext_frc_fixed_tod & + , ext_frc_volc_type & + , ext_frc_volc_cycle_yr & , xactive_prates & , exo_coldens_file & , tuv_xsect_file & diff --git a/components/eam/src/chemistry/mozart/mo_chemini.F90 b/components/eam/src/chemistry/mozart/mo_chemini.F90 index ff0eb0df15f1..564855d76dc5 100644 --- a/components/eam/src/chemistry/mozart/mo_chemini.F90 +++ b/components/eam/src/chemistry/mozart/mo_chemini.F90 @@ -43,6 +43,8 @@ subroutine chemini & , ext_frc_cycle_yr & , ext_frc_fixed_ymd & , ext_frc_fixed_tod & + , ext_frc_volc_type & + , ext_frc_volc_cycle_yr & , xactive_prates & , exo_coldens_file & , tuv_xsect_file & @@ -133,6 +135,8 @@ subroutine chemini & integer, intent(in) :: ext_frc_cycle_yr integer, intent(in) :: ext_frc_fixed_ymd integer, intent(in) :: ext_frc_fixed_tod + character(len=*), intent(in) :: ext_frc_volc_type + integer, intent(in) :: ext_frc_volc_cycle_yr character(len=*), intent(in) :: srf_emis_type integer, intent(in) :: srf_emis_cycle_yr integer, intent(in) :: srf_emis_fixed_ymd @@ -183,7 +187,8 @@ subroutine chemini & ! ... initialize external forcings module !----------------------------------------------------------------------- call setext_inti() - call extfrc_inti(ext_frc_specifier, ext_frc_type, ext_frc_cycle_yr, ext_frc_fixed_ymd, ext_frc_fixed_tod) + call extfrc_inti(ext_frc_specifier, ext_frc_type, ext_frc_cycle_yr, ext_frc_fixed_ymd, ext_frc_fixed_tod, & + ext_frc_volc_type, ext_frc_volc_cycle_yr) if (masterproc) write(iulog,*) 'chemini: after extfrc_inti on node ',iam !----------------------------------------------------------------------- diff --git a/components/eam/src/chemistry/mozart/mo_extfrc.F90 b/components/eam/src/chemistry/mozart/mo_extfrc.F90 index 23b3f480882e..d00e0d5673e9 100644 --- a/components/eam/src/chemistry/mozart/mo_extfrc.F90 +++ b/components/eam/src/chemistry/mozart/mo_extfrc.F90 @@ -46,7 +46,8 @@ module mo_extfrc contains - subroutine extfrc_inti( extfrc_specifier, extfrc_type, extfrc_cycle_yr, extfrc_fixed_ymd, extfrc_fixed_tod) + subroutine extfrc_inti( extfrc_specifier, extfrc_type, extfrc_cycle_yr, extfrc_fixed_ymd, extfrc_fixed_tod, & + extfrc_volc_type, extfrc_volc_cycle_yr) !----------------------------------------------------------------------- ! ... initialize the surface forcings @@ -71,6 +72,8 @@ subroutine extfrc_inti( extfrc_specifier, extfrc_type, extfrc_cycle_yr, extfrc_f integer , intent(in) :: extfrc_cycle_yr integer , intent(in) :: extfrc_fixed_ymd integer , intent(in) :: extfrc_fixed_tod + character(len=*), intent(in) :: extfrc_volc_type + integer , intent(in) :: extfrc_volc_cycle_yr !----------------------------------------------------------------------- ! ... local variables @@ -204,6 +207,14 @@ subroutine extfrc_inti( extfrc_specifier, extfrc_type, extfrc_cycle_yr, extfrc_f else if( extfrc_type == 'CYCLICAL' ) then write(iulog,*) ' cycle year = ',extfrc_cycle_yr end if + if (extfrc_volc_type /= 'NULL' ) then + write(iulog,*) ' ' + write(iulog,*) 'Volcanic SO2 type = ',extfrc_volc_type + if (extfrc_volc_type == 'CYCLICAL' ) then + write(iulog,*) ' ' + write(iulog,*) 'Volcanic SO2 cycle year = ',extfrc_volc_cycle_yr + end if + end if write(iulog,*) ' ' write(iulog,*) 'there are ',extfrc_cnt,' species with external forcing files' do m = 1,extfrc_cnt @@ -264,11 +275,19 @@ subroutine extfrc_inti( extfrc_specifier, extfrc_type, extfrc_cycle_yr, extfrc_f allocate(forcings(m)%file%in_pbuf(size(forcings(m)%sectors))) forcings(m)%file%in_pbuf(:) = .false. - call trcdata_init( forcings(m)%sectors, & - forcings(m)%filename, filelist, datapath, & - forcings(m)%fields, & - forcings(m)%file, & - rmv_file, extfrc_cycle_yr, extfrc_fixed_ymd, extfrc_fixed_tod, extfrc_type) + if (trim(forcings(m)%species) == 'SO2' .and. extfrc_volc_type /= 'NULL') then + call trcdata_init( forcings(m)%sectors, & + forcings(m)%filename, filelist, datapath, & + forcings(m)%fields, & + forcings(m)%file, rmv_file, extfrc_volc_cycle_yr, & + extfrc_fixed_ymd, extfrc_fixed_tod, extfrc_volc_type) + else + call trcdata_init( forcings(m)%sectors, & + forcings(m)%filename, filelist, datapath, & + forcings(m)%fields, & + forcings(m)%file, & + rmv_file, extfrc_cycle_yr, extfrc_fixed_ymd, extfrc_fixed_tod, extfrc_type) + end if enddo frcing_loop diff --git a/components/eam/src/chemistry/mozart/tracer_data.F90 b/components/eam/src/chemistry/mozart/tracer_data.F90 new file mode 100644 index 000000000000..62ccbcef8a25 --- /dev/null +++ b/components/eam/src/chemistry/mozart/tracer_data.F90 @@ -0,0 +1,2884 @@ +module tracer_data +!----------------------------------------------------------------------- +! module used to read (and interpolate) offline tracer data (sources and +! mixing ratios) +! Created by: Francis Vitt -- 2 May 2006 +! Modified by : Jim Edwards -- 10 March 2009 +! Modified by : Cheryl Craig and Chih-Chieh (Jack) Chen -- February 2010 +! Modified by : Jinbo Xie --March 2023 +! Added a new option in interpolate_trcdata to work on linoz +! inputdata. A new UCI interpolation that better conserves +! mass in implemented and added a new function. It is called +! when linoz data are used in interpolate_trcdata. +! +!----------------------------------------------------------------------- + + use perf_mod, only : t_startf, t_stopf + use shr_kind_mod, only : r8 => shr_kind_r8,r4 => shr_kind_r4, shr_kind_cl, SHR_KIND_CS + use time_manager, only : get_curr_date, get_step_size, get_curr_calday + use spmd_utils, only : masterproc + use ppgrid, only : pcols, pver, pverp, begchunk, endchunk + use cam_abortutils, only : endrun + use cam_logfile, only : iulog + + use physics_buffer, only : physics_buffer_desc, pbuf_get_field, pbuf_get_index + use time_manager, only : set_time_float_from_date, set_date_from_time_float + use pio, only : file_desc_t, var_desc_t, & + pio_seterrorhandling, pio_internal_error, pio_bcast_error, & + pio_setdebuglevel, & + pio_char, pio_noerr, & + pio_inq_dimid, pio_inq_varid, & + pio_def_dim, pio_def_var, & + pio_put_att, pio_put_var, & + pio_get_var, pio_get_att, pio_nowrite, pio_inq_dimlen, & + pio_inq_vardimid, pio_inq_dimlen, pio_closefile, & + pio_inquire_variable + + implicit none + + private ! all unless made public + save + + public :: trfld, input3d, input2d, trfile + public :: trcdata_init + public :: advance_trcdata + public :: get_fld_data + public :: put_fld_data + public :: get_fld_ndx + public :: write_trc_restart + public :: read_trc_restart + public :: init_trc_restart + public :: incr_filename + !added public for linoz new function diagnostic + public :: read_next_trcdata + public :: interpolate_trcdata + public :: get_model_time + !!PUBLIC MEMBERS + + type input3d + real(r8), dimension(:,:,:), pointer :: data => null() + endtype input3d + + type input2d + real(r8), dimension(:,:), pointer :: data => null() + endtype input2d + + type trfld + real(r8), dimension(:,:,:), pointer :: data => null() + type(input3d), dimension(4) :: input + character(len=32) :: srcnam + character(len=32) :: fldnam + character(len=32) :: units + type(var_desc_t) :: var_id + integer :: coords(4) ! LATDIM | LONDIM | LEVDIM | TIMDIM + integer :: order(4) ! LATDIM | LONDIM | LEVDIM | TIMDIM + logical :: srf_fld = .false. + integer :: pbuf_ndx = -1 + endtype trfld + + type trfile + type(input2d), dimension(4) :: ps_in + character(len=shr_kind_cl) :: pathname = ' ' + character(len=shr_kind_cl) :: curr_filename = ' ' + character(len=shr_kind_cl) :: next_filename = ' ' + type(file_desc_t) :: curr_fileid + type(file_desc_t) :: next_fileid + + type(var_desc_t), pointer :: currfnameid => null() ! pio restart file var id + type(var_desc_t), pointer :: nextfnameid => null() ! pio restart file var id + + character(len=shr_kind_cl) :: filenames_list = '' + real(r8) :: datatimem = -1.e36_r8 ! time of prv. values read in + real(r8) :: datatimep = -1.e36_r8 ! time of nxt. values read in + real(r8) :: datatimes(4) + integer :: interp_recs + real(r8), pointer, dimension(:) :: curr_data_times => null() + real(r8), pointer, dimension(:) :: next_data_times => null() + logical :: remove_trc_file = .false. ! delete file when finished with it + real(r8) :: offset_time + integer :: cyc_ndx_beg + integer :: cyc_ndx_end + integer :: cyc_yr = 0 + real(r8) :: one_yr = 0 + real(r8) :: curr_mod_time ! model time - calendar day + real(r8) :: next_mod_time ! model time - calendar day - next time step + integer :: nlon + integer :: nlat + integer :: nlev + integer :: nilev + integer :: ps_coords(3) ! LATDIM | LONDIM | TIMDIM + integer :: ps_order(3) ! LATDIM | LONDIM | TIMDIM + real(r8), pointer, dimension(:) :: lons => null() + real(r8), pointer, dimension(:) :: lats => null() + real(r8), pointer, dimension(:) :: levs => null() + real(r8), pointer, dimension(:) :: ilevs => null() + real(r8), pointer, dimension(:) :: hyam => null() + real(r8), pointer, dimension(:) :: hybm => null() + real(r8), pointer, dimension(:,:) :: ps => null() + real(r8), pointer, dimension(:) :: hyai => null() + real(r8), pointer, dimension(:) :: hybi => null() + real(r8), pointer, dimension(:,:) :: weight_x => null(), weight_y => null() + integer, pointer, dimension(:) :: count_x => null(), count_y => null() + integer, pointer, dimension(:,:) :: index_x => null(), index_y => null() + real(r8) :: p0 + type(var_desc_t) :: ps_id + logical, allocatable, dimension(:) :: in_pbuf + logical :: has_ps = .false. + logical :: zonal_ave = .false. + logical :: alt_data = .false. + logical :: cyclical = .false. + logical :: cyclical_list = .false. + logical :: weight_by_lat = .false. + logical :: conserve_column = .false. + logical :: fill_in_months = .false. + logical :: fixed = .false. + logical :: initialized = .false. + logical :: top_bndry = .false. + logical :: stepTime = .false. ! Do not interpolate in time, but use stepwise times + logical :: linoz_v3 = .false. !set for linoz_v3 interpolation only + logical :: linoz_v2 = .false. !set for linoz_v2 interpolation only + endtype trfile + + integer, public, parameter :: MAXTRCRS = 100 + + integer, parameter :: LONDIM = 1 + integer, parameter :: LATDIM = 2 + integer, parameter :: LEVDIM = 3 + integer, parameter :: TIMDIM = 4 + + integer, parameter :: PS_TIMDIM = 3 + + integer, parameter :: ZA_LATDIM = 1 + integer, parameter :: ZA_LEVDIM = 2 + integer, parameter :: ZA_TIMDIM = 3 + + integer, parameter :: nm=1 ! array index for previous (minus) data + integer, parameter :: np=2 ! array index for next (plus) data + + integer :: plon, plat + +contains + +!-------------------------------------------------------------------------- +!-------------------------------------------------------------------------- + subroutine trcdata_init( specifier, filename, filelist, datapath, flds, file, & + rmv_file, data_cycle_yr, data_fixed_ymd, data_fixed_tod, data_type ) + + use mo_constants, only : d2r + use cam_control_mod, only : nsrest + use dyn_grid, only : get_dyn_grid_parm + use string_utils, only : to_upper + use horizontal_interpolate, only : xy_interp_init +#if ( defined SPMD ) + use mpishorthand, only: mpicom, mpir8, mpiint +#endif + + implicit none + + character(len=*), intent(in) :: specifier(:) + character(len=*), intent(in) :: filename + character(len=*), intent(in) :: filelist + character(len=*), intent(in) :: datapath + type(trfld), dimension(:), pointer :: flds + type(trfile), intent(inout) :: file + logical, intent(in) :: rmv_file + integer, intent(in) :: data_cycle_yr + integer, intent(in) :: data_fixed_ymd + integer, intent(in) :: data_fixed_tod + character(len=*), intent(in) :: data_type + + integer :: f, mxnflds, astat + integer :: str_yr, str_mon, str_day + integer :: lon_dimid, lat_dimid, lev_dimid, tim_dimid, old_dimid + integer :: dimids(4), did + type(var_desc_t) :: varid + integer :: idx + integer :: ierr + integer :: errcode + real(r8) :: start_time, time1, time2 + integer :: i1,i2,j1,j2 + integer :: nvardims, vardimids(4) + + character(len=80) :: data_units + + call specify_fields( specifier, flds ) + + file%datatimep=-1.e36_r8 + file%datatimem=-1.e36_r8 + + mxnflds = 0 + if (associated(flds)) mxnflds = size( flds ) + + if (mxnflds < 1) return + + file%remove_trc_file = rmv_file + file%pathname = trim(datapath) + file%filenames_list = trim(filelist) + + file%fill_in_months = .false. + file%cyclical = .false. + file%cyclical_list = .false. + +! does not work when compiled with pathf90 +! select case ( to_upper(data_type) ) + select case ( data_type ) + case( 'FIXED' ) + file%fixed = .true. + case( 'INTERP_MISSING_MONTHS' ) + file%fill_in_months = .true. + case( 'CYCLICAL' ) + file%cyclical = .true. + file%cyc_yr = data_cycle_yr + case( 'CYCLICAL_LIST' ) + file%cyclical_list = .true. + file%cyc_yr = data_cycle_yr + case( 'SERIAL' ) + case default + write(iulog,*) 'trcdata_init: invalid data type: '//trim(data_type)//' file: '//trim(filename) + write(iulog,*) 'trcdata_init: valid data types: SERIAL | CYCLICAL | CYCLICAL_LIST | FIXED | INTERP_MISSING_MONTHS ' + call endrun('trcdata_init: invalid data type: '//trim(data_type)//' file: '//trim(filename)) + endselect + + if ( (.not.file%fixed) .and. ((data_fixed_ymd>0._r8) .or.(data_fixed_tod>0._r8))) then + call endrun('trcdata_init: Cannot specify data_fixed_ymd or data_fixed_tod if data type is not FIXED') + endif + if ( (.not.file%cyclical) .and. (data_cycle_yr>0._r8) ) then + call endrun('trcdata_init: Cannot specify data_cycle_yr if data type is not CYCLICAL') + endif + + if (masterproc) then + write(iulog,*) 'trcdata_init: data type: '//trim(data_type)//' file: '//trim(filename) + endif + + ! if there is no list of files (len_trim(file%filenames_list)<1) then + ! -> set curr_filename from namelist rather from restart data + if ( len_trim(file%curr_filename)<1 .or. len_trim(file%filenames_list)<1 .or. file%fixed ) then ! initial run + file%curr_filename = trim(filename) + + call get_model_time(file) + + if ( file%fixed ) then + str_yr = data_fixed_ymd/10000 + str_mon = (data_fixed_ymd - str_yr*10000)/100 + str_day = data_fixed_ymd - str_yr*10000 - str_mon*100 + call set_time_float_from_date( start_time, str_yr, str_mon, str_day, data_fixed_tod ) + file%offset_time = start_time - file%curr_mod_time + else + file%offset_time = 0 + endif + endif + + call set_time_float_from_date( time2, 2, 1, 1, 0 ) + call set_time_float_from_date( time1, 1, 1, 1, 0 ) + file%one_yr = time2-time1 + + if ( file%cyclical .or. file%cyclical_list) then + file%cyc_ndx_beg = -1 + file%cyc_ndx_end = -1 + if ( file%cyc_yr /= 0 ) then + call set_time_float_from_date( time1, file%cyc_yr , 1, 1, 0 ) + call set_time_float_from_date( time2, file%cyc_yr+1, 1, 1, 0 ) + file%one_yr = time2-time1 + endif + + call open_trc_datafile( file%curr_filename, file%pathname, file%curr_fileid, file%curr_data_times, & + cyc_ndx_beg=file%cyc_ndx_beg, cyc_ndx_end=file%cyc_ndx_end, cyc_yr=file%cyc_yr ) + else + call open_trc_datafile( file%curr_filename, file%pathname, file%curr_fileid, file%curr_data_times ) + file%curr_data_times = file%curr_data_times - file%offset_time + endif + + call pio_seterrorhandling(File%curr_fileid, PIO_BCAST_ERROR) + ierr = pio_inq_dimid( file%curr_fileid, 'lon', idx ) + call pio_seterrorhandling(File%curr_fileid, PIO_INTERNAL_ERROR) + + file%zonal_ave = (ierr/=PIO_NOERR) + + plon = get_dyn_grid_parm('plon') + plat = get_dyn_grid_parm('plat') + + if ( .not. file%zonal_ave ) then + + call get_dimension( file%curr_fileid, 'lon', file%nlon, dimid=old_dimid, data=file%lons ) + + file%lons = file%lons * d2r + + lon_dimid = old_dimid + + endif + + ierr = pio_inq_dimid( file%curr_fileid, 'time', old_dimid) + + ! Hack to work with weird netCDF and old gcc or NAG bug. + tim_dimid = old_dimid + + call get_dimension( file%curr_fileid, 'lat', file%nlat, dimid=old_dimid, data=file%lats ) + file%lats = file%lats * d2r + + lat_dimid = old_dimid + + allocate( file%ps(file%nlon,file%nlat), stat=astat ) + if( astat /= 0 ) then + write(iulog,*) 'trcdata_init: file%ps allocation error = ',astat + call endrun('trcdata_init: failed to allocate x array') + end if + + call pio_seterrorhandling(File%curr_fileid, PIO_BCAST_ERROR) + ierr = pio_inq_varid( file%curr_fileid, 'PS', file%ps_id ) + file%has_ps = (ierr==PIO_NOERR) + ierr = pio_inq_dimid( file%curr_fileid, 'altitude', idx ) + file%alt_data = (ierr==PIO_NOERR) + + call pio_seterrorhandling(File%curr_fileid, PIO_INTERNAL_ERROR) + + if ( file%has_ps) then + ierr = pio_inq_vardimid (file%curr_fileid, file%ps_id, dimids(1:3)) + do did = 1,3 + if ( dimids(did) == lon_dimid ) then + file%ps_coords(LONDIM) = did + file%ps_order(did) = LONDIM + else if ( dimids(did) == lat_dimid ) then + file%ps_coords(LATDIM) = did + file%ps_order(did) = LATDIM + else if ( dimids(did) == tim_dimid ) then + file%ps_coords(PS_TIMDIM) = did + file%ps_order(did) = PS_TIMDIM + endif + enddo + endif + + if (masterproc) then + write(iulog,*) 'trcdata_init: file%has_ps = ' , file%has_ps + endif ! masterproc + + if (file%alt_data) then + call get_dimension( file%curr_fileid, 'altitude_int', file%nilev, data=file%ilevs ) + call get_dimension( file%curr_fileid, 'altitude', file%nlev, dimid=old_dimid, data=file%levs ) + else + call get_dimension( file%curr_fileid, 'lev', file%nlev, dimid=old_dimid, data=file%levs ) + !!added for Linoz_v3 + call get_dimension( file%curr_fileid, 'ilev', file%nilev, data=file%ilevs ) + if (old_dimid>0) then + file%levs = file%levs*100._r8 ! mbar->pascals + endif + endif + + ! For some bizarre reason, netCDF with older gcc is keeping a pointer to the dimid, and overwriting it later! + ! Hackish workaround is to make a copy... + lev_dimid = old_dimid + + if (file%has_ps) then + + allocate( file%hyam(file%nlev), file%hybm(file%nlev), stat=astat ) + if( astat /= 0 ) then + write(iulog,*) 'trcdata_init: file%hyam,file%hybm allocation error = ',astat + call endrun('trcdata_init: failed to allocate file%hyam and file%hybm arrays') + end if + + allocate( file%hyai(file%nlev+1), file%hybi(file%nlev+1), stat=astat ) + if( astat /= 0 ) then + write(iulog,*) 'trcdata_init: file%hyai,file%hybi allocation error = ',astat + call endrun('trcdata_init: failed to allocate file%hyai and file%hybi arrays') + end if + + call pio_seterrorhandling(File%curr_fileid, PIO_BCAST_ERROR) + ierr = pio_inq_varid( file%curr_fileid, 'P0', varid) + call pio_seterrorhandling(File%curr_fileid, PIO_INTERNAL_ERROR) + + if ( ierr == PIO_NOERR ) then + ierr = pio_get_var( file%curr_fileid, varid, file%p0 ) + else + file%p0 = 100000._r8 + endif + ierr = pio_inq_varid( file%curr_fileid, 'hyam', varid ) + ierr = pio_get_var( file%curr_fileid, varid, file%hyam ) + ierr = pio_inq_varid( file%curr_fileid, 'hybm', varid ) + ierr = pio_get_var( file%curr_fileid, varid, file%hybm ) + if (file%conserve_column) then + ierr = pio_inq_varid( file%curr_fileid, 'hyai', varid ) + ierr = pio_get_var( file%curr_fileid, varid, file%hyai ) + ierr = pio_inq_varid( file%curr_fileid, 'hybi', varid ) + ierr = pio_get_var( file%curr_fileid, varid, file%hybi ) + endif + + allocate( file %ps (pcols,begchunk:endchunk), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate file%ps array; error = ',astat + call endrun + end if + allocate( file%ps_in(1)%data(pcols,begchunk:endchunk), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate file%ps_in(1)%data array; error = ',astat + call endrun + end if + allocate( file%ps_in(2)%data(pcols,begchunk:endchunk), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate file%ps_in(2)%data array; error = ',astat + call endrun + end if + if( file%fill_in_months ) then + allocate( file%ps_in(3)%data(pcols,begchunk:endchunk), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate file%ps_in(3)%data array; error = ',astat + call endrun + end if + allocate( file%ps_in(4)%data(pcols,begchunk:endchunk), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate file%ps_in(4)%data array; error = ',astat + call endrun + end if + end if + endif + + flds_loop: do f = 1,mxnflds + + ! initialize the coordinate values to -1, + ! to defend against fields that do not have certain dimension, e.g., zonal average surface fields + ! and check against it when assigning value to cnt3 for that dimension + + do did = 1,4 + flds(f)%coords(did) = -1 + end do + + ! get netcdf variable id for the field + ierr = pio_inq_varid( file%curr_fileid, flds(f)%srcnam, flds(f)%var_id ) + + ! determine if the field has a vertical dimension + if (lev_dimid>0) then + ierr = pio_inquire_variable( file%curr_fileid, flds(f)%var_id, ndims=nvardims ) + ierr = pio_inquire_variable( file%curr_fileid, flds(f)%var_id, dimids=vardimids(:nvardims) ) + flds(f)%srf_fld = .not.any(vardimids(:nvardims)==lev_dimid) + else + flds(f)%srf_fld = .true. + endif + + ! allocate memory only if not already in pbuf2d + + if ( .not. file%in_pbuf(f) ) then + if ( flds(f)%srf_fld .or. file%top_bndry ) then + allocate( flds(f) %data(pcols,1,begchunk:endchunk), stat=astat ) + else + allocate( flds(f) %data(pcols,pver,begchunk:endchunk), stat=astat ) + endif + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate flds(f)%data array; error = ',astat + call endrun + end if + else + flds(f)%pbuf_ndx = pbuf_get_index(flds(f)%fldnam,errcode) + endif + + if (flds(f)%srf_fld) then + allocate( flds(f)%input(1)%data(pcols,1,begchunk:endchunk), stat=astat ) + else + allocate( flds(f)%input(1)%data(pcols,file%nlev,begchunk:endchunk), stat=astat ) + endif + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate flds(f)%input(1)%data array; error = ',astat + call endrun + end if + if (flds(f)%srf_fld) then + allocate( flds(f)%input(2)%data(pcols,1,begchunk:endchunk), stat=astat ) + else + allocate( flds(f)%input(2)%data(pcols,file%nlev,begchunk:endchunk), stat=astat ) + endif + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate flds(f)%input(2)%data array; error = ',astat + call endrun + end if + + if( file%fill_in_months ) then + if (flds(f)%srf_fld) then + allocate( flds(f)%input(3)%data(pcols,1,begchunk:endchunk), stat=astat ) + else + allocate( flds(f)%input(3)%data(pcols,file%nlev,begchunk:endchunk), stat=astat ) + endif + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate flds(f)%input(3)%data array; error = ',astat + call endrun + end if + if (flds(f)%srf_fld) then + allocate( flds(f)%input(4)%data(pcols,1,begchunk:endchunk), stat=astat ) + else + allocate( flds(f)%input(4)%data(pcols,file%nlev,begchunk:endchunk), stat=astat ) + endif + if( astat/= 0 ) then + write(iulog,*) 'trcdata_init: failed to allocate flds(f)%input(4)%data array; error = ',astat + call endrun + end if + endif + + if ( file%zonal_ave ) then + ierr = pio_inq_vardimid (file%curr_fileid, flds(f)%var_id, dimids(1:3)) + do did = 1,3 + if ( dimids(did) == lat_dimid ) then + flds(f)%coords(ZA_LATDIM) = did + flds(f)%order(did) = ZA_LATDIM + else if ( dimids(did) == lev_dimid ) then + flds(f)%coords(ZA_LEVDIM) = did + flds(f)%order(did) = ZA_LEVDIM + else if ( dimids(did) == tim_dimid ) then + flds(f)%coords(ZA_TIMDIM) = did + flds(f)%order(did) = ZA_TIMDIM + endif + enddo + else if ( flds(f)%srf_fld ) then + ierr = pio_inq_vardimid (file%curr_fileid, flds(f)%var_id, dimids(1:3)) + do did = 1,3 + if ( dimids(did) == lon_dimid ) then + flds(f)%coords(LONDIM) = did + flds(f)%order(did) = LONDIM + else if ( dimids(did) == lat_dimid ) then + flds(f)%coords(LATDIM) = did + flds(f)%order(did) = LATDIM + else if ( dimids(did) == tim_dimid ) then + flds(f)%coords(PS_TIMDIM) = did + flds(f)%order(did) = PS_TIMDIM + endif + enddo + else + ierr = pio_inq_vardimid (file%curr_fileid, flds(f)%var_id, dimids) + do did = 1,4 + if ( dimids(did) == lon_dimid ) then + flds(f)%coords(LONDIM) = did + flds(f)%order(did) = LONDIM + else if ( dimids(did) == lat_dimid ) then + flds(f)%coords(LATDIM) = did + flds(f)%order(did) = LATDIM + else if ( dimids(did) == lev_dimid ) then + flds(f)%coords(LEVDIM) = did + flds(f)%order(did) = LEVDIM + else if ( dimids(did) == tim_dimid ) then + flds(f)%coords(TIMDIM) = did + flds(f)%order(did) = TIMDIM + endif + enddo + endif + + ierr = pio_get_att( file%curr_fileid, flds(f)%var_id, 'units', data_units) + data_units = trim(data_units) + flds(f)%units = data_units(1:32) + + enddo flds_loop + +! if weighting by latitude, compute weighting for horizontal interpolation + if( file%weight_by_lat ) then +! get dimensions of CAM resolution + plon = get_dyn_grid_parm('plon') + plat = get_dyn_grid_parm('plat') + +! weight_x & weight_y are weighting function for x & y interpolation + allocate(file%weight_x(plon,file%nlon)) + allocate(file%weight_y(plat,file%nlat)) + allocate(file%count_x(plon)) + allocate(file%count_y(plat)) + allocate(file%index_x(plon,file%nlon)) + allocate(file%index_y(plat,file%nlat)) + file%weight_x(:,:) = 0.0_r8 + file%weight_y(:,:) = 0.0_r8 + file%count_x(:) = 0 + file%count_y(:) = 0 + file%index_x(:,:) = 0 + file%index_y(:,:) = 0 + + if(masterproc) then +! compute weighting + call xy_interp_init(file%nlon,file%nlat,file%lons,file%lats,plon,plat,file%weight_x,file%weight_y) + + do i2=1,plon + file%count_x(i2) = 0 + do i1=1,file%nlon + if(file%weight_x(i2,i1).gt.0.0_r8 ) then + file%count_x(i2) = file%count_x(i2) + 1 + file%index_x(i2,file%count_x(i2)) = i1 + endif + enddo + enddo + + do j2=1,plat + file%count_y(j2) = 0 + do j1=1,file%nlat + if(file%weight_y(j2,j1).gt.0.0_r8 ) then + file%count_y(j2) = file%count_y(j2) + 1 + file%index_y(j2,file%count_y(j2)) = j1 + endif + enddo + enddo + endif + +#if ( defined SPMD) + call mpibcast(file%weight_x, plon*file%nlon, mpir8 , 0, mpicom) + call mpibcast(file%weight_y, plat*file%nlat, mpir8 , 0, mpicom) + call mpibcast(file%count_x, plon, mpiint , 0, mpicom) + call mpibcast(file%count_y, plat, mpiint , 0, mpicom) + call mpibcast(file%index_x, plon*file%nlon, mpiint , 0, mpicom) + call mpibcast(file%index_y, plat*file%nlat, mpiint , 0, mpicom) +#endif + endif + + end subroutine trcdata_init + +!----------------------------------------------------------------------- +! Reads more data if needed and interpolates data to current model time +!----------------------------------------------------------------------- + subroutine advance_trcdata( flds, file, state, pbuf2d ) + use physics_types,only : physics_state + use physics_buffer, only : physics_buffer_desc + use ppgrid, only : pver,pcols + + implicit none + + type(trfile), intent(inout) :: file + type(trfld), intent(inout) :: flds(:) + type(physics_state), intent(in) :: state(begchunk:endchunk) + + type(physics_buffer_desc), optional, pointer :: pbuf2d(:,:) + integer :: ncol + real(r8) :: data_time + real(r8) :: t(pcols,pver) ! input temperature (K) + real(r8) :: rho(pcols,pver) ! input temperature (K) + real(r8) :: pmid(pcols,pver) ! pressure at layer midpoints (pa) +!--------------------------------------BLH----------------------------------- + + call t_startf('advance_trcdata') + if ( .not.( file%fixed .and. file%initialized ) ) then + + call get_model_time(file) + + data_time = file%datatimep + + if ( file%cyclical .or. file%cyclical_list ) then + ! wrap around + if ( (file%datatimepfile%datatimem) ) then + data_time = data_time + file%one_yr + endif + endif + + ! For stepTime need to advance if the times are equal + ! Should not impact other runs? + if ( file%curr_mod_time >= data_time ) then + call t_startf('read_next_trcdata') + call read_next_trcdata(state, flds, file ) + call t_stopf('read_next_trcdata') + if(masterproc) write(iulog,*) 'READ_NEXT_TRCDATA ', flds%fldnam,data_time + end if + + endif + + ! need to interpolate the data, regardless + ! each mpi task needs to interpolate + call t_startf('interpolate_trcdata') + if(present(pbuf2d)) then + call interpolate_trcdata( state, flds, file, pbuf2d ) + else + call interpolate_trcdata( state, flds, file ) + endif + call t_stopf('interpolate_trcdata') + + file%initialized = .true. + + call t_stopf('advance_trcdata') + + end subroutine advance_trcdata + +!------------------------------------------------------------------- +!------------------------------------------------------------------- + subroutine get_fld_data( flds, field_name, data, ncol, lchnk, pbuf ) + + use physics_buffer, only : physics_buffer_desc, pbuf_get_field + + implicit none + + type(trfld), intent(inout) :: flds(:) + character(len=*), intent(in) :: field_name + real(r8), intent(out) :: data(:,:) + integer, intent(in) :: lchnk + integer, intent(in) :: ncol + type(physics_buffer_desc), pointer :: pbuf(:) + + + integer :: f, nflds + real(r8),pointer :: tmpptr(:,:) + + data(:,:) = 0._r8 + nflds = size(flds) + + do f = 1, nflds + if ( trim(flds(f)%fldnam) == trim(field_name) ) then + if ( flds(f)%pbuf_ndx>0 ) then + call pbuf_get_field(pbuf, flds(f)%pbuf_ndx, tmpptr) + data(:ncol,:) = tmpptr(:ncol,:) + else + data(:ncol,:) = flds(f)%data(:ncol,:,lchnk) + endif + endif + enddo + + end subroutine get_fld_data + +!------------------------------------------------------------------- +!------------------------------------------------------------------- + subroutine put_fld_data( flds, field_name, data, ncol, lchnk, pbuf ) + + use physics_buffer, only : physics_buffer_desc, pbuf_get_field + + implicit none + + type(trfld), intent(inout) :: flds(:) + character(len=*), intent(in) :: field_name + real(r8), intent(in) :: data(:,:) + integer, intent(in) :: lchnk + integer, intent(in) :: ncol + type(physics_buffer_desc), pointer :: pbuf(:) + + + integer :: f, nflds + real(r8),pointer :: tmpptr(:,:) + + nflds = size(flds) + + do f = 1, nflds + if ( trim(flds(f)%fldnam) == trim(field_name) ) then + if ( flds(f)%pbuf_ndx>0 ) then + call pbuf_get_field(pbuf, flds(f)%pbuf_ndx, tmpptr) + tmpptr(:ncol,:) = data(:ncol,:) + else + flds(f)%data(:ncol,:,lchnk) = data(:ncol,:) + endif + endif + enddo + + end subroutine put_fld_data + +!------------------------------------------------------------------- +!------------------------------------------------------------------- + subroutine get_fld_ndx( flds, field_name, idx ) + + implicit none + + type(trfld), intent(in) :: flds(:) + character(len=*), intent(in) :: field_name + integer, intent(out) :: idx + integer :: f, nflds + + idx = -1 + nflds = size(flds) + + do f = 1, nflds + if ( trim(flds(f)%fldnam) == trim(field_name) ) then + idx = f + return + endif + enddo + + end subroutine get_fld_ndx + +!------------------------------------------------------------------------------ +!------------------------------------------------------------------------------ + subroutine get_model_time(file) + implicit none + type(trfile), intent(inout) :: file + + integer yr, mon, day, ncsec ! components of a date + + call get_curr_date(yr, mon, day, ncsec) + + if ( file%cyclical .or. file%cyclical_list) yr = file%cyc_yr + call set_time_float_from_date( file%curr_mod_time, yr, mon, day, ncsec ) + file%next_mod_time = file%curr_mod_time + get_step_size()/86400._r8 + + end subroutine get_model_time + +!------------------------------------------------------------------------------ +!------------------------------------------------------------------------------ + subroutine check_files( file, fids, itms, times_found) + + implicit none + + type(trfile), intent(inout) :: file + type(file_desc_t), intent(out) :: fids(2) ! ids of files that contains these recs + integer, optional, intent(out) :: itms(2) + logical, optional, intent(inout) :: times_found + + !----------------------------------------------------------------------- + ! ... local variables + !----------------------------------------------------------------------- + logical :: list_cycled + + list_cycled = .false. + + !----------------------------------------------------------------------- + ! If next time beyond the end of the time list, + ! then increment the filename and move on to the next file + !----------------------------------------------------------------------- + if ((file%next_mod_time > file%curr_data_times(size(file%curr_data_times))).or.file%cyclical_list) then + if (file%cyclical_list) then + if ( associated(file%next_data_times) ) then + if ((file%curr_mod_time > file%datatimep)) then + + call advance_file(file) + + endif + endif + + endif + if ( .not. associated(file%next_data_times) ) then + ! open next file if not already opened... + if (file%cyclical_list) then + file%next_filename = incr_filename( file%curr_filename, filenames_list=file%filenames_list, datapath=file%pathname ,& + cyclical_list=file%cyclical_list, list_cycled=list_cycled) + else + file%next_filename = incr_filename( file%curr_filename, filenames_list=file%filenames_list, datapath=file%pathname) + endif + call open_trc_datafile( file%next_filename, file%pathname, file%next_fileid, file%next_data_times ) + file%next_data_times = file%next_data_times - file%offset_time + endif + endif + + !----------------------------------------------------------------------- + ! If using next_data_times and the current is greater than or equal to the next, then + ! close the current file, and set up for next file. + !----------------------------------------------------------------------- + if ( associated(file%next_data_times) ) then + if (file%cyclical_list .and. list_cycled) then ! special case - list cycled + + file%datatimem = file%curr_data_times(size(file%curr_data_times)) + itms(1)=size(file%curr_data_times) + fids(1)=file%curr_fileid + + file%datatimep = file%next_data_times(1) + itms(2)=1 + fids(2) = file%next_fileid + + times_found = .true. + + else if (file%curr_mod_time >= file%next_data_times(1)) then + + call advance_file(file) + + endif + endif + + end subroutine check_files + +!----------------------------------------------------------------------- +!----------------------------------------------------------------------- + function incr_filename( filename, filenames_list, datapath, cyclical_list, list_cycled ) + + !----------------------------------------------------------------------- + ! ... Increment or decrement a date string withing a filename + ! the filename date section is assumed to be of the form + ! yyyy-dd-mm + !----------------------------------------------------------------------- + + use string_utils, only : incstr + use shr_file_mod, only : shr_file_getunit, shr_file_freeunit + + implicit none + + + character(len=*), intent(in) :: filename ! present dynamical dataset filename + character(len=*), optional, intent(in) :: filenames_list + character(len=*), optional, intent(in) :: datapath + logical , optional, intent(in) :: cyclical_list ! If true, allow list to cycle + logical , optional, intent(out) :: list_cycled + character(len=shr_kind_cl) :: incr_filename ! next filename in the sequence + + + ! set new next_filename ... + + !----------------------------------------------------------------------- + ! ... local variables + !----------------------------------------------------------------------- + integer :: pos, pos1, istat + character(len=shr_kind_cl) :: fn_new, line, filepath + character(len=6) :: seconds + character(len=5) :: num + integer :: ios,unitnumber + + if (present(list_cycled)) list_cycled = .false. + + if (( .not. present(filenames_list)) .or.(len_trim(filenames_list) == 0)) then + !----------------------------------------------------------------------- + ! ... ccm type filename + !----------------------------------------------------------------------- + pos = len_trim( filename ) + fn_new = filename(:pos) + if ( masterproc ) write(iulog,*) 'incr_flnm: old filename = ',trim(fn_new) + if( fn_new(pos-2:) == '.nc' ) then + pos = pos - 3 + end if + istat = incstr( fn_new(:pos), 1 ) + if( istat /= 0 ) then + write(iulog,*) 'incr_flnm: incstr returned ', istat + write(iulog,*) ' while trying to decrement ',trim( fn_new ) + call endrun + end if + + else + + !------------------------------------------------------------------- + ! ... open filenames_list + !------------------------------------------------------------------- + if ( masterproc ) write(iulog,*) 'incr_flnm: old filename = ',trim(filename) + if ( masterproc ) write(iulog,*) 'incr_flnm: open filenames_list : ',trim(filenames_list) + unitnumber = shr_file_getUnit() + if ( present(datapath) ) then + filepath = trim(datapath) //'/'// trim(filenames_list) + else + filepath = trim(datapath) + endif + + open( unit=unitnumber, file=filepath, iostat=ios, status="OLD") + if (ios /= 0) then + call endrun('not able to open filenames_list file: '//trim(filepath)) + endif + + !------------------------------------------------------------------- + ! ... read file names + !------------------------------------------------------------------- + read( unit=unitnumber, fmt='(A)', iostat=ios ) line + if (ios /= 0) then + call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) + endif + + !------------------------------------------------------------------- + ! If current filename is '', then initialize with the first filename read in + ! and skip this section. + !------------------------------------------------------------------- + if (filename /= '') then + + !------------------------------------------------------------------- + ! otherwise read until find current filename + !------------------------------------------------------------------- + do while( trim(line) /= trim(filename) ) + read( unit=unitnumber, fmt='(A)', iostat=ios ) line + if (ios /= 0) then + call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) + endif + enddo + + !------------------------------------------------------------------- + ! Read next filename + !------------------------------------------------------------------- + read( unit=unitnumber, fmt='(A)', iostat=ios ) line + + !--------------------------------------------------------------------------------- + ! If cyclical_list, then an end of file is not an error, but rather + ! a signal to rewind and start over + !--------------------------------------------------------------------------------- + + if (ios /= 0) then + if (present(cyclical_list)) then + if (cyclical_list) then + list_cycled=.true. + rewind(unitnumber) + read( unit=unitnumber, fmt='(A)', iostat=ios ) line + ! Error here should never happen, but check just in case + if (ios /= 0) then + call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) + endif + else + call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) + endif + else + call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) + endif + endif + + endif + + !--------------------------------------------------------------------------------- + ! Assign the current filename and close the filelist + !--------------------------------------------------------------------------------- + fn_new = trim(line) + + close(unit=unitnumber) + call shr_file_freeUnit(unitnumber) + endif + + !--------------------------------------------------------------------------------- + ! return the current filename + !--------------------------------------------------------------------------------- + incr_filename = trim(fn_new) + if ( masterproc ) write(iulog,*) 'incr_flnm: new filename = ',trim(incr_filename) + + end function incr_filename + +!------------------------------------------------------------------------------ +!------------------------------------------------------------------------------ + subroutine find_times( itms, fids, time, file, datatimem, datatimep, times_found ) + + implicit none + + type(trfile), intent(in) :: file + real(r8), intent(out) :: datatimem, datatimep + + integer, intent(out) :: itms(2) ! record numbers that bracket time + type(file_desc_t), intent(out) :: fids(2) ! ids of files that contains these recs + + real(r8), intent(in) :: time ! time of interest + logical, intent(inout) :: times_found + + integer :: np1 ! current forward time index of dataset + integer :: n,i ! + integer :: curr_tsize, next_tsize, all_tsize + integer :: astat + integer :: cyc_tsize + + real(r8), allocatable, dimension(:):: all_data_times + + curr_tsize = size(file%curr_data_times) + next_tsize = 0 + if ( associated(file%next_data_times)) next_tsize = size(file%next_data_times) + + all_tsize = curr_tsize + next_tsize + + allocate( all_data_times( all_tsize ), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'find_times: failed to allocate all_data_times array; error = ',astat + call endrun + end if + + all_data_times(:curr_tsize) = file%curr_data_times(:) + if (next_tsize > 0) all_data_times(curr_tsize+1:all_tsize) = file%next_data_times(:) + + if ( .not. file%cyclical ) then + if ( all( all_data_times(:) > time ) ) then + write(iulog,*) 'FIND_TIMES: ALL data times are after ', time + write(iulog,*) 'FIND_TIMES: data times: ',all_data_times(:) + write(iulog,*) 'FIND_TIMES: time: ',time + call endrun('find_times: all(all_data_times(:) > time) '// trim(file%curr_filename) ) + endif + + ! find bracketing times + find_times_loop : do n=1, all_tsize-1 + np1 = n + 1 + datatimem = all_data_times(n) !+ file%offset_time + datatimep = all_data_times(np1) !+ file%offset_time + ! When stepTime, datatimep may not equal the time (as only datatimem is used) + ! Should not break other runs? + if ( (time .ge. datatimem) .and. (time .lt. datatimep) ) then + times_found = .true. + exit find_times_loop + endif + enddo find_times_loop + + else ! file%cyclical + + cyc_tsize = file%cyc_ndx_end - file%cyc_ndx_beg + 1 + + if ( cyc_tsize > 1 ) then + + call findplb(all_data_times(file%cyc_ndx_beg:file%cyc_ndx_end),cyc_tsize, time, n ) + + if (n == cyc_tsize) then + np1 = 1 + else + np1 = n+1 + endif + + datatimem = all_data_times(n +file%cyc_ndx_beg-1) + datatimep = all_data_times(np1+file%cyc_ndx_beg-1) + times_found = .true. + + endif + endif + + if ( .not. times_found ) then + if (masterproc) then + write(iulog,*)'FIND_TIMES: Failed to find dates bracketing desired time =', time + write(iulog,*)' datatimem = ',file%datatimem + write(iulog,*)' datatimep = ',file%datatimep + write(iulog,*)' all_data_times = ',all_data_times + !call endrun() + return + endif + endif + + deallocate( all_data_times, stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'find_times: failed to deallocate all_data_times array; error = ',astat + call endrun + end if + + if ( .not. file%cyclical ) then + itms(1) = n + itms(2) = np1 + else + itms(1) = n +file%cyc_ndx_beg-1 + itms(2) = np1 +file%cyc_ndx_beg-1 + endif + + fids(:) = file%curr_fileid + + do i=1,2 + if ( itms(i) > curr_tsize ) then + itms(i) = itms(i) - curr_tsize + fids(i) = file%next_fileid + endif + enddo + + end subroutine find_times + +!------------------------------------------------------------------------ +!------------------------------------------------------------------------ + subroutine read_next_trcdata(state, flds, file ) + + use shr_const_mod, only:pi => shr_const_pi + use physics_types,only : physics_state + use ppgrid, only: pcols, pver, pverp,begchunk,endchunk + use physconst, only: rair + use iop_data_mod + use rad_constituents, only: rad_cnst_get_info, rad_cnst_get_aer_props, & + rad_cnst_get_mode_props, rad_cnst_get_mode_num + + implicit none + + type (trfile), intent(inout) :: file + type (trfld),intent(inout) :: flds(:) + type(physics_state), intent(in) :: state(begchunk:endchunk) + + integer :: recnos(4),i,f,nflds ! + integer :: cnt4(4) ! array of counts for each dimension + integer :: strt4(4) ! array of starting indices + integer :: cnt3(3) ! array of counts for each dimension + integer :: strt3(3) ! array of starting indices + type(file_desc_t) :: fids(4) + logical :: times_found + + integer :: cur_yr, cur_mon, cur_day, cur_sec, yr1, yr2, mon, date, sec + real(r8) :: series1_time, series2_time + type(file_desc_t) :: fid1, fid2 + + integer :: nspec,nmodes,n,ii,kk,l1,k,lchnk + real(r8) :: profile_p(pver),pp(pver),meanP,meanO,sumii,volfrac,specdens,aero_den(6) + real(r8) :: q_a(3,6) + real(r8) :: rho(pcols,pver) ! air density (kg m-3) + character(len=20) :: aername + character(len=3) :: arnam(7) = (/'so4','pom','soa','bc ','dst','ncl','num'/) + + nflds = size(flds) + times_found = .false. + + if(single_column .and. scm_observed_aero) then + + call ver_profile_aero(pp) + ! The following do loop gets the species properties and calculates the aerosol + ! mass mixing ratio of each species from observed total number and size distribution + ! properties in the unit kg/m^3 for the 3 modes. Data is read from the forcing file. + ! For mode 1 (accumulation mode) q_a(1,1)=q(so4),q_a(1,2)=q(pom),q_a(1,3)=q(soa), + ! q_a(1,4)=q(bc),q_a(1,5)=q(dst),q_a(1,6)=q(ncl). + ! For mode 2 (aitken mode) q_a(2,1)=q(so4),q_a(2,2)=q(soa),q_a(2,3)=q(ncl). + ! For mode 3 (coarse mode) q_a(3,1)=q(dst),q_a(3,2)=q(ncl),q_a(3,3)=q(so4). + + call rad_cnst_get_info(0, nmodes=nmodes) + do n=1, nmodes + call rad_cnst_get_info(0, n, nspec=nspec) + do l1 = 1, nspec + call rad_cnst_get_aer_props(0, n,l1,density_aer=specdens) + call rad_cnst_get_aer_props(0, n, l1, aername=aername) + aero_den(l1)=specdens + q_a(n,l1) = specdens*scm_div(n,l1)*scm_num(n)*((pi/6.0_r8)*( & + scm_dgnum(n)**3)*exp(4.5_r8*(log(scm_std(n))**2) )) + enddo + enddo + + do k = 1, pver + do ii = 1, state(begchunk)%ncol + rho(ii,k) = state(begchunk)%pmid(ii,k)/(rair*state(begchunk)%t(ii,k)) + enddo + enddo + end if + + do while( .not. times_found ) + call find_times( recnos, fids, file%curr_mod_time,file,file%datatimem,file%datatimep, times_found ) + if ( .not. times_found ) then + call check_files( file, fids, recnos, times_found ) + endif + enddo + + ! If single column do not interpolate aerosol data, just use the step function. + ! The exception is if we are trying to "replay" a column from the full model + if(single_column .and. .not. use_replay) then + file%stepTime = .true. + endif + + if (file%stepTime) then + file%interp_recs = 1 + else + file%interp_recs = 2 + end if + + if ( file%fill_in_months ) then + + if( file%datatimep-file%datatimem > file%one_yr ) then + + call get_curr_date(cur_yr, cur_mon, cur_day, cur_sec) + + call set_date_from_time_float(file%datatimem, yr1, mon, date, sec ) + call set_date_from_time_float(file%datatimep, yr2, mon, date, sec ) + + call set_time_float_from_date( series1_time, yr1, cur_mon, cur_day, cur_sec ) + call set_time_float_from_date( series2_time, yr2, cur_mon, cur_day, cur_sec ) + + fid1 = fids(1) + fid2 = fids(2) + file%cyclical = .true. + call set_cycle_indices( fid1, file%cyc_ndx_beg, file%cyc_ndx_end, yr1) + call find_times( recnos(1:2), fids(1:2), series1_time, file, file%datatimes(1), file%datatimes(2), times_found ) + + if ( .not. times_found ) then + call endrun('read_next_trcdata: time not found for series1_time') + endif + call set_cycle_indices( fid2, file%cyc_ndx_beg, file%cyc_ndx_end, yr2) + + if ( fid1%fh /= fid2%fh ) then + file%cyc_ndx_beg = file%cyc_ndx_beg + size(file%curr_data_times) + file%cyc_ndx_end = file%cyc_ndx_end + size(file%curr_data_times) + endif + call find_times( recnos(3:4), fids(3:4), series2_time, file, file%datatimes(3), file%datatimes(4), times_found ) + if ( .not. times_found ) then + call endrun('read_next_trcdata: time not found for series2_time') + endif + file%cyclical = .false. + file%interp_recs = 4 + + call set_date_from_time_float( file%datatimes(1), yr1, mon, date, sec ) + call set_time_float_from_date( file%datatimem, cur_yr, mon, date, sec ) + if (file%datatimes(1) > file%datatimes(2) ) then ! wrap around + if ( cur_mon == 1 ) then + call set_time_float_from_date( file%datatimem, cur_yr-1, mon, date, sec ) + endif + endif + + call set_date_from_time_float( file%datatimes(2), yr1, mon, date, sec ) + call set_time_float_from_date( file%datatimep, cur_yr, mon, date, sec ) + if (file%datatimes(1) > file%datatimes(2) ) then ! wrap around + if ( cur_mon == 12 ) then + call set_time_float_from_date( file%datatimep, cur_yr+1, mon, date, sec ) + endif + endif + + endif + + endif + + ! + ! Set up hyperslab corners + ! + strt4(:) = 1 + strt3(:) = 1 + + do i=1,file%interp_recs + + do f = 1,nflds + if ( file%zonal_ave ) then + ! Defend against zonal mean surface fields that do not set the value via dimension match + if (flds(f)%coords(ZA_LATDIM) .gt. 0) cnt3(flds(f)%coords(ZA_LATDIM)) = file%nlat + if (flds(f)%srf_fld) then + ! Defend against zonal mean surface fields that do not set the value via dimension match + if (flds(f)%coords(ZA_LEVDIM) .gt. 0) cnt3(flds(f)%coords(ZA_LEVDIM)) = 1 + else + cnt3(flds(f)%coords(ZA_LEVDIM)) = file%nlev + endif + cnt3(flds(f)%coords(ZA_TIMDIM)) = 1 + strt3(flds(f)%coords(ZA_TIMDIM)) = recnos(i) + !! + if (file%linoz_v3 .or. file%linoz_v2) then + !!check if these are the surface variables + !!no need to do interpolate since only used + !!in preprocessing, + !!clim +57 is the correspondent srf variable + if (index(flds(f)%fldnam,"_clim") .gt.0.and.& + index(flds(f)%fldnam,"P_clim") .le.0.and.& + index(flds(f)%fldnam,"L_clim") .le.0.and.& + index(flds(f)%fldnam,"_srf") .le.0)then + call read_za_trc_linoz( fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt3, cnt3, file, & + (/ flds(f)%order(ZA_LATDIM),flds(f)%order(ZA_LEVDIM)/), & + vid_srf=flds(f+57)%var_id ) + elseif (index(flds(f)%fldnam,"_srf").gt.0) then + if (index(flds(f)%fldnam,"ch4_avg_srf").gt.0) then + cnt3(1)=1!set 1st dim since no ZA_LATDIM + endif + call read_zasrf_trc_linoz(fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt3, cnt3, file) + else + call read_za_trc_linoz( fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt3, cnt3, file, & + (/ flds(f)%order(ZA_LATDIM),flds(f)%order(ZA_LEVDIM) /) ) + endif + else + call read_za_trc( fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt3, cnt3, file, & + (/ flds(f)%order(ZA_LATDIM),flds(f)%order(ZA_LEVDIM) /) ) + endif + else if ( flds(f)%srf_fld ) then + cnt3( flds(f)%coords(LONDIM)) = file%nlon + cnt3( flds(f)%coords(LATDIM)) = file%nlat + cnt3( flds(f)%coords(PS_TIMDIM)) = 1 + strt3(flds(f)%coords(PS_TIMDIM)) = recnos(i) + call read_2d_trc( fids(i), flds(f)%var_id, flds(f)%input(i)%data(:,1,:), strt3, cnt3, file, & + (/ flds(f)%order(LONDIM),flds(f)%order(LATDIM) /) ) + else + cnt4(flds(f)%coords(LONDIM)) = file%nlon + cnt4(flds(f)%coords(LATDIM)) = file%nlat + cnt4(flds(f)%coords(LEVDIM)) = file%nlev + cnt4(flds(f)%coords(TIMDIM)) = 1 + strt4(flds(f)%coords(TIMDIM)) = recnos(i) + call read_3d_trc( fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt4, cnt4, file, & + (/ flds(f)%order(LONDIM),flds(f)%order(LATDIM),flds(f)%order(LEVDIM) /)) + + ! + ! This section sets the observed aersol mass and number mixing ratios in the + ! appropriate variables. The observed aerosol inforamtion is read from the + ! forcing file as total number and size distribution parameters. Then the total + ! volume is calculated.Using the desnsity of each species, the mass is calculated. + ! Finally the number is partition among the each species using the species fraction + ! data read from the forcing file. + ! + if(single_column .and. scm_observed_aero) then + kk=index(trim(flds(f)%fldnam),'_')-1 + if(index(trim(flds(f)%fldnam),'1') > 0 .and.index(trim(flds(f)%fldnam),'log') < 1) then + if(flds(f)%fldnam(1:kk).eq.arnam(1).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(1),flds(f)%input(i)%data, & + rho,pp,q_a(1,1),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(2).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(2),flds(f)%input(i)%data, & + rho,pp,q_a(1,2),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(3).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(3),flds(f)%input(i)%data, & + rho,pp,q_a(1,3),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(4).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(4),flds(f)%input(i)%data, & + rho,pp,q_a(1,4),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(5).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(5),flds(f)%input(i)%data, & + rho,pp,q_a(1,5),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(6).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(6),flds(f)%input(i)%data, & + rho,pp,q_a(1,6),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(7).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(7),flds(f)%input(i)%data, & + rho,pp,scm_num(1),state(begchunk)%ncol) + endif + elseif(index(trim(flds(f)%fldnam),'2') > 0 .and.index(trim(flds(f)%fldnam),'log') < 1) then + if(flds(f)%fldnam(1:kk).eq.arnam(1).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(1),flds(f)%input(i)%data, & + rho,pp,q_a(2,1),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(3).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(3),flds(f)%input(i)%data, & + rho,pp,q_a(2,2),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(6).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(6),flds(f)%input(i)%data, & + rho,pp,q_a(2,3),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(7).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(7),flds(f)%input(i)%data, & + rho,pp,scm_num(2),state(begchunk)%ncol) + endif + elseif(index(trim(flds(f)%fldnam),'3') > 0 .and.index(trim(flds(f)%fldnam),'log') < 1) then + if(flds(f)%fldnam(1:kk).eq.arnam(1).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(1),flds(f)%input(i)%data, & + rho,pp,q_a(3,3),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(5).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(5),flds(f)%input(i)%data, & + rho,pp,q_a(3,1),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(6).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(6),flds(f)%input(i)%data, & + rho,pp,q_a(3,2),state(begchunk)%ncol) + elseif(flds(f)%fldnam(1:kk).eq.arnam(7).and.index(trim(flds(f)%fldnam),'log') < 1) then + call replace_aero_data(flds(f)%fldnam,arnam(7),flds(f)%input(i)%data, & + rho,pp,scm_num(3),state(begchunk)%ncol) + endif + endif + endif !scm_observed_aero + endif + + enddo + + if ( file%has_ps ) then + cnt3(file%ps_coords(LONDIM)) = file%nlon + cnt3(file%ps_coords(LATDIM)) = file%nlat + cnt3(file%ps_coords(PS_TIMDIM)) = 1 + strt3(file%ps_coords(PS_TIMDIM)) = recnos(i) + call read_2d_trc( fids(i), file%ps_id, file%ps_in(i)%data, strt3, cnt3, file, & + (/ file%ps_order(LONDIM),file%ps_order(LATDIM) /) ) + endif + + enddo + + end subroutine read_next_trcdata + +!-------------------------------------------------------------------------------- +!This subroutine replaces the climatological aerosol information by the observed +!once after they are read +! + subroutine replace_aero_data(aerofulnam,spnam,aero_q_data,rho,pp,q_mix,ncoli) + use ppgrid, only: pcols,pver,begchunk,endchunk + + implicit none + real(r8), intent(inout) :: aero_q_data(pcols,pver,begchunk:endchunk) + real(r8), intent(in) :: rho(pcols,pver),pp(pver) + real(r8) :: sumii,meanO + real(r8), intent(in) :: q_mix + character(len=32),intent(in) ::aerofulnam + character(len=3),intent(in) :: spnam + character(len=32) ::aerosubnam + integer, intent(in) :: ncoli + integer :: ii,k,countj + + if(trim(aerofulnam(1:2)).eq.'bc') then + aerosubnam=aerofulnam(1:4) + else + aerosubnam=aerofulnam(1:5) + endif + + if((trim(aerosubnam).eq.(trim(spnam)//'_a').or.trim(aerosubnam).eq.(trim(spnam)//'_c')) & + .and.index(trim(aerofulnam),'log') < 1) then + + aero_q_data=q_mix + sumii=0._r8 + countj =0 + do ii = 1, ncoli + do k = 1, pver + if(pp(k).gt.0._r8) then + countj=countj+1 + endif + if(trim(spnam).ne.'num') then + aero_q_data(ii,k,begchunk)=(aero_q_data(ii,k,begchunk)*pp(k)) /rho(ii,k) + endif + sumii=sumii + aero_q_data(ii,k,begchunk) + enddo + enddo + meanO=sumii/countj + do k = 1, pver + if(meanO.ne.0.) then + aero_q_data(1,k,begchunk)=pp(k)*meanO + else + aero_q_data(1,k,begchunk)=0._r8 + endif + enddo + endif + + end subroutine replace_aero_data + +!--------------------------------------------------------------------------- +!This subroutine generates a heavyside type profiles for the observed aerosol. +!This setting is constant profile in the lower atmosphere and then exponentially +!decreasing to zero at the top of the atmosphere. The level of initial decay is +!controlled by "initial_val". Larger than -3.5 pushes the decay point up and smaller +!brings it closer to the surface. +! + subroutine ver_profile_aero(vertprof_aero) + use mo_constants, only : pi + use ppgrid, only: pcols,pver + + implicit none + real(r8), intent(inout) :: vertprof_aero(pver) + real(r8) :: initial_val = -3.5_r8 + integer :: k + + do k=1,pver + if(k==1) then + vertprof_aero(k)=0._r8 + elseif(k==2) then + vertprof_aero(k)=1._r8/(1._r8 + exp(-2._r8*initial_val * pi)) + else + vertprof_aero(k)=1._r8/(1._r8 + exp(-2._r8*(initial_val * pi + pi/4._r8*(k-2)))) + endif + enddo + end subroutine ver_profile_aero + +!------------------------------------------------------------------------ + + + subroutine read_2d_trc( fid, vid, loc_arr, strt, cnt, file, order ) + use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish + + use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p, get_lon_all_p, get_lat_all_p + use mo_constants, only : pi + use dycore, only: dycore_is + use polar_avg, only: polar_average + use horizontal_interpolate, only : xy_interp + + implicit none + type(file_desc_t), intent(in) :: fid + type(var_desc_t), intent(in) :: vid + integer, intent(in) :: strt(:), cnt(:), order(2) + real(r8),intent(out) :: loc_arr(:,:) + type (trfile), intent(in) :: file + + real(r8) :: to_lats(pcols), to_lons(pcols), wrk(pcols) + real(r8), allocatable, target :: wrk2d(:,:) + real(r8), pointer :: wrk2d_in(:,:) + + integer :: tsize, c, i, j, ierr, ncols + real(r8), parameter :: zero=0_r8, twopi=2_r8*pi + type(interp_type) :: lon_wgts, lat_wgts + integer :: lons(pcols), lats(pcols) + + nullify(wrk2d_in) + allocate( wrk2d(cnt(1),cnt(2)), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'read_2d_trc: wrk2d allocation error = ',ierr + call endrun + end if + + if(order(1)/=1 .or. order(2)/=2 .or. cnt(1)/=file%nlon .or. cnt(2)/=file%nlat) then + allocate( wrk2d_in(file%nlon, file%nlat), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'read_2d_trc: wrk2d_in allocation error = ',ierr + call endrun + end if + end if + + + + ierr = pio_get_var( fid, vid, strt, cnt, wrk2d ) + if(associated(wrk2d_in)) then + wrk2d_in = reshape( wrk2d(:,:),(/file%nlon,file%nlat/), order=order ) + deallocate(wrk2d) + else + wrk2d_in => wrk2d + end if + + j=1 + +! if weighting by latitude, the perform horizontal interpolation by using weight_x, weight_y + + if(file%weight_by_lat) then + + call t_startf('xy_interp') + + do c = begchunk,endchunk + ncols = get_ncols_p(c) + call get_lon_all_p(c,ncols,lons) + call get_lat_all_p(c,ncols,lats) + + call xy_interp(file%nlon,file%nlat,1,plon,plat,pcols,ncols,file%weight_x,file%weight_y,wrk2d_in,loc_arr(:,c-begchunk+1), & + lons,lats,file%count_x,file%count_y,file%index_x,file%index_y) + enddo + + call t_stopf('xy_interp') + + else + do c=begchunk,endchunk + ncols = get_ncols_p(c) + call get_rlat_all_p(c, pcols, to_lats) + call get_rlon_all_p(c, pcols, to_lons) + + call lininterp_init(file%lons, file%nlon, to_lons, ncols, 2, lon_wgts, zero, twopi) + call lininterp_init(file%lats, file%nlat, to_lats, ncols, 1, lat_wgts) + + call lininterp(wrk2d_in, file%nlon, file%nlat, loc_arr(1:ncols,c-begchunk+1), ncols, lon_wgts, lat_wgts) + + call lininterp_finish(lon_wgts) + call lininterp_finish(lat_wgts) + end do + endif + + if(allocated(wrk2d)) then + deallocate(wrk2d) + else + deallocate(wrk2d_in) + end if + if(dycore_is('LR')) call polar_average(loc_arr) + end subroutine read_2d_trc + +!------------------------------------------------------------------------ + + subroutine read_za_trc( fid, vid, loc_arr, strt, cnt, file, order ) + use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish + use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p + use mo_constants, only : pi + use dycore, only : dycore_is + use polar_avg, only : polar_average + + implicit none + type(file_desc_t), intent(in) :: fid + type(var_desc_t), intent(in) :: vid + integer, intent(in) :: strt(:), cnt(:) + integer, intent(in) :: order(2) + real(r8), intent(out):: loc_arr(:,:,:) + type (trfile), intent(in) :: file + + type(interp_type) :: lat_wgts + real(r8) :: to_lats(pcols), to_lons(pcols), wrk(pcols) + real(r8), allocatable, target :: wrk2d(:,:) + real(r8), pointer :: wrk2d_in(:,:) + integer :: c, k, ierr, ncols + + nullify(wrk2d_in) + allocate( wrk2d(cnt(1),cnt(2)), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'read_2d_trc: wrk2d allocation error = ',ierr + call endrun + end if + + if(order(1)/=1 .or. order(2)/=2 .or. cnt(1)/=file%nlat .or. cnt(2)/=file%nlev) then + allocate( wrk2d_in(file%nlat, file%nlev), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'read_2d_trc: wrk2d_in allocation error = ',ierr + call endrun + end if + end if + + + ierr = pio_get_var( fid, vid, strt, cnt, wrk2d ) + if(associated(wrk2d_in)) then + wrk2d_in = reshape( wrk2d(:,:),(/file%nlat,file%nlev/), order=order ) + deallocate(wrk2d) + else + wrk2d_in => wrk2d + end if + + + + do c=begchunk,endchunk + ncols = get_ncols_p(c) + call get_rlat_all_p(c, pcols, to_lats) + + call lininterp_init(file%lats, file%nlat, to_lats, ncols, 1, lat_wgts) + do k=1,file%nlev + call lininterp(wrk2d_in(:,k), file%nlat, wrk(1:ncols), ncols, lat_wgts) + loc_arr(1:ncols,k,c-begchunk+1) = wrk(1:ncols) + end do + call lininterp_finish(lat_wgts) + end do + + if(allocated(wrk2d)) then + deallocate(wrk2d) + else + deallocate(wrk2d_in) + end if +! if(dycore_is('LR')) call polar_average(loc_arr) + end subroutine read_za_trc +!------------------------------------------------------------------------ + subroutine read_za_trc_linoz( fid, vid, loc_arr, strt, cnt, file, order ,vid_srf) + use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish + use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p + use mo_constants, only : pi + use dycore, only : dycore_is + use polar_avg, only : polar_average + + implicit none + type(file_desc_t), intent(in) :: fid + type(var_desc_t), intent(in) :: vid + integer, intent(in) :: strt(:), cnt(:) + integer, intent(in) :: order(2) + real(r8), intent(out):: loc_arr(:,:,:) + type (trfile), intent(in) :: file + !! + type(var_desc_t), intent(in), optional :: vid_srf + integer :: cnt_srf(2) + integer :: strt_srf(2) + !! + type(interp_type) :: lat_wgts + real(r8) :: to_lats(pcols), to_lons(pcols), wrk(pcols) + real(r8), allocatable, target :: wrk2d(:,:) + real(r8), allocatable, target :: wrksrf(:) + real(r8), pointer :: wrk2d_in(:,:) + integer :: c, k, ierr, ncols + + nullify(wrk2d_in) + allocate( wrk2d(cnt(1),cnt(2)), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'read_2d_trc: wrk2d allocation error = ',ierr + call endrun + end if + + if(order(1)/=1 .or. order(2)/=2 .or. cnt(1)/=file%nlat .or. cnt(2)/=file%nlev) then + allocate( wrk2d_in(file%nlat, file%nlev), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'read_2d_trc: wrk2d_in allocation error = ',ierr + call endrun + end if + end if + !! + ierr = pio_get_var( fid, vid, strt, cnt, wrk2d ) + !! + if (file%linoz_v3) then + !!since io reads in global data for every thread + !!and interpolate to local chunk, we can do surface, + !!polar, and surface padding preprocessfor every variable + !!before interpolation + !!it is put here rather than the data formation process + !!to prevent forget when producing forcing data + !read in order of 0.1hPa~985hPa, pad top with 2nd layer + wrk2d(:,1)=wrk2d(:,2) + !both N/S poles need to pad by nearest data + wrk2d(1,:)=wrk2d(2,:) + wrk2d(file%nlat,:)=wrk2d(file%nlat-1,:) + !!read in srf data to pad surface + !!they are inputs from CMIP forcing and other references + cnt_srf(1)=cnt(1) + cnt_srf(2)=cnt(3) + strt_srf(1)=strt(1) + strt_srf(2)=strt(3) + !!check if vid_srf is present to determine clim variables + if (present(vid_srf)) then + !!padding for clim terms + allocate(wrksrf(cnt(1)), stat=ierr ) + ierr = pio_get_var( fid, vid_srf, strt_srf, cnt_srf, wrksrf ) + !!surface padding + wrk2d(:,file%nlev)=wrksrf + deallocate(wrksrf) + endif + !! + endif + !! + if(associated(wrk2d_in)) then + wrk2d_in = reshape( wrk2d(:,:),(/file%nlat,file%nlev/), order=order ) + deallocate(wrk2d) + else + wrk2d_in => wrk2d + end if + + do c=begchunk,endchunk + ncols = get_ncols_p(c) + call get_rlat_all_p(c, pcols, to_lats) + call lininterp_init(file%lats, file%nlat, to_lats, ncols, 1, lat_wgts) + do k=1,file%nlev + call lininterp(wrk2d_in(:,k), file%nlat, wrk(1:ncols), ncols, lat_wgts) + loc_arr(1:ncols,k,c-begchunk+1) = wrk(1:ncols) + end do + call lininterp_finish(lat_wgts) + end do + + if(allocated(wrk2d)) then + deallocate(wrk2d) + else + deallocate(wrk2d_in) + end if + !! + if(allocated(wrksrf)) then + deallocate(wrksrf) + end if +! if(dycore_is('LR')) call polar_average(loc_arr) + end subroutine read_za_trc_linoz + +!------------------------------------------------------------------------ + subroutine read_zasrf_trc_linoz( fid, vid, loc_arr, strt, cnt, file) + use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish + use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p + !! + implicit none + type(file_desc_t), intent(in) :: fid + type(var_desc_t), intent(in) :: vid + integer, intent(in) :: strt(:), cnt(:) + real(r8), intent(out):: loc_arr(:,:,:) + type (trfile), intent(in) :: file + !! + real(r8), allocatable, target :: wrk(:) + real(r8), pointer :: wrk_in(:) + real(r8) :: wrk_out(pcols) + type(interp_type) :: lat_wgts + real(r8) :: to_lats(pcols), to_lons(pcols) + integer :: c, k, ierr, ncols + integer :: cnt_srf(2) + integer :: strt_srf(2) + !! + nullify(wrk_in) + allocate( wrk(cnt(1)), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'read_zasrf_trc_linoz: wrk allocation error = ',ierr + call endrun + end if + !! + cnt_srf(1)=cnt(1) + cnt_srf(2)=cnt(3) + strt_srf(1)=strt(1) + strt_srf(2)=strt(3) + !! + !for surface variable with the dimension of (time,lat) or (time,1) + !for (time) it is also set like (time,1) + ierr = pio_get_var( fid, vid, strt_srf, cnt_srf, wrk ) + !! + if(associated(wrk_in)) then + wrk_in = reshape( wrk(:),(/file%nlat/)) + deallocate(wrk) + else + wrk_in => wrk + end if + !! + do c=begchunk,endchunk + ncols = get_ncols_p(c) + call get_rlat_all_p(c, pcols, to_lats) + call lininterp_init(file%lats, file%nlat, to_lats, ncols, 1, lat_wgts) + !! + if (cnt(1).eq.1) then!!for single timeseries + do k=1,1 + loc_arr(1:ncols,k,c-begchunk+1) = wrk_in(1) + end do + else + do k=1,1 + call lininterp(wrk_in(:), file%nlat, wrk_out(1:ncols), ncols, lat_wgts) + loc_arr(1:ncols,k,c-begchunk+1) = wrk_out(1:ncols) + end do + end if + !! + call lininterp_finish(lat_wgts) + end do + !! + if(allocated(wrk)) then + deallocate(wrk) + else + deallocate(wrk_in) + end if + !! + end subroutine read_zasrf_trc_linoz + +!------------------------------------------------------------------------ + + subroutine read_3d_trc( fid, vid, loc_arr, strt, cnt, file, order) + use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish + use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p, get_lon_all_p,& + get_lat_all_p + use mo_constants, only : pi + use dycore, only : dycore_is + use polar_avg, only : polar_average + use dycore, only : dycore_is + use horizontal_interpolate, only : xy_interp + + implicit none + + type(file_desc_t), intent(in) :: fid + type(var_desc_t), intent(in) :: vid + integer, intent(in) :: strt(:), cnt(:), order(3) + real(r8),intent(out) :: loc_arr(:,:,:) + + type (trfile), intent(in) :: file + + integer :: i,j,k, astat, c, ncols + integer :: lons(pcols), lats(pcols) + + integer :: jlim(2), jl, ju, ierr + integer :: gndx + + real(r8), allocatable, target :: wrk3d(:,:,:) + real(r8), pointer :: wrk3d_in(:,:,:) + real(r8) :: to_lons(pcols), to_lats(pcols) + real(r8), parameter :: zero=0_r8, twopi=2_r8*pi + type(interp_type) :: lon_wgts, lat_wgts + + loc_arr(:,:,:) = 0._r8 + nullify(wrk3d_in) + allocate(wrk3d(cnt(1),cnt(2),cnt(3)), stat=ierr) + if( ierr /= 0 ) then + write(iulog,*) 'read_3d_trc: wrk3d allocation error = ',ierr + call endrun + end if + + ierr = pio_get_var( fid, vid, strt, cnt, wrk3d ) + + if(order(1)/=1 .or. order(2)/=2 .or. order(3)/=3 .or. & + cnt(1)/=file%nlon.or.cnt(2)/=file%nlat.or.cnt(3)/=file%nlev) then + allocate(wrk3d_in(file%nlon,file%nlat,file%nlev),stat=ierr) + if( ierr /= 0 ) then + write(iulog,*) 'read_3d_trc: wrk3d allocation error = ',ierr + call endrun + end if + wrk3d_in = reshape( wrk3d(:,:,:),(/file%nlon,file%nlat,file%nlev/), order=order ) + deallocate(wrk3d) + else + wrk3d_in => wrk3d + end if + + j=1 + +! If weighting by latitude, then perform horizontal interpolation by using weight_x, weight_y + + if(file%weight_by_lat) then + + call t_startf('xy_interp') + + do c = begchunk,endchunk + ncols = get_ncols_p(c) + call get_lon_all_p(c,ncols,lons) + call get_lat_all_p(c,ncols,lats) + + call xy_interp(file%nlon,file%nlat,file%nlev,plon,plat,pcols,ncols,file%weight_x,file%weight_y,wrk3d_in, & + loc_arr(:,:,c-begchunk+1), lons,lats,file%count_x,file%count_y,file%index_x,file%index_y) + enddo + + call t_stopf('xy_interp') + + else + do c=begchunk,endchunk + ncols = get_ncols_p(c) + call get_rlat_all_p(c, pcols, to_lats) + call get_rlon_all_p(c, pcols, to_lons) + + call lininterp_init(file%lons, file%nlon, to_lons(1:ncols), ncols, 2, lon_wgts, zero, twopi) + call lininterp_init(file%lats, file%nlat, to_lats(1:ncols), ncols, 1, lat_wgts) + + + call lininterp(wrk3d_in, file%nlon, file%nlat, file%nlev, loc_arr(:,:,c-begchunk+1), ncols, pcols, lon_wgts, lat_wgts) + + + call lininterp_finish(lon_wgts) + call lininterp_finish(lat_wgts) + end do + endif + + if(allocated(wrk3d)) then + deallocate( wrk3d, stat=astat ) + else + deallocate( wrk3d_in, stat=astat ) + end if + if( astat/= 0 ) then + write(iulog,*) 'read_3d_trc: failed to deallocate wrk3d array; error = ',astat + call endrun + endif + if(dycore_is('LR')) call polar_average(file%nlev, loc_arr) + end subroutine read_3d_trc + +!------------------------------------------------------------------------------ + + subroutine interpolate_trcdata( state, flds, file, pbuf2d ) + use mo_util, only : rebin + use physics_types,only : physics_state + use physconst, only : cday + use physics_buffer, only : physics_buffer_desc, pbuf_get_field + + implicit none + + type(physics_state), intent(in) :: state(begchunk:endchunk) + type (trfld), intent(inout) :: flds(:) + type (trfile), intent(inout) :: file + + type(physics_buffer_desc), optional, pointer :: pbuf2d(:,:) + + + real(r8) :: fact1, fact2 + real(r8) :: deltat + integer :: f,nflds,c,ncol, i,k + real(r8) :: ps(pcols) + real(r8) :: datain(pcols,file%nlev) + real(r8) :: pin(pcols,file%nlev) + real(r8) :: pint(pcols,file%nilev) + + real(r8) :: model_z(pverp) + real(r8), parameter :: m2km = 1.e-3_r8 + real(r8), pointer :: data_out3d(:,:,:) + real(r8), pointer :: data_out(:,:) + integer :: chnk_offset + + nflds = size(flds) + + if ( file%interp_recs == 4 ) then + deltat = file%datatimes(3) - file%datatimes(1) + fact1 = (file%datatimes(3) - file%datatimem)/deltat + fact2 = 1._r8-fact1 +!$OMP PARALLEL DO PRIVATE (C, NCOL, F) + do c = begchunk,endchunk + ncol = state(c)%ncol + if ( file%has_ps ) then + file%ps_in(1)%data(:ncol,c) = fact1*file%ps_in(1)%data(:ncol,c) + fact2*file%ps_in(3)%data(:ncol,c) + endif + do f = 1,nflds + flds(f)%input(1)%data(:ncol,:,c) = fact1*flds(f)%input(1)%data(:ncol,:,c) + fact2*flds(f)%input(3)%data(:ncol,:,c) + enddo + enddo + + deltat = file%datatimes(4) - file%datatimes(2) + fact1 = (file%datatimes(4) - file%datatimep)/deltat + fact2 = 1._r8-fact1 + +!$OMP PARALLEL DO PRIVATE (C, NCOL, F) + do c = begchunk,endchunk + ncol = state(c)%ncol + if ( file%has_ps ) then + file%ps_in(2)%data(:ncol,c) = fact1*file%ps_in(2)%data(:ncol,c) + fact2*file%ps_in(4)%data(:ncol,c) + endif + do f = 1,nflds + flds(f)%input(2)%data(:ncol,:,c) = fact1*flds(f)%input(2)%data(:ncol,:,c) + fact2*flds(f)%input(4)%data(:ncol,:,c) + enddo + enddo + + endif + !------------------------------------------------------------------------- + ! If file%interp_recs=1 then no time interpolation -- set + ! fact1=1 and fact2=0 and will just use first value unmodified + !------------------------------------------------------------------------- + + if (file%interp_recs == 1) then + fact1=1._r8 + fact2=0._r8 + else + file%interp_recs = 2 + + deltat = file%datatimep - file%datatimem + + if ( file%cyclical .and. (deltat < 0._r8) ) then + deltat = deltat+file%one_yr + if ( file%datatimep >= file%curr_mod_time ) then + fact1 = (file%datatimep - file%curr_mod_time)/deltat + else + fact1 = (file%datatimep+file%one_yr - file%curr_mod_time)/deltat + endif + else + fact1 = (file%datatimep - file%curr_mod_time)/deltat + endif + + ! this assures that FIXED data are b4b on restarts + if ( file%fixed ) then + fact1 = dble(int(fact1*cday+.5_r8))/dble(cday) + endif + fact2 = 1._r8-fact1 + endif + + chnk_offset=-begchunk+1 + + fld_loop: do f = 1,nflds + + if (flds(f)%pbuf_ndx<=0) then + data_out3d => flds(f)%data(:,:,:) + endif + +!$OMP PARALLEL DO PRIVATE (C, NCOL, PS, I, K, PIN, DATAIN, MODEL_Z, DATA_OUT) + do c = begchunk,endchunk + if (flds(f)%pbuf_ndx>0) then + if(.not.present(pbuf2d)) then + call endrun ('tracer_data.F90(subr interpolate_trcdata):' // & + 'pbuf2d must be passed as an argument for pbuf_get_field subr call') + endif + call pbuf_get_field(pbuf2d, c, flds(f)%pbuf_ndx, data_out) + else + data_out => data_out3d(:,:,c+chnk_offset) + endif + ncol = state(c)%ncol + if (file%alt_data) then + + if (fact2 == 0) then ! This needed as %data is not set if fact2=0 (and lahey compiler core dumps) + datain(:ncol,:) = fact1*flds(f)%input(nm)%data(:ncol,:,c) + else + datain(:ncol,:) = fact1*flds(f)%input(nm)%data(:ncol,:,c) + fact2*flds(f)%input(np)%data(:ncol,:,c) + end if + do i = 1,ncol + model_z(1:pverp) = m2km * state(c)%zi(i,pverp:1:-1) + call rebin( file%nlev, pver, file%ilevs, model_z, datain(i,:), data_out(i,:) ) + enddo + + else + + if ( file%nlev>1 ) then + if ( file%has_ps ) then + if (fact2 == 0) then ! This needed as %data is not set if fact2=0 (and lahey compiler core dumps) + ps(:ncol) = fact1*file%ps_in(nm)%data(:ncol,c) + else + ps(:ncol) = fact1*file%ps_in(nm)%data(:ncol,c) + fact2*file%ps_in(np)%data(:ncol,c) + end if + do i = 1,ncol + do k = 1,file%nlev + pin(i,k) = file%p0*file%hyam(k) + ps(i)*file%hybm(k) + enddo + enddo + else + do k = 1,file%nlev + pin(:,k) = file%levs(k) + enddo + !!Currently designed for linoz_v2/v3 use + if (file%linoz_v3 .or. file%linoz_v2) then + do k = 1,file%nilev + pint(:,k) = file%ilevs(k) + enddo + endif + endif + endif + + if (flds(f)%srf_fld) then + do i = 1,ncol + if (fact2 == 0) then ! This needed as %data is not set if fact2=0 (and lahey compiler core dumps) + data_out(i,1) = & + fact1*flds(f)%input(nm)%data(i,1,c) + else + data_out(i,1) = & + fact1*flds(f)%input(nm)%data(i,1,c) + fact2*flds(f)%input(np)%data(i,1,c) + endif + enddo + else + if (fact2 == 0) then ! This needed as %data is not set if fact2=0 (and lahey compiler core dumps) + datain(:ncol,:) = fact1*flds(f)%input(nm)%data(:ncol,:,c) + else + datain(:ncol,:) = fact1*flds(f)%input(nm)%data(:ncol,:,c) + fact2*flds(f)%input(np)%data(:ncol,:,c) + end if + if ( file%top_bndry ) then + call vert_interp_ub(ncol, file%nlev, file%levs, datain(:ncol,:), data_out(:ncol,:) ) + else if(file%conserve_column) then + call vert_interp_mixrat(ncol,file%nlev,pver,state(c)%pint, & + datain, data_out(:,:), & + file%p0,ps,file%hyai,file%hybi) + else if(file%linoz_v3 .or. file%linoz_v2) then + !!uci chemistry for linoz that better conserves mass + !!uci interpolation for 55 out of 57 variables in linoz_v3 + !!excluding t_clim, o3col_clim + !!for linoz_v2 it is simiar with less variables + if (flds(f)%fldnam.ne.'t_clim ' & + .and.flds(f)%fldnam.ne.'o3col_clim ') then + !!file ilevs is in hPa, while model level in Pa, so times 100 + call vert_interp_uci(ncol, file%nlev, 100*file%ilevs, state(c)%pint, datain, data_out(:,:) ) + else + call vert_interp(ncol, file%nlev, pin, state(c)%pmid, datain, data_out(:,:) ) + endif + !! + else + call vert_interp(ncol, file%nlev, pin, state(c)%pmid, datain, data_out(:,:) ) + endif + endif + + endif + enddo + + enddo fld_loop + + end subroutine interpolate_trcdata + +!----------------------------------------------------------------------- +!----------------------------------------------------------------------- + subroutine get_dimension( fid, dname, dsize, dimid, data ) + implicit none + type(file_desc_t), intent(inout) :: fid + character(*), intent(in) :: dname + integer, intent(out) :: dsize + + integer, optional, intent(out) :: dimid + real(r8), optional, pointer, dimension(:) :: data + + integer :: vid, ierr, id + + call pio_seterrorhandling( fid, PIO_BCAST_ERROR) + ierr = pio_inq_dimid( fid, dname, id ) + call pio_seterrorhandling( fid, PIO_INTERNAL_ERROR) + + if ( ierr==PIO_NOERR ) then + + ierr = pio_inq_dimlen( fid, id, dsize ) + + if ( present(dimid) ) then + dimid = id + endif + + if ( present(data) ) then + if ( associated(data) ) then + deallocate(data, stat=ierr) + if( ierr /= 0 ) then + write(iulog,*) 'get_dimension: data deallocation error = ',ierr + call endrun('get_dimension: failed to deallocate data array') + end if + endif + allocate( data(dsize), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'get_dimension: data allocation error = ',ierr + call endrun('get_dimension: failed to allocate data array') + end if + + ierr = pio_inq_varid( fid, dname, vid ) + ierr = pio_get_var( fid, vid, data ) + endif + else + dsize = 1 + if ( present(dimid) ) then + dimid = -1 + endif + endif + + end subroutine get_dimension + +!----------------------------------------------------------------------- +!----------------------------------------------------------------------- + subroutine set_cycle_indices( fileid, cyc_ndx_beg, cyc_ndx_end, cyc_yr ) + + implicit none + + type(file_desc_t), intent(inout) :: fileid + integer, intent(out) :: cyc_ndx_beg + integer, intent(out) :: cyc_ndx_end + integer, intent(in) :: cyc_yr + + integer, allocatable , dimension(:) :: dates, datesecs + integer :: timesize, i, astat, year, ierr + type(var_desc_T) :: dateid + call get_dimension( fileid, 'time', timesize ) + cyc_ndx_beg=-1 + + allocate( dates(timesize), stat=astat ) + if( astat/= 0 ) then + write(*,*) 'set_cycle_indices: failed to allocate dates array; error = ',astat + call endrun + end if + + ierr = pio_inq_varid( fileid, 'date', dateid ) + ierr = pio_get_var( fileid, dateid, dates ) + + do i=1,timesize + year = dates(i) / 10000 + if ( year == cyc_yr ) then + if (cyc_ndx_beg < 0) then + cyc_ndx_beg = i + endif + cyc_ndx_end = i + endif + enddo + deallocate( dates, stat=astat ) + if( astat/= 0 ) then + write(*,*) 'set_cycle_indices: failed to deallocate dates array; error = ',astat + call endrun + end if + if (cyc_ndx_beg < 0) then + write(*,*) 'set_cycle_indices: cycle year not found : ' , cyc_yr + call endrun('set_cycle_indices: cycle year not found') + endif + + end subroutine set_cycle_indices +!----------------------------------------------------------------------- + +!----------------------------------------------------------------------- + subroutine open_trc_datafile( fname, path, piofile, times, cyc_ndx_beg, cyc_ndx_end, cyc_yr ) + + use ioFileMod, only: getfil + use cam_pio_utils, only: cam_pio_openfile + + implicit none + + character(*), intent(in) :: fname + character(*), intent(in) :: path + type(file_desc_t), intent(inout) :: piofile + real(r8), pointer :: times(:) + + integer, optional, intent(out) :: cyc_ndx_beg + integer, optional, intent(out) :: cyc_ndx_end + integer, optional, intent(in) :: cyc_yr + + character(len=shr_kind_cl) :: filen, filepath + integer :: year, month, day, dsize, i, timesize + integer :: dateid,secid + integer, allocatable , dimension(:) :: dates, datesecs + integer :: astat, ierr + logical :: need_first_ndx + + if (len_trim(path) == 0) then + filepath = trim(fname) + else + filepath = trim(path) // '/' // trim(fname) + end if + ! + ! open file and get fileid + ! + call getfil( filepath, filen, 0 ) + call cam_pio_openfile( piofile, filen, PIO_NOWRITE) + if(masterproc) write(iulog,*)'open_trc_datafile: ',trim(filen) + + call get_dimension(piofile, 'time', timesize) + + if ( associated(times) ) then + deallocate(times, stat=ierr) + if( ierr /= 0 ) then + write(iulog,*) 'open_trc_datafile: data deallocation error = ',ierr + call endrun('open_trc_datafile: failed to deallocate data array') + end if + endif + allocate( times(timesize), stat=ierr ) + if( ierr /= 0 ) then + write(iulog,*) 'open_trc_datafile: data allocation error = ',ierr + call endrun('open_trc_datafile: failed to allocate data array') + end if + + allocate( dates(timesize), stat=astat ) + if( astat/= 0 ) then + if(masterproc) write(iulog,*) 'open_trc_datafile: failed to allocate dates array; error = ',astat + call endrun + end if + allocate( datesecs(timesize), stat=astat ) + if( astat/= 0 ) then + if(masterproc) write(iulog,*) 'open_trc_datafile: failed to allocate datesec array; error = ',astat + call endrun + end if + + ierr = pio_inq_varid( piofile, 'date', dateid ) + call pio_seterrorhandling( piofile, PIO_BCAST_ERROR) + ierr = pio_inq_varid( piofile, 'datesec', secid ) + call pio_seterrorhandling( piofile, PIO_INTERNAL_ERROR) + + if(ierr==PIO_NOERR) then + ierr = pio_get_var( piofile, secid, datesecs ) + else + datesecs=0 + end if + + ierr = pio_get_var( piofile, dateid, dates ) + need_first_ndx=.true. + + do i=1,timesize + year = dates(i) / 10000 + month = mod(dates(i),10000)/100 + day = mod(dates(i),100) + call set_time_float_from_date( times(i), year, month, day, datesecs(i) ) + if ( present(cyc_yr) ) then + if ( year == cyc_yr ) then + if ( present(cyc_ndx_beg) .and. need_first_ndx ) then + cyc_ndx_beg = i + need_first_ndx = .false. + endif + if ( present(cyc_ndx_end) ) then + cyc_ndx_end = i + endif + endif + endif + enddo + + deallocate( dates, stat=astat ) + if( astat/= 0 ) then + if(masterproc) write(iulog,*) 'open_trc_datafile: failed to deallocate dates array; error = ',astat + call endrun + end if + deallocate( datesecs, stat=astat ) + if( astat/= 0 ) then + if(masterproc) write(iulog,*) 'open_trc_datafile: failed to deallocate datesec array; error = ',astat + call endrun + end if + + if ( present(cyc_yr) .and. present(cyc_ndx_beg) ) then + if (cyc_ndx_beg < 0) then + write(iulog,*) 'open_trc_datafile: cycle year not found : ' , cyc_yr + call endrun('open_trc_datafile: cycle year not found') + endif + endif + + end subroutine open_trc_datafile + +!-------------------------------------------------------------------------- +!-------------------------------------------------------------------------- + subroutine specify_fields( specifier, fields ) + + implicit none + + character(len=*), intent(in) :: specifier(:) + type(trfld), pointer, dimension(:) :: fields + + integer :: fld_cnt, astat + integer :: i,j + character(len=shr_kind_cl) :: str1, str2 + character(len=32), allocatable, dimension(:) :: fld_name, src_name + integer :: nflds + + nflds = size(specifier) + + allocate(fld_name(nflds), src_name(nflds), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'specify_fields: failed to allocate fld_name, src_name arrays; error = ',astat + call endrun + end if + + fld_cnt = 0 + + count_cnst: do i = 1, nflds + + if ( len_trim( specifier(i) ) == 0 ) then + exit count_cnst + endif + + j = scan( specifier(i),':') + + if (j > 0) then + str1 = trim(adjustl( specifier(i)(:j-1) )) + str2 = trim(adjustl( specifier(i)(j+1:) )) + fld_name(i) = trim(adjustl( str1 )) + src_name(i) = trim(adjustl( str2 )) + else + fld_name(i) = trim(adjustl( specifier(i) )) + src_name(i) = trim(adjustl( specifier(i) )) + endif + + fld_cnt = fld_cnt + 1 + + enddo count_cnst + + if( fld_cnt < 1 ) then + nullify(fields) + return + end if + + !----------------------------------------------------------------------- + ! ... allocate field type array + !----------------------------------------------------------------------- + allocate( fields(fld_cnt), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'specify_fields: failed to allocate fields array; error = ',astat + call endrun + end if + + do i = 1,fld_cnt + fields(i)%fldnam = fld_name(i) + fields(i)%srcnam = src_name(i) + enddo + + deallocate(fld_name, src_name) + + end subroutine specify_fields + +!------------------------------------------------------------------------------ + + subroutine init_trc_restart( whence, piofile, tr_file ) + + implicit none + character(len=*), intent(in) :: whence + type(file_desc_t), intent(inout) :: piofile + type(trfile), intent(inout) :: tr_file + + character(len=32) :: name + integer :: ioerr, mcdimid, maxlen + + + ! Dimension should already be defined in restart file + call pio_seterrorhandling(pioFile, PIO_BCAST_ERROR) + ioerr = pio_inq_dimid(pioFile,'max_chars', mcdimid) + call pio_seterrorhandling(pioFile, PIO_INTERNAL_ERROR) + ! but define it if nessasary + if(ioerr/= PIO_NOERR) then + ioerr = pio_def_dim(pioFile, 'max_chars', SHR_KIND_CL, mcdimid) + end if + + if(len_trim(tr_file%curr_filename)>1) then + allocate(tr_file%currfnameid) + name = trim(whence)//'_curr_fname' + ioerr = pio_def_var(pioFile, name,pio_char, (/mcdimid/), tr_file%currfnameid) + ioerr = pio_put_att(pioFile, tr_file%currfnameid, 'offset_time', tr_file%offset_time) + maxlen = len_trim(tr_file%curr_filename) + ioerr = pio_put_att(pioFile, tr_file%currfnameid, 'actual_len', maxlen) + else + nullify(tr_file%currfnameid) + end if + + if(len_trim(tr_file%next_filename)>1) then + allocate(tr_file%nextfnameid) + name = trim(whence)//'_next_fname' + ioerr = pio_def_var(pioFile, name,pio_char, (/mcdimid/), tr_file%nextfnameid) + maxlen = len_trim(tr_file%next_filename) + ioerr = pio_put_att(pioFile, tr_file%nextfnameid, 'actual_len', maxlen) + else + nullify(tr_file%nextfnameid) + end if + end subroutine init_trc_restart +!------------------------------------------------------------------------- +! writes file names to restart file +!------------------------------------------------------------------------- + subroutine write_trc_restart( piofile, tr_file ) + + implicit none + + type(file_desc_t), intent(inout) :: piofile + type(trfile), intent(inout) :: tr_file + + integer :: ioerr, slen ! error status + if(associated(tr_file%currfnameid)) then + ioerr = pio_put_var(pioFile, tr_file%currfnameid, tr_file%curr_filename) + deallocate(tr_file%currfnameid) + nullify(tr_file%currfnameid) + end if + if(associated(tr_file%nextfnameid)) then + ioerr = pio_put_var(pioFile, tr_file%nextfnameid, tr_file%next_filename) + deallocate(tr_file%nextfnameid) + nullify(tr_file%nextfnameid) + end if + end subroutine write_trc_restart + +!------------------------------------------------------------------------- +! reads file names from restart file +!------------------------------------------------------------------------- + subroutine read_trc_restart( whence, piofile, tr_file ) + + implicit none + + character(len=*), intent(in) :: whence + type(file_desc_t), intent(inout) :: piofile + type(trfile), intent(inout) :: tr_file + type(var_desc_t) :: vdesc + character(len=64) :: name + integer :: ioerr ! error status + integer :: slen + + call PIO_SetErrorHandling(piofile, PIO_BCAST_ERROR) + name = trim(whence)//'_curr_fname' + ioerr = pio_inq_varid(piofile, name, vdesc) + if(ioerr==PIO_NOERR) then + tr_file%curr_filename=' ' + ioerr = pio_get_att(piofile, vdesc, 'offset_time', tr_file%offset_time) + ioerr = pio_get_att(piofile, vdesc, 'actual_len', slen) + ioerr = pio_get_var(piofile, vdesc, tr_file%curr_filename) + if(slen P2 (decreasing up) +!--- integrates (p-avg) the value F0 from F on the grid P +!--- assumes model pressure increases from top to bottom. +!---NOTE reverse order in P's +!---Assume that the quantity is constant over range halfway to layer above/below +!--- and calculate box edges from model top to model bottom +! +!---For a model level between pressure range P1 > P2 (decreasing up) +!---calculate the SOM Z-moments of the loss freq at std z* (log-p) intervals +!-------- the pressure levels BETWEEN z* values are: +! P(i) < P(i+1) bounds z*(i) +!-------- The MOMENTS for a square-wave or 'bar': F(x)=F0 b<=x<=c, =0.0 else +!----- S0 = f0 (x) [from x=b to x=c] +!----------------------------------------------------------------------- + implicit none + integer, intent(in) :: NL + real(r8), intent(in) :: P1,P2,P(NL+1),F(NL) + real(r8), intent(out):: F0 + integer I + real(r8) XB,XC,PC,PB,SGNF0,PF1,PF2 +!----------------------------------------------------------------------- + F0 = 0._r8 + ! + do I = 1,NL + PF1=P(I) + PF2=P(I+1) + ! + PC = min(P1,PF2) + PB = max(P2,PF1) + ! + if (PC .gt. PB) then +!--- have condition: P1 .ge. PC .gt. PB .ge. P2 +!--- and 0 .le. XB .lt. XC .le. 1 + XC = (PC-P2)/(P1-P2) + XB = (PB-P2)/(P1-P2) +!-------- assume that the quantity, F, is constant over interval [XLO,XUP], +!-------- F0: (c-b), +!-------- calculate its contribution to the moments in the interval [0,1] + F0 = F0 +F(I) *(XC -XB) + endif + enddo +!---limiter on Z-moments: force monotonicity (tables can be + or -) + SGNF0 = sign(1._r8, F0) + F0 = abs(F0) + F0 = SGNF0 * F0 + END SUBROUTINE vert_interp_uci_single +!------------------------------------------------------------------------------ + subroutine vert_interp_ub( ncol, nlevs, plevs, datain, dataout ) + use ref_pres, only : ptop_ref + + + !----------------------------------------------------------------------- + ! + ! Interpolate data from current time-interpolated values to top interface pressure + ! -- from mo_tgcm_ubc.F90 + !-------------------------------------------------------------------------- + implicit none + ! Arguments + ! + integer, intent(in) :: ncol + integer, intent(in) :: nlevs + real(r8), intent(in) :: plevs(nlevs) + real(r8), intent(in) :: datain(ncol,nlevs) + real(r8), intent(out) :: dataout(ncol) + + ! + ! local variables + ! + integer :: i,ku,kl,kk + real(r8) :: pinterp, delp + + pinterp = ptop_ref + + if( pinterp <= plevs(1) ) then + kl = 1 + ku = 1 + delp = 0._r8 + else if( pinterp >= plevs(nlevs) ) then + kl = nlevs + ku = nlevs + delp = 0._r8 + else + + do kk = 2,nlevs + if( pinterp <= plevs(kk) ) then + ku = kk + kl = kk - 1 + delp = log( pinterp/plevs(kk) ) / log( plevs(kk-1)/plevs(kk) ) + exit + end if + end do + + end if + + do i = 1,ncol + dataout(i) = datain(i,kl) + delp * (datain(i,ku) - datain(i,kl)) + end do + + end subroutine vert_interp_ub +!------------------------------------------------------------------------------ + +!------------------------------------------------------------------------------ +!------------------------------------------------------------------------------ + subroutine advance_file(file) + + !------------------------------------------------------------------------------ + ! This routine advances to the next file + !------------------------------------------------------------------------------ + + use shr_sys_mod, only: shr_sys_system + use ioFileMod, only: getfil + + implicit none + + type(trfile), intent(inout) :: file + + !----------------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------------- + character(len=shr_kind_cl) :: ctmp + character(len=shr_kind_cl) :: loc_fname + integer :: istat, astat + + !----------------------------------------------------------------------- + ! close current file ... + !----------------------------------------------------------------------- + call pio_closefile( file%curr_fileid ) + + !----------------------------------------------------------------------- + ! remove if requested + !----------------------------------------------------------------------- + if( file%remove_trc_file ) then + call getfil( file%curr_filename, loc_fname, 0 ) + write(iulog,*) 'advance_file: removing file = ',trim(loc_fname) + ctmp = 'rm -f ' // trim(loc_fname) + write(iulog,*) 'advance_file: fsystem issuing command - ' + write(iulog,*) trim(ctmp) + call shr_sys_system( ctmp, istat ) + end if + + !----------------------------------------------------------------------- + ! Advance the filename and file id + !----------------------------------------------------------------------- + file%curr_filename = file%next_filename + file%curr_fileid = file%next_fileid + + !----------------------------------------------------------------------- + ! Advance the curr_data_times + !----------------------------------------------------------------------- + deallocate( file%curr_data_times, stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'advance_file: failed to deallocate file%curr_data_times array; error = ',astat + call endrun + end if + allocate( file%curr_data_times( size( file%next_data_times ) ), stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'advance_file: failed to allocate file%curr_data_times array; error = ',astat + call endrun + end if + file%curr_data_times(:) = file%next_data_times(:) + + !----------------------------------------------------------------------- + ! delete information about next file (as was just assigned to current) + !----------------------------------------------------------------------- + file%next_filename = '' + + deallocate( file%next_data_times, stat=astat ) + if( astat/= 0 ) then + write(iulog,*) 'advance_file: failed to deallocate file%next_data_times array; error = ',astat + call endrun + end if + nullify( file%next_data_times ) + + end subroutine advance_file + +end module tracer_data From 104dd9408cbfbc53d24037dcb187d93fba106299 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Sun, 25 Aug 2024 23:49:33 -0500 Subject: [PATCH 558/904] Add compsets and support files for v3 single forcing simulations --- cime_config/allactive/config_compsets.xml | 72 ++++++++++ ...R_eam_CMIP6-GHG_chemUCI-Linoz-mam5-vbs.xml | 121 +++++++++++++++++ ..._eam_CMIP6-LULC_chemUCI-Linoz-mam5-vbs.xml | 124 ++++++++++++++++++ ...R_eam_CMIP6-aer_chemUCI-Linoz-mam5-vbs.xml | 119 +++++++++++++++++ ...R_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml | 122 +++++++++++++++++ ...eam_CMIP6-ozone_chemUCI-Linoz-mam5-vbs.xml | 121 +++++++++++++++++ ..._eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml | 123 +++++++++++++++++ ...CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml | 120 +++++++++++++++++ .../eam/cime_config/config_component.xml | 9 ++ .../use_cases/20thC_CMIP6xLULC_transient.xml | 48 +++++++ .../elm/cime_config/config_component.xml | 9 +- .../cime_config/config_component_e3sm.xml | 6 + 12 files changed, 993 insertions(+), 1 deletion(-) create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-GHG_chemUCI-Linoz-mam5-vbs.xml create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-LULC_chemUCI-Linoz-mam5-vbs.xml create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-aer_chemUCI-Linoz-mam5-vbs.xml create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-ozone_chemUCI-Linoz-mam5-vbs.xml create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml create mode 100644 components/elm/bld/namelist_files/use_cases/20thC_CMIP6xLULC_transient.xml diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index 21032c483714..63883bff15d9 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -86,6 +86,78 @@ 20TRSOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + WCYCL20TR-GHG + 20TRSOI_EAM%CMIP6-GHG_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + hist-GHG + 20TRSOI_EAM%CMIP6-GHG_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + WCYCL20TR-aer + 20TRSOI_EAM%CMIP6-AER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + hist-aer + 20TRSOI_EAM%CMIP6-AER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + WCYCL20TR-xGHG-xaer + 20TRSOI_EAM%CMIP6-xGHG-xAER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + hist-all-xGHG-xaer + 20TRSOI_EAM%CMIP6-xGHG-xAER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + WCYCL20TR-nat + 20TRSOI_EAM%CMIP6-NAT_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + hist-nat + 20TRSOI_EAM%CMIP6-NAT_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + WCYCL20TR-ozone + 20TRSOI_EAM%CMIP6-OZONE_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + hist-ozone + 20TRSOI_EAM%CMIP6-OZONE_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + WCYCL20TR-lulc + 20TRSOI_EAM%CMIP6-LULC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + hist-lulc + 20TRSOI_EAM%CMIP6-LULC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + WCYCL20TR-volc + 20TRSOI_EAM%CMIP6-VOLC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + hist-volc + 20TRSOI_EAM%CMIP6-VOLC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-GHG_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-GHG_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..cd16d6a0d9a6 --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-GHG_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,121 @@ + + + + +.true. + + +.true. +.true. +.true. + + +atm/cam/solar/Solar_1850control_input4MIPS_c20181106.nc +18500101 +FIXED + + +atm/cam/ggas/GHG_CMIP-1-2-0_Annual_Global_0000-2014_c20180105.nc +RAMPED + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc + +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_1850_2014_avg_so2_elev_strat_1850.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160416.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +CYCLICAL +1849 +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +18500101 +FIXED +1850 +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +CYCLICAL + + +SERIAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-LULC_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-LULC_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..e2f53df4dc60 --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-LULC_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,124 @@ + + + + +.true. + + +.true. +.true. +.true. + + +atm/cam/solar/Solar_1850control_input4MIPS_c20181106.nc +18500101 +FIXED + + + +808.249e-9 +273.0211e-9 +32.1102e-12 +0.0 + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc + +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_1850_2014_avg_so2_elev_strat_1850.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160416.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +CYCLICAL +1849 +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +18500101 +FIXED +1850 +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +CYCLICAL + + +SERIAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-aer_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-aer_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..8aedc762728b --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-aer_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,119 @@ + + + + +.true. + + +.true. +.true. +.true. + + +atm/cam/solar/Solar_1850control_input4MIPS_c20181106.nc +18500101 +FIXED + + + +808.249e-9 +273.0211e-9 +32.1102e-12 +0.0 + + +INTERP_MISSING_MONTHS +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1850-2014_c180205_hist-aero_1850-volcano.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +INTERP_MISSING_MONTHS +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +INTERP_MISSING_MONTHS +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +18500101 +FIXED +1850 +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +CYCLICAL + + +SERIAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..8f3a2644527d --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,122 @@ + + + + +.true. + + +.true. +.true. +.true. + + +atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc +SERIAL + + + +808.249e-9 +273.0211e-9 +32.1102e-12 +0.0 + + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1850-2014_c180205_1850-aero_hist-volcano.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160416.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +CYCLICAL +1849 +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +18500101 +FIXED +1850 +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +INTERP_MISSING_MONTHS + + +SERIAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-ozone_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-ozone_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..58f69d8fe97f --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-ozone_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,121 @@ + + + + +.true. + + +.true. +.true. +.true. + + +atm/cam/solar/Solar_1850control_input4MIPS_c20181106.nc +18500101 +FIXED + + + +808.249e-9 +273.0211e-9 +32.1102e-12 +0.0 + + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_1850_2014_avg_so2_elev_strat_1850.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160416.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +CYCLICAL +1849 +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +SERIAL +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +INTERP_MISSING_MONTHS + + +SERIAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..6f0754423592 --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,123 @@ + + + + +.true. + + +.true. +.true. +.true. + + +atm/cam/solar/Solar_1850control_input4MIPS_c20181106.nc +18500101 +FIXED + + + +808.249e-9 +273.0211e-9 +32.1102e-12 +0.0 + + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1850-2014_c180205_1850-aero_hist-volcano.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160416.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +CYCLICAL +1849 +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +18500101 +FIXED +1850 +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +CYCLICAL + + +SERIAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..f70010a82c44 --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,120 @@ + + + + +.true. + + +.true. +.true. +.true. + + +atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc +SERIAL + + + +808.249e-9 +273.0211e-9 +32.1102e-12 +0.0 + + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1850-2014_c180205_1850-aero_hist-volcano.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160416.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +CYCLICAL +1849 +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +SERIAL +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +INTERP_MISSING_MONTHS + + +SERIAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 1a961b84affe..8f1e68161cb7 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -53,6 +53,7 @@ -phys default &eamv3_phys_defaults; &eamv3_chem_defaults; &eamv3_phys_defaults; &eamv3_chem_defaults; + &eamv3_phys_defaults; &eamv3_chem_defaults; &eamv3_phys_defaults; &eamv3_chem_defaults; -bc_dep_to_snow_updates -co2_cycle @@ -125,6 +126,14 @@ 1850_E3SMv1_superfast_ar5-emis 1850S_E3SMv1_superfast_ar5-emis 20TR_eam_CMIP6_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6-GHG_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6-aer_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6-ozone_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6-lulc_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs 20TR_eam_CMIP6_chemUCI-Linoz-mam5-vbs SSP585_eam_CMIP6_chemUCI-Linoz-mam5-vbs SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs diff --git a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6xLULC_transient.xml b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6xLULC_transient.xml new file mode 100644 index 000000000000..7a6247bbe27f --- /dev/null +++ b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6xLULC_transient.xml @@ -0,0 +1,48 @@ + + + + +Historical single forcing simulation with land held at 1850 condition +Simulate transient land-use, aerosol deposition, and Nitrogen deposition changes from 1850 to 2005 +Simulate transient land-use, aerosol deposition, and Nitrogen deposition changes from 1850 to 2005 +Simulate transient land-use, aerosol deposition, and Nitrogen deposition changes from 1850 to 2005 + +1850 + +1850-2000 + +arb_ic + + +1850 +1850 +1850 + + +2000 +2000 +2000 + +1850 +1850 +1850 + + + + +lnd/clm2/surfdata_map/surfdata_ne30np4_simyr1850_2015_c171018.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne30np4_hist_simyr1850_2015_c20171018.nc +lnd/clm2/initdata_map/20180316.DECKv1b_A1.ne30_oEC.edison.clm2.r.1980-01-01-00000.8575c3f_c20190904.nc + +lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1850_c190730.nc + +lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c200609_with_TOP.nc + + +lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc + +.false. +.false. + + + diff --git a/components/elm/cime_config/config_component.xml b/components/elm/cime_config/config_component.xml index da060584cfb8..b191a0a2248d 100755 --- a/components/elm/cime_config/config_component.xml +++ b/components/elm/cime_config/config_component.xml @@ -72,7 +72,14 @@ 1850_CMIP6bgc_control 1850_SCMIP6_control 1850_SCMIP6_control - 20thC_CMIP6_transient + 20thC_CMIP6_transient + 20thC_CMIP6xLULC_transient + 20thC_CMIP6xLULC_transient + 20thC_CMIP6xLULC_transient + 20thC_CMIP6xLULC_transient + 20thC_CMIP6xLULC_transient + 20thC_CMIP6_transient + 20thC_CMIP6_transient 20thC_CMIP6bgc_transient 20thC_CMIP6bgc_transient 20thC_bgc_transient diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index 08758f935313..acdca60ae2a6 100755 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -753,6 +753,12 @@ 312.821 388.717 388.717 + 284.317 + 284.317 + 284.317 + 284.317 + 284.317 + 284.317 0.000001 0.000001 284.317 From 82fd0061a9b32bdf209b4a173ecb47d3f7d09587 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:00:22 +0000 Subject: [PATCH 559/904] Bump tj-actions/changed-files from 44 to 45 Bumps [tj-actions/changed-files](https://github.com/tj-actions/changed-files) from 44 to 45. - [Release notes](https://github.com/tj-actions/changed-files/releases) - [Changelog](https://github.com/tj-actions/changed-files/blob/main/HISTORY.md) - [Commits](https://github.com/tj-actions/changed-files/compare/v44...v45) --- updated-dependencies: - dependency-name: tj-actions/changed-files dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-md-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 8ee8b7a7f13c..507c2c3745ea 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: tj-actions/changed-files@v44 + - uses: tj-actions/changed-files@v45 id: changed-files with: files: '**/*.md' From 624d9ac5757e18db32477702dba04f442e24d247 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 26 Aug 2024 06:07:59 -0500 Subject: [PATCH 560/904] Revert changes to .gitignore --- .gitignore | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.gitignore b/.gitignore index d2fcc80bbb71..d7a1fabfd6bc 100644 --- a/.gitignore +++ b/.gitignore @@ -51,16 +51,3 @@ components/eamxx/docs/common/eamxx_params.md components/eamxx/src/python/build components/eamxx/src/python/build_src components/eamxx/src/python/dist - -# OS generated files # -###################### -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -ehthumbs.db -Thumbs.db - -# idea folder -.idea/ From 4ee10629129d1fa52b0f525b9d079807713c3eae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:41:45 +0000 Subject: [PATCH 561/904] Bump tj-actions/changed-files from 44 to 45 Bumps [tj-actions/changed-files](https://github.com/tj-actions/changed-files) from 44 to 45. - [Release notes](https://github.com/tj-actions/changed-files/releases) - [Changelog](https://github.com/tj-actions/changed-files/blob/main/HISTORY.md) - [Commits](https://github.com/tj-actions/changed-files/compare/v44...v45) --- updated-dependencies: - dependency-name: tj-actions/changed-files dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-md-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 8ee8b7a7f13c..507c2c3745ea 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: tj-actions/changed-files@v44 + - uses: tj-actions/changed-files@v45 id: changed-files with: files: '**/*.md' From 9ed3ead2038dd963a242243a018f49895a52f60f Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 26 Aug 2024 14:13:17 -0400 Subject: [PATCH 562/904] clarify comment for cmake option --- components/eamxx/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index a60fc1e342fe..5715e5be1745 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -198,7 +198,7 @@ set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C inst set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) -set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for all components") +set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them") set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) From a7ec7e67c6793b500dc83106995bf6038ae90e96 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 26 Aug 2024 12:28:59 -0600 Subject: [PATCH 563/904] Update CIME submodule ... to af3eab5b8a213e79923551e7f9d9f56edb944c0f Changes: 1) Generalize _hists_match for mom6 files 2) support append in config_pes.xml overrides field 3) Add case git: Adds a case interface to git. It creates a local git repository when case.setup is run and updates that repository with each action that triggers an update of the CaseStatus file. Optionally a remote repository can be attached by setting CASE_GIT_REPOSITORY to the name of the remote repository. 4) xmlchange: Add documentation for setting values with commas 5) Remove mct for cesm 6) remove load_balancing_tool, broken and unsupported Fixes 1) Fix/rest n in tests: move the computation of REST_N into the python 2) cprnc: need to make sure the directory exists before linking to it 3) check_lockedfiles call in xmlchange needs quiet=True 4) fixes detection of pfunit_path by making sure it exists 5) Fixes handling cprnc output 6) Fixes creating new environment with specific python version 7) Fixes `query_config` tool. - Adds longname to `--grids` - Fixes duplicate and empty choices 8) Fixes check_lockedfiles call in xmlchange 9) Fix list_e3sm_tests tool [BFB] --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index f903115718eb..af3eab5b8a21 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit f903115718ebc30669ce557f511abaef231a1d88 +Subproject commit af3eab5b8a213e79923551e7f9d9f56edb944c0f From aa08e0c2b115604f79f6b87ae321689504caaa33 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 26 Aug 2024 13:10:55 -0600 Subject: [PATCH 564/904] Disable git interface --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index af3eab5b8a21..eacb1b7f5ed9 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit af3eab5b8a213e79923551e7f9d9f56edb944c0f +Subproject commit eacb1b7f5ed984f0c124076c78fd1e80dbd9b7a1 From 24d9107e254be2cb41421980d31ca34e368b30a4 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Wed, 7 Feb 2024 06:50:31 -0700 Subject: [PATCH 565/904] Add relativeVorticityAtSurface to highFrequencyStats --- components/mpas-ocean/cime_config/buildnml | 1 + .../src/analysis_members/Registry_high_frequency_output.xml | 5 ++++- .../src/analysis_members/mpas_ocn_high_frequency_output.F | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 4c7ed2c0a311..9ee046ae552d 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -922,6 +922,7 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') diff --git a/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml b/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml index 5ab40af19234..67f0dcf60c49 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml @@ -27,6 +27,9 @@ + @@ -304,12 +307,12 @@ + - diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F index 60a5dececcb3..60e7f47fa33a 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F @@ -183,7 +183,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ integer, dimension(:,:), pointer :: edgesOnCell, cellsOnEdge, edgesOnEdge real (kind=RKIND) :: invAreaCell1, layerThicknessEdge1, coeff, weightedNormalVel, cellArea - real (kind=RKIND), dimension(:), pointer :: refBottomDepth, kineticEnergyAt250m, kineticEnergyAtSurface, relativeVorticityAt250m + real (kind=RKIND), dimension(:), pointer :: refBottomDepth, kineticEnergyAt250m, kineticEnergyAtSurface, relativeVorticityAt250m, relativeVorticityAtSurface real (kind=RKIND), dimension(:), pointer :: divergenceAt250m, relativeVorticityVertexAt250m real (kind=RKIND), dimension(:), pointer :: divergenceAtBottom,relativeVorticityAtBottom,kineticEnergyAtBottom real (kind=RKIND), dimension(:), pointer :: vertVelAt250m @@ -263,6 +263,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(highFrequencyOutputAMPool, 'kineticEnergyAt250m', kineticEnergyAt250m) call mpas_pool_get_array(highFrequencyOutputAMPool, 'kineticEnergyAtSurface', kineticEnergyAtSurface) call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAt250m', relativeVorticityAt250m) + call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAtSurface', relativeVorticityAtSurface) call mpas_pool_get_array(highFrequencyOutputAMPool, 'divergenceAt250m', divergenceAt250m) call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAtBottom', relativeVorticityAtBottom) call mpas_pool_get_array(highFrequencyOutputAMPool, 'divergenceAtBottom', divergenceAtBottom) @@ -418,6 +419,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ divergenceAt250m(:) = divergence(iLevel0250,:) relativeVorticityVertexAt250m(:) = relativeVorticity(iLevel0250,:) kineticEnergyAtSurface(:) = kineticEnergyCell(1,:) + relativeVorticityAtSurface(:) = relativeVorticityCell(1,:) activeTracersAtSurface(1,:) = activeTracers(1,1,:) activeTracersAtSurface(2,:) = activeTracers(2,1,:) activeTracersAt250m(1,:) = activeTracers(1,iLevel0250,:) From 8d324b21cd3bb90b1f8d28c5631e1a1b02788d42 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Mon, 26 Aug 2024 19:04:55 -0400 Subject: [PATCH 566/904] Add libfabric/1.15.2.0 module to frontier-scream-gpu. --- cime_config/machines/config_machines.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index d0e015349b3a..889fc03834e1 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1436,11 +1436,10 @@ craype-accel-amd-gfx90a rocm/5.4.0 libunwind/1.6.2 - - cce/15.0.1 - craype craype/2.7.20 - cray-mpich cray-mpich/8.1.26 + libfabric/1.15.2.0 + craype/2.7.20 + cray-mpich/8.1.26 cray-python/3.9.13.1 subversion/1.14.1 git/2.36.1 From dabae5c6e1d2c36dc32a75bbbae33f709a55d0f1 Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Mon, 13 Nov 2023 11:59:09 -0500 Subject: [PATCH 567/904] 3 initial tests added into 'e3sm_lnd_developer' for user-defined PFT codes, but NOT YET fully work. Two tests are actually similary as 'elm_usrdat' test, but with 42_FLUXNETSITES, so those 2 shall produce same outputs. The 3rd one, usrpft_arctic_I1850CNPRDCTCBC, is for testing an actual Arctic Site (See Sulman et al. 2021), however which would not produce same results as the paper because ELM developments in arctic PFT physiology not yet included. NOTE: there are a few needed inputs are included in each tests, which if not in $DIN_LOC_ROOT user can copy them into to allow it work. --- cime_config/tests.py | 3 ++- .../clm_params_c180524-sub12_updated20240201.nc | Bin 0 -> 82964 bytes .../shell_commands | 11 +++++++++++ .../usrpft_arctic_I1850CNPRDCTCBC/user_nl_elm | 7 +++++++ .../clm_params_c211124vpft.nc | Bin 0 -> 93128 bytes .../shell_commands | 10 ++++++++++ .../usrpft_codetest_I1850CNPRDCTCBC/user_nl_elm | 10 ++++++++++ .../shell_commands | 11 +++++++++++ .../usrpft_default_I1850CNPRDCTCBC/user_nl_elm | 2 ++ 9 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_arctic_I1850CNPRDCTCBC/clm_params_c180524-sub12_updated20240201.nc create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_arctic_I1850CNPRDCTCBC/shell_commands create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_arctic_I1850CNPRDCTCBC/user_nl_elm create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_codetest_I1850CNPRDCTCBC/clm_params_c211124vpft.nc create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_codetest_I1850CNPRDCTCBC/shell_commands create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_codetest_I1850CNPRDCTCBC/user_nl_elm create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_default_I1850CNPRDCTCBC/shell_commands create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_default_I1850CNPRDCTCBC/user_nl_elm diff --git a/cime_config/tests.py b/cime_config/tests.py index 556f80ec9702..d6512d88ede1 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -76,7 +76,6 @@ ) }, - "e3sm_land_developer" : { "share" : True, "time" : "0:45:00", @@ -95,6 +94,8 @@ "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", "ERS.r05_r05.IELM.elm-lnd_rof_2way", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", "ERS.r05_r05.IELM.elm-V2_ELM_MOSART_features", "ERS.ELM_USRDAT.IELM.elm-surface_water_dynamics" ) diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_arctic_I1850CNPRDCTCBC/clm_params_c180524-sub12_updated20240201.nc b/components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_arctic_I1850CNPRDCTCBC/clm_params_c180524-sub12_updated20240201.nc new file mode 100644 index 0000000000000000000000000000000000000000..2ec124ef77968589cf6c89b76f65993234444ed6 GIT binary patch literal 82964 zcmeHw3!EH9o&Ro&%GlK8Gk9dj#r)NavQ1lD}ii+_6{_4@w)7vxMvpuun zK4*WDt?r(#s&7^O>Q}$|)vtavuyJ#3jkuZ+)ZzckR3uAp=HmanOg0@&jQV4GVl+EO zZ)<9r@xLLa?;lMk!t{J9{?E=Nqp@TM2jbC0G8%5tqM1+xh_eBdRLdr{psoQ>AQ;oN z1Hi2%^aI-9Wt%p&_NUQoA#LmSRcNfRo@t`$98B)lI@f8)Oy$?v)2pTR)^Jh_hzOan zbS_AZoN3yX(?yJKiqYNKyLt^(K4K(7dox;VYa;0n1VdWu4&2B0`qO&IAC4z9P^3)F z-_e!`neQj3+0|vF8Ax?$tr?2`3jBZN`VMnU%0)oriB3+aBcG5JT$s{q=|H8%{iy&d zGvg0+_H?Z7T4jYtm9$Y05zTdV^>(c7?Osdp)>4rPUO@_LJ9;{LCb)O$s%$Sk-@N@(mGco*WS)`y{maaj4Y9OI+F|fv&ocy zG?okoV)4K@=s1}6$J0o@v!l%}eZn<4S969M`qi4=jy1iVyjjf|BJJ_?dj6MkXj`qf z_E0IaAwRg4p2@5??pAad#i#PB-;D~8FOW?4nhw+iQUU8orq}bo^qN}M7ODMPqrxQq z)xF)5D$Inxn&>@aU)8&6bpd^$cFMv&<(lj0=w02@+qJGRcXVWsaxR{tuFR>~Ov!B> zE4vDbt&qqp#6?+vxNNha-eOBGD=;LO+TBeCtL(~9Mh0erBFw9MSM`*YWFcP8nfi01 zn%1LX(CWg_Wl1ihMbgP+)*nkIM%TymK!o3!QUP~b)3LU*lk?is8RzVcdInrI#f7nCL`!k;EDNa&5pW@m+T+NZM96t@Jq5hkklnfOod?b^OA6bocBdPYOL@6U_qVMvteXfOxK z8wAV+MvWv!qykL5eUWia)KjTE>+$_aP0fjMgr|pfh`ygsWe zjXn6Lr@L|CduyGZ8rDbCx}Nb5=&?*R$NRS(&gSjWL^Pg@V`>w_+$O8VlZotDY(gV; zX7totnyAucS^h})XYCEW)av+;w^2PENG8LgD-^VU1AcR&+hRQKtic_Fex9x-{_>fh zn7~{4711H_Cw}=(h>(eZ{@S(C62;3!>al1!vGOOUJgl@bHK_qY`y`zN%AmK9g$pq}A9gUts$+Z@cr7d0B zrB@+(eC~c=t4~Ct<8n->6fG_VjkHp_T%Csjbx_^6@|?WX+o^+^N=$XX>8`gwEuuCx zG-G(cWv)BoC&hT*8NonWZ|97u9im5yG;Cb(MjR(T;-%gUp2+h9b8x{Er$T#F_K}*J zKaC@-<0?bk;Bs3(%OPFo(t((tWVKW}IhsgjpiT5sUHKzYBka%5J*rt}OBK({ubb>#4Hc284~6uOzc$oVU4Zv!l28o_xc(_wPhp zYuV~MXzok3WZzb8SPKZ|&?O+z(_|?Ii9Ug;yA}zApxe|iqG%k8n5FY{ocf3=%X9d; z0sJAF2&E{ZtFoBpT-LJCd}(xuL$DnKn|NN@K<|*oKXQrlFIVI1#z@>B&Y(qc?+T}f zI$yatiiL`_9;1aO2v_MW=G55MI6AH9U`{@EARs0Kb~3aZ-Zx9}PIvw5`9)MmK_YJ4 z;BwX0;#2wqUt1_18ceMi|9lJJNxdHiTyQD@5oUkRY7$5~3FK8eX z+8hh>5yNd69!8c85h{9!Q?u7HaNaRkyc~1S!-+BJxXuU;E#;7+(GPtg2dZ$l_~vHSYuB@lnCxSi^A z_GQHPsX|43C!BK3`0EjT-UOitiwcVUx$C1P=$B(HLBnfI-O$i#VVx8U(4k}?4f-ZL z+JHu*KfNVnnn=)Pxn7u$ay4QxDl-;H@5jBFE~X-jIq{tpsbm`el!_f0N5fbIr`#c* zpj+}ac0OS8269^t8p!K1@l+DhauSGtod+i{xT4 zG(r+~1?Ui=STZ9NWPs$h9LMl&BXqPeITW~R4^d?P*V-SP+M^q5@i8nks4;-pJ(kx| z0T?<68aH7=OHHCWZ!E*H4H6t}OFCGG3l6zya*So^*K?ZAaA@|0j`*kIRxow9N zR1OT^WL88YhpTenJ9U*ySB#HaCI%Kt>S&ly6ureX5~q>}(20r=nY*&p)(in;V>lCo z?kW()y{odB=*wER`uMB5+D8Q+wWGcEU1RBFZgflw1ro{B1SE@e@;W_4+zU#NfCdpK zn(5WT!`u4X!`u3|QXRT2%MpCb`B;)-c`r)^_!wEaLGloIT$ZDDK0dF@rgCFp5zlLz zsQ7m6p?-6VEd@w(xPd?@;ZFv+{HN0JvhH>ot=P&jc~FCyTvRk-ArHb_u`Jo6w)PnC zsx_c>xOauM0(yn!?UJ)rYz?S(MhEb7x(MI~m#ear=&LGz1?XbNA>gT}!7s{1QFqvQ zDa0QuA8{Ic+qpg}8i#IlvOX%$6Nj+8k;GCApOCtw@fjxosS(V6zl|I*6TdmFY1!51 zC6u1D`jk^fZZ<&6#a-N#A=e%V@g+gG{BGtD;gsu}L>sC-OZLMO>zgy;T}Z`Vr*zgX z((Ng60mA$N%xJ-|>3f&;I`as=ZIZ@Cn3+*jcbH4iu+uzXm1!8Y%mYju$Cw4s zT$X9pe3YRfJ(i3mqI~7vRU4EjPovToAYD5;I-AP|X&uw(5KNpyI?ZVU>DH~JD8%I! zcM9LuM<$|rObjxfaXZ)twWxBDR#oWqvSSeYqZtkQksPF}iTp&SjB*^qw=?z$`wp*V zQ1R{RUH#^=e(O4b%OtyRS%qhnM#=tZq76G7N_O9}%IJ($klkzR4gqgH#6sBh;;ww` zR(|dz7y5>kL6Z=ENfvB|5V%|ekpKjdO(~I)FuHA<6Zp1~t0Thk@d}i!R!nzo`|G)i zWg7N2dJ2Z{R2r-jaq-D5?)o8MZHQ#jwC;slcl1ob*?iJx!w?GX5GHZV zy{$@Z{SjcBtz)#K^&lY*6ksL2kkc4yr5~$3g&pRLHo)#K4V^n|IQSxmfr0B=!C!*h zPRp%cy4dCs?ORw z2^(vMv*}zYyE_#wtF0SSzyq3u@@$%Py2D&g=TWCy+i;9<$0AsxnVxm51zfT~cBJ^q zFK4->m&Oj@ot5$LCly^K#*qs$G0aVg5A6?uMZK2bg%a3o1D21~ip*G)S{8YlY#X&) zJQ>plc632xZO2X+l!TXR5Gqwz#wLghUZq-DoXFWhX* z$tB?;<5&)=S*SeavKHZRiGTJGzQuY4tbW7-jJe&5;-?JcW#xHJbHrw`UP0NJ)Mk0E zXPz9oEyEFDk$i)XY_7^+lIg5va9VRYm5W2N`sK=tORP=vfwevu3?@@tT<}aUs;qXj zqJDFWEk)P@TqxqLtKyfGjro`2kF_41XrXr}E%%rvgXV$Im>3q*rn;u_relCt$G5Hs zoUY1Mf<9FI3e@?W4nUd}3FT;_s?sf4-z9K4%~P*4RkcyQy$0K$fd_LwwNTEdf}=?t zd-#&r;gwEfFu{Znw&_@^n1F7Wv^?3d7K~4EEJ0$IF?~oI^{{J}SF}sIn{xCxtlbdWrI)xkmu7F@*(|aA!vc z?#(nZl{uJGpI^NZlSP^e(4vl9sOCL7qbIo43t(sy&{om!VaztL0-WTM4EiJt)#5^g zIZ{*(vdaqiL)W=2M+u&<=%)f>(Y#~8Qy(DKS4N*Jfz4_Bzm&^Sra{U5 z_|N4$mvw#$^|K}_R(^S`px+gxA5+=v>USQc3F|V`(pbUV%q&N`#d?U^AQD#UBzHJV z(D{rpt^&86OU&c?B-ml`=_NA8b{<$!H-0MbYxN_)8E-Y$Yb?mWm)n09BOn^R-88gy0A{0M-85B+UN7e2l$Oi7I(3Nhg<503et^V+g8Feo zdy~d8(?rj%j)qNLs)(FN)>X8bK~oBUZiCUB8jB?{R0J?Iq*1$wtS3Ta6Y)S9w$-(# z9zhw2SI1sDyqXuUOkO>IKFU!Si^DLDTFraARghQDr><5%ng+R(_Q^XoCtiK_Wn9{a zu%Agq1;dagRy=#RePRAFd_FUhsK~lp1?zJ`2XHlTwVSZ;P-#zfE~et!^-uMiOa8Io z5U@20_Q4gSD*jZl@o{NLw3{8iQ+RLT?4e8>|Jns7^7R;?9K2#x@$GP`-(1$?X;frY zk0d~-_@3%CqA0yn^KyJEhBn0j({8?nMt822Agv&m>umiYfjEf1Xxvq=KnI;*2k{sn=AHgxV z^T~5EO@^La^b?*&cwO7=(fQ&D`l-gekJxS54k20{->=M5@XVh|tv{YK026gsc|hK! zeVRhs6OLlKPfqYe{%&zoiRa4kNf{uT+s2KIYscSDGmS&$NtSv2YYKC$Glqb7Ar{#D zgIn^`7rDl2#%)JaR2KEkBzk%|uI<}5Zp1g%dB&TuorL=y*xfeVrqPZx=uFha1k)Hi z!{{c&{nU(-BBdYdRhp9Y!cS53dL4f!;8QOHF0N1}9n}9q33+ z?U{!W){}lkm@s(9N_$#M$ZCxot60o)iSaBzEiZ4vHKApz?vxiBKTVQ91 zoTKpMNl0AFEN6}2TkPlLldx)IzG$=BYl+n*YENyDwq)+i!PRRhsp*O9C%D5iw{?CL z*lFIINFKnQcN(6idG8_(Sm(m+Y9^VmT_2i??%$35RqLd69*!XpE4tDHI&x`4QOp+W zOuk+bJ{efyRLB_2R4$Ur;7(H;XJ@1N1M2S z%YJk=1MYchoS!D$ZJW~Fc6M;(tqXPSBsppt^tERnM%~nLr>eNY<&{S^@Uf@3UFXGo zZ#Nlqa#@R3(_y(qhU8{-o513-eHYW$+%72=2-3aRGB}M*+9bOqWE_Q$X_KznE%p(C z9V=_oV+VMOY1ct{J}qh&9|hL>_)F5BXNgAjNc$kBOFrQ;lxO*5$wA3M$H(oOgr?*-lLo=XBbMh7q-b&Fv5xChPXR?_nW=b$^9TO7| zyE&D^Ja#yl^E8iLdRVmQwCVh8c9Re}IfSq0?%Cddwe%=JB5Ju~wOt>Wie6#v z=urD8=Tie*SaDm16I2GN|4YdY$dxFAS^qa=GP0PY|D#I7G3=UJdKgOfS}?}RgYxWiQC~juRuPqcf{lt`(a?O zgPRJuWy)Q>B96MO_3B6`N=MCQ&3@XFxRraJ&u<}^ym1Pdxh#)Hxa#8L@E1dO&U8|0 zJz$n}-)Pqcm#T6I4^t$7a<`4p{2RV%4BH*y#LT=k!m&J3=dO*Q472toUt&J0Yqdkb zMQhm8<$tl&*`Cwte$laxrLI~#fwG)p8p8+q&SGcUq4-X|@29f2c<$)nRL1f1X)?h0Urp5Dk}GI z%VJ@FQ$e}sg89J9&ol`BZ#>hW_JS1fgEwdG>`aN@+0rN8tIImpl=8Q@0{mIqmA5B# zaJT4|S6pxmV7R_`ntW5(>WfX@DO`?=7j^(Q{J3#iC-VR^Zzx*#C3u6AjB+9I25CDr zJ`s~fsaW7nXQ^W>S5{MQ@gD+a=_5&X)w0WA$HUA%k}B9&FFXOfe2!wBD_H8jk~Rwe zP++4#76Z|8trYrGX^WKtw_8{}$x1=kBj9t3odQ4CE+d35yZY|pK8B#Bv!C~PwNbce z2=&`Q{_(I>iCcFxQo{b;X)oF(Zv5j(EG}vNqJzLakBptMr6=MKVy_8a;PTpMon-S% zcOp;n=}KAVclWmNS?K_`Dj%d{HeqiBcaoUr!M9RY8q^a$x8*yIZ`HV4QF|lPxLZMc zql+2viv1@B>nXb}?D~S-e`01kllPwxLE~~+w-+DAx3fvjX7m8>bi;Z501~FOZO$p% zV^Xsja~|OrmXMen!iq&&t+}<6pWBn6y}?>|0>_@ggpGe2FgCFAEemeTy^OHcb3e%B zanWU&`Iq94Rkm>ID+6NwYM_O?20MR}N`fxB3EE}5E*k+%zQ)Z@4iuQZ@`Ebc|13KS zh~%eqx_rgE@l(p#15p{Uxvcj~G<@42cTD28px;+xPPK~UCf*Z@?)S9a?rF7GJc;wF zOAex*W=GTM=qQ%N!&Ga9_2y+SHT(qa@*8Pu#vSG1cF(FE4?A1h*=$v?4aX#uGSYTw zQ)A^LRvRC;baxzi)?bGOHR-U_YhPFK?aHoxbBV2)0h^jjq;Rj|d(wBxan26kDW-9~ z%=P`|!@wu%F@QVoSXEi=aH`*2)*~O_Xnj93O>;>b`$N-A(lt+9K7x8Ae-Cndpu6_> zAY;4O%WYVI(5kZA`agwjh{4%_+I#~>+w#(knRnveAaB=&koyi_52LWBpsR;3EY1Q z79~FryswPv32|JU*r6w!d2<|Y@gKz3>O2XO0bJqdIfy^y@<29BnhlfU!+!o>iSr~{ zj{{;|HUM7%0;{=fGx5r)j>;|DdhQ>9mb8WWMsGS0#?j1V#Q}Y?ICPv(QKfYUv3^qO z!{J0koo`Pv$4E_W8$)<1_FZk?Iyk(0=Vt$q>1=VrPG^@9jnTdB`j3hJ?dZHi9j`)g zhcwKcah{Jl%|{+Jo`6pgoP(1~gRW?+O`-i!elHdnq7>qz$<)krvLbElge`B&PBUxlwo8n{tvR@^L5^YW=)9T#oD9aI`*PTq z$@K`TK6|J>v7gBw6K9!vtZA_#fR^% zOt(Kc%6q4GI&@lpK8=njq;^KX^rt=&ntG&S&rxpn8(r1bLoMacA%5r zdOBo|j+V&4RUqaTpiA(5v^dTXqHATWjs(gk4C}V-jbkGsEYL)y8>AQJmE2Iu%Hc zMbSbE80SGA+_-6A$MzvuQm?`bT;*c5Yx8EaEQpk$tAdO1Ae|^1jmL3=DKe)3R@jd7gcG{EDe`)@;IqGRvshw8sHXVft<=^kb}U~1f>rh&+boEhTl8UX3r4ZjzIbV zd*~Eq-LX)P++k>!ZQKa8AO=!wSi`|{&3c5-k0T;L-Bkf`>HgS>&&pb{J^#o3u4PlfvsaL*xGB{a5*xCtK1 zripU#o=Ngbo^>g%MOhp5c^mGVa9%_*-9;OV`ET@xD;*B^Y5mksIF2$f{ z5j0er-w!Niek_+j%Esr+k7*^uEw7u*kBUEXFF{5ksvNR+)Hr= zEtpa=Bc@wiTu`DU4Q8tDzK(msM4b0zVxs&NrenCL`JOy7&@(QlK777m&*yHw_b~7+ zNbBL`I1Uu@tBDO>;__ODrTXoLT{K0ZLyVefpT=?;yZGcC+7{C!nwlBDRpDxh%c{hb zAnKC8eCa4h)$?)ST@1P0ABbi7tPF>w2&IOSFua$iw!|4&>(o4(>t3j2fJn=7p}lA7ceF)9E-x(2#lwS@tbcCR|B|LtflzCTAT4(oSZHW zP%w6J(V5**l554*b*fK=dp_Nh9LPN!l;3B{t=sdI&Z>lG5ru6Lu+^d2D>lz|SkoTb zlZ-(f07(M|ocUwu!#XT7NOY6O42mkw2H@*T17{K%1ry&Ty#j104{&oaRTd3pnG*}d z!vQ~zQWsvF%cTuF{&We^9GBs^1%nsfuXL(s1Al)PF8PehI^Vd<0~9`t88|9O4aK8H1yG#qGo&E`(U0cRa0h z8l)%aqLGO!49pmNB*xe@Vq9j6QR-X@VH&)E0VmhD3YClZ_AjAtZJDHRaj9rWNqhct;V=+8ii6xIzf^2e_VM=-V5`idA<%)NOP{CkHWZ*XRlbD>anCADy-)2 zh8~Jf%T?Wu17Aa2_d_i|J?m22lW;OjIvI>B`Z%@-VZB#sIS2z*)#<<~oaC6%n)7-F z^SYGpATZa_8W7h8d#~T}*MCIDst#{bc%R7U+Y}xy zv7e0g5*aHzhk>UtbRevpHhHZdr~bKYTcL|4BK=Sj?sCO}064iu55n~}Q04i^ow#oR z8;<03zK-t|Bhi$w{IlnIVvjB5W>UOEm!BMN+`5ZnFSkAI<{sx5 zx)Bf*tx*-s=Xo}h^dXkDDc{%eSp(tPyp=?fvfI%P*98WyLY>WE7szNluhS}AH_Q8E zR_%HWcuo@vv2-qh(bhkRvu#B`DYyQtbE8uFunil<8SrF{Z~$7cF->F-o%1+cvvYGz zlVm{hz@SFRsO80{RF#HY^)T+64EaT@sd&X-N`7f^E5Fc6hY9C#+&Auv#7Lb=3sPP& zmYRQV;)dR+FPe3d$ww=0Mxh^-rgf^yn+cDfUeeTaxtgGotxYr}NWrM*CxDsj_PQ#k z+be%8E|d*U0F#^_8x|{G>(yzzDXl;2Qfuj4Xi$CaIpC}hWaC(l#GUuHZ{Ri+#cj}b z_GjZ{^oK>ckVYmG(mD=GNVwnxUC`PcbSp4Gk4CfvOqd`H?vK(jZZJ+G9RQ2KQd;d`4U-(CzqLrm(-om4ut0G7w+TaiS)qUc+*vIUZjQe_L3b zOhN$2WYppsc)76169#2aW(8pv#fxmxd{OySSsUpdHZheNi=F6e&46itGT@5Ce>Jsd zIW$4Nl+cLrTHlw;^L&MQJ}?*PYa?RAiPv^2#dEricFM1}M$)=|Jt-HJ7Oc3gmMsG2 z*+fr%e9WaX?{w7wO?@&E=A6uqwk|L*BTr?4MU$#T6)8C0m-Sn z*$v*sfN3_jF;^03`E+X6Zx&%amD>)RgibYcFw30{EBp9}kGJKwyH%!lqQ@(Ca^V6*$c>(rs*y%3AZq21Xf)j-A@`G3yY4Rmtw4#!4+S86Ijm^GaF!DbuX`WSL<)LdKkR7e7w?#7p{G+Oc5Qw7pmd+aRs4L0&nr;%p+kswhBPu2Y6+Am7DibJ znRjfZbP})7TV-At`q6?MBA-)m^*LBrfke=z4U><2Ftb=7O*^9#<8h3xD!tekkimhxU#6)n9s43(~vpe>Nv=u8TEMtkF$+&MEq85d;=S>RkSri z)^&!#I0o^I{m9}xp4?Am;QN}oAmdf045ikHFPXH~u4K9{KJKZV7=%R8z){?r?UCj5 z*fY=eJU=G3f_(z$j5#_|(v11*c$iD`X!!CA(Oxb*SDQA@Rr7Y4B`hxCDT?++g`_l= zuhV0|W5__Gfp|PX=&O&h5;D-kz&RUzl9s^b_D0WjTS|xAy%N( z^hkB7==V78=VA_pWww}_L{*y>mKUEx4P1o{B^V6FmO>#G8CahHR)a3Wdd%!2(~IrD z(^ydnx;O#s%62%Oi)EwatRrn05aZYC)A!}Zyo&U)YT}K>eLf6gt1%WV!k|O{p#P%L4f%h_0Sa;jM6qsmFSPZlI z;1q3CtvZp~D|k_D>$sVpH3;7$n0=9W3KatQI_1t+Wb!7pNJ7!>MYx|&pQzn-qzJ`3 zukXdTTG#620I6{Zm?*Ep+k(f_(wSV)PusIcW65A3 z1`fqP@`|~VXE`dotAK1)Fl?O1>t+0LvI{<jBO%K3HHjpp>M*p)kD+_k13{>ZHxX zsagy?8*xuQy-2vA_g>r4$@XOVKr#*1LA!UL$sVP{i1>n*AVRfcLW7J!X!1C>(LLYy zD<)Cp;N(|o9fk|Ek80Z;!?=r;(TP|F$Zel2h=f74PX@88`gt$%X~N#UXKSUTL-N@^*Ol5oy`=vG!r%-bs^rlf9mp0Xgax`d@=&=NoIpez6g# z0+=f;YupG-GY^EQ>v*nDr@p`FDme6l!XO^Z!Q~ej`$}Gbyj7dujCdFabn@kXR$dp)o%N^@kn&dT z`C;5!)+PC>v)8tDqBXbcl42D)kN4xi%h!B6r_YAeaUI8Sds1Le-T!G|rM)A>G`-fR zlZ;qu|2pNb+WYf#Pkwolg0XtV<>ZrReOW$vc65RLSY7fM*uPBJG4HCB?VA≧jaH z+B-B;b^0yfBA!S!>a{+d+S!38@lvF=uF26&uB;%Wi(4epC4`#CMI@ zL&yEr;@)fCPJOnk#2z}~JJ)vmoY9U;2`{XLuJ>cmn+CjXILHkoj`eQj$NL9m4mWyS z4`5!jp^tcXF|b|)eDFb_hUTi*ZT{?OGD^>@K>4x-xj)AVRB~xN?*I-M6)m*jKJQ1Z zLb8y9-vpB%HNLZ4?cOiLkJ^S+z&)S)sqxcb?ng^R$?bf#Y5D4%lo8dgT)l#~9I{~x z!%^udj*jE=MVtM>vftHydf6_!ns!d*o`%}677}<{HJx6*iTZQb$ zHf#jGMwsGEWUx9>fjTd>#((i-zYkQ6F?ivFbvbQnmYaf1^|uZ8vn4*U``SBQl)|{o z5hHi;0#x}n-i7-@-5@;LPLpo1yFe!8!l%W&hp$bLhuyH&%r!}7`^F8|0=w9c8seKh zz0#%A__26EkHvD>ux7H%wi{Q&gbU0r*%gaQ_fb>!r|pDko3b|uB+%R8Uo%u_&?a>4 zP`N_7H))ddCJ8DQ4Iw7Fh~d6T%782`zCxsyLbyYc^uU8W^7tUif7f;16< zeHuwpSl8x57CE(r!%BDnqdhP&I_k#k99)fRice*Mr;&JF-o%yTB)Q88r}fO8FYi_d zOHiIT6p`H5W;M7q%wj(yPMzRvp7TQta-(gUwly;OH5q4e=)|BSD1qKWWq{jTI+0gF z6_&^R7@xd>7TMw<)nQ?UD4gS9t^0tE*d5w3c>HX}-v1b+&Hegv`x#ps%57KwLCA%| zYXB6p%6mCU(;8N0gqKHxCOt}|$D-GZjbgNkikd4Li{V&Oz7-4SoP>$Aun&!XOlW$X zOhkzCA$QQYE5s2_D`py2ZT1>`H&S72a^dQ z`~u&e8cwF;aJdgfEtU+#so-Au=&2Pa*cJ~?Ui^~Z1!4P@8k>N0S>%%6=`a@^#a>K!bs0;; zS|fZQkdBz66!{J_vY7cZO@0S!k!UyvS27OiS-*qdxPs)0rRO%^KrD=*#dJx0khg-S zx07#GInon;>_;Lg48o($CBHiWpT#A=LmSiOnjXaGH8pI$xT|!i(x2{-9bEIv`|e49 zlifTVdT8BPj@|T!haX&g!<*QRTONPxs(}scH9z^~```MG!z}f-jsJ3c^Y_>_55NDS z+keE^x+Q;k-7B8`1e^7Z8$bQ7wsXtHebVxc-1EpqHwFL5GRMvv+j{V;Y~TLwhM&Fc z5f&S{dHaL6Zf4=m+QwVsA7oc8KmEezH!NoxzSe$w=gMc;`G;CQJ@cD2zTYmKc+JyC z-&}Ti|CaCH>bV+p^?$AZo}}`s{?4Rxa~2hESf3?aA)kwd%jECY@Uiiu^b4z2_~4tK z?#6ZW7Dn0BR61l&hes&gYK4@x-Nocfx1n7c=65O`AMGUgKXgr3c~w5v@TF4KEO(qk zDHayP$z;Ep6s{F9nA@;OIx1h^uij19dyMdB^4V=@jjx_1y|dTR5*Ye%g~RE>Z>6hB zJuCeGTSInB4~Jf@5Q*2;%)q>Q7UtCTm`^v>%)xA$=F%tSGTQz7H0pJC`+hD%MeoN2 zEcO1RfHYURYc?ap|FS!k^@#Qn6QO%3h+;VO6vk!o46^6dYBm8n3TH~u{ zh27wHs{Ot%+tY2R;#oqK{{Qyy=rH*@hjr} zQ1Y{4;g7#X*SCGe!XojEdaBa+tngPW^!@Rkh1dV=vU{u{BP}~Fx1lw@dKRgm{y&&_ z*IRe|^itm^E)8F^^y7`bx1@ge*ahEA`QCT`l~2Cm-1~j+e0cZYfA8O0eD8QqI`g3g zzxUnNckVwOJGjPo`_dz!SAB4<@!g>x-}tBXL%V%4oUZ!!S>^Meq(1bcruF-LZbNH) z`E1mWl-~`x$MUSca|*5Toxb<_d$sFdm51HHIMhmqs{Cy8N6sZmG{RMaE*ZO($aAM- zXpJwQRr=KqeaA%(l@68r}&-1Zw-A?PIntx<6EDrMW5`ohj~`j3Vm+%7gPL>EhRBC z_2FuT)zS;7bk+F5KAIBkQ0cJB^LK^cO0Sdj9ZFwOW~kDgH2xKtFHD+;sPBr5rz-tw zhsHeon8>M8&kDEiAvs>D(8_TIHp-TUYvVW?uT>L7YHB{;QK1{Rhg|Du3$hbM6 z$|WDNVR7#^%*U+EU8UzCMtUEh>DZOO@ma$kK62)pR?e8~a~oRY%V$-3)$divwERP3fXLMZ?JF`8t-?D1lbh_`3>8A`s&-0z8ly&`M&qRf4H^og#_!nrSMp53oJldZq^pD+9UKksI}zdPlQ+n0n@C}KmN|2_>HANr9Iep%9= z+t34Uyc4gc{wXaXbr(m4IF*;g!7a^$~?`thmI zIuEJvv%>Ebs&;Y;t?8)mS(5ReDjll)eE%->W-1&CZvT&`?t0$mHdOH}VWs58V(Ij4 znt0n|9XIz@D>UlIX*`(f?~)#imCy703q%Q6l|rL_K2NKD*Dm*|@RO3Ss?a*VTAx)s zOK7BTp{q&sI@RA*{i}5+6>gXHW_?!gi-kLn@`y}^3V*p_x$=VB?-dTW-uK2r0~Y?-|)&Kckk?T z8x~2&z%Ti(l5ZxJr^x)%_|6(%@$rXf%i-k9ZNF9DO-`Zk4STbAHtc?__PrO$-;0Gs z#^b_Vs`oUBue+7=)OW`Own^#8cCaEZcb&}|Up=z}hxsjg?Wr$&`nxZC)(Ep5;=S9@ z8ecxEe98?Cx^$x3GJjV%e82U*=C`+wWhx!A4<1!n6ohW)`RcoZ$fgJO56Afv3$w!T zV6%938(QPbXP$`7AK!!B{4(^w-;(mF3co?aIqYVSi^zuFS-qE`P103)#T1`Vb}+Ov z6qmstWxsNIW;_L-Lf z9;;HQ+QE06rgzPMFv5rBv)j-bUp^b@Tj|8@I->f{8ecy5rN8>y=K} ze(!yKZo?w!DExm5OV@z!F3E2z9G1@2`>*{oLR`|RR;Umfqbc&x@{hp-qzw(Lh2?`pl~Eo{x}-ZlE>tH#-?5B>gAf4Sj(tn=|l zKlz>okFb_K-(9-tbNktaJ1>Z)x1`v@dmgy=^p}6g_rmP6AK&=Oe&3^av*3S!rLEkM z?b-Op+a~JHVER*U?Ya1g8(H|gN0P6;{WUC*-4b8EcL{sN2U5FNg#MB3{NbyYeK$Uv zZCnxVjUBj}wQl+89ltuIh0T8PwvohF?7{p@DAU=Es-a%bFW5doucqt*mZ$&5rYqJymvivdVk1=xPJ6=kK$fvYv-{ zJzvbW@p|rIn`J%s@p|rO9lV}D%og!_?(#h=>bcyI4Lbd`_TN1<|AEt3=;fcf>2qKEA{+U7y#8S7A$HBJ!4*Z$jG^R|z^;qPAD!3LgK^}@UB`&r8q54`Ep$U!#y?v4Jg%dRRr zJX!f}yZL|a>iJ|VyY~FQ_V4|dJ?!P*`rXK%{dciFBe%Y)=AT~4uDq{*-^bp$f^Av$ zv6L8rT){?ex$qZT&WW;X*4+Emr$6;ecI8i7KJe!IRs6`Xy}N*1`Rs?|q!bpTF!^ zufF>WZ1i*geaga`PBzjO>e+E$H@kA@wRi4$?>x5dlJ{Spe)K2o%w5m#d}#0sWrrs# z-{AA9544v;mP99f6I4T#=*Y!MZbRIhu95$eIHId zv+MEIeJ@*l?wz0RxxTN*|MvfP{Y`WGx-b81VDERj`&JCLU-jnKf3I(8^N+4*`|?A5 zv(o-|Z(j4l`p1?%|JHdAK3Z&kUF`k(PbR)`$<|-~-tf;O*W1*-PqqKW_py)vY2lY{ z53*$1BJtI3^t~V$>{R?t`DAyiAKO)y7 z77od%9(b@7}k+(BZ%9wX9a?%f#1teLf2w<*%3~e(C$v z-uhoW`^k4IJ65ed(qz2Rj$t>|4CH z=Rdx?^lqiUJ1UaR>w8rB#d1!Kp*OKS`@SRhzr9@IdWS5(vNwL>O0h3a+B;kGQ@D+G zlJWXJ|Au{E9=m@1m*$C$T|T+4VU+jh9S12#LocQFSE_cBVc(LAe)Z^;4>zm&Fus52 zTmpWcDwppWv5wUD>ot!(xpbga)gzPTQaF7w-uk!vA~)x*{`}R0243kuO8KEUD)mkxHsU`i=uFWs-v8P7P5HFQin?X1#4jv;nBS~_ z_zUenx%1X$-#?LM>(Ql7vEG(L@|JfGR2aI@64~#$mg^zam zv_pH@XIl@9?CE9yrO#G))!1PaP$SuL^%0*YGU`=7hKdGqonZ<0)!R+(SZ^X_~1 z-E;1_=bn4+x#!+jzjS$?L;Q*X3h-Cx4hEzCM4ax@vk^$p^;GW?y&-2ZnUt8;&>-AJuxohUs z&aJNTx_$ndxpV#Vi4}z%fp{Vk;}yZXh1D7-#2wbE7iiVh3oGX?tf~YN)pHhVK5rm_ zcG1wBVRs1Z@oV8o!nILw(dX3?k@Op`WZ2gk^?AT6{z7v^OLVvs8k)Dg-51k9bgT@e zfx%v%mhibiDdq7|@ab>B`L(M~6nZK}RJ~hq!g<9C-j#dL@$Q5fb%sg$=Ps=2#V`Z?LrMLhX`i!j&fGqDZ%_MxIagV^aBeNST{`yF zye*TVXw2sk(rlFE^2*uO>B5#SWG3Mv6c8>;6x3TR#bp+>r=i`mK`_Uv4*fu26eyy3 z?!q~>{VFn@-r|;|Pg|U9*R)y<`n?)j-c(#%^T#64sKH3MeGzFaeq|^Hx@BJF{HiLh zBcNt#Dr>-q^0xKa}p$Cfo!FymfMP`%7jJC=`kLlf#6jxe153a_fLL1yM zO{>ws?wYv^=R$o;t-U_3mDSi*pzjQ>YOtS%VaI5g7*#Z$%E6KD(O67&hq6`p*qsaq zlpR6oB1#9LIB`9S+TBGXB@+12^QlXHaj48FH%c(zM!A`ifOW#NP@WO{9FAR0NRBp) zd-PP2Pi2mMPIlv=W`fKT2EDO6H5sJTh)IlogA#1;8;;b_7-qF2nIaLfDHV zsm*1X`R(-;p&BIc16`wcIvkJD^(ayVZZat-mEM*_Q@%-#MpRXVD4!LL$JeiF(5O5z z-gGV!OS#43OGsnfu3aVH_JBsmNKdrG4QP}xE4t`8{jBIx&$7h(Y&Xg&0>43ZHMN}) zH*aqfPsV$F?J=J(?yC0%eqo8uXG88j6Gy9lX+f5zyu2O6==a-DYJr$X@$$2ksfnCFeEub;Vf^(W<;&rUF9PjQ z@#OgqRAsbbPDwS(mM@dkL==iB9@l~a2swEQ4n$u&E)Y&&J|8pAhe3hghbeO?B|^Q% z+kW>qg-5o4>~H@^9+FY|^qaVrJ>PhL!@J;F35=6u%pEM7$SIBhRdCm$AZraMja{lv@why0%K$5ACtYCT?SKyF;V5 zw350-+FY1_WQl0f5Sc03%#OFRPwd@dpqu6xM!E-M_sm}#g@upa4F@9G+6GI9Cp&!F zCU-2PTSQt(ViR0UoO(Ojt?i&|nW(uwzP`{zb%X64Npx^4)L1vGHYuniFC%fmx-H7# z4{b|Aq+#8XVIPEb8^-iX^SeE8E^*Hv$0cS`yc|nfd#-i9e&sPc*C2;%hZm>dx)A+4 z5cWjrMI)`GC(Pp|+4k*JLHw+AMlml-t6$ip@jMF8UZjfrA(xlT99$dm41_1Mmj`0L zvY0POPHT)Qu>|I4i7q2cyl|Rp5}hMUe!R3DwB>thqH*EIPNS3RlF63Lub-pU%Q=so z99h|q+3MsYn>r$iNW2Tiybn4KvzYH}+ z{pHI#qnOiT@qj$qLT1UguLZ(dedFR_Fyavl7UaVA$B=;}9SbgWU%24Oal~n{>o z^EbtRQXXHSxmJFg8!l{d-&jX0dCaR9`K)Aq_7aq%wWQ`$rKw3Mtq1Umx;>udK`+<1 zc6E?6zDgJi{Zw@imqt4pdEaczOy{&!C<Q(?~B8!#I0v zZPhGg?~f0mxMCOu@QNz+WXnTNYacQ=&(Mu@4+s`=e48+2xMPq{!l%`1G&<5Gey+I?0u62m{`21LODFWkvgI>u*on@xd42d!HL zcyp>T>$Mi(&X0$paStx=Gtxn_U>>s$3kf;3>bqslAdsG5BrcpKC_hVn{b@S#7QtT@l&aK7rWcPyUW}D&RJ3%&BFXj+&EpP7qFop{W08ya>N3kH~^Apiz zhc`PqiRbjQ;-`9+CGJ08k8(!y6*^DY6=~!4iLp&H_36XWan+d8s9{MZAefxqhcQ=> zW2bDBx6cPY)rEdQQI_wA(vx zpqv6y{&H+I;Kbt!_f+-NaQ zQoXGBbu#7m#Skh316EFhV-b*#rKf6|PPZj1+#bHXq3nitFC{#3t)&vzc04;QRE=EL z_$RlJ??Vk|Fy=O4#sY;!`3J<8-s>svEk#+ya`jxze~dI+$(Ws;IL9!InCwJDkJhb3 zntX68(XLLif4@HlG#AD?BEfKgukzW^TcqZDc+aYxr}D8c=EF3|gK#yEk8E0ZtZXGY zG4RWluHPbWfxj!@3wrr`M!E)KyOt`t$*UxM9y|;z5Z54$NsRtoDMOZU;}15>UvCCH zY=c-wGtywymh_x{R(w&gL{Jml-mPSiHo}1@f=CwyFnogAKR^)!-PD~w# z#`R8zyN$;bWb4<;+4>d8&Q=Egxe9p;E{0%n2}o~?krwsuO9m?Ixvk5^ z)M(15hFJvhHYZ|9PhxGE`=PC5osV-(qH{ zyzoaNiaaZ=v38NioAFR|`$tRBw+U91}A|5DZ&5Y`2SM0-&b4p?thWu-|5tZaO{J~?+OCL3gGl2cbM^za@R_l5b!DxlH2Fq^06 zURb+Wfr;>u++M#GA`n9Do1?az5haJvY)QR>m0AdK6IvW`uoPh-_o`(pdve?e%*abpI6FN&>-fNzgS9CqmNpS&gm*=7^=u~514#vk(=L27X zn^aHlK4HZpJ6q=zZYLO&diLWZw^Qu&k8IoG6N$e>cFO&o)}KkXtoSpQ-a+(&!S!ug zzcTy2^NQaf9{Ig$I-iEeibscaa2J|2em8#r%mZhA|9pI7(|ml?8qh8H+vUhRViF*HD=rK#JYv9(S9W8|QQT&dz7>>-NCbTX zOn9@D^CNywbS2!$T&>{=x)Rvg6h~Bc?^V1k={oWj&~;d6kgAd<|09t$`wZ=LGN-A6 zPfj8+3ZxMcs5IB7B_3N|SU^BpAj4SG=(n;X`dU9Yx*j2N zSY?F0L;D=WY}XsWbcmM2IR&=#OS-1aLH03Hzp4Nbqa0?ke-Ah4QKJ!T^luw|fePP# z;F|;oxvOzS%KVgT97eoWwr9~f_%$>vU5YZy_8Z9ny-zH>J$!#~J53}P!#zYM=R$164qWWR|6N*DWuoI0EgtD=qYaj)Cvn(_ zxD(}}wwpR|k_U`oZy7S5g2+ZvCz`m6888gx#>mEULBG{}qq;%Hx1`a7k;b$nNNNTv0Q687aAw{gW`M}Re+>0^{7&+N`g$j!Xrm_49nOx)*=N5Yo=T`OIi&H8;=FF!2DfgOsOJh9Nt zS*`(Q@;4vJ{ms4baDHdmPRDLTZqdJ@{5?B4vYS(k;_v4z#nwn2jERUKiY+hj;(Gk zr2TS@nd5R!;(LcEAcv3TwAzG~e+qOnx(fuZEZgt+9%Se1;z4&Cy=f=kcDh^ZwJwf= z1HrUK@hoXOVT+&*p6%Skd5kkeEvh|-S<-T16KE+24a<7+NmGaq6epRh@s*G9gC1W` z+6%fyUR)WbvJGc! zCqByfbzIuYx}B{wHYMv>@B5N=+sQPV@VQO%!85m+*G#7xy5nRa_nLE+B1)S48GdYo z)Y`hVcmS3(LXSFxVQXbe+RQ_mBgsDJAtkkjOBu+UAt93NTAQVIE2g+C@YdNbW0 z(@SOAj&D+~W_m$aft!34eq5!?IL+_+>-$*9?F7R@MD}ZI9KRhu&2w|A2aQ4MC+jtf ze7j@TH7q_`?kFRDY#gg{Iqw_O-c-@f^+Y`%j1>*!(^dYDsN4XP4Gl*BN0VU@u$TXX zDutiKH=F#plW6Vjpwtnd%j}mKkdBaRe#(aAJ`~+oVW%TXH-kroA{HmBwGKO9UzI)% zi9$U-8jUnD*kl}m?EOfxO-uKI-hBAus^{VYKcoCu&HGeeu+p#UnOz@~xXbPYz9N5W z0JOtxH~t-)HvU;}6wi{j@=nl}k8Of7WIBy61JPSm$_K`6oy8RLzmoj5hA6q-fzzPD zj@~TwRRyE^ggUX`78m##d2Lmnk@AQYbNQ4v+jh2+ky8{O#145fu=ueo@J#A^f%Uaz zama!(G?brF2%lhO5$G7XK5~H1Ys;%7UpTE15O@$2j2vKcnnf`ETszbQN~7S;qZ!!6bEM z+kRx8j}Ku?R`o-mXADJ2ZSs2hHE#3BDzX0Ck8!)^8sOvSSGFCD=U3*$Evy0k>dWY9PJIX^miF{2ioaoPwvrHqMp|OGwC1W9_@a} z`g|JRxQ%MI1##b2?+noz-G8O6(b0-opuekhi=#2qDji?9o6FbjjH`5FJr3DT>va6! z`nVYUt@=$h-$M;Ahqc{nw?#L3y5Zy|A+ZIle-VeK~1 zIF^FB5VGrcwIS9NH}EskO7hs>Iq4~DP$r#8L`9}{x0Lc(Lp_grYsA6HI4p)fivT+w zd%F`dppWp&zPz9V*J@no*Lt>LTU^HS87F2nR8#SEgu zPCr}r8G2FvWc^Ovf;3+v;z!?$hsOS5*=|F(zPyOUiq6r=)mePmoTJ0L7fA_lTk@^9 z?Oitoc?)GkKXJ{-vw_&Wb7)-VlA3Wpg5`DuJ2^Ojc5mHY@L^OShL6o)$;eAwN#_g6 zT&A^C?1OG`>F;X#6_sIEN6UC8dXavHcqc1eH&w0^uT*5~8$-qG*{%tj#}xA#wd`-s z?I^b3A~?L|fjUOHvFdB|oPJict7mrnk+@kD~UK!qT%r$7rr&>DaxhN+i>GMtn8vw-SBd zmOQEtAC%H?z$fkXz~;LIKEI{$UEX-tk{^Ub{_OhIVDwJb`d(E-Za*{Y8KW;n%Gojv z^kc0Ep5$1s*xiBHET!CL;l*KFhc9IT8>|zf6VHxsc6?X$*oS%)bab`hC@J0|MtwDq zdd#7iC9>m}iqoN&(s|Le(MR`84(WdN<#$rMQ0#J!*3~Uzm-`=6b6O|p8s*3Q1;-u6 zvOJHjaDBUmQxL1EVXVfclgLj)Wd?$XLgibO+pyLfjbd)h&zw!*b1UR*#m`P&q&%!( z$V>Zf3#{~~8Xv9bQ_r%*y|M_n3ur&MmrfT=*F#pgjs4OJKepE+$d*3oqfx0` z3~AEBj}@(NT)GZ8vZZY|@^Ze2q4q%X<@6f~x1R(D7x)?bs3l));xdk39tj3$pB|UR zc3)T=mo6gU#bIh3PnNtnqZPT;d6zT>8tEH|4q?qS^GwafGR)r&zt4dIx@rX@s zSlQUTcFl5Eli^!RG(Hpm$KWs1SY4@(IHgsZCJplsoO!EGyp^9;3S(a+&YDfez6Sf{twHngjEgpTwF*6F&Rs_K1V?2S!^<6ts;M>m@CV zo0my$rOtV=19`}_#Qtf3P=L_k~vAEBJtd{!L*@8%WRkr9Y5&^bSVZDTZPnByK z-BA?^bkbmmwr^hDK;;shMUK4lI{*tKc-RqWqx)IVR#SoFf+(+*uA8-zO5&dqC2S!d zj<|qDCO6j8DT#>hiF9_Mw^I>19n@a=(G}o2zims0o$uv1j7eOsqASF(--{4|NAZEs z7Hp`hbjiKWc6_kKe{ zRB#i0=hYpSdzS3#YS;Ehr}i|W_lugRJUtw}F8AAz9L>+a$A@C1^xNUjU*E3p9kJr~ z8LR0$eMlQ@UE19vPw}E%K6LgT9W+kIL&k9&1?)4(1@szP1m|E!X>$-st*)?$TGc+= zaXo6iD}+xE$BnTElqZueD}t^7v1POq!HU+p27H0Z?F$CtWeo_1feip` zshyIJcvYO+%!8%l?kYtD$EA4D0~*G{URb)Uesx0=qNyl=06#M=R&7#Zv=w;N zfd19RKq!RIFT%`M+&7Mc`G^?^iSDJ#%r4*@ouZ>@Ks4~3RoT*zW~cBmZa0t?V`534 zU~)yX(`RGZK4(4w~@4ld^1 zU5JOGYcJ>;O=F>_V-EF3BzwtCf9W}r#=}(CR=t{v*7>my*Tpy!G7_t%%_IDoSzh~# zcb;)?l{`3$+AD&`5S@rqydVIQpuW6cs zx$5|U5se4%X==`-Ui`U$aE|bXqXY-rUelK1;CiqCpA`^;fN9{AZE`^oa2fTRbX)eq zZ&lZH{YKu)Y;{X7{dU0w&@pnoiz|A&y60C_AU=+`WIYy6U&@t01Rt~2hd&x^NXwIQ zS9PC)Yr;vIY$$P3`3k3oYqHPfG4Hl`?fSrRdW=F=_W1=Rz&|nO^F})P*`0C$9yjr` z<3V3?zodmsQabpzgkl$Gl1|{0A^LC-YU zaV~c-!L4i@d@3e^YQ(gsx34|-3)H-Z&qaxHkU*|(&wBAGHV~SIkt!7Nh{Fv*o1(K8 z*G06~nP$fpJhbXJvvs>bU!mt3A;T2eVG|JHr)Xff=JtM7_V&J=T#Yr_`(miq z@vQ*9;`J`lFUr)ml^yh1ITN?;vnQzbNY`3vrYq3uG!P5OMvd3R`ABakyLR>M+cwk< zRE|sW{cO-MUW^}n%Z*D8Aa=C$)sN;|j344_M&eLJJu;ZSvlAdYNo@Bq@}?CDJ_qN! zzU=#n9dC|RbMQWPGb)~R0Czr`CF^;Ojc>DQT_lL<6-F`yrKCRX{dj)r!5G`wI@ z!j1js_zIewp&9fJ&C?J+vWYLTD_WzVb#%}j^5Prj0=B0W;?Af*;&|_47EXv!7^3x? zb>n(8^{wi-u2Jk{+D?W}fLWG0EKAJ+`G`!Kl~2HoG;EPH1anBk3`-j19JD%(44oHS zK?nEWh$(0<`RLU?efV!md--p;L`$!Q@-mcw7cv861L;PU~EJxU)B3gTo>W2(+G|_QnAFim2qxh*AI?T_L>_$ zByrv!+-WFY9x9^n?gWm}Bp!UmAP(WQtMiD4CO6S$rL)TPamAoTQy<9V&KNo|BQ=_& zc+i6~$LRBGj5|o%(PGDkKIhe5HFweH)qGl_c(V)m3qw8^+;F+^^EO0q@Dj}#p%;Cf z2wN3jEG4E84y@wPA;dWS9-m51&7+Kd-V1hvrhM|ba_^BnfBNrl6$L9E>{0lnpUAE~ z`j1Z(tnlqs_@phEdwil`g>Rq2$1Sd*;HM~9;d>kSiaeXVY9q5<{p{p>#;Wvf%OqOz z!=1Ee0H>+?+Hl+#u%`Oi2e>YTn(-%nd|k+n25M9JNh`}fFZL}^X$D6c23wCi7VFa7 zFac8E)503og`5_c^f-HpE^YgGK~;qu?W}dqxCZ?*&Sb~)G9|w+EJBLg7&QmN#;8df z+!YNIKtloMZjCEjIBjNrTj|lG^|>1wlzh5r^A%o9ulZU?_ECj*3bGXv-sLOLrZ=*| zd%TV}ea_fejb`jj&56_f$mwSmQyXi*cet2P$C6=adsidAhau$9Oo!RF>1#f{WU2V> z5v@*b!YsN&6UDWS2f5=b)#j+B@=c1crn$J`Q zJgt2OJff*|KGlLIKKHNAe(vA@u^?^ixsU;;oFC{bjCOpn)0chEzpK^S4DU&*ZB~Ge z0(T+=_d2fZ+oq7Op=DlkU$Zz7q96sF!|2gyiA=8cLwVuY6SN!omNEB3P*%XNg|Q?I zV_;*zix$LUJk7B{j9vggteVrM=xsu(2;VP-D=RA-s;}>t#?$3Fq%4?}Nk{$4hg0}r z)#xjMC85wv>;dNOCL#>P0y`$lZQQSqDj8RHGsJY|uv__n;&7<9Z&6UtO!38Yc+9R>&} z#hU_as;#29Hm{FG&GwMn%sK@Eo30d6L?>j6G~xi?$M^cK)p5cj*nvcU%1 zUXz>QPa`jxzFUgy7~U*;-`)zGqj=1hvQT=?Mb5Obi9WP`2iY)gL+&ejN}0uP`i)v}OI%Qnz5LC$yi)=#;dJXm^Wn%0ty zOUWFMaS<6bC>hv}>%+r1dNc-k;m!g+BH?cDb}`#*c5-^g8a}u|2zU^aaz&7~1;#Rt zl7>EhQ7VGhx#XtY7kud#FY-x)qWMnHRuuEN!ZHSbu)I4NZHY-bKS+=8%a4}w#^k)y z!ur={6b(B;!`N8hBG>v@1nWqMJ|vC}o@eu$1EFZpR|bzfK5|2T1Jj2ow35X!cL(3( zx{ja!r&TRjFxTiWxUdIwjNM2JT(DkTDmuWPwqg|rR3QvOtA03nwLaj}yo(}zcqk*) zUGZiYu6c}4b#7v7Y-4GMpbUcU7)6_l)dqTrzg|X%tw-@GDC(p{o07GCxW-%#A6(+$yY$GG zZoAmshIMN1IOnO!hpva+LH3miDFX6vTx|kT+Z!t=3)%1Z`l3(cB^W zAK3BOPRGt$))|F_;D{>N0kEf0w2%f7wL%j>TG@(YWJYh*Y3H~V4?jSv5Ox5hm`{cW zOViO-b^r?Jq3ewtwc>}1QloS|DxM7%fBh01_fY1i=m0$c41-YITC>{fQ&$nu$m49Q zVMAm`V_$0&C-+*bQ8q(0*Xt^Bkub?Na!tUsF`h_HKWt@v(DM)KmWa!PuqWvldLvC! zkQhVr6r8W%q1a)h2itbXsCkbp5)mEPz070e^bfu%o@l_QkHPKkP{@rN_zjkZ9E`!P z63{UUGEYk^GF;u3uD*OsH4XBdJl>tf#5pw|Cq3Cah@E($cvXq(F|dK*h7Kwtc(8n$ znb`)eKMMhtQ0C&B%3@g7X{iM*db{DPTgG!ih~`YOadx^e7wyJ?U&RK6lEFlPHps>F z03jmJwjJ&0JYBbo$*-d(Hdq|uYMj3AZiq%AHgfk5?Q7{3OT_6>ZM*_0qmD4{)1-`< z_*$4-pMa~0a7kHHPP_vqYf?Weo}A5Tk#@Q~f@ZL^I>Fk4$1b1g$r&_VIdLHBxJTZZ@dEa#@0f9$`>EadI5%xS08L*^Q!q@qBOVG+#L0O9%qI3NHz_$EUis{I}UJaqLp-{SsDl+`;WT^|6CX-$By$k_4TuC23O6A&vjdi%Fhz4>}cRqoagOLlI z4A3!cN)|KuKYyIk^-h0xB4MCjv zAU@G*@L#I>Zv)Oz62I8_V&86k(J;f72EKh0SW|7V9oOl;37qbeBi}?#noYxv7t_)& z_bt+CLSC(e&xDP!BdvFWHnCR$A8N+6om|+-^t5_kFqlLzwjnCaYFyY!c%cj;)xiLF ziX%PHezZ!hmAS*{EjSCslOAj!d}(mmz`9er1>qJ*EvKosNJWAUB|AO1E*7@8a!KQk zpp*DyH9nouVsJUbyDQ??$L5cZ6Nj+VQ9cO`JBkyCrN=3UT#a#k043dCu4ya%y>C-s z3nz`#w;vSxN{L1s0YqQ0P^5sP2!Z3rx~3N2DGmaqHj4-kv9T3vF`ImNyHcBh(^*Qe zFDeLYWutGVwZ&bIdRg(e4R(jvQH2&%Cz+A8g-NH(qt2n+&uaX zG*Vr5Ir3Wf0FGQQs>1Cer0wc5UKJblCrHZhN-w?*MV*coL3#>%?@ECV)N6$1$a8PQ z^@U` zV%%n#WeOc+Zskd0D-F6=x}@klZQlzT$55aH*_bYTR~BjFsU2Sn`;e2q=-h3llLi!J z$Wf1nZI*zk8>jy4!@)uPl;n;sZwzPsVla^P;Pd%XEl}<&=b8*bsT6ie=Py|WS%^HM z;lK=1r5lq$3TmRmw%{2MElUSjaYGI2lAk01B4D3kXe;ShuT^Lr>zR>NrCL=v4ZyPB zt#@GShfA7!*gl^O`#Ph1Ify!xX847DK5x)RAKGyF5PH*&eFI_s{cRAG@VQVAk=-(1 z8v+;I!Bi%cYsiPy$$&SBZK*Dcd~LCa8;H}<^4&w}1zP5lW1)dOEtM~YR_sQ~RGuMU zmgQMg!+DtDpGnvLE!+vdeGW^ZjpKx{bx2zVL!gyKZFXop4(ezn19tYmc0M{-;PVX%VN)$KXiZREiCl=2UeVW z%&(aDynof+`(%Kv{nfc&xo<{@IdAp6b;;hpu<280es%r6i~3z(|5x%S-?aJ>wsEOz zZ}p0GEV2Ia3znVqNw(qMnJu6G<@+o+>b}BTUpt$5j=ug)ZRyjjWn=eWg25eZ(GS}{ zdsaz=O>^)3XX2dxqIIzUwE7O0Jp1o>_QWl1e`E38$9Amjewc08SX21Isn4=t>opAz zUbmcitMZDj3w@KFHS-fE>|ZjIE%`;o%~iACW5;bRy=TNP9nLo=bzS;y&(-~HVD_Je zw|m|z_udivCA+5C^VEWlB-`@WJr7R1>@#fhidSDet9}W)^!Zo5{<-V7vFPWQ{?E-N zzh>v|`TB`BKf~C9qyBmMdGCLZjeKPDJvWpe({DZ}?oNFAq(81=S4e)*F4RiHuM6`2u}$89%0=nvbaT6#3i?)pIkPA@iG| z`TfDBU+nB5OzcpH4}Rv|8vJ-=P@f*g@IUG$@3RV?bm;9R(>*enpyyh~GpPLL`|7#6 zua~FF&lP;v+e(D6LmYBu;y*R-9w=n8ATyM=4;aWq{{bW5KNOLf@Bg7}=y$cvd{gOu z7j%fcpz1eJsH(}X(fB31qDR2wUJliN>w^)BQ~44D!BWjq~@9((f_nUU5f<>b@O(;VbmJ$!UhK$@?}?(J@e{& z75r?sNM9~iDcR<2BCYtQ?yIzVuI?XT$ce*$S+~d2e*SYRsM^Wg4k~Ta4o>-8(J@e{ z+T%b$MW-2`y04)5xm{YlH{8(qBiWAz2#1?~g`a8SNN3mlxqHxAK#yQ42CO9TK4tIRQG0Aa+<4DJ3$CU0o zj_@G(kF69$3>2y=IG?1MWKE3@ACWTtB)^@y-_Nuv|33xl{Oc#5)O(6g4~*A(=-L*J zKAXStfEi9Zwwu3ih7)$}<@bje5{`r6=X_h*QUip{aqamf@Bi#t=UbnAy#DmBPISKQ zEBeEqPW}$Ty?NueJAZxJ9g{A8VFiDF;iC3lX_fC)X&acJDu1YI)!QG z(4o3-2Q$g_0kUOOJ%$^yNrD0$D&J=&C<$gJ!kzWE&8l2m?bhMhUrhYuqEG$9dHTE; zU!PuIro+~@hTpt%+kU4Bnu;5?WY612bKJJ@g~=kH8NOHFO%Hm^@GlcZ{vn2oi$u7= zp$^$H&MpUA_HGURScA|ztYIn*=r^&YWBA;Xt$mn3V9R^>3v9W}zmmUJ!&a;k`4-Ff z>t(rSd*k#Oq8uHs%s1Gyngh-g<@E^QP*9)qR}-0!KBlLYk6Y%~@5}Pk`@MYH3SX|_ zJJ)kc*dY!jJBJ+%)Xvr48?L!Zn~*%Fpl2`VB~!3%n@DF1)!GZ~a=?!b6q?5|nHHrU zxc__2{r3vo&eu9(cWMva-0S(<^0|aeB=xlyl|E2VqIc-NALRp5Q2B!FV5a=){t(0A zn#((17vqi&P3^1SH|NWcR_$@1pz}5{UpC{*bl;pWQ`-4AnqHiChx3(h{IciMheuf9 zb<`n#uAuXQ*RMQg=b3M)=jOD^$7n|gerBkbukZ{J6xiWQ-AuKabcwTD+Os!Gdv}Wo zer@NOR5Pa;s(j}A*`{SXsdjPRCHa>n9IpOh#_JU3T8$3h5O$C=6a2f7Pp5=>dps-e z|Bl4y^(&nUs{3~E`RKQvDPFX}X@<(jZ37h@gN5etL6w*3zIv{p%BRx*sgR8nHCIrV zFSER<^gfZ_DWO?Thb_%N9|nD=pbYC~nP4m96=4!8d0<;R#C;hx%r2ApzR)A?U!w`A z<_({TinPk7o}2Hh=jy(KIbP%9#Na&KpAt<1tos1v$7LWNfdXTTj2j^#NlF zjwS{124mNe!d#=n<=+*HGep>n^%Rkh#Kkh#YUM{(Cq(EX}jktUnaOf;u$D>aEqY- zK*5(X;LCJBOS%LXhW~%3S#F+`a+L}06*Q}UtjAmyiTj6YTD62(|JmJF&lObft8{;% zdVio$;Zrdc!vz%`2MQ{FWx|)~xw@YTW-9McyRXj!GtA%3^>IG(75d$zpn9GuCdB;Q zoL29t`@;>LJv5z*-r{^v>N^G1eFc9c`JkY>|8uM^!k%=Rp^0ybe`Yw?`aA#3mut^I zVDL4`2Q%JG_s#h-qzB7~gXMk3x;ZJVR#3ziO1MD-$#o#MSUx|k zn_sYMnO3k=rWM{CK~XB^QK}xp1Qi_vh356JVM}j_iFHFA9wyTYw#l@D>G@UtmHr$s z+$%~_cAF|km2U?#jgQ&hSLGk-dY64_8?VZt4pki1u*2O#GW7VIex})UF^(VV@KBG( zQ1mN(z`i5>gNG_S#N!xL9ERQ=nfx(>ysz}S9n}3y1H`AwKh*aqsCxXnz=7)RgXKNd z4$2BHJ_RHxx@B!W=sPu&xxI`ozjb+vL&*H^8Kvyzdb-aHt(^; z-MnC?d@*Y6sZQWy3aa~NX!qQlKgVf7CHVi4)QP!%8>M>zsGDwC?c!>;vbIe$jR74@QsI;f~JW z@g1X{a{jMa>vulT`P+ZJd+%}`sxeNLr_v91|M$u>uh`-I4I<3KH+)Wq&#@nN&j~&3 z%of_n+i=T!w)jd+F)UpI6N@l@q3`#r)(kh>NZoaRctNRLWs1)z#2=c=4pR-BE4CWfD{xZaH*nF+mGSMCiZn#_eRu#NL zrX^JQ6x{Hld@f+!vaZj+Sb5FDLmldJwOd>rEOW1Z(;@ZlK_g#EZu$$47wy0f6r7L& zU#9z6(xuB)rkG+i?%F~1zB%7dlg4vxGZb4t22MaXE&Gbq?$=x>k zxe;|&eDxozX8qqc>(2bvUH9Glqi@#LPk-{;oq2EQaLuMy{pY{ir9U^PRsL0P?|QJ{ zc2Auds`7^zvTs!KCSdoRpMMO+L;q;cDvCeV;a#J*zYo_YyY0!hX$6km66X7N*zMEH z>6`#P-{#xCg5OQd1T(d-j#uFyB+P`TfA>echHvB;{_@IjlnVYvHi$n_n)4O8M`XZI z`ifr}9>~U!D-vydsKepf`)a~#mNDJ`r2J4OXwHAIr1i5OezEYw@3D{S z8n!kb`PZVy0oVNE^@kp=Yi@qbH~X%3JTF>%`fnEZN~?UEpS(0OdD1g=o1eVwf;)Gt z(V;4Th@of=w1?VfF;J-DB+XFT?8=9g2`V}U3fWVa@>kgl=iS0}{9Dg#CT}uR@Rwgd zj`pjvXP!KQkN3}=yp@mZ&wQzyUexoM#&I>S^TI9u)A1_&gMFFHrZ*UwWKYi^(t#PQh)l5+3lTcM-$b6OuYQ81=p}y}?`G|7_6`$g8BhQAS4!j8vZ-uvQDU>L4FG79hD>nZE;=Tp4rVu{)L7A>Pq z`CHoNvkXFFmdD%XR0sBz%)r;X2|@1$&jN%E=96-|XQPVOuT| ze)#_?PKTTLp(2m5%dQi1%1!^|lW)DxsQji_2`1B-_^jhq_y-IBYp%Lsn=gRHky-(l9Cz32NOFX~gP8y0guekpp_oQrBb6VYhMCLO=#ixIFsBCpr z&w-{@4EmuCRs9e77<&0>OLmAuT`u>K$GhaO&RoO2vfLqtdOg%0R<-9F~_*(fejR z{UtpA=3Te?^6JcKmA`BBRqvnj>#x4s}R%pr76j1LVhWzb|Sr{Qomaf`WE@>9sC8x0P zFW>*?WnW`euReFzO%tDGrR$!UzU;>v*$Hco55!hP*`zyucK0XF`Ihs;QBz-Cdck7n zb30huAAernU&z)i{o9RQ`A0C{JD;mPY2Rk%J@(niB{yHn+=&&Tnd^^Y=Y1o(c9!Q0 zY|Yb`%y=R+iY=YxT^QVSHY;24hg)A8SIS0xbYpAy;amF+*L?o*7Vopyv9^_Gj6C*+ zLe~28xz(qh^<8$s%P0^ zrHl5vS>tQFx8M8RYQ{eM_0#;3l0V{zd`c46B|AO1by zV&^^I68rlb4eab$&qW@r`Wb81KK$s#rGH_kAF*QGxhrbeDX+h|sAlI|?8N0K-tmpI zZ)JHO?OgJ+J*)Z+*9PwX@K2jxWWK4Nd}qSX4riWozQ5(izqp^Z?g|xjN1tNnUe`A3 za@TIQ=4i)~-<|LptM@#8)n87`XQd@uPTM=##fm<<`TQTP*xhe9TzJ% z&gAd0^OpC#bMp^xWovHy&Xu41Xf>eJ5ZHykd! ztFC$Lw%WVO*!jo(Yw`O3S;x+K^v|vTaNWk%wO)6jU-z1`s2QH#xZyAWwUp! zFR6~a&Wd)dIPcO&Zs<4kX!|Ejtr*K<&#tZcbiq+(Uo@)GQzu|Dm(S>-etrTR&=LXI3pM4!(Ff ztNq*7`))2gfgL$(&6@e+kLWiXF8%y5-UBiY)>TaT(^cPMm(|tX5`M4c)wy-2PCI7% zJ+&9t)w;g;tBbdcsjE5tKKJ^^YwBh-R-AS9r+-~Hz2uoQ${%{FZe+~$mF4q3T=e3M z{hu5A;B)=;1T4BM{K&~GU-=7D`+nvAHGQ4DNR;UO?(Kh{^uWz+Y=%s~%EL4K`dRo) z?z{aBgb}3e;N|!4`jYzIwO+nDXW=bb zpK_63#rCQEiaysKDDzW?idPsAtG z{YieKP|~4)TS3M^I^}zdKKpWV^lgzpF(Lv#b(YYgm;d`aFS#FI<^1KbXYc;U!`(W) z>iw65JoQL^insB;jQ`Nj)RwV``g8f7Ga>lP)ET3(Z9L1OJwL5@e*1ML&M(txA}jAx z_0`YfD&9fAR;SuuxmT2|^F#7W<#*nVFI0W?+b^*fuICx`v%{r4*ZsWQ7oPIsYV-TL z{H=TMgvTEE`sZ|cc%O$stx)|yfA8J;&iz0APPJNR)#X8u=U8&sHkXELXOp*m>gLAGLyS#eQ`#(Ox`Kybrt2HB9t_{?2H4OMOG*2;bEhba8G9tT(Z{kydX8!yMb zLlmm^__!ViM?3zzw8sO7t?s6}JfM|s<8TU&bi?m~$yMDPj@vBYgo}H4dQwk%T7HZ8 m0sXzlM83%qs{BI@Kdy)1`^W7e Date: Mon, 26 Mar 2018 10:06:23 -0400 Subject: [PATCH 568/904] Allow namelist 'maxpft' workable for natural-PFT parameter data read. --- components/elm/bld/ELMBuildNamelist.pm | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/components/elm/bld/ELMBuildNamelist.pm b/components/elm/bld/ELMBuildNamelist.pm index 68afd92e132d..cbf067e67607 100755 --- a/components/elm/bld/ELMBuildNamelist.pm +++ b/components/elm/bld/ELMBuildNamelist.pm @@ -1552,13 +1552,19 @@ sub setup_cmdl_maxpft { $nl_flags->{'maxpft'} = $val; if ( ($nl_flags->{'bgc_mode'} ne "sp") && ($nl_flags->{'maxpft'} != $maxpatchpft{$nl_flags->{'use_crop'}}) ) { - fatal_error("** For CN or BGC mode you MUST set max patch PFT's to $maxpatchpft{$nl_flags->{'use_crop'}}\n" . - "**\n" . - "** When the crop model is on then it must be set to $maxpatchpft{'crop'} otherwise to $maxpatchpft{'nocrop'}\n" . - "** Set the bgc mode, crop and maxpft by the following means from highest to lowest precedence:\n" . - "** * by the command-line options -bgc, -crop and -maxpft\n" . - "** * by a default configuration file, specified by -defaults\n" . - "**\n"); + if ($opts->{$var} eq "default") { + fatal_error("** For CN or BGC mode you MUST set max patch PFT $val to $maxpatchpft{$nl_flags->{'use_crop'}}\n" . + "**\n" . + "** When the crop model is on then it must be set to $maxpatchpft{'crop'} otherwise to $maxpatchpft{'nocrop'}\n" . + "** Set the bgc mode, crop and maxpft by the following means from highest to lowest precedence:\n" . + "** * by the command-line options -bgc, -crop and -maxpft\n" . + "** * by a default configuration file, specified by -defaults\n" . + "**\n"); + } else { + message("running with maxpft NOT equal to $maxpatchpft{$nl_flags->{'use_crop'}} is " . + "NOT validated / scientifically supported.\n"); + + } } if ( $nl_flags->{'maxpft'} > $maxpatchpft{$nl_flags->{'use_crop'}} ) { fatal_error("** Max patch PFT's can NOT exceed $maxpatchpft{$nl_flags->{'use_crop'}}\n" . From 98be2ae3c832a825c764ca2bea9a0770f1140dc2 Mon Sep 17 00:00:00 2001 From: Fengming Yuan Date: Mon, 26 Mar 2018 10:06:23 -0400 Subject: [PATCH 569/904] Allow read flexible-length PFT parameters and names, if user-defined. NOTES for when user-defined: (1) explicitly define a few PFT properties (flags) as following: nonvasular (0, 1=moss, or 2=lichen); woody (0, 1=tree, or 2=shrub); needleleaf ( 0=broadleaf, or 1=needleleaf); graminoid ( 0=woody/crop/nonvascular, or 1=graminoid); generic_crop ( 0=non-crop/prognostic crop, 1=generic_crop. NOTE that this is for not having conflicts with crop module development). climatezone (0=not-specific, 1=tropical, 2=temperate, 3=boreal, or 4=arctic) (2) in case's env_run.xml, add ' -maxpft 12' in which 12 is an example number of pfts, in "ELM_BLDNML_OPTS'" like the following: char CLM build-namelist options --- .../elm/src/data_types/VegetationType.F90 | 5 + components/elm/src/main/elm_varpar.F90 | 9 +- components/elm/src/main/pftvarcon.F90 | 354 +++++++++++++++++- 3 files changed, 352 insertions(+), 16 deletions(-) diff --git a/components/elm/src/data_types/VegetationType.F90 b/components/elm/src/data_types/VegetationType.F90 index a3711f7c117b..dd8bd7d842d4 100644 --- a/components/elm/src/data_types/VegetationType.F90 +++ b/components/elm/src/data_types/VegetationType.F90 @@ -33,6 +33,11 @@ module VegetationType ! 24 => irrigated soybean ! -------------------------------------------------------- ! + + ! ----------------------user-defined parameter file --------------------------------------------------------------------- + ! NOTE: if user provides own parameter file, with different 'pft_name' or order, + ! the above ilist of default PFTs will be replaced, and arrays of sizes as below will be changed as well. + use shr_kind_mod , only : r8 => shr_kind_r8 use elm_varcon , only : ispval, spval diff --git a/components/elm/src/main/elm_varpar.F90 b/components/elm/src/main/elm_varpar.F90 index 21e8f659210f..1f82d65ec7eb 100644 --- a/components/elm/src/main/elm_varpar.F90 +++ b/components/elm/src/main/elm_varpar.F90 @@ -48,20 +48,23 @@ module elm_varpar integer, parameter :: ndst = 4 ! number of dust size classes (BGC only) integer, parameter :: dst_src_nbr = 3 ! number of size distns in src soil (BGC only) integer, parameter :: sz_nbr = 200 ! number of sub-grid bins in large bin of dust size distribution (BGC only) - integer, parameter :: mxpft = 50 ! maximum number of PFT's for any mode; + !integer, parameter :: mxpft = 50 ! maximum number of PFT's for any mode; + integer :: mxpft = 50 ! maximum number of PFT's for any mode; + ! can be modified from reading pft-physiology in 'main/pftvarcon.F90:pftconrd' ! FIX(RF,032414) might we set some of these automatically from reading pft-physiology? integer, parameter :: numveg = 16 ! number of veg types (without specific crop) integer, parameter :: nlayer = 3 ! number of VIC soil layer --Added by AWang integer :: nlayert ! number of VIC soil layer + 3 lower thermal layers - integer :: numpft = mxpft ! actual # of patches (without bare), a total that spans LUs + integer :: numpft = 50 ! actual # of patches (without bare), a total that spans LUs integer :: numcft = 36 ! actual # of crops logical :: crop_prog = .true. ! If prognostic crops is turned on integer :: maxpatch_urb= 5 ! max number of urban patches (columns) in urban landunit integer :: mxpft_nc ! maximum number of PFT's when use_crop=False; - integer :: maxpatch_pft ! max number of plant functional types in naturally vegetated landunit (namelist setting) + integer :: maxpatch_pft ! max number of plant functional types in naturally vegetated landunit (namelist setting: maxpft) + ! This number must be exactly matched with 'natpft' in surfdata.nc integer, parameter :: nsoilorder = 15 ! number of soil orders diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index 2e2a00f4b4ec..cbd5946bb4e6 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -9,11 +9,19 @@ module pftvarcon use shr_kind_mod, only : r8 => shr_kind_r8 use shr_log_mod , only : errMsg => shr_log_errMsg use abortutils , only : endrun - use elm_varpar , only : mxpft, numrad, ivis, inir, cft_lb, cft_ub + use elm_varpar , only : mxpft, numrad, ivis, inir use elm_varpar , only: mxpft_nc use elm_varctl , only : iulog, use_vertsoilc use elm_varpar , only : nlevdecomp_full, nsoilorder use elm_varctl , only : nu_com + !------------------------------------------------------------------------------------------- + use elm_varpar , only : crop_prog + use elm_varpar , only : natpft_size, natpft_lb, natpft_ub + use elm_varpar , only : cft_size, cft_lb, cft_ub + use elm_varpar , only : surfpft_size, surfpft_lb, surfpft_ub + use elm_varpar , only : numpft, numcft, maxpatch_pft, max_patch_per_col, maxpatch_urb + use elm_varctl , only : create_crop_landunit + !------------------------------------------------------------------------------------------- ! ! !PUBLIC TYPES: implicit none @@ -116,6 +124,7 @@ module pftvarcon real(r8), allocatable :: grpnow(:) !growth respiration parameter real(r8), allocatable :: rootprof_beta(:) !CLM rooting distribution parameter for C and N inputs [unitless] + ! add pft dependent parameters for phosphorus -X.YANG real(r8), allocatable :: leafcp(:) !leaf C:P [gC/gP] real(r8), allocatable :: lflitcp(:) !leaf litter C:P (gC/gP) @@ -303,6 +312,16 @@ module pftvarcon real(r8), allocatable :: gcbr_p(:) !effectiveness of roots in reducing rainfall-driven erosion real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion + ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, + ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. + real(r8), allocatable :: climatezone(:) ! distributed climate zone (0 = any zone, 1 = tropical, 2 = temperate, 3 = boreal, 4 = arctic) + real(r8), allocatable :: nonvascular(:) ! nonvascular lifeform flag (0 = vascular, 1 = moss, 2 = lichen) + real(r8), allocatable :: needleleaf(:) ! needleleaf lifeform flag (0 = broadleaf, 1 = needleleaf) + real(r8), allocatable :: graminoid(:) ! graminoid lifeform flag (0 = nonvascular+woody+crop+percrop, 1 = graminoid) + real(r8), allocatable :: generic_crop(:) ! generic_crop (0 = non_crop or prognostic crop, 1 = generic crop, i.e. crop when use_crop=false) + real(r8), allocatable :: nfixer(:) ! nitrogen fixer flag (0 = inable, 1 = able to nitrogen fixation from atm. N2) + + ! ! !PUBLIC MEMBER FUNCTIONS: public :: pftconrd ! Read and initialize vegetation (PFT) constants @@ -347,6 +366,9 @@ subroutine pftconrd integer :: dimid ! netCDF dimension id integer :: npft ! number of pfts on pft-physiology file logical :: readv ! read variable in or not + logical :: PFT_DEFAULT ! pft names are default, i.e. NOT user-defined + integer :: ncft0, ncft ! crop pft index of first/last when 'create_crop_landunit' is true + integer :: noncropmax ! max non-crop pft index (to check when 'create_crop_landunit' is true) character(len=32) :: subname = 'pftconrd' ! subroutine name ! ! Expected PFT names: The names expected on the paramfile file and the order they are expected to be in. @@ -410,6 +432,24 @@ subroutine pftconrd expected_pftnames(49) = 'willow ' expected_pftnames(50) = 'irrigated_willow ' + ! read actual 'npft' from parameter file + if (masterproc) then + write(iulog,*) 'Attempting to read PFT physiological data .....' + end if + call getfil (paramfile, locfn, 0) + call ncd_pio_openfile (ncid, trim(locfn), 0) + call ncd_inqdid(ncid,'pft',dimid) + call ncd_inqdlen(ncid,dimid,npft) + + ! now 'mxpft' in 'elm_varpar' updated by npft here + mxpft = npft - 1 + mxpft_nc = npft - 1 ! when .not.use_crop, so here temporarily set and may be changed after read-through PFT-physiology + + ! NOTES: + ! In parameter file, 'npft' (mxpft) [number of pfts in parameter file] can be greater than 'maxpatch_pft' [number of pfts in surfdata file] which also set by namelist 'maxpft', + ! but cannot be less, i.e. there cannot be more pfts in the surface file than there are set of pft parameters, if running the non-crop version of model. + if (npft=1 .or. percrop(i)>=1) .and. generic_crop(i)==0) then + ! at this moment, read-in 'generic_crop' is actually 'iscft' (i.e. crop as CFT or not) + ! So must re-assign + generic_crop(i) = 1 + else + generic_crop(i) = 0 + end if + end do + end if + + ! woody=2 for shrub NOT YET ready in rest of ELM code + do i = 0, npft-1 + if (woody(i)>1) woody(i) == 1 + end do call ncd_pio_closefile(ncid) + + if ( PFT_DEFAULT ) then + ! if still reading in default PFT physiology file, + ! pft indexing will be as old way + do i = 0, mxpft if(.not. use_crop .and. i > mxpft_nc) EXIT ! exit the do loop @@ -1032,13 +1167,13 @@ subroutine pftconrd ! on non-fates columns. For now, they are incompatible, and this check is warranted (rgk 04-2017) ! avd - this should be independent of FATES because it fails for non-crop config otherwise - - if(.not. use_crop .and. i > mxpft_nc) EXIT ! exit the do loop + if(.not. use_fates)then if ( trim(adjustl(pftname(i))) /= trim(expected_pftnames(i)) )then write(iulog,*)'pftconrd: pftname is NOT what is expected, name = ', & trim(pftname(i)), ', expected name = ', trim(expected_pftnames(i)) call endrun(msg='pftconrd: bad name for pft on paramfile dataset'//errMsg(__FILE__, __LINE__)) end if + end if if ( trim(pftname(i)) == 'not_vegetated' ) noveg = i if ( trim(pftname(i)) == 'needleleaf_evergreen_temperate_tree' ) ndllf_evr_tmp_tree = i @@ -1103,12 +1238,204 @@ subroutine pftconrd nppercropmax = nwillowirrig ! last prognostic perennial crop in list end if + !------------------------------------------------------------------------------------------- + ! the following is initialized as 0 above for all PFT. + ! here the hard-coded values (or flags) for default ELM PFT physiology will be working as original + ! when not using those indexing of PFT orders anymore in other codes than here. + needleleaf(noveg+1:ndllf_dcd_brl_tree) = 1 + graminoid(nc3_arctic_grass:nc4_grass) = 1 + generic_crop(nc3crop:nc3irrig) = 1 + nfixer(nsoybean) = 1 + nfixer(nsoybeanirrig) = 1 + + climatezone(ndllf_evr_tmp_tree) = 2 + climatezone(ndllf_evr_brl_tree) = 3 + climatezone(ndllf_dcd_brl_tree) = 3 + climatezone(nbrdlf_evr_trp_tree) = 1 + climatezone(nbrdlf_evr_tmp_tree) = 2 + climatezone(nbrdlf_dcd_trp_tree) = 1 + climatezone(nbrdlf_dcd_tmp_tree) = 2 + climatezone(nbrdlf_dcd_brl_tree) = 3 + climatezone(nbrdlf_dcd_tmp_shrub)= 2 + climatezone(nbrdlf_dcd_brl_shrub)= 3 + climatezone(nc3_arctic_grass) = 4 + !------------------------------------------------------------------------------------------- + + + + ! NOT default PFT file + else + + ! not vegetated checking + noveg = -1 + + ! when user-defined PFTs, if crop included, it must be the default way: + ! cropts must be in one block and after nat-pft if 'create_crop_landunit' or 'use_crop' is true + npcropmin = -1 ! first prognostic crop + npcropmax = -1 ! last prognostic crop in list + nppercropmin = -1 ! first prognostic perennial crop + nppercropmax = -1 ! last prognostic perennial crop in list + + numcft = 0 + ncft = -1 + ncft0 = -1 + noncropmax = 0 + do i = 0, npft-1 + if (crop(i)>=1 .or. percrop(i)>=1 .or. generic_crop(i)>=1) then + numcft = numcft + 1 + + if(use_crop) then + ! if 'generic_crop' specifically defined, + ! 'crop' will not be counted into prognostic + if (crop(i)>=1 .and. generic_crop(i)==0) then + npcropmax = i + if(npcropmin<=0) npcropmin = i + end if + + ! NOTE: there is a misunderstanding in pft physiology parameter file: 'perennial crop' IS NOT 'crop', + ! but still counted into 'numcft' + if(percrop(i)==1) then + nppercropmax = i + if(nppercropmin<=0) nppercropmin = i + end if + + else + if(crop(i)>=1 .or. generic_crop(i)>=1) then + ! in case either 'crop' or 'generic_crop' or both defined when not use_crop=.true. + generic_crop(i) = 1 + else + generic_crop(i) = 0 + end if + end if + + ! + if (create_crop_landunit) then + ! make sure all crop-pft are in one block and following nat-pft SO THAT creating crop landunit is possible + ncft = ncft + 1 + if (ncft0<0) ncft0=i + end if + + ! + else if (create_crop_landunit) then + noncropmax = i + + end if + + ! need to check 'noveg' + if ( woody(i)<=0 .and. graminoid(i)<=0 .and. nonvascular(i)<=0 .and. & + generic_crop(i)<=0 .and. crop(i)<=0 .and. percrop(i)<=0) then + if (noveg>=0) then + ! not yet support multiple non-vegetated PFT + ! this also will catch error of no actual PFT if npft>1 + call endrun(msg=' ERROR: more than 1 not vegetated in physiology parameter nc file.'//errMsg(__FILE__, __LINE__)) + else + noveg = i + end if + end if + + ! + end do + + ! make sure non-generic crop indices always beyond natural-pft, even if not available - used in filterMod.F90) + if (npcropmin < 0 .and. npcropmax < 0) then + npcropmin = npft + npcropmax = npft + end if + + ! MUST re-do some constants which already set in 'elm_varpar.F90:elm_varpar_init()' + mxpft_nc = min(maxpatch_pft,npft) - 1 ! user-defined is what max. + numpft = min(maxpatch_pft,npft) - 1 ! actual # of patches (without bare) + + if (create_crop_landunit) then + if (ncft0 /= noncropmax+1) then + call endrun(msg=' ERROR: when create_crop_landunit is true, crop must be following non-crop PFT .'//errMsg(__FILE__, __LINE__)) + end if + if (ncft /= npft-1) then + call endrun(msg=' ERROR: when create_crop_landunit is true, last crop must be the last one of all PFTs .'//errMsg(__FILE__, __LINE__)) + end if + + natpft_size = (numpft + 1) - numcft ! note that numpft doesn't include bare ground -- thus we add 1 + cft_size = numcft + else + natpft_size = numpft + 1 ! note that numpft doesn't include bare ground -- thus we add 1 + cft_size = 0 + end if + natpft_lb = 0 + natpft_ub = natpft_lb + natpft_size - 1 + cft_lb = natpft_ub + 1 + cft_ub = max(cft_lb, cft_lb + cft_size - 1) ! NOTE: if cft_size is ZERO, could be issue (but so far so good) + surfpft_lb = natpft_lb + surfpft_ub = natpft_ub + surfpft_size = natpft_size + max_patch_per_col= max(numpft+1, numcft, maxpatch_urb) + + + end if ! end if 'PFT_DEFAULT' + + ! checking of pft flags' conflict + if ( .not. use_fates ) then + do i = 0, mxpft + if (i == noveg) then + if ( (nonvascular(i)+woody(i)+graminoid(i)+generic_crop(i)+crop(i)+percrop(i)) >= 1 .or. & + (needleleaf(i)+evergreen(i)+stress_decid(i)+season_decid(i)+nfixer(i)) >= 1 ) then + print *, 'ERROR: Incorrect not-vegetated PFT flags: ', i, ' ', trim(pftname(i)) + call endrun(msg=' ERROR: not_vegetated has at least one positive PFT flag '//errMsg(__FILE__, __LINE__)) + end if + + else if ( (nonvascular(i)+woody(i)+graminoid(i)+generic_crop(i)+crop(i)+percrop(i)) >= 1) then + if (nonvascular(i) >= 1 .and. (woody(i)+graminoid(i)+generic_crop(i)+crop(i)+percrop(i)) >= 1) then + print *, 'ERROR: Incorrect nonvasculr PFT flags: ', i, ' ', trim(pftname(i)) + call endrun(msg=' ERROR: nonvascular PFT cannot be any of woody/graminoid/crop type '//errMsg(__FILE__, __LINE__)) + else if (woody(i) >= 1 .and. (nonvascular(i)+graminoid(i)+generic_crop(i)+crop(i)+percrop(i)) >= 1) then + print *, 'ERROR: Incorrect woody PFT flags: ', i, ' ', trim(pftname(i)) + call endrun(msg=' ERROR: woody PFT cannot be any of nonvascular/graminoid/crop type - '//errMsg(__FILE__, __LINE__)) + else if (graminoid(i) >= 1 .and. (nonvascular(i)+woody(i)+generic_crop(i)+crop(i)+percrop(i)) >=1 ) then + print *, 'ERROR: Incorrect graminoid PFT flags: ', i, ' ', trim(pftname(i)) + call endrun(msg=' ERROR: graminoid PFT cannot be any of nonvascular/woody/crop type - '//errMsg(__FILE__, __LINE__)) + else if ( (generic_crop(i)+crop(i)+percrop(i)) >= 1 .and. (nonvascular(i)+woody(i)+graminoid(i)) >= 1) then + print *, 'ERROR: Incorrect crop PFT flags: ', i, ' ', trim(pftname(i)) + call endrun(msg=' ERROR: crop PFT cannot be any of nonvascular/woody/graminoid type - '//errMsg(__FILE__, __LINE__)) + end if + + if( (stress_decid(i)*season_decid(i)) >= 1 ) then + print *, 'ERROR: Incorrect stress_decid or season_decid flags: ', i, ' ', trim(pftname(i)) + call endrun(msg=' ERROR: stress_decid AND season_decid cannot be both 1 - '//errMsg(__FILE__, __LINE__)) + elseif( (evergreen(i)*(stress_decid(i)+season_decid(i)) ) >= 1 ) then + print *, 'ERROR: Incorrect evergreen AND season_/stress_decid flags: ', i, ' ', trim(pftname(i)) + call endrun(msg=' ERROR: evergreen AND (stress_decid OR season_decid) cannot be both 1 - '//errMsg(__FILE__, __LINE__)) + end if + + else + + call endrun(msg=' ERROR: not_vegetated AND none of vegetation type for PFT - '//errMsg(__FILE__, __LINE__)) + + end if + + end do + end if + + ! information + if (masterproc) then + write(iulog,*) + write(iulog,*) 'Using PFT physiological parameters from: ', paramfile + write(iulog,*) ' -- index -- name -- climate zone -- -- woody -- -- needleleaf -- -- evergreen -- -- stress_decid -- -- season_decid -- -- graminoid-- -- generic_crop -- -- crop -- -- perennial crop -- -- nfixer --' + do i = 0, npft-1 + write(iulog,*) i, pftname(i), int(climatezone(i)), int(woody(i)), int(needleleaf(i)), & + int(evergreen(i)), int(stress_decid(i)), int(season_decid(i)), & + int(graminoid(i)), int(generic_crop(i)), int(crop(i)), int(percrop(i)), int(nfixer(i)) + end do + write(iulog,*) + end if + + + !------------------------------------------------------------------------------------------- + call set_is_pft_known_to_model() - call set_num_cfts_known_to_model() + if (cft_size>0) call set_num_cfts_known_to_model() if( .not. use_fates ) then if( .not. use_crop) then - if ( npcropmax /= mxpft_nc )then + if ( npcropmax /= mxpft_nc .and. crop_prog)then call endrun(msg=' ERROR: npcropmax is NOT the last value'//errMsg(__FILE__, __LINE__)) end if else @@ -1118,6 +1445,7 @@ subroutine pftconrd end if do i = 0, mxpft if(.not. use_crop .and. i > mxpft_nc) EXIT ! exit the do loop + if(.not.PFT_DEFAULT) EXIT ! no checking of indexing PFTs for user-defined if( .not. use_crop) then if ( irrigated(i) == 1.0_r8 .and. (i == nc3irrig .or. & i == ncornirrig .or. & From a8b486bc426d5f69971230ed8312fdb45b97c489 Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Tue, 31 Oct 2023 14:41:12 -0400 Subject: [PATCH 570/904] Distinguish woody types between tree (1) and shrub (2). --- .../elm/src/biogeochem/AllocationMod.F90 | 26 +++++++-------- .../src/biogeochem/CNAllocationBetrMod.F90 | 22 ++++++------- .../elm/src/biogeochem/CNCarbonStateType.F90 | 2 +- .../src/biogeochem/CNNStateUpdate1BeTRMod.F90 | 8 ++--- .../src/biogeochem/CNNitrogenStateType.F90 | 2 +- .../elm/src/biogeochem/CNPhenologyBeTRMod.F90 | 32 +++++++++---------- .../src/biogeochem/CarbonStateUpdate1Mod.F90 | 18 +++++------ .../elm/src/biogeochem/ComputeSeedMod.F90 | 2 +- .../elm/src/biogeochem/GrowthRespMod.F90 | 2 +- .../elm/src/biogeochem/MaintenanceRespMod.F90 | 2 +- .../biogeochem/NitrogenStateUpdate1Mod.F90 | 8 ++--- .../src/biogeochem/PhenologyFluxLimitMod.F90 | 22 ++++++------- .../elm/src/biogeochem/PhenologyMod.F90 | 32 +++++++++---------- .../biogeochem/PhosphorusStateUpdate1Mod.F90 | 8 ++--- .../elm/src/biogeochem/VegStructUpdateMod.F90 | 2 +- .../elm/src/data_types/VegetationDataType.F90 | 6 ++-- .../data_types/VegetationPropertiesType.F90 | 17 +++++----- components/elm/src/main/pftvarcon.F90 | 11 +++---- 18 files changed, 111 insertions(+), 111 deletions(-) diff --git a/components/elm/src/biogeochem/AllocationMod.F90 b/components/elm/src/biogeochem/AllocationMod.F90 index 2795fd42c8ed..a31a07cd8f6c 100644 --- a/components/elm/src/biogeochem/AllocationMod.F90 +++ b/components/elm/src/biogeochem/AllocationMod.F90 @@ -583,7 +583,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp ! These fluxes should already be in gC/m2/s mr = leaf_mr(p) + froot_mr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) else if (ivt(p) >= npcropmin) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) @@ -815,7 +815,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp ! determine N requirements ! determine P requirements -X. YANG - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then c_allometry(p) = (1._r8+g1)*(1._r8+f1+f3*(1._r8+f2)) n_allometry(p) = 1._r8/cnl + f1/cnfr + (f3*f4*(1._r8+f2))/cnlw + & (f3*(1._r8-f4)*(1._r8+f2))/cndw @@ -2372,7 +2372,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & endif mr = leaf_mr(p) + froot_mr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) else if (ivt(p) >= npcropmin) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) @@ -2466,7 +2466,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & plant_calloc(p) = availc(p) ! here no down-regulation on allocatable C here, NP limitation is implemented in leaf-level NP control on GPP - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then c_allometry(p) = (1._r8+g1)*(1._r8+f1+f3*(1._r8+f2)) n_allometry(p) = 1._r8/cnl + f1/cnfr + (f3*f4*(1._r8+f2))/cnlw + & (f3*(1._r8-f4)*(1._r8+f2))/cndw @@ -2549,7 +2549,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * (1._r8 - fcur) * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f1 * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f1 * (1._r8 - fcur) * (1 + g1) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * (1._r8 - fcur) * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * (1._r8 - f4) * fcur * (1 + g1) @@ -2580,7 +2580,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_leafc_storage(p) = nlc * (1._r8 - fcur) cpool_to_frootc(p) = nlc * f1 * fcur cpool_to_frootc_storage(p) = nlc * f1 * (1._r8 - fcur) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -2633,7 +2633,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_leafn_storage(p) = (nlc / cnl) * (1._r8 - fcur) npool_to_frootn(p) = (nlc * f1 / cnfr) * fcur npool_to_frootn_storage(p) = (nlc * f1 / cnfr) * (1._r8 - fcur) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then npool_to_livestemn(p) = (nlc * f3 * f4 / cnlw) * fcur npool_to_livestemn_storage(p) = (nlc * f3 * f4 / cnlw) * (1._r8 - fcur) npool_to_deadstemn(p) = (nlc * f3 * (1._r8 - f4) / cndw) * fcur @@ -2677,7 +2677,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_leafp_storage(p) = (nlc / cpl) * (1._r8 - fcur) ppool_to_frootp(p) = (nlc * f1 / cpfr) * fcur ppool_to_frootp_storage(p) = (nlc * f1 / cpfr) * (1._r8 - fcur) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then ppool_to_livestemp(p) = (nlc * f3 * f4 / cplw) * fcur ppool_to_livestemp_storage(p) = (nlc * f3 * f4 / cplw) * (1._r8 -fcur) ppool_to_deadstemp(p) = (nlc * f3 * (1._r8 - f4) / cpdw) *fcur @@ -2711,7 +2711,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ! growth is assigned here. gresp_storage = cpool_to_leafc_storage(p) + cpool_to_frootc_storage(p) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) gresp_storage = gresp_storage + cpool_to_deadstemc_storage(p) gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) @@ -2748,7 +2748,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_frootn(p) = cpool_to_frootc(p) / cnfr npool_to_frootn_storage(p) = cpool_to_frootc_storage(p) / cnfr - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then supplement_to_plantn(p) = supplement_to_plantn(p) + cpool_to_livestemc(p) / cnlw - npool_to_livestemn(p) supplement_to_plantn(p) = supplement_to_plantn(p) + cpool_to_livestemc_storage(p) / cnlw & - npool_to_livestemn_storage(p) @@ -2815,7 +2815,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_frootp(p) = cpool_to_frootc(p) / cpfr ppool_to_frootp_storage(p) = cpool_to_frootc_storage(p) / cpfr - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then supplement_to_plantp(p) = supplement_to_plantp(p) + max(cpool_to_livestemc(p) / cplw & - ppool_to_livestemp(p),0._r8) @@ -3891,7 +3891,7 @@ subroutine dynamic_plant_alloc( nutrient_scalar, water_scalar, laindex, alloc_le alloc_froot = min(alloc_froot, 0.4_r8) ! stem allocation - if (woody == 1.0_r8) then + if (woody >= 1.0_r8) then alloc_stem = alloc_s0 * 3.0_r8 * min(nu_scalar,w_scalar) / (2.0_r8 * light_scalar + min(nu_scalar,w_scalar)) else alloc_stem = 0.0_r8 @@ -3908,7 +3908,7 @@ subroutine dynamic_plant_alloc( nutrient_scalar, water_scalar, laindex, alloc_le ! if lai greater than laimax then no allocation to leaf; leaf allocation goes to stem or fine root if (laindex > laimax) then - if (woody == 1.0_r8) then + if (woody >= 1.0_r8) then alloc_stem = alloc_stem + alloc_leaf/2._r8 - 0.005_r8 alloc_froot = alloc_froot + alloc_leaf/2._r8 - 0.005_r8 else diff --git a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 index 9701eebda607..42fad84a9d1d 100644 --- a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 +++ b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 @@ -643,7 +643,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp ! These fluxes should already be in gC/m2/s mr = leaf_mr(p) + froot_mr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) else if (ivt(p) >= npcropmin) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) @@ -852,7 +852,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp ! based on available C, use constant allometric relationships to ! determine N requirements - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then c_allometry(p) = (1._r8+g1)*(1._r8+f1+f3*(1._r8+f2)) n_allometry(p) = 1._r8/cnl + f1/cnfr + (f3*f4*(1._r8+f2))/cnlw + & (f3*(1._r8-f4)*(1._r8+f2))/cndw @@ -1434,7 +1434,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & plant_palloc(p) = sminp_to_ppool(p) + retransp_to_ppool(p) mr = leaf_mr(p) + froot_mr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) else if (ivt(p) >= npcropmin) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) @@ -1523,7 +1523,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & plant_calloc(p) = availc(p) ! here no down-regulation on allocatable C here, NP limitation is implemented in leaf-level NP control on GPP - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then c_allometry(p) = (1._r8+g1)*(1._r8+f1+f3*(1._r8+f2)) n_allometry(p) = 1._r8/cnl + f1/cnfr + (f3*f4*(1._r8+f2))/cnlw + & (f3*(1._r8-f4)*(1._r8+f2))/cndw @@ -1604,7 +1604,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * (1._r8 - fcur) * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f1 * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f1 * (1._r8 - fcur) * (1 + g1) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * (1._r8 - fcur) * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * (1._r8 - f4) * fcur * (1 + g1) @@ -1635,7 +1635,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_leafc_storage(p) = nlc * (1._r8 - fcur) cpool_to_frootc(p) = nlc * f1 * fcur cpool_to_frootc_storage(p) = nlc * f1 * (1._r8 - fcur) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -1688,7 +1688,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_leafn_storage(p) = (nlc / cnl) * (1._r8 - fcur) npool_to_frootn(p) = (nlc * f1 / cnfr) * fcur npool_to_frootn_storage(p) = (nlc * f1 / cnfr) * (1._r8 - fcur) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then npool_to_livestemn(p) = (nlc * f3 * f4 / cnlw) * fcur npool_to_livestemn_storage(p) = (nlc * f3 * f4 / cnlw) * (1._r8 - fcur) npool_to_deadstemn(p) = (nlc * f3 * (1._r8 - f4) / cndw) * fcur @@ -1732,7 +1732,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_leafp_storage(p) = (nlc / cpl) * (1._r8 - fcur) ppool_to_frootp(p) = (nlc * f1 / cpfr) * fcur ppool_to_frootp_storage(p) = (nlc * f1 / cpfr) * (1._r8 - fcur) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then ppool_to_livestemp(p) = (nlc * f3 * f4 / cplw) * fcur ppool_to_livestemp_storage(p) = (nlc * f3 * f4 / cplw) * (1._r8 -fcur) ppool_to_deadstemp(p) = (nlc * f3 * (1._r8 - f4) / cpdw) *fcur @@ -1766,7 +1766,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ! growth is assigned here. gresp_storage = cpool_to_leafc_storage(p) + cpool_to_frootc_storage(p) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) gresp_storage = gresp_storage + cpool_to_deadstemc_storage(p) gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) @@ -1797,7 +1797,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_frootn(p) = cpool_to_frootc(p) / cnfr npool_to_frootn_storage(p) = cpool_to_frootc_storage(p) / cnfr - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then supplement_to_sminn_vr(c,1) = supplement_to_sminn_vr(c,1) + cpool_to_livestemc(p) / cnlw - npool_to_livestemn(p) supplement_to_sminn_vr(c,1) = supplement_to_sminn_vr(c,1) + cpool_to_livestemc_storage(p) / cnlw & - npool_to_livestemn_storage(p) @@ -1864,7 +1864,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_frootp(p) = cpool_to_frootc(p) / cpfr ppool_to_frootp_storage(p) = cpool_to_frootc_storage(p) / cpfr - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then supplement_to_sminp_vr(c,1) = supplement_to_sminp_vr(c,1) + max(cpool_to_livestemc(p) / cplw & - ppool_to_livestemp(p),0._r8) supplement_to_sminp_vr(c,1) = supplement_to_sminp_vr(c,1) + max(cpool_to_livestemc_storage(p) / cplw & diff --git a/components/elm/src/biogeochem/CNCarbonStateType.F90 b/components/elm/src/biogeochem/CNCarbonStateType.F90 index 2ed04f03231c..8f258bbb6a9a 100644 --- a/components/elm/src/biogeochem/CNCarbonStateType.F90 +++ b/components/elm/src/biogeochem/CNCarbonStateType.F90 @@ -477,7 +477,7 @@ subroutine InitCold(this, bounds, ratio, c12_carbonstate_vars) this%livestemc_storage_patch(p) = 0._r8 this%livestemc_xfer_patch(p) = 0._r8 - if (veg_vp%woody(veg_pp%itype(p)) == 1._r8) then + if (veg_vp%woody(veg_pp%itype(p)) >= 1.0_r8) then this%deadstemc_patch(p) = 0.1_r8 * ratio else this%deadstemc_patch(p) = 0._r8 diff --git a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 index 6eee5f923da0..95d884de490f 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 @@ -90,7 +90,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%frootn(p) = veg_ns%frootn(p) + veg_nf%frootn_xfer_to_frootn(p)*dt veg_ns%frootn_xfer(p) = veg_ns%frootn_xfer(p) - veg_nf%frootn_xfer_to_frootn(p)*dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) - veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%deadstemn(p) = veg_ns%deadstemn(p) + veg_nf%deadstemn_xfer_to_deadstemn(p)*dt @@ -116,7 +116,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%leafn_to_retransn(p)*dt ! live wood turnover and retranslocation fluxes - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_deadstemn(p)*dt veg_ns%deadstemn(p) = veg_ns%deadstemn(p) + veg_nf%livestemn_to_deadstemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_retransn(p)*dt @@ -153,7 +153,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_frootn_storage(p)*dt veg_ns%frootn_storage(p) = veg_ns%frootn_storage(p) + veg_nf%npool_to_frootn_storage(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%npool_to_livestemn(p)*dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn_storage(p)*dt @@ -189,7 +189,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%frootn_storage(p) = veg_ns%frootn_storage(p) - veg_nf%frootn_storage_to_xfer(p)*dt veg_ns%frootn_xfer(p) = veg_ns%frootn_xfer(p) + veg_nf%frootn_storage_to_xfer(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ns%livestemn_storage(p) = veg_ns%livestemn_storage(p) - veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) + veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%deadstemn_storage(p) = veg_ns%deadstemn_storage(p) - veg_nf%deadstemn_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/CNNitrogenStateType.F90 b/components/elm/src/biogeochem/CNNitrogenStateType.F90 index 003e34e132d9..dda6477899a7 100644 --- a/components/elm/src/biogeochem/CNNitrogenStateType.F90 +++ b/components/elm/src/biogeochem/CNNitrogenStateType.F90 @@ -588,7 +588,7 @@ subroutine InitCold(this, bounds, & ! tree types need to be initialized with some stem mass so that ! roughness length is not zero in canopy flux calculation - if (veg_vp%woody(veg_pp%itype(p)) == 1._r8) then + if (veg_vp%woody(veg_pp%itype(p)) >= 1.0_r8) then this%deadstemn_patch(p) = deadstemc_patch(p) / veg_vp%deadwdcn(veg_pp%itype(p)) else this%deadstemn_patch(p) = 0._r8 diff --git a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 index 4bc23896427c..23f461ad3055 100644 --- a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 @@ -693,7 +693,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & frootn_xfer_to_frootn(p) = 0.0_r8 leafp_xfer_to_leafp(p) = 0.0_r8 frootp_xfer_to_frootp(p) = 0.0_r8 - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer_to_livestemc(p) = 0.0_r8 deadstemc_xfer_to_deadstemc(p) = 0.0_r8 livecrootc_xfer_to_livecrootc(p) = 0.0_r8 @@ -714,7 +714,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & frootc_xfer(p) = 0.0_r8 frootn_xfer(p) = 0.0_r8 frootp_xfer(p) = 0.0_r8 - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer(p) = 0.0_r8 livestemn_xfer(p) = 0.0_r8 livestemp_xfer(p) = 0.0_r8 @@ -777,7 +777,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & ! set carbon fluxes for shifting storage pools to transfer pools leafc_storage_to_xfer(p) = fstor2tran * leafc_storage(p)/dt frootc_storage_to_xfer(p) = fstor2tran * frootc_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_storage_to_xfer(p) = fstor2tran * livestemc_storage(p)/dt deadstemc_storage_to_xfer(p) = fstor2tran * deadstemc_storage(p)/dt livecrootc_storage_to_xfer(p) = fstor2tran * livecrootc_storage(p)/dt @@ -788,7 +788,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & ! set nitrogen fluxes for shifting storage pools to transfer pools leafn_storage_to_xfer(p) = fstor2tran * leafn_storage(p)/dt frootn_storage_to_xfer(p) = fstor2tran * frootn_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemn_storage_to_xfer(p) = fstor2tran * livestemn_storage(p)/dt deadstemn_storage_to_xfer(p) = fstor2tran * deadstemn_storage(p)/dt livecrootn_storage_to_xfer(p) = fstor2tran * livecrootn_storage(p)/dt @@ -798,7 +798,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & ! set phosphorus fluxes for shifting storage pools to transfer pools leafp_storage_to_xfer(p) = fstor2tran * leafp_storage(p)/dt frootp_storage_to_xfer(p) = fstor2tran * frootp_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemp_storage_to_xfer(p) = fstor2tran * livestemp_storage(p)/dt deadstemp_storage_to_xfer(p) = fstor2tran * deadstemp_storage(p)/dt livecrootp_storage_to_xfer(p) = fstor2tran * livecrootp_storage(p)/dt @@ -1040,7 +1040,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & frootn_xfer_to_frootn(p) = 0._r8 leafp_xfer_to_leafp(p) = 0._r8 frootp_xfer_to_frootp(p) = 0._r8 - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer_to_livestemc(p) = 0._r8 deadstemc_xfer_to_deadstemc(p) = 0._r8 livecrootc_xfer_to_livecrootc(p) = 0._r8 @@ -1061,7 +1061,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & frootc_xfer(p) = 0._r8 frootn_xfer(p) = 0._r8 frootp_xfer(p) = 0._r8 - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer(p) = 0._r8 livestemn_xfer(p) = 0._r8 livestemp_xfer(p) = 0._r8 @@ -1148,7 +1148,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set carbon fluxes for shifting storage pools to transfer pools leafc_storage_to_xfer(p) = fstor2tran * leafc_storage(p)/dt frootc_storage_to_xfer(p) = fstor2tran * frootc_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_storage_to_xfer(p) = fstor2tran * livestemc_storage(p)/dt deadstemc_storage_to_xfer(p) = fstor2tran * deadstemc_storage(p)/dt livecrootc_storage_to_xfer(p) = fstor2tran * livecrootc_storage(p)/dt @@ -1159,7 +1159,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set nitrogen fluxes for shifting storage pools to transfer pools leafn_storage_to_xfer(p) = fstor2tran * leafn_storage(p)/dt frootn_storage_to_xfer(p) = fstor2tran * frootn_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemn_storage_to_xfer(p) = fstor2tran * livestemn_storage(p)/dt deadstemn_storage_to_xfer(p) = fstor2tran * deadstemn_storage(p)/dt livecrootn_storage_to_xfer(p) = fstor2tran * livecrootn_storage(p)/dt @@ -1169,7 +1169,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set phosphorus fluxes for shifting storage pools to transfer pools leafp_storage_to_xfer(p) = fstor2tran * leafp_storage(p)/dt frootp_storage_to_xfer(p) = fstor2tran * frootp_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemp_storage_to_xfer(p) = fstor2tran * livestemp_storage(p)/dt deadstemp_storage_to_xfer(p) = fstor2tran * deadstemp_storage(p)/dt livecrootp_storage_to_xfer(p) = fstor2tran * livecrootp_storage(p)/dt @@ -1268,7 +1268,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & leafc_storage_to_xfer(p) = leafc_storage(p) * bgtr(p) frootc_storage_to_xfer(p) = frootc_storage(p) * bgtr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_storage_to_xfer(p) = livestemc_storage(p) * bgtr(p) deadstemc_storage_to_xfer(p) = deadstemc_storage(p) * bgtr(p) livecrootc_storage_to_xfer(p) = livecrootc_storage(p) * bgtr(p) @@ -1279,7 +1279,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set nitrogen fluxes for shifting storage pools to transfer pools leafn_storage_to_xfer(p) = leafn_storage(p) * bgtr(p) frootn_storage_to_xfer(p) = frootn_storage(p) * bgtr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemn_storage_to_xfer(p) = livestemn_storage(p) * bgtr(p) deadstemn_storage_to_xfer(p) = deadstemn_storage(p) * bgtr(p) livecrootn_storage_to_xfer(p) = livecrootn_storage(p) * bgtr(p) @@ -1290,7 +1290,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set phosphorus fluxes for shifting storage pools to transfer pools leafp_storage_to_xfer(p) = leafp_storage(p) * bgtr(p) frootp_storage_to_xfer(p) = frootp_storage(p) * bgtr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemp_storage_to_xfer(p) = livestemp_storage(p) * bgtr(p) deadstemp_storage_to_xfer(p) = deadstemp_storage(p) * bgtr(p) livecrootp_storage_to_xfer(p) = livecrootp_storage(p) * bgtr(p) @@ -2362,7 +2362,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & frootn_xfer_to_frootn(p) = t1 * frootn_xfer(p) leafp_xfer_to_leafp(p) = t1 * leafp_xfer(p) frootp_xfer_to_frootp(p) = t1 * frootp_xfer(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer_to_livestemc(p) = t1 * livestemc_xfer(p) deadstemc_xfer_to_deadstemc(p) = t1 * deadstemc_xfer(p) livecrootc_xfer_to_livecrootc(p) = t1 * livecrootc_xfer(p) @@ -2391,7 +2391,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & frootn_xfer_to_frootn(p) = frootn_xfer(p) / dt leafp_xfer_to_leafp(p) = leafp_xfer(p) / dt frootp_xfer_to_frootp(p) = frootp_xfer(p) / dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer_to_livestemc(p) = livestemc_xfer(p) / dt deadstemc_xfer_to_deadstemc(p) = deadstemc_xfer(p) / dt livecrootc_xfer_to_livecrootc(p) = livecrootc_xfer(p) / dt @@ -2909,7 +2909,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp, & p = filter_soilp(fp) ! only calculate these fluxes for woody types - if (woody(ivt(p)) > 0._r8) then + if (woody(ivt(p)) >= 1.0_r8) then if ( nu_com .eq. 'RD') then ! live stem to dead stem turnover diff --git a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 index 057fd0e56b6e..a371a44c537a 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 @@ -274,7 +274,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%leafc_xfer(p) = veg_cs%leafc_xfer(p) - veg_cf%leafc_xfer_to_leafc(p)*dt veg_cs%frootc(p) = veg_cs%frootc(p) + veg_cf%frootc_xfer_to_frootc(p)*dt veg_cs%frootc_xfer(p) = veg_cs%frootc_xfer(p) - veg_cf%frootc_xfer_to_frootc(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%livestemc_xfer_to_livestemc(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) - veg_cf%livestemc_xfer_to_livestemc(p)*dt veg_cs%deadstemc(p) = veg_cs%deadstemc(p) + veg_cf%deadstemc_xfer_to_deadstemc(p)*dt @@ -297,7 +297,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%frootc(p) = veg_cs%frootc(p) - veg_cf%frootc_to_litter(p)*dt ! livewood turnover fluxes - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%livestemc(p) = veg_cs%livestemc(p) - veg_cf%livestemc_to_deadstemc(p)*dt veg_cs%deadstemc(p) = veg_cs%deadstemc(p) + veg_cf%livestemc_to_deadstemc(p)*dt veg_cs%livecrootc(p) = veg_cs%livecrootc(p) - veg_cf%livecrootc_to_deadcrootc(p)*dt @@ -315,7 +315,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_xsmrpool(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%leaf_curmr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%froot_curmr(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livestem_curmr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livecroot_curmr(p)*dt end if @@ -333,7 +333,7 @@ subroutine CarbonStateUpdate1(bounds, & if (nu_com .ne. 'RD') then veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%xsmrpool_turnover(p)*dt end if - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livestem_xsmr(p)*dt veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livecroot_xsmr(p)*dt end if @@ -355,7 +355,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%frootc(p) = veg_cs%frootc(p) + veg_cf%cpool_to_frootc(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_frootc_storage(p)*dt veg_cs%frootc_storage(p) = veg_cs%frootc_storage(p) + veg_cf%cpool_to_frootc_storage(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_livestemc(p)*dt veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%cpool_to_livestemc(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_livestemc_storage(p)*dt @@ -387,7 +387,7 @@ subroutine CarbonStateUpdate1(bounds, & ! growth respiration fluxes for current growth veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_leaf_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_froot_gr(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livestem_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_deadstem_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livecroot_gr(p)*dt @@ -401,7 +401,7 @@ subroutine CarbonStateUpdate1(bounds, & ! growth respiration for transfer growth veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_leaf_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_froot_gr(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_livestem_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_deadstem_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_livecroot_gr(p)*dt @@ -415,7 +415,7 @@ subroutine CarbonStateUpdate1(bounds, & ! growth respiration at time of storage veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_leaf_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_froot_storage_gr(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livestem_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_deadstem_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livecroot_storage_gr(p)*dt @@ -435,7 +435,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%leafc_xfer(p) = veg_cs%leafc_xfer(p) + veg_cf%leafc_storage_to_xfer(p)*dt veg_cs%frootc_storage(p) = veg_cs%frootc_storage(p) - veg_cf%frootc_storage_to_xfer(p)*dt veg_cs%frootc_xfer(p) = veg_cs%frootc_xfer(p) + veg_cf%frootc_storage_to_xfer(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%livestemc_storage(p) = veg_cs%livestemc_storage(p) - veg_cf%livestemc_storage_to_xfer(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) + veg_cf%livestemc_storage_to_xfer(p)*dt veg_cs%deadstemc_storage(p) = veg_cs%deadstemc_storage(p) - veg_cf%deadstemc_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/ComputeSeedMod.F90 b/components/elm/src/biogeochem/ComputeSeedMod.F90 index 61b916d64f35..9b7ebf8838dc 100644 --- a/components/elm/src/biogeochem/ComputeSeedMod.F90 +++ b/components/elm/src/biogeochem/ComputeSeedMod.F90 @@ -124,7 +124,7 @@ subroutine ComputeSeedAmounts(p, & if (pft_type /= noveg) then my_leaf_seed = leafc_seed_param * & SpeciesTypeMultiplier(species, pft_type, COMPONENT_LEAF) - if (veg_vp%woody(pft_type) == 1._r8) then + if (veg_vp%woody(pft_type) >= 1.0_r8) then my_deadstem_seed = deadstemc_seed_param * & SpeciesTypeMultiplier(species, pft_type, COMPONENT_DEADWOOD) end if diff --git a/components/elm/src/biogeochem/GrowthRespMod.F90 b/components/elm/src/biogeochem/GrowthRespMod.F90 index f499dedb2e0f..a5002d108e51 100644 --- a/components/elm/src/biogeochem/GrowthRespMod.F90 +++ b/components/elm/src/biogeochem/GrowthRespMod.F90 @@ -133,7 +133,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) transfer_froot_gr(p) = frootc_xfer_to_frootc(p) * grperc(ivt(p)) * & (1._r8 - grpnow(ivt(p))) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then cpool_livestem_gr(p) = cpool_to_livestemc(p) * grperc(ivt(p)) cpool_livestem_storage_gr(p) = cpool_to_livestemc_storage(p) * & grperc(ivt(p)) * grpnow(ivt(p)) diff --git a/components/elm/src/biogeochem/MaintenanceRespMod.F90 b/components/elm/src/biogeochem/MaintenanceRespMod.F90 index ef83ca92ff60..eab60926ab60 100644 --- a/components/elm/src/biogeochem/MaintenanceRespMod.F90 +++ b/components/elm/src/biogeochem/MaintenanceRespMod.F90 @@ -182,7 +182,7 @@ subroutine MaintenanceResp(bounds, & end if - if (woody(ivt(p)) == 1) then + if (woody(ivt(p)) >= 1.0_r8) then livestem_mr(p) = livestemn(p)*br_mr*tc livecroot_mr(p) = livecrootn(p)*br_mr*tc else if (ivt(p) >= npcropmin .and. livestemn(p) .gt. 0._r8) then diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 index 1eefadd7bead..14d834245f5a 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 @@ -286,7 +286,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%frootn(p) = veg_ns%frootn(p) + veg_nf%frootn_xfer_to_frootn(p)*dt veg_ns%frootn_xfer(p) = veg_ns%frootn_xfer(p) - veg_nf%frootn_xfer_to_frootn(p)*dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) - veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%deadstemn(p) = veg_ns%deadstemn(p) + veg_nf%deadstemn_xfer_to_deadstemn(p)*dt @@ -312,7 +312,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%leafn_to_retransn(p)*dt ! live wood turnover and retranslocation fluxes - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_deadstemn(p)*dt veg_ns%deadstemn(p) = veg_ns%deadstemn(p) + veg_nf%livestemn_to_deadstemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_retransn(p)*dt @@ -354,7 +354,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_frootn_storage(p)*dt veg_ns%frootn_storage(p) = veg_ns%frootn_storage(p) + veg_nf%npool_to_frootn_storage(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%npool_to_livestemn(p)*dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn_storage(p)*dt @@ -390,7 +390,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%frootn_storage(p) = veg_ns%frootn_storage(p) - veg_nf%frootn_storage_to_xfer(p)*dt veg_ns%frootn_xfer(p) = veg_ns%frootn_xfer(p) + veg_nf%frootn_storage_to_xfer(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ns%livestemn_storage(p) = veg_ns%livestemn_storage(p) - veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) + veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%deadstemn_storage(p) = veg_ns%deadstemn_storage(p) - veg_nf%deadstemn_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 index b6d05460c0bb..786efbe6323a 100644 --- a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 +++ b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 @@ -630,7 +630,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_frootc) = veg_cs%frootc(p) ystates(s_frootc_xfer) = veg_cs%frootc_xfer(p) ystates(s_frootc_storage) = veg_cs%frootc_storage(p) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then ystates(s_livestemc) = veg_cs%livestemc(p) ystates(s_livestemc_xfer) = veg_cs%livestemc_xfer(p) ystates(s_livestemc_storage) = veg_cs%livestemc_storage(p) @@ -662,7 +662,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& + veg_cf%cpool_froot_gr(p) & + veg_cf%cpool_leaf_storage_gr(p) & + veg_cf%cpool_froot_storage_gr(p) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then ar_p = ar_p & + veg_cf%livestem_curmr(p) & + veg_cf%livecroot_curmr(p) & @@ -692,7 +692,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_cpool_to_xsmrpool) = veg_cf%cpool_to_xsmrpool(p) rfluxes(f_cpool_to_gresp_storage) = veg_cf%cpool_to_gresp_storage(p) rfluxes(f_cpool_to_ar) = ar_p - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then rfluxes(f_cpool_to_livestemc) = veg_cf%cpool_to_livestemc(p) rfluxes(f_cpool_to_livestemc_storage) = veg_cf%cpool_to_livestemc_storage(p) rfluxes(f_cpool_to_deadstemc) = veg_cf%cpool_to_deadstemc(p) @@ -746,7 +746,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_cpool_to_xsmrpool) , veg_cf%cpool_to_xsmrpool(p)) call fpmax(rfluxes(f_cpool_to_gresp_storage) , veg_cf%cpool_to_gresp_storage(p)) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then call fpmax(rfluxes(f_cpool_to_livestemc) , veg_cf%cpool_to_livestemc(p)) call fpmax(rfluxes(f_cpool_to_livestemc_storage) , veg_cf%cpool_to_livestemc_storage(p)) call fpmax(rfluxes(f_cpool_to_deadstemc) , veg_cf%cpool_to_deadstemc(p)) @@ -794,7 +794,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& call ascal(veg_cf%cpool_froot_gr(p) , rscal) call ascal(veg_cf%cpool_leaf_storage_gr(p) , rscal) call ascal(veg_cf%cpool_froot_storage_gr(p) , rscal) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then call ascal(veg_cf%livestem_curmr(p) , rscal) call ascal(veg_cf%livecroot_curmr(p) , rscal) call ascal(veg_cf%cpool_livestem_gr(p) , rscal) @@ -864,7 +864,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_frootn) = veg_ns%frootn(p) ystates(s_frootn_xfer) = veg_ns%frootn_xfer(p) ystates(s_frootn_storage) = veg_ns%frootn_storage(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then ystates(s_livestemn) = veg_ns%livestemn(p) ystates(s_livestemn_xfer) = veg_ns%livestemn_xfer(p) ystates(s_livestemn_storage) = veg_ns%livestemn_storage(p) @@ -894,7 +894,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_npool_to_leafn_storage) = veg_nf%npool_to_leafn_storage(p) rfluxes(f_npool_to_frootn) = veg_nf%npool_to_frootn(p) rfluxes(f_npool_to_frootn_storage) = veg_nf%npool_to_frootn_storage(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then rfluxes(f_npool_to_livestemn) = veg_nf%npool_to_livestemn(p) rfluxes(f_npool_to_livestemn_storage) = veg_nf%npool_to_livestemn_storage(p) rfluxes(f_npool_to_livecrootn) = veg_nf%npool_to_livecrootn(p) @@ -950,7 +950,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_npool_to_leafn_storage) , veg_nf%npool_to_leafn_storage(p)) call fpmax(rfluxes(f_npool_to_frootn) , veg_nf%npool_to_frootn(p)) call fpmax(rfluxes(f_npool_to_frootn_storage) , veg_nf%npool_to_frootn_storage(p)) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then call fpmax(rfluxes(f_npool_to_livestemn) , veg_nf%npool_to_livestemn(p)) call fpmax(rfluxes(f_npool_to_livestemn_storage) , veg_nf%npool_to_livestemn_storage(p)) call fpmax(rfluxes(f_npool_to_livecrootn) , veg_nf%npool_to_livecrootn(p)) @@ -1049,7 +1049,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_frootn) = veg_ps%frootp(p) ystates(s_frootn_xfer) = veg_ps%frootp_xfer(p) ystates(s_frootn_storage) = veg_ps%frootp_storage(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then ystates(s_livestemn) = veg_ps%livestemp(p) ystates(s_livestemn_xfer) = veg_ps%livestemp_xfer(p) ystates(s_livestemn_storage) = veg_ps%livestemp_storage(p) @@ -1079,7 +1079,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_npool_to_leafn_storage) = veg_pf%ppool_to_leafp_storage(p) rfluxes(f_npool_to_frootn) = veg_pf%ppool_to_frootp(p) rfluxes(f_npool_to_frootn_storage) = veg_pf%ppool_to_frootp_storage(p) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then rfluxes(f_npool_to_livestemn) = veg_pf%ppool_to_livestemp(p) rfluxes(f_npool_to_livestemn_storage) = veg_pf%ppool_to_livestemp_storage(p) rfluxes(f_npool_to_livecrootn) = veg_pf%ppool_to_livecrootp(p) @@ -1135,7 +1135,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_npool_to_leafn_storage) , veg_pf%ppool_to_leafp_storage(p)) call fpmax(rfluxes(f_npool_to_frootn) , veg_pf%ppool_to_frootp(p)) call fpmax(rfluxes(f_npool_to_frootn_storage) , veg_pf%ppool_to_frootp_storage(p)) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then call fpmax(rfluxes(f_npool_to_livestemn) , veg_pf%ppool_to_livestemp(p)) call fpmax(rfluxes(f_npool_to_livestemn_storage) , veg_pf%ppool_to_livestemp_storage(p)) call fpmax(rfluxes(f_npool_to_livecrootn) , veg_pf%ppool_to_livecrootp(p)) diff --git a/components/elm/src/biogeochem/PhenologyMod.F90 b/components/elm/src/biogeochem/PhenologyMod.F90 index 9f51301acb74..d5ddfc2f44ed 100644 --- a/components/elm/src/biogeochem/PhenologyMod.F90 +++ b/components/elm/src/biogeochem/PhenologyMod.F90 @@ -771,7 +771,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) frootn_xfer_to_frootn(p) = 0.0_r8 leafp_xfer_to_leafp(p) = 0.0_r8 frootp_xfer_to_frootp(p) = 0.0_r8 - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer_to_livestemc(p) = 0.0_r8 deadstemc_xfer_to_deadstemc(p) = 0.0_r8 livecrootc_xfer_to_livecrootc(p) = 0.0_r8 @@ -792,7 +792,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) frootc_xfer(p) = 0.0_r8 frootn_xfer(p) = 0.0_r8 frootp_xfer(p) = 0.0_r8 - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer(p) = 0.0_r8 livestemn_xfer(p) = 0.0_r8 livestemp_xfer(p) = 0.0_r8 @@ -855,7 +855,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) ! set carbon fluxes for shifting storage pools to transfer pools leafc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * leafc_storage(p)/dt frootc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * frootc_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livestemc_storage(p)/dt deadstemc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * deadstemc_storage(p)/dt livecrootc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livecrootc_storage(p)/dt @@ -866,7 +866,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) ! set nitrogen fluxes for shifting storage pools to transfer pools leafn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * leafn_storage(p)/dt frootn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * frootn_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livestemn_storage(p)/dt deadstemn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * deadstemn_storage(p)/dt livecrootn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livecrootn_storage(p)/dt @@ -876,7 +876,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) ! set phosphorus fluxes for shifting storage pools to transfer pools leafp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * leafp_storage(p)/dt frootp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * frootp_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livestemp_storage(p)/dt deadstemp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * deadstemp_storage(p)/dt livecrootp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livecrootp_storage(p)/dt @@ -1116,7 +1116,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & frootn_xfer_to_frootn(p) = 0._r8 leafp_xfer_to_leafp(p) = 0._r8 frootp_xfer_to_frootp(p) = 0._r8 - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer_to_livestemc(p) = 0._r8 deadstemc_xfer_to_deadstemc(p) = 0._r8 livecrootc_xfer_to_livecrootc(p) = 0._r8 @@ -1137,7 +1137,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & frootc_xfer(p) = 0._r8 frootn_xfer(p) = 0._r8 frootp_xfer(p) = 0._r8 - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer(p) = 0._r8 livestemn_xfer(p) = 0._r8 livestemp_xfer(p) = 0._r8 @@ -1229,7 +1229,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set carbon fluxes for shifting storage pools to transfer pools leafc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * leafc_storage(p)/dt frootc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * frootc_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livestemc_storage(p)/dt deadstemc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * deadstemc_storage(p)/dt livecrootc_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livecrootc_storage(p)/dt @@ -1240,7 +1240,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set nitrogen fluxes for shifting storage pools to transfer pools leafn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * leafn_storage(p)/dt frootn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * frootn_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livestemn_storage(p)/dt deadstemn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * deadstemn_storage(p)/dt livecrootn_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livecrootn_storage(p)/dt @@ -1250,7 +1250,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set phosphorus fluxes for shifting storage pools to transfer pools leafp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * leafp_storage(p)/dt frootp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * frootp_storage(p)/dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livestemp_storage(p)/dt deadstemp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * deadstemp_storage(p)/dt livecrootp_storage_to_xfer(p) = PhenolParamsInst%fstor2tran * livecrootp_storage(p)/dt @@ -1352,7 +1352,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & leafc_storage_to_xfer(p) = leafc_storage(p) * bgtr(p) frootc_storage_to_xfer(p) = frootc_storage(p) * bgtr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_storage_to_xfer(p) = livestemc_storage(p) * bgtr(p) deadstemc_storage_to_xfer(p) = deadstemc_storage(p) * bgtr(p) livecrootc_storage_to_xfer(p) = livecrootc_storage(p) * bgtr(p) @@ -1363,7 +1363,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set nitrogen fluxes for shifting storage pools to transfer pools leafn_storage_to_xfer(p) = leafn_storage(p) * bgtr(p) frootn_storage_to_xfer(p) = frootn_storage(p) * bgtr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemn_storage_to_xfer(p) = livestemn_storage(p) * bgtr(p) deadstemn_storage_to_xfer(p) = deadstemn_storage(p) * bgtr(p) livecrootn_storage_to_xfer(p) = livecrootn_storage(p) * bgtr(p) @@ -1374,7 +1374,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & ! set phosphorus fluxes for shifting storage pools to transfer pools leafp_storage_to_xfer(p) = leafp_storage(p) * bgtr(p) frootp_storage_to_xfer(p) = frootp_storage(p) * bgtr(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemp_storage_to_xfer(p) = livestemp_storage(p) * bgtr(p) deadstemp_storage_to_xfer(p) = deadstemp_storage(p) * bgtr(p) livecrootp_storage_to_xfer(p) = livecrootp_storage(p) * bgtr(p) @@ -2694,7 +2694,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & frootn_xfer_to_frootn(p) = t1 * frootn_xfer(p) leafp_xfer_to_leafp(p) = t1 * leafp_xfer(p) frootp_xfer_to_frootp(p) = t1 * frootp_xfer(p) - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer_to_livestemc(p) = t1 * livestemc_xfer(p) deadstemc_xfer_to_deadstemc(p) = t1 * deadstemc_xfer(p) livecrootc_xfer_to_livecrootc(p) = t1 * livecrootc_xfer(p) @@ -2723,7 +2723,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & frootn_xfer_to_frootn(p) = frootn_xfer(p) / dt leafp_xfer_to_leafp(p) = leafp_xfer(p) / dt frootp_xfer_to_frootp(p) = frootp_xfer(p) / dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then livestemc_xfer_to_livestemc(p) = livestemc_xfer(p) / dt deadstemc_xfer_to_deadstemc(p) = deadstemc_xfer(p) / dt livecrootc_xfer_to_livecrootc(p) = livecrootc_xfer(p) / dt @@ -3315,7 +3315,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp) p = filter_soilp(fp) ! only calculate these fluxes for woody types - if (woody(ivt(p)) > 0._r8) then + if (woody(ivt(p)) >= 1.0_r8) then if ( nu_com .eq. 'RD') then ! live stem to dead stem turnover diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 index 63390dc419e2..feea9fd86fb3 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 @@ -244,7 +244,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%frootp(p) = veg_ps%frootp(p) + veg_pf%frootp_xfer_to_frootp(p)*dt veg_ps%frootp_xfer(p) = veg_ps%frootp_xfer(p) - veg_pf%frootp_xfer_to_frootp(p)*dt - if (woody(ivt(p)) == 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ps%livestemp(p) = veg_ps%livestemp(p) + veg_pf%livestemp_xfer_to_livestemp(p)*dt veg_ps%livestemp_xfer(p) = veg_ps%livestemp_xfer(p) - veg_pf%livestemp_xfer_to_livestemp(p)*dt veg_ps%deadstemp(p) = veg_ps%deadstemp(p) + veg_pf%deadstemp_xfer_to_deadstemp(p)*dt @@ -270,7 +270,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%retransp(p) = veg_ps%retransp(p) + veg_pf%leafp_to_retransp(p)*dt ! live wood turnover and retranslocation fluxes - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ps%livestemp(p) = veg_ps%livestemp(p) - veg_pf%livestemp_to_deadstemp(p)*dt veg_ps%deadstemp(p) = veg_ps%deadstemp(p) + veg_pf%livestemp_to_deadstemp(p)*dt veg_ps%livestemp(p) = veg_ps%livestemp(p) - veg_pf%livestemp_to_retransp(p)*dt @@ -312,7 +312,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_frootp_storage(p)*dt veg_ps%frootp_storage(p) = veg_ps%frootp_storage(p) + veg_pf%ppool_to_frootp_storage(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_livestemp(p)*dt veg_ps%livestemp(p) = veg_ps%livestemp(p) + veg_pf%ppool_to_livestemp(p)*dt veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_livestemp_storage(p)*dt @@ -348,7 +348,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%frootp_storage(p) = veg_ps%frootp_storage(p) - veg_pf%frootp_storage_to_xfer(p)*dt veg_ps%frootp_xfer(p) = veg_ps%frootp_xfer(p) + veg_pf%frootp_storage_to_xfer(p)*dt - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_ps%livestemp_storage(p) = veg_ps%livestemp_storage(p) - veg_pf%livestemp_storage_to_xfer(p)*dt veg_ps%livestemp_xfer(p) = veg_ps%livestemp_xfer(p) + veg_pf%livestemp_storage_to_xfer(p)*dt veg_ps%deadstemp_storage(p) = veg_ps%deadstemp_storage(p) - veg_pf%deadstemp_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 6796ab1993ad..9a7db79154b4 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -159,7 +159,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & tsai_min = tsai_min * 0.5_r8 tsai(p) = max(tsai_alpha*tsai_old+max(tlai_old-tlai(p),0._r8),tsai_min) - if (woody(ivt(p)) == 1._r8) then + if (woody(ivt(p)) >= 1.0_r8) then ! trees and shrubs diff --git a/components/elm/src/data_types/VegetationDataType.F90 b/components/elm/src/data_types/VegetationDataType.F90 index 45dc063b8533..eea30fc55fef 100644 --- a/components/elm/src/data_types/VegetationDataType.F90 +++ b/components/elm/src/data_types/VegetationDataType.F90 @@ -2465,7 +2465,7 @@ subroutine veg_cs_init(this, begp, endp, carbon_type, ratio) this%livestemc_storage(p) = 0._r8 this%livestemc_xfer(p) = 0._r8 - if (veg_vp%woody(veg_pp%itype(p)) == 1._r8) then + if (veg_vp%woody(veg_pp%itype(p)) >= 1.0_r8) then this%deadstemc(p) = 0.1_r8 * ratio else this%deadstemc(p) = 0._r8 @@ -3937,7 +3937,7 @@ subroutine veg_ns_init(this, begp, endp, veg_cs) ! tree types need to be initialized with some stem mass so that ! roughness length is not zero in canopy flux calculation - if (veg_vp%woody(veg_pp%itype(p)) == 1._r8) then + if (veg_vp%woody(veg_pp%itype(p)) >= 1.0_r8) then this%deadstemn(p) = veg_cs%deadstemc(p) / veg_vp%deadwdcn(veg_pp%itype(p)) else this%deadstemn(p) = 0._r8 @@ -4621,7 +4621,7 @@ subroutine veg_ps_init(this, begp, endp, veg_cs) ! tree types need to be initialized with some stem mass so that ! roughness length is not zero in canopy flux calculation - if (veg_vp%woody(veg_pp%itype(p)) == 1._r8) then + if (veg_vp%woody(veg_pp%itype(p)) >= 1.0_r8) then this%deadstemp(p) = veg_cs%deadstemc(p) / veg_vp%deadwdcp(veg_pp%itype(p)) else this%deadstemp(p) = 0._r8 diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 205477791956..2cb4e01c8a46 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -17,7 +17,7 @@ module VegetationPropertiesType ! !PUBLIC TYPES: type, public :: vegetation_properties_type integer , pointer :: noveg (:) => null() ! value for not vegetated - integer , pointer :: tree (:) => null() ! tree or not? + !integer , pointer :: tree (:) => null() ! tree or not? real(r8), pointer :: smpso (:) => null() ! soil water potential at full stomatal opening (mm) real(r8), pointer :: smpsc (:) => null() ! soil water potential at full stomatal closure (mm) real(r8), pointer :: fnitr (:) => null() ! foliage nitrogen limitation factor (-) @@ -159,7 +159,7 @@ subroutine veg_vp_init(this) ! ! !USES: use elm_varpar, only : numrad, numpft - use pftvarcon , only : ntree, smpso, smpsc, fnitr + use pftvarcon , only : smpso, smpsc, fnitr use pftvarcon , only : z0mr, displar, dleaf, rhol, rhos, taul, taus, xl use pftvarcon , only : c3psn, slatop, dsladlai, leafcn, flnr, woody use pftvarcon , only : lflitcn, frootcn, livewdcn, deadwdcn, froot_leaf, stem_leaf, croot_stem @@ -190,7 +190,7 @@ subroutine veg_vp_init(this) !------------------------------------------------------------------------ allocate(this%noveg (0:numpft)) ; this%noveg (:) =huge(1) - allocate(this%tree (0:numpft)) ; this%tree (:) =huge(1) + !allocate(this%tree (0:numpft)) ; this%tree (:) =huge(1) allocate(this%smpso (0:numpft)) ; this%smpso (:) =spval allocate(this%smpsc (0:numpft)) ; this%smpsc (:) =spval allocate(this%fnitr (0:numpft)) ; this%fnitr (:) =spval @@ -308,11 +308,12 @@ subroutine veg_vp_init(this) do m = 0,numpft - if (m <= ntree) then - this%tree(m) = 1 - else - this%tree(m) = 0 - end if + ! not needed anymore: woody(m)=1 for tree, 2 for shrub, or 0 for any other + !if (woody(m) == 1) then + ! this%tree(m) = 1 + !else + ! this%tree(m) = 0 + !end if do ib = 1,numrad this%rhol(m,ib) = rhol(m,ib) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index cbd5946bb4e6..1b06a49cf2f6 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -115,7 +115,7 @@ module pftvarcon real(r8), allocatable :: dsladlai(:) !dSLA/dLAI [m^2/gC] real(r8), allocatable :: leafcn(:) !leaf C:N [gC/gN] real(r8), allocatable :: flnr(:) !fraction of leaf N in Rubisco [no units] - real(r8), allocatable :: woody(:) !woody lifeform flag (0 or 1) + real(r8), allocatable :: woody(:) !woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) real(r8), allocatable :: lflitcn(:) !leaf litter C:N (gC/gN) real(r8), allocatable :: frootcn(:) !fine root C:N (gC/gN) real(r8), allocatable :: livewdcn(:) !live wood (phloem and ray parenchyma) C:N (gC/gN) @@ -1146,11 +1146,6 @@ subroutine pftconrd end do end if - ! woody=2 for shrub NOT YET ready in rest of ELM code - do i = 0, npft-1 - if (woody(i)>1) woody(i) == 1 - end do - call ncd_pio_closefile(ncid) @@ -1239,6 +1234,10 @@ subroutine pftconrd end if !------------------------------------------------------------------------------------------- + ! default: for tree and shrub always 1, now hard-coded as following + woody(ndllf_evr_tmp_tree:nbrdlf_dcd_brl_tree) = 1 + woody(nbrdlf_evr_shrub:nbrdlf_dcd_brl_shrub) = 2 + ! the following is initialized as 0 above for all PFT. ! here the hard-coded values (or flags) for default ELM PFT physiology will be working as original ! when not using those indexing of PFT orders anymore in other codes than here. From 475eac8d4a33bb80cefd0537e1925ee0d8a4000e Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Mon, 29 Jan 2024 12:03:52 -0500 Subject: [PATCH 571/904] avoid using hard coded PFT total number and indices in PFT physiology parameter file. --- components/elm/src/biogeochem/CH4Mod.F90 | 5 +- .../src/biogeochem/CNNStateUpdate1BeTRMod.F90 | 10 +-- .../src/biogeochem/CarbonStateUpdate1Mod.F90 | 31 ++++++---- .../elm/src/biogeochem/DryDepVelocity.F90 | 56 +++++++++-------- components/elm/src/biogeochem/FireMod.F90 | 61 ++++++++++++++----- .../elm/src/biogeochem/GapMortalityMod.F90 | 7 ++- .../biogeochem/NitrogenStateUpdate1Mod.F90 | 14 +++-- .../biogeochem/PhosphorusStateUpdate1Mod.F90 | 14 +++-- .../src/biogeochem/PrecisionControlMod.F90 | 10 ++- .../src/biogeochem/SatellitePhenologyMod.F90 | 4 +- .../elm/src/biogeochem/VOCEmissionMod.F90 | 38 ++++++------ .../elm/src/biogeochem/VegStructUpdateMod.F90 | 13 ++-- .../elm/src/biogeophys/CanopyFluxesMod.F90 | 8 ++- .../elm/src/biogeophys/PhotosynthesisMod.F90 | 2 - components/elm/src/biogeophys/SedYieldMod.F90 | 10 ++- .../elm/src/dyn_subgrid/dynHarvestMod.F90 | 5 +- components/elm/src/main/surfrdUtilsMod.F90 | 32 +++++----- 17 files changed, 192 insertions(+), 128 deletions(-) diff --git a/components/elm/src/biogeochem/CH4Mod.F90 b/components/elm/src/biogeochem/CH4Mod.F90 index ce2378049fd5..3b3c1fa80611 100644 --- a/components/elm/src/biogeochem/CH4Mod.F90 +++ b/components/elm/src/biogeochem/CH4Mod.F90 @@ -2426,7 +2426,7 @@ subroutine ch4_aere (bounds, & ! !USES: !$acc routine seq use elm_varcon , only : rpi - use pftvarcon , only : nc3_arctic_grass, crop, nc3_nonarctic_grass, nc4_grass, noveg + use pftvarcon , only : graminoid, crop, noveg use CH4varcon , only : transpirationloss, usefrootc, use_aereoxid_prog ! ! !ARGUMENTS: @@ -2563,8 +2563,7 @@ subroutine ch4_aere (bounds, & is_vegetated = .false. end if - if (veg_pp%itype(p) == nc3_arctic_grass .or. crop(veg_pp%itype(p)) == 1 .or. & - veg_pp%itype(p) == nc3_nonarctic_grass .or. veg_pp%itype(p) == nc4_grass) then + if (graminoid(veg_pp%itype(p)) == 1 .or. crop(veg_pp%itype(p)) == 1) then poros_tiller = 0.3_r8 ! Colmer 2003 else poros_tiller = 0.3_r8 * CH4ParamsInst%nongrassporosratio diff --git a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 index 95d884de490f..4c7fd83d2108 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 @@ -10,7 +10,7 @@ module CNNStateUpdate1BeTRMod use elm_varpar , only : crop_prog, i_met_lit, i_cel_lit, i_lig_lit, i_cwd use elm_varctl , only : iulog use elm_varcon , only : nitrif_n2o_loss_frac - use pftvarcon , only : npcropmin, nc3crop + use pftvarcon , only : crop, generic_crop use VegetationPropertiesType , only : veg_vp use CNDecompCascadeConType , only : decomp_cascade_con use CNStateType , only : cnstate_type @@ -101,7 +101,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) - veg_nf%deadcrootn_xfer_to_deadcrootn(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (crop(ivt(p)) >= 1 .and. generic_crop(ivt(p)) < 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) - veg_nf%livestemn_xfer_to_livestemn(p)*dt @@ -126,7 +126,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%livecrootn(p) = veg_ns%livecrootn(p) - veg_nf%livecrootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%livecrootn_to_retransn(p)*dt end if - if (ivt(p) >= npcropmin) then ! Beth adds retrans from froot + if (crop(ivt(p)) >= 1 .and. generic_crop(ivt(p)) < 1) then ! Beth adds retrans from froot veg_ns%frootn(p) = veg_ns%frootn(p) - veg_nf%frootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%frootn_to_retransn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_litter(p)*dt @@ -172,7 +172,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_storage(p) = veg_ns%deadcrootn_storage(p) + veg_nf%npool_to_deadcrootn_storage(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (crop(ivt(p)) >= 1 .and. generic_crop(ivt(p)) < 1) then ! skip 2 generic crops veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%npool_to_livestemn(p)*dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn_storage(p)*dt @@ -200,7 +200,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) + veg_nf%deadcrootn_storage_to_xfer(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (crop(ivt(p)) >= 1 .and. generic_crop(ivt(p)) < 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn_storage(p) = veg_ns%livestemn_storage(p) - veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) + veg_nf%livestemn_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 index a371a44c537a..25455dbbb20e 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 @@ -11,7 +11,7 @@ module CarbonStateUpdate1Mod use elm_varcon , only : dzsoi_decomp use elm_varctl , only : nu_com, use_c13, use_c14 use elm_varctl , only : use_pflotran, pf_cmode, use_fates - use pftvarcon , only : npcropmin, nc3crop + use pftvarcon , only : crop, generic_crop, percrop use CNDecompCascadeConType , only : decomp_cascade_type use CNStateType , only : cnstate_type use CNDecompCascadeConType , only : decomp_cascade_con @@ -274,7 +274,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%leafc_xfer(p) = veg_cs%leafc_xfer(p) - veg_cf%leafc_xfer_to_leafc(p)*dt veg_cs%frootc(p) = veg_cs%frootc(p) + veg_cf%frootc_xfer_to_frootc(p)*dt veg_cs%frootc_xfer(p) = veg_cs%frootc_xfer(p) - veg_cf%frootc_xfer_to_frootc(p)*dt - if (woody(ivt(p)) >= 1.0_r8) then + if (woody(ivt(p)) >= 1.0_r8) then veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%livestemc_xfer_to_livestemc(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) - veg_cf%livestemc_xfer_to_livestemc(p)*dt veg_cs%deadstemc(p) = veg_cs%deadstemc(p) + veg_cf%deadstemc_xfer_to_deadstemc(p)*dt @@ -284,7 +284,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%deadcrootc(p) = veg_cs%deadcrootc(p) + veg_cf%deadcrootc_xfer_to_deadcrootc(p)*dt veg_cs%deadcrootc_xfer(p) = veg_cs%deadcrootc_xfer(p) - veg_cf%deadcrootc_xfer_to_deadcrootc(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%livestemc_xfer_to_livestemc(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) - veg_cf%livestemc_xfer_to_livestemc(p)*dt @@ -303,7 +304,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%livecrootc(p) = veg_cs%livecrootc(p) - veg_cf%livecrootc_to_deadcrootc(p)*dt veg_cs%deadcrootc(p) = veg_cs%deadcrootc(p) + veg_cf%livecrootc_to_deadcrootc(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_cs%livestemc(p) = veg_cs%livestemc(p) - veg_cf%livestemc_to_litter(p)*dt veg_cs%grainc(p) = veg_cs%grainc(p) - veg_cf%grainc_to_food(p)*dt @@ -319,7 +321,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livestem_curmr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livecroot_curmr(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livestem_curmr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%grain_curmr(p)*dt end if @@ -337,7 +340,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livestem_xsmr(p)*dt veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livecroot_xsmr(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livestem_xsmr(p)*dt veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%grain_xsmr(p)*dt if (harvdate(p) < 999) then ! beginning at harvest, send to atm @@ -373,7 +377,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_deadcrootc_storage(p)*dt veg_cs%deadcrootc_storage(p) = veg_cs%deadcrootc_storage(p) + veg_cf%cpool_to_deadcrootc_storage(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_livestemc(p)*dt veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%cpool_to_livestemc(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_livestemc_storage(p)*dt @@ -393,7 +398,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livecroot_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_deadcroot_gr(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livestem_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_grain_gr(p)*dt end if @@ -407,7 +413,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_livecroot_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_deadcroot_gr(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_livestem_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_grain_gr(p)*dt end if @@ -421,7 +428,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livecroot_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_deadcroot_storage_gr(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livestem_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_grain_storage_gr(p)*dt end if @@ -447,7 +455,8 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%gresp_storage(p) = veg_cs%gresp_storage(p) - veg_cf%gresp_storage_to_xfer(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) + veg_cf%gresp_storage_to_xfer(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_cs%livestemc_storage(p) = veg_cs%livestemc_storage(p) - veg_cf%livestemc_storage_to_xfer(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) + veg_cf%livestemc_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/DryDepVelocity.F90 b/components/elm/src/biogeochem/DryDepVelocity.F90 index 009de700b455..293713da54a7 100644 --- a/components/elm/src/biogeochem/DryDepVelocity.F90 +++ b/components/elm/src/biogeochem/DryDepVelocity.F90 @@ -145,14 +145,9 @@ subroutine depvel_compute( bounds, & use seq_drydep_mod , only : rcls, h2_a, h2_b, h2_c, ri, rac, rclo, rlu, rgss, rgso use landunit_varcon, only : istsoil, istice, istice_mec, istdlak, istwet use elm_varctl , only : iulog - use pftvarcon , only : noveg, ndllf_evr_tmp_tree, ndllf_evr_brl_tree - use pftvarcon , only : ndllf_dcd_brl_tree, nbrdlf_evr_trp_tree - use pftvarcon , only : nbrdlf_evr_tmp_tree, nbrdlf_dcd_trp_tree - use pftvarcon , only : nbrdlf_dcd_tmp_tree, nbrdlf_dcd_brl_tree - use pftvarcon , only : nbrdlf_evr_shrub, nbrdlf_dcd_tmp_shrub - use pftvarcon , only : nbrdlf_dcd_brl_shrub,nc3_arctic_grass - use pftvarcon , only : nc3_nonarctic_grass, nc4_grass, nc3crop - use pftvarcon , only : nc3irrig, npcropmin, npcropmax + use pftvarcon , only : noveg, nonvascular + use pftvarcon , only : woody, graminoid, generic_crop, crop, percrop + use pftvarcon , only : needleleaf ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -275,24 +270,33 @@ subroutine depvel_compute( bounds, & !map ELM veg type into Wesely veg type wesveg = wveg_unset - if (elmveg == noveg ) wesveg = 8 - if (elmveg == ndllf_evr_tmp_tree ) wesveg = 5 - if (elmveg == ndllf_evr_brl_tree ) wesveg = 5 - if (elmveg == ndllf_dcd_brl_tree ) wesveg = 5 - if (elmveg == nbrdlf_evr_trp_tree ) wesveg = 4 - if (elmveg == nbrdlf_evr_tmp_tree ) wesveg = 4 - if (elmveg == nbrdlf_dcd_trp_tree ) wesveg = 4 - if (elmveg == nbrdlf_dcd_tmp_tree ) wesveg = 4 - if (elmveg == nbrdlf_dcd_brl_tree ) wesveg = 4 - if (elmveg == nbrdlf_evr_shrub ) wesveg = 11 - if (elmveg == nbrdlf_dcd_tmp_shrub ) wesveg = 11 - if (elmveg == nbrdlf_dcd_brl_shrub ) wesveg = 11 - if (elmveg == nc3_arctic_grass ) wesveg = 3 - if (elmveg == nc3_nonarctic_grass ) wesveg = 3 - if (elmveg == nc4_grass ) wesveg = 3 - if (elmveg == nc3crop ) wesveg = 2 - if (elmveg == nc3irrig ) wesveg = 2 - if (elmveg >= npcropmin .and. elmveg <= npcropmax ) wesveg = 2 + if (elmveg == noveg ) wesveg = 8 + if (nonvascular(elmveg) == 2.0_r8 ) wesveg = 8 ! assuming lichen like bare-ground + !if (elmveg == ndllf_evr_tmp_tree ) wesveg = 5 + !if (elmveg == ndllf_evr_brl_tree ) wesveg = 5 + !if (elmveg == ndllf_dcd_brl_tree ) wesveg = 5 + if (woody(elmveg) == 1.0_r8 .and. needleleaf(elmveg) == 1) wesveg = 5 + !if (elmveg == nbrdlf_evr_trp_tree ) wesveg = 4 + !if (elmveg == nbrdlf_evr_tmp_tree ) wesveg = 4 + !if (elmveg == nbrdlf_dcd_trp_tree ) wesveg = 4 + !if (elmveg == nbrdlf_dcd_tmp_tree ) wesveg = 4 + !if (elmveg == nbrdlf_dcd_brl_tree ) wesveg = 4 + if (woody(elmveg) == 1.0_r8 .and. needleleaf(elmveg) == 0) wesveg = 4 + !if (elmveg == nbrdlf_evr_shrub ) wesveg = 11 + !if (elmveg == nbrdlf_dcd_tmp_shrub ) wesveg = 11 + !if (elmveg == nbrdlf_dcd_brl_shrub ) wesveg = 11 + if (woody(elmveg) == 2.0_r8 ) wesveg = 11 + !if (elmveg == nc3_arctic_grass ) wesveg = 3 + !if (elmveg == nc3_nonarctic_grass ) wesveg = 3 + !if (elmveg == nc4_grass ) wesveg = 3 + if (graminoid(elmveg) == 1.0_r8 ) wesveg = 3 + if (nonvascular(elmveg) == 1.0_r8 ) wesveg = 3 ! assuming moss like grass + !if (elmveg == nc3crop ) wesveg = 2 + !if (elmveg == nc3irrig ) wesveg = 2 + if (generic_crop(elmveg) == 1.0_r8 ) wesveg = 2 + if (crop(elmveg) == 1.0_r8 ) wesveg = 2 + if (percrop(elmveg) == 1.0_r8 ) wesveg = 2 + !if (elmveg >= npcropmin .and. elmveg <= npcropmax ) wesveg = 2 #ifndef _OPENACC if (wesveg == wveg_unset )then write(iulog,*) 'elmveg = ', elmveg, 'lun_pp%itype = ', lun_pp%itype(l) diff --git a/components/elm/src/biogeochem/FireMod.F90 b/components/elm/src/biogeochem/FireMod.F90 index 0dec0c08a50b..743141b20182 100644 --- a/components/elm/src/biogeochem/FireMod.F90 +++ b/components/elm/src/biogeochem/FireMod.F90 @@ -126,8 +126,8 @@ subroutine FireArea (bounds, & use elm_varcon , only: secspday, spval use elm_varctl , only: use_nofire, spinup_state, spinup_mortality_factor use dynSubgridControlMod , only: run_has_transient_landcover - use pftvarcon , only: nc4_grass, nc3crop, ndllf_evr_tmp_tree - use pftvarcon , only: nbrdlf_evr_trp_tree, nbrdlf_dcd_trp_tree, nbrdlf_evr_shrub + use pftvarcon , only: noveg, woody, graminoid, generic_crop, crop, percrop + use pftvarcon , only: climatezone, needleleaf, evergreen ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -305,11 +305,16 @@ subroutine FireArea (bounds, & if (pi <= col_pp%npfts(c)) then p = col_pp%pfti(c) + pi - 1 ! For crop veg types - if( veg_pp%itype(p) > nc4_grass )then + if( generic_crop(veg_pp%itype(p)) == 1 .or. & + crop(veg_pp%itype(p)) == 1 .or. & + percrop(veg_pp%itype(p)) == 1 )then cropf_col(c) = cropf_col(c) + veg_pp%wtcol(p) end if ! For natural vegetation (non-crop and non-bare-soil) - if( veg_pp%itype(p) >= ndllf_evr_tmp_tree .and. veg_pp%itype(p) <= nc4_grass )then + if( veg_pp%itype(p) /= noveg .and. & + (generic_crop(veg_pp%itype(p)) == 0 .and. & + crop(veg_pp%itype(p)) == 0 .and. & + percrop(veg_pp%itype(p)) == 0) )then lfwt(c) = lfwt(c) + veg_pp%wtcol(p) end if end if @@ -331,7 +336,10 @@ subroutine FireArea (bounds, & ! column-level litter carbon ! is available, so we use leaf carbon to estimate the ! litter carbon for crop PFTs - if( veg_pp%itype(p) > nc4_grass .and. veg_pp%wtcol(p) > 0._r8 .and. leafc_col(c) > 0._r8 )then + if( (generic_crop(veg_pp%itype(p)) == 1 .or. & + crop(veg_pp%itype(p)) == 1 .or. & + percrop(veg_pp%itype(p)) == 1) .and. & + veg_pp%wtcol(p) > 0._r8 .and. leafc_col(c) > 0._r8 )then fuelc_crop(c)=fuelc_crop(c) + (leafc(p) + leafc_storage(p) + & leafc_xfer(p))*veg_pp%wtcol(p)/cropf_col(c) + & totlitc(c)*leafc(p)/leafc_col(c)*veg_pp%wtcol(p)/cropf_col(c) @@ -368,21 +376,37 @@ subroutine FireArea (bounds, & p = col_pp%pfti(c) + pi - 1 ! For non-crop -- natural vegetation and bare-soil - if( veg_pp%itype(p) < nc3crop .and. cropf_col(c) < 1.0_r8 )then + if( (generic_crop(veg_pp%itype(p)) == 0 .and. & + crop(veg_pp%itype(p)) == 0 .and. & + percrop(veg_pp%itype(p)) == 0) .and. & + cropf_col(c) < 1.0_r8 ) then if( btran2(p) .ne. spval) then if (btran2(p) <= 1._r8 ) then btran_col(c) = btran_col(c)+btran2(p)*veg_pp%wtcol(p) wtlf(c) = wtlf(c)+veg_pp%wtcol(p) end if end if - if( veg_pp%itype(p) == nbrdlf_evr_trp_tree .and. veg_pp%wtcol(p) > 0._r8 )then + ! broadleaf evergreen tropical tree + if( (needleleaf(veg_pp%itype(p)) == 0 .and. & + evergreen(veg_pp%itype(p)) == 1 .and. & + climatezone(veg_pp%itype(p)) == 1 .and. & + woody(veg_pp%itype(p)) == 1.0_r8) .and. & + veg_pp%wtcol(p) > 0._r8 )then trotr1_col(c)=trotr1_col(c)+veg_pp%wtcol(p)*col_pp%wttopounit(c) end if - if( veg_pp%itype(p) == nbrdlf_dcd_trp_tree .and. veg_pp%wtcol(p) > 0._r8 )then + ! broadleaf deciduous tropical tree + if( (needleleaf(veg_pp%itype(p)) == 0 .and. & + evergreen(veg_pp%itype(p)) == 0 .and. & + climatezone(veg_pp%itype(p)) == 1 .and. & + woody(veg_pp%itype(p)) == 1.0_r8) .and. & + veg_pp%wtcol(p) > 0._r8 ) then trotr2_col(c)=trotr2_col(c)+veg_pp%wtcol(p)*col_pp%wttopounit(c) end if if (transient_landcover) then !true when landuse data is used - if( veg_pp%itype(p) == nbrdlf_evr_trp_tree .or. veg_pp%itype(p) == nbrdlf_dcd_trp_tree )then + ! broadleaf tropical tree + if(needleleaf(veg_pp%itype(p)) == 0 .and. & + climatezone(veg_pp%itype(p)) == 1 .and. & + woody(veg_pp%itype(p)) == 1.0_r8)then if(lfpftd(p) > 0._r8)then dtrotr_col(c)=dtrotr_col(c)+lfpftd(p)*col_pp%wttopounit(c) end if @@ -405,7 +429,8 @@ subroutine FireArea (bounds, & ! For NOT bare-soil if( veg_pp%itype(p) /= noveg )then ! For shrub and grass (crop already excluded above) - if( veg_pp%itype(p) >= nbrdlf_evr_shrub )then !for shurb and grass + if( woody(veg_pp%itype(p)) == 2.0_r8 .or. & + graminoid(veg_pp%itype(p)) == 1 )then !for shurb and grass lgdp_col(c) = lgdp_col(c) + (0.1_r8 + 0.9_r8* & exp(-1._r8*SHR_CONST_PI* & (gdp_lf(c)/8._r8)**0.5_r8))*veg_pp%wtcol(p) & @@ -416,7 +441,7 @@ subroutine FireArea (bounds, & lpop_col(c) = lpop_col(c) + (0.2_r8 + 0.8_r8* & exp(-1._r8*SHR_CONST_PI* & (hdmlf/450._r8)**0.5_r8))*veg_pp%wtcol(p)/lfwt(c) - else ! for trees + else if (woody(veg_pp%itype(p)) == 1.0_r8) then ! for trees if( gdp_lf(c) > 20._r8 )then lgdp_col(c) =lgdp_col(c)+0.39_r8*veg_pp%wtcol(p)/(1.0_r8 - cropf_col(c)) else @@ -491,7 +516,10 @@ subroutine FireArea (bounds, & if (pi <= col_pp%npfts(c)) then p = col_pp%pfti(c) + pi - 1 ! For crop - if( forc_t(t) >= SHR_CONST_TKFRZ .and. veg_pp%itype(p) > nc4_grass .and. & + if( forc_t(t) >= SHR_CONST_TKFRZ .and. & + (generic_crop(veg_pp%itype(p))>=1 .or. & + crop(veg_pp%itype(p))>=1 .or. & + percrop(veg_pp%itype(p))>=1) .and. & kmo == abm_lf(c) .and. forc_rain(t)+forc_snow(t) == 0._r8 .and. & burndate(p) >= 999 .and. veg_pp%wtcol(p) > 0._r8 )then ! catch crop burn time @@ -661,7 +689,8 @@ subroutine FireFluxes (num_soilc, filter_soilc, num_soilp, filter_soilp, & ! !USES: !$acc routine seq use pftvarcon , only: cc_leaf,cc_lstem,cc_dstem,cc_other,fm_leaf,fm_lstem,fm_other,fm_root,fm_lroot,fm_droot - use pftvarcon , only: nc3crop,lf_flab,lf_fcel,lf_flig,fr_flab,fr_fcel,fr_flig + use pftvarcon , only: lf_flab,lf_fcel,lf_flig,fr_flab,fr_fcel,fr_flig + use pftvarcon , only: generic_crop, crop, percrop use elm_varpar , only: max_patch_per_col use elm_varctl , only: spinup_state, spinup_mortality_factor use dynSubgridControlMod , only: get_flanduse_timeseries @@ -706,7 +735,7 @@ subroutine FireFluxes (num_soilc, filter_soilc, num_soilp, filter_soilp, & is_cwd => decomp_cascade_con%is_cwd , & ! Input: [logical (:) ] TRUE => pool is a cwd pool is_litter => decomp_cascade_con%is_litter , & ! Input: [logical (:) ] TRUE => pool is a litter pool - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform (1=woody, 0=not woody) + !woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform (1=woody, 0=not woody) cropf_col => cnstate_vars%cropf_col , & ! Input: [real(r8) (:) ] cropland fraction in veg column croot_prof => cnstate_vars%croot_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of coarse roots @@ -965,7 +994,9 @@ subroutine FireFluxes (num_soilc, filter_soilc, num_soilp, filter_soilp, & c = veg_pp%column(p) itype = veg_pp%itype(p) - if( itype < nc3crop .and. cropf_col(c) < 1.0_r8)then + if( (generic_crop(veg_pp%itype(p)) == 0 .and. crop(veg_pp%itype(p)) == 0 .and. & + percrop(veg_pp%itype(p)) == 0 ) .and. & + cropf_col(c) < 1.0_r8)then ! For non-crop (bare-soil and natural vegetation) if (transient_landcover) then !true when landuse data is used f = (fbac(c)-baf_crop(c))/(1.0_r8-cropf_col(c)) diff --git a/components/elm/src/biogeochem/GapMortalityMod.F90 b/components/elm/src/biogeochem/GapMortalityMod.F90 index 495c46ed0dfc..31cd45814d62 100644 --- a/components/elm/src/biogeochem/GapMortalityMod.F90 +++ b/components/elm/src/biogeochem/GapMortalityMod.F90 @@ -564,7 +564,7 @@ subroutine mortality_rate_soilorder(& ! USES !$acc routine seq - use pftvarcon , only: nbrdlf_evr_trp_tree, nbrdlf_dcd_trp_tree + use pftvarcon , only: woody, needleleaf, climatezone use soilorder_varcon, only: r_mort_soilorder ! @@ -586,7 +586,10 @@ subroutine mortality_rate_soilorder(& do fp = 1,num_soilp p = filter_soilp(fp) c = veg_pp%column(p) - if( veg_pp%itype(p) == nbrdlf_evr_trp_tree .or. veg_pp%itype(p) == nbrdlf_dcd_trp_tree )then + ! broadleaf tropical trees + if( woody(veg_pp%itype(p)) == 1.0_r8 .and. & + needleleaf(veg_pp%itype(p)) == 0 .and. & + climatezone(veg_pp%itype(p)) == 1 )then r_mort_cal(p) = r_mort_soilorder( isoilorder(c) ) else r_mort_cal(p) = 0.02_r8 ! Default mortality rate diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 index 14d834245f5a..d72dc4dc598e 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 @@ -9,7 +9,7 @@ module NitrogenStateUpdate1Mod use elm_varpar , only : crop_prog, i_met_lit, i_cel_lit, i_lig_lit, i_cwd use elm_varctl , only : iulog use elm_varcon , only : nitrif_n2o_loss_frac - use pftvarcon , only : npcropmin, nc3crop + use pftvarcon , only : crop, generic_crop, percrop use VegetationPropertiesType , only : veg_vp use CNDecompCascadeConType , only : decomp_cascade_con use CNStateType , only : cnstate_type @@ -297,7 +297,8 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) - veg_nf%deadcrootn_xfer_to_deadcrootn(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) - veg_nf%livestemn_xfer_to_livestemn(p)*dt @@ -322,7 +323,8 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%livecrootn(p) = veg_ns%livecrootn(p) - veg_nf%livecrootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%livecrootn_to_retransn(p)*dt end if - if (ivt(p) >= npcropmin) then ! Beth adds retrans from froot + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! Beth adds retrans from froot veg_ns%frootn(p) = veg_ns%frootn(p) - veg_nf%frootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%frootn_to_retransn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_litter(p)*dt @@ -373,7 +375,8 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_storage(p) = veg_ns%deadcrootn_storage(p) + veg_nf%npool_to_deadcrootn_storage(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%npool_to_livestemn(p)*dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn_storage(p)*dt @@ -401,7 +404,8 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) + veg_nf%deadcrootn_storage_to_xfer(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn_storage(p) = veg_ns%livestemn_storage(p) - veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) + veg_nf%livestemn_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 index feea9fd86fb3..e29074eb332f 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 @@ -8,7 +8,7 @@ module PhosphorusStateUpdate1Mod use elm_varpar , only : nlevdecomp, ndecomp_pools, ndecomp_cascade_transitions use elm_varpar , only : crop_prog, i_met_lit, i_cel_lit, i_lig_lit, i_cwd use elm_varctl , only : iulog - use pftvarcon , only : npcropmin, nc3crop + use pftvarcon , only : crop, generic_crop, percrop use soilorder_varcon , only : smax,ks_sorption use VegetationPropertiesType , only : veg_vp use CNDecompCascadeConType , only : decomp_cascade_con @@ -255,7 +255,8 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_xfer(p) = veg_ps%deadcrootp_xfer(p) - veg_pf%deadcrootp_xfer_to_deadcrootp(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ps%livestemp(p) = veg_ps%livestemp(p) + veg_pf%livestemp_xfer_to_livestemp(p)*dt veg_ps%livestemp_xfer(p) = veg_ps%livestemp_xfer(p) - veg_pf%livestemp_xfer_to_livestemp(p)*dt @@ -280,7 +281,8 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%livecrootp(p) = veg_ps%livecrootp(p) - veg_pf%livecrootp_to_retransp(p)*dt veg_ps%retransp(p) = veg_ps%retransp(p) + veg_pf%livecrootp_to_retransp(p)*dt end if - if (ivt(p) >= npcropmin) then ! Beth adds retrans from froot + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! Beth adds retrans from froot veg_ps%frootp(p) = veg_ps%frootp(p) - veg_pf%frootp_to_retransp(p)*dt veg_ps%retransp(p) = veg_ps%retransp(p) + veg_pf%frootp_to_retransp(p)*dt veg_ps%livestemp(p) = veg_ps%livestemp(p) - veg_pf%livestemp_to_litter(p)*dt @@ -331,7 +333,8 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_storage(p) = veg_ps%deadcrootp_storage(p) + veg_pf%ppool_to_deadcrootp_storage(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_livestemp(p)*dt veg_ps%livestemp(p) = veg_ps%livestemp(p) + veg_pf%ppool_to_livestemp(p)*dt veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_livestemp_storage(p)*dt @@ -359,7 +362,8 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_xfer(p) = veg_ps%deadcrootp_xfer(p) + veg_pf%deadcrootp_storage_to_xfer(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & + generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ps%livestemp_storage(p) = veg_ps%livestemp_storage(p) - veg_pf%livestemp_storage_to_xfer(p)*dt veg_ps%livestemp_xfer(p) = veg_ps%livestemp_xfer(p) + veg_pf%livestemp_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/PrecisionControlMod.F90 b/components/elm/src/biogeochem/PrecisionControlMod.F90 index 89e30c83fd1e..8767d300b244 100644 --- a/components/elm/src/biogeochem/PrecisionControlMod.F90 +++ b/components/elm/src/biogeochem/PrecisionControlMod.F90 @@ -39,7 +39,7 @@ subroutine PrecisionControl(num_soilc, filter_soilc, num_soilp, filter_soilp) !$acc routine seq use elm_varctl , only : iulog, use_c13, use_c14, use_fates use elm_varpar , only : nlevdecomp_full, crop_prog - use pftvarcon , only : nc3crop + use pftvarcon , only : crop, generic_crop, percrop use tracer_varcon , only : is_active_betr_bgc use CNDecompCascadeConType , only : decomp_cascade_con ! @@ -211,7 +211,9 @@ subroutine PrecisionControl(num_soilc, filter_soilc, num_soilp, filter_soilp) veg_ps%frootp_xfer(p) = 0._r8 end if - if ( crop_prog .and. veg_pp%itype(p) >= nc3crop )then + if ( crop_prog .and. & + (crop(veg_pp%itype(p)) >= 1 .or. & + percrop(veg_pp%itype(p)) >= 1) )then ! grain C and N if (abs(veg_cs%grainc(p)) < ccrit) then pc = pc + veg_cs%grainc(p) @@ -513,7 +515,9 @@ subroutine PrecisionControl(num_soilc, filter_soilc, num_soilp, filter_soilp) endif end if - if ( crop_prog .and. veg_pp%itype(p) >= nc3crop )then + if ( crop_prog .and. & + (crop(veg_pp%itype(p)) >= 1 .or. & + percrop(veg_pp%itype(p)) >= 1) )then ! xsmrpool (C only) if (abs(veg_cs%xsmrpool(p)) < ccrit) then pc = pc + veg_cs%xsmrpool(p) diff --git a/components/elm/src/biogeochem/SatellitePhenologyMod.F90 b/components/elm/src/biogeochem/SatellitePhenologyMod.F90 index b5d9b1bd3a4b..688b254bd839 100644 --- a/components/elm/src/biogeochem/SatellitePhenologyMod.F90 +++ b/components/elm/src/biogeochem/SatellitePhenologyMod.F90 @@ -304,7 +304,7 @@ subroutine SatellitePhenology(bounds, num_filter, filter, & ! Calculates leaf areas (tlai, elai), stem areas (tsai, esai) and height (htop). ! ! !USES: - use pftvarcon, only : noveg, nbrdlf_dcd_brl_shrub + use pftvarcon, only : woody use elm_varctl, only : use_fates_sp ! ! !ARGUMENTS: @@ -371,7 +371,7 @@ subroutine SatellitePhenology(bounds, num_filter, filter, & ! snow burial fraction for short vegetation (e.g. grasses) as in ! Wang and Zeng, 2007. - if (veg_pp%itype(p) > noveg .and. veg_pp%itype(p) <= nbrdlf_dcd_brl_shrub ) then + if (woody(veg_pp%itype(p)) >= 1.0_r8) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) else diff --git a/components/elm/src/biogeochem/VOCEmissionMod.F90 b/components/elm/src/biogeochem/VOCEmissionMod.F90 index cb58227b5235..64a72fe78b8e 100644 --- a/components/elm/src/biogeochem/VOCEmissionMod.F90 +++ b/components/elm/src/biogeochem/VOCEmissionMod.F90 @@ -9,13 +9,9 @@ module VOCEmissionMod use shr_log_mod , only : errMsg => shr_log_errMsg use elm_varctl , only : iulog use elm_varpar , only : numpft, nlevcan - use pftvarcon , only : ndllf_evr_tmp_tree, ndllf_evr_brl_tree - use pftvarcon , only : ndllf_dcd_brl_tree, nbrdlf_evr_trp_tree - use pftvarcon , only : nbrdlf_evr_tmp_tree, nbrdlf_dcd_brl_shrub - use pftvarcon , only : nbrdlf_dcd_trp_tree, nbrdlf_dcd_tmp_tree - use pftvarcon , only : nbrdlf_dcd_brl_tree, nbrdlf_evr_shrub - use pftvarcon , only : nc3_arctic_grass , nc3crop - use pftvarcon , only : nc4_grass, noveg + use elm_varpar , only : numveg ! fixed as 16, while numpft above may be variable + use pftvarcon , only : woody, graminoid, generic_crop, crop, percrop + use pftvarcon , only : needleleaf, evergreen use shr_megan_mod , only : shr_megan_megcomps_n, shr_megan_megcomp_t, shr_megan_linkedlist use shr_megan_mod , only : shr_megan_mechcomps_n, shr_megan_mechcomps, shr_megan_mapped_emisfctrs use MEGANFactorsMod , only : Agro, Amat, Anew, Aold, betaT, ct1, ct2, LDF, Ceo @@ -114,7 +110,7 @@ subroutine InitAllocate(this, bounds) ! !LOCAL VARIABLES: integer :: i, imeg integer :: class_num - real(r8) :: factors(numpft) + real(r8) :: factors(numveg) real(r8) :: molec_wght integer :: begg, endg integer :: begp, endp @@ -128,7 +124,8 @@ subroutine InitAllocate(this, bounds) meg_cmp => shr_megan_linkedlist do while(associated(meg_cmp)) - allocate(meg_cmp%emis_factors(numpft)) + !allocate(meg_cmp%emis_factors(numpft)) + allocate(meg_cmp%emis_factors(numveg)) ! TODO: VOCEmission input is with a hard-wired num of pft (16) call megan_factors_get( trim(meg_cmp%name), factors, class_num, molec_wght ) meg_cmp%emis_factors = factors meg_cmp%class_number = class_num @@ -693,21 +690,21 @@ function get_map_EF(ivt_in, g_in, ti_in, vocemis_vars) get_map_EF = 0._r8 - if ( ivt_in == ndllf_evr_tmp_tree & - .or. ivt_in == ndllf_evr_brl_tree) then !fineleaf evergreen + if (woody(ivt_in) == 1.0_r8 .and. evergreen(ivt_in) == 1 & + .and. needleleaf(ivt_in) == 1) then !fineleaf evergreen trees get_map_EF = vocemis_vars%efisop_grc(2,g_in, ti_in) - else if (ivt_in == ndllf_dcd_brl_tree) then !fineleaf deciduous + else if (woody(ivt_in) == 1.0_r8 .and. evergreen(ivt_in) == 0 & + .and. needleleaf(ivt_in) == 1) then !fineleaf deciduous trees get_map_EF = vocemis_vars%efisop_grc(3,g_in, ti_in) - else if (ivt_in >= nbrdlf_evr_trp_tree & - .and. ivt_in <= nbrdlf_dcd_brl_tree) then !broadleaf trees + else if (woody(ivt_in) == 1.0_r8 & + .and. needleleaf(ivt_in) == 0) then !broadleaf trees get_map_EF = vocemis_vars%efisop_grc(1,g_in,ti_in) - else if (ivt_in >= nbrdlf_evr_shrub & - .and. ivt_in <= nbrdlf_dcd_brl_shrub) then !shrubs + else if (woody(ivt_in) == 2.0_r8) then !shrubs get_map_EF = vocemis_vars%efisop_grc(4,g_in, ti_in) - else if (ivt_in >= nc3_arctic_grass & - .and. ivt_in <= nc4_grass) then !grass + else if (graminoid(ivt_in) == 1) then !grass get_map_EF = vocemis_vars%efisop_grc(5,g_in, ti_in) - else if (ivt_in >= nc3crop) then !crops + else if (generic_crop(ivt_in) == 1 .or. crop(ivt_in) == 1 & + .or. percrop(ivt_in) == 1) then !crops get_map_EF = vocemis_vars%efisop_grc(6,g_in, ti_in) end if @@ -970,7 +967,8 @@ function get_gamma_A(ivt_in, elai240_in,elai_in,nclass_in) real(r8) :: elai_prev ! lai for previous timestep real(r8) :: fnew, fgro, fmat, fold ! fractions of leaves at different phenological stages !----------------------------------------------------------------------- - if ( (ivt_in == ndllf_dcd_brl_tree) .or. (ivt_in >= nbrdlf_dcd_trp_tree) ) then ! non-evergreen + !if ( (ivt_in == ndllf_dcd_brl_tree) .or. (ivt_in >= nbrdlf_dcd_trp_tree) ) then ! non-evergreen + if ( (woody(ivt_in) == 1.0_r8) .and. (evergreen(ivt_in) <= 0) ) then ! non-evergreen tree if ( (elai240_in > 0.0_r8) .and. (elai240_in < 1.e30_r8) )then elai_prev = 2._r8*elai240_in-elai_in ! have accumulated average lai over last timestep diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 9a7db79154b4..cd982139eef4 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -37,8 +37,8 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! vegetation structure (LAI, SAI, height) ! ! !USES: - use pftvarcon , only : noveg, nc3crop, nc3irrig, nbrdlf_evr_shrub, nbrdlf_dcd_brl_shrub - use pftvarcon , only : ncorn, ncornirrig, npcropmin, ztopmx, laimx + use pftvarcon , only : noveg, woody, generic_crop, crop, percrop + use pftvarcon , only : ncorn, ncornirrig, ztopmx, laimx use pftvarcon , only : nmiscanthus, nmiscanthusirrig, nswitchgrass, nswitchgrassirrig use elm_time_manager , only : get_rad_step_size use elm_varctl , only : spinup_state, spinup_mortality_factor @@ -148,7 +148,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! alpha are set by PFT, and alpha is scaled to CLM time step by multiplying by ! dt and dividing by dtsmonth (seconds in average 30 day month) ! tsai_min scaled by 0.5 to match MODIS satellite derived values - if (ivt(p) == nc3crop .or. ivt(p) == nc3irrig) then ! generic crops + if (generic_crop(ivt(p)) == 1) then ! generic crops tsai_alpha = 1.0_r8-1.0_r8*dt/dtsmonth tsai_min = 0.1_r8 @@ -164,7 +164,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! trees and shrubs ! if shrubs have a squat taper - if (ivt(p) >= nbrdlf_evr_shrub .and. ivt(p) <= nbrdlf_dcd_brl_shrub) then + if (woody(ivt(p)) == 2.0_r8) then taper = 10._r8 ! otherwise have a tall taper else @@ -194,7 +194,8 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & hbot(p) = max(0._r8, min(3._r8, htop(p)-1._r8)) - else if (ivt(p) >= npcropmin) then ! prognostic crops + else if ( generic_crop(ivt(p)) <1 .and. & + (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) ) then ! prognostic crops if (tlai(p) >= laimx(ivt(p))) peaklai(p) = 1 ! used in CNAllocation @@ -248,7 +249,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! adjust lai and sai for burying by snow. ! snow burial fraction for short vegetation (e.g. grasses) as in ! Wang and Zeng, 2007. - if (ivt(p) > noveg .and. ivt(p) <= nbrdlf_dcd_brl_shrub ) then + if (woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) else diff --git a/components/elm/src/biogeophys/CanopyFluxesMod.F90 b/components/elm/src/biogeophys/CanopyFluxesMod.F90 index 6c96e70e172d..fd6cd3ab163e 100755 --- a/components/elm/src/biogeophys/CanopyFluxesMod.F90 +++ b/components/elm/src/biogeophys/CanopyFluxesMod.F90 @@ -17,7 +17,7 @@ module CanopyFluxesMod use elm_varctl , only : use_hydrstress use elm_varpar , only : nlevgrnd, nlevsno use elm_varcon , only : namep - use pftvarcon , only : nbrdlf_dcd_tmp_shrub, nsoybean , nsoybeanirrig + use pftvarcon , only : crop, nfixer use decompMod , only : bounds_type use PhotosynthesisMod , only : Photosynthesis, PhotosynthesisTotal, Fractionation, PhotoSynthesisHydraulicStress use SoilMoistStressMod , only : calc_effective_soilporosity, calc_volumetric_h2oliq @@ -869,7 +869,8 @@ subroutine CanopyFluxes(bounds, num_nolakeurbanp, filter_nolakeurbanp, & p = filterp(f) c = veg_pp%column(p) if(.not.veg_pp%is_fates(p)) then - if (veg_pp%itype(p) == nsoybean .or. veg_pp%itype(p) == nsoybeanirrig) then + ! soybean (crop with N fixation) + if (crop(veg_pp%itype(p)) >= 1 .and. nfixer(veg_pp%itype(p)) == 1) then btran(p) = min(1._r8, btran(p) * 1.25_r8) end if @@ -909,7 +910,8 @@ subroutine CanopyFluxes(bounds, num_nolakeurbanp, filter_nolakeurbanp, & do f = 1, fn p = filterp(f) c = veg_pp%column(p) - if (veg_pp%itype(p) == nsoybean .or. veg_pp%itype(p) == nsoybeanirrig) then + ! soybean (crop with N fixation) + if (crop(veg_pp%itype(p)) >= 1 .and. nfixer(veg_pp%itype(p)) == 1) then btran(p) = min(1._r8, btran(p) * 1.25_r8) end if end do diff --git a/components/elm/src/biogeophys/PhotosynthesisMod.F90 b/components/elm/src/biogeophys/PhotosynthesisMod.F90 index 65ddbbb71553..f080c457fa3c 100644 --- a/components/elm/src/biogeophys/PhotosynthesisMod.F90 +++ b/components/elm/src/biogeophys/PhotosynthesisMod.F90 @@ -225,7 +225,6 @@ subroutine Photosynthesis ( bounds, fn, filterp, & !$acc routine seq use elm_varcon , only : rgas, tfrz use elm_varctl , only : carbon_only - use pftvarcon , only : nbrdlf_dcd_tmp_shrub, nsoybean, nsoybeanirrig, npcropmin use pftvarcon , only : vcmax_np1, vcmax_np2, vcmax_np3, vcmax_np4, jmax_np1, jmax_np2, jmax_np3 ! ! !ARGUMENTS: @@ -1594,7 +1593,6 @@ subroutine PhotosynthesisHydraulicStress ( bounds, fn, filterp, & use elm_varctl , only : carbon_only !use elm_varctl , only : lnc_opt, reduce_dayl_factor, vcmax_opt use elm_varpar , only : nlevsoi - use pftvarcon , only : nbrdlf_dcd_tmp_shrub, npcropmin use pftvarcon , only : vcmax_np1, vcmax_np2, vcmax_np3, vcmax_np4, jmax_np1, jmax_np2, jmax_np3 use ColumnType , only : col_pp diff --git a/components/elm/src/biogeophys/SedYieldMod.F90 b/components/elm/src/biogeophys/SedYieldMod.F90 index 10b0db728149..e50544edde6c 100644 --- a/components/elm/src/biogeophys/SedYieldMod.F90 +++ b/components/elm/src/biogeophys/SedYieldMod.F90 @@ -63,7 +63,7 @@ subroutine SoilErosion (bounds, num_soilc, filter_soilc, & use elm_time_manager, only : get_step_size use landunit_varcon , only : istcrop, istsoil, istice use pftvarcon , only : gcbc_p, gcbc_q, gcbr_p, gcbr_q - use pftvarcon , only : nc4_grass + use pftvarcon , only : generic_crop, crop, percrop ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -218,7 +218,9 @@ subroutine SoilErosion (bounds, num_soilc, filter_soilc, & (veg_cs%livecrootc(p)+veg_cs%deadcrootc(p))*croot_prof(p,1) ) fgndcov = exp( -gcbc_p(veg_pp%itype(p))*PCT_gnd - & gcbr_p(veg_pp%itype(p))*Broot ) - if( veg_pp%itype(p) > nc4_grass )then + if( generic_crop(veg_pp%itype(p)) >= 1 .or. & + crop(veg_pp%itype(p)) >= 1 .or. & + percrop(veg_pp%itype(p)) >=1 )then Es_Pcrp = Es_Pcrp + pfactor(c) * ftillage * flitho * & fgndcov * veg_pp%wtcol(p) * K * (KE_DT+KE_LD) @@ -266,7 +268,9 @@ subroutine SoilErosion (bounds, num_soilc, filter_soilc, & nh = 0.03_r8 + 0.05_r8*max(Crsd,Clai) fsr = fsr + veg_pp%wtcol(p) * (0.03_r8/nh)**0.6_r8 - if ( veg_pp%itype(p) > nc4_grass ) then + if ( generic_crop(veg_pp%itype(p)) >= 1 .or. & + crop(veg_pp%itype(p)) >= 1 .or. & + percrop(veg_pp%itype(p)) >=1 ) then ftillage_tc = ftillage_tc + ftillage * veg_pp%wtcol(p) Es_Q = Es_Q + 19.1_r8 * qfactor(c) * 2._r8/COH * flitho * fslp * & diff --git a/components/elm/src/dyn_subgrid/dynHarvestMod.F90 b/components/elm/src/dyn_subgrid/dynHarvestMod.F90 index 5256f415efe2..07455f77ec6e 100644 --- a/components/elm/src/dyn_subgrid/dynHarvestMod.F90 +++ b/components/elm/src/dyn_subgrid/dynHarvestMod.F90 @@ -202,7 +202,8 @@ subroutine CNHarvest (num_soilc, filter_soilc, num_soilp, filter_soilp, cnstate_ ! Harvest mortality routine for coupled carbon-nitrogen code (CN) ! !USES: - use pftvarcon , only : noveg, nbrdlf_evr_shrub, pprodharv10 + use pftvarcon , only : pprodharv10 + use pftvarcon , only : woody use elm_varcon , only : secspday use elm_time_manager, only : get_days_per_year use GridcellType , only : grc_pp @@ -377,7 +378,7 @@ subroutine CNHarvest (num_soilc, filter_soilc, num_soilp, filter_soilp, cnstate_ ! If this is a tree pft, then ! get the annual harvest "mortality" rate (am) from harvest array ! and convert to rate per second - if (ivt(p) > noveg .and. ivt(p) < nbrdlf_evr_shrub) then + if (woody(ivt(p)) == 1.0_r8) then if (do_harvest) then am = 0._r8 diff --git a/components/elm/src/main/surfrdUtilsMod.F90 b/components/elm/src/main/surfrdUtilsMod.F90 index a32570c15a29..1e58b5174afa 100644 --- a/components/elm/src/main/surfrdUtilsMod.F90 +++ b/components/elm/src/main/surfrdUtilsMod.F90 @@ -20,7 +20,7 @@ module surfrdUtilsMod ! !PUBLIC MEMBER FUNCTIONS: public :: check_sums_equal_1_3d ! Confirm that sum(arr(n,t,:)) == 1 for all n public :: check_sums_equal_1_2d ! Confirm that sum(arr(n,:)) == 1 for all n - public :: convert_cft_to_pft ! Conversion of crop CFT to natural veg PFT:w + public :: convert_cft_to_pft ! Conversion of crop CFT to natural veg PFT public :: collapse_crop_types ! Collapse unused crop types into types used in this run public :: collapse_crop_var ! Collapse crop variables according to cft weights determined in previous "collapse" subroutines public :: convert_pft_to_cft ! Conversion of crops from natural veg to CFT @@ -124,20 +124,21 @@ subroutine convert_cft_to_pft( begg, endg, cftsize, wt_cft ) ! !USES: use elm_varsur , only : wt_lunit, wt_nat_patch use elm_varpar , only : cft_size, surfpft_size - use pftvarcon , only : nc3crop + use elm_varpar , only : natpft_size use landunit_varcon , only : istsoil, istcrop use topounit_varcon , only : max_topounits ! !ARGUMENTS: implicit none integer , intent(in) :: begg, endg - integer , intent(in) :: cftsize ! CFT size + integer , intent(in) :: cftsize ! CFT size ! this could be wrong if by input real(r8) , intent(inout) :: wt_cft(begg:,:,:) ! CFT weights ! ! !LOCAL VARIABLES: integer :: g, t ! index !----------------------------------------------------------------------- - SHR_ASSERT_ALL((ubound(wt_cft ) == (/endg,max_topounits, cftsize /)), errMsg(__FILE__, __LINE__)) - SHR_ASSERT_ALL((ubound(wt_nat_patch) == (/endg,max_topounits, nc3crop+cftsize-1/)), errMsg(__FILE__, __LINE__)) + ! note (01-29-2024 by fmyuan@ornl.gov, cftsize --> elm_varpar:cft_size) + SHR_ASSERT_ALL((ubound(wt_cft ) == (/endg,max_topounits, cft_size /)), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_ALL((ubound(wt_nat_patch) == (/endg,max_topounits, natpft_size+cft_size/)), errMsg(__FILE__, __LINE__)) do g = begg, endg do t = 1, max_topounits @@ -145,12 +146,12 @@ subroutine convert_cft_to_pft( begg, endg, cftsize, wt_cft ) ! Move CFT over to PFT and do weighted average of the crop and soil parts wt_nat_patch(g,t,:) = wt_nat_patch(g,t,:) * wt_lunit(g,t,istsoil) wt_cft(g,t,:) = wt_cft(g,t,:) * wt_lunit(g,t,istcrop) - wt_nat_patch(g,t,nc3crop:) = wt_cft(g,t,:) ! Add crop CFT's to end of natural veg PFT's - wt_lunit(g,t,istsoil) = (wt_lunit(g,t,istsoil) + wt_lunit(g,t,istcrop)) ! Add crop landunit to soil landunit + wt_nat_patch(g,t,natpft_size:) = wt_cft(g,t,:) ! Add crop CFT's to end of natural veg PFT's + wt_lunit(g,t,istsoil) = (wt_lunit(g,t,istsoil) + wt_lunit(g,t,istcrop)) ! Add crop landunit to soil landunit wt_nat_patch(g,t,:) = wt_nat_patch(g,t,:) / wt_lunit(g,t,istsoil) - wt_lunit(g,t,istcrop) = 0.0_r8 ! Zero out crop CFT's + wt_lunit(g,t,istcrop) = 0.0_r8 ! Zero out crop CFT's else - wt_nat_patch(g,t,nc3crop:) = 0.0_r8 ! Make sure generic crops are zeroed out + wt_nat_patch(g,t,natpft_size:) = 0.0_r8 ! Make sure generic crops are zeroed out end if end do end do @@ -171,7 +172,6 @@ subroutine convert_pft_to_cft( begg, endg ) use elm_varsur , only : wt_lunit, wt_nat_patch, wt_cft use elm_varpar , only : cft_size, surfpft_size use elm_varpar , only : cft_size, cft_lb, cft_ub, surfpft_lb, surfpft_ub - use pftvarcon , only : nc3crop use landunit_varcon , only : istsoil, istcrop use topounit_varcon , only : max_topounits ! !ARGUMENTS: @@ -309,8 +309,8 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose ! !USES: use elm_varctl , only : irrigate use elm_varpar , only : cft_lb, cft_ub, cft_size - use pftvarcon , only : nc3crop, nc3irrig, npcropmax, mergetoelmpft use pftvarcon , only: is_pft_known_to_model + use pftvarcon , only : npcropmax, mergetoelmpft, npcropmin use topounit_varcon , only : max_topounits ! TKT use GridcellType , only : grc_pp ! TKT ! @@ -369,10 +369,12 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose ! stride 2 ! where stride 2 means "every other" - do t = 1,max_topounits - wt_cft(g,t, nc3crop:npcropmax-1:2) = & - wt_cft(g,t, nc3crop:npcropmax-1:2) + wt_cft(g,t, nc3irrig:npcropmax:2) ! TKT - wt_cft(g,t, nc3irrig:npcropmax:2) = 0._r8 + do t = grc_pp%topi(g), grc_pp%topf(g) ! TKT + t2 = t - grc_pp%topi(g) + 1 + + wt_cft(g,t2, cft_lb:cft_ub-1:2) = & + wt_cft(g,t2, cft_lb:cft_ub-1:2) + wt_cft(g,t2, cft_lb+1:cft_ub:2) ! TKT + wt_cft(g,t2, cft_lb+1:cft_ub:2) = 0._r8 end do ! TKT end do From 4314a504beed21046772e05080cdbd9ddbb84a5c Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Tue, 30 Jan 2024 10:00:41 -0500 Subject: [PATCH 572/904] All PFT read-in constants are passed into VegetationProperties, i.e. veg_pp. Only difference may be actual number of PFTs due to surface data 'natpft' length less than physiology parameter length ('npft'). So it's shall be exactly same by 'use pftvarcon, only: ', or by 'veg_pp%'. This commit try to use either in one code file of .F90. --- components/elm/src/biogeochem/FireMod.F90 | 2 -- .../elm/src/biogeochem/GapMortalityMod.F90 | 1 - .../elm/src/biogeochem/VegStructUpdateMod.F90 | 1 - .../data_types/VegetationPropertiesType.F90 | 24 +++++++++++++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/elm/src/biogeochem/FireMod.F90 b/components/elm/src/biogeochem/FireMod.F90 index 743141b20182..24e34e37c555 100644 --- a/components/elm/src/biogeochem/FireMod.F90 +++ b/components/elm/src/biogeochem/FireMod.F90 @@ -735,8 +735,6 @@ subroutine FireFluxes (num_soilc, filter_soilc, num_soilp, filter_soilp, & is_cwd => decomp_cascade_con%is_cwd , & ! Input: [logical (:) ] TRUE => pool is a cwd pool is_litter => decomp_cascade_con%is_litter , & ! Input: [logical (:) ] TRUE => pool is a litter pool - !woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform (1=woody, 0=not woody) - cropf_col => cnstate_vars%cropf_col , & ! Input: [real(r8) (:) ] cropland fraction in veg column croot_prof => cnstate_vars%croot_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of coarse roots stem_prof => cnstate_vars%stem_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of stems diff --git a/components/elm/src/biogeochem/GapMortalityMod.F90 b/components/elm/src/biogeochem/GapMortalityMod.F90 index 31cd45814d62..3257a6365b65 100644 --- a/components/elm/src/biogeochem/GapMortalityMod.F90 +++ b/components/elm/src/biogeochem/GapMortalityMod.F90 @@ -110,7 +110,6 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform croplive => crop_vars%croplive_patch & ! Input: [logical (:) ] flag, true if planted, not harvested ) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index cd982139eef4..de7062b810d5 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -83,7 +83,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) slatop => veg_vp%slatop , & ! Input: [real(r8) (:) ] specific leaf area at top of canopy, projected area basis [m^2/gC] dsladlai => veg_vp%dsladlai , & ! Input: [real(r8) (:) ] dSLA/dLAI, projected area basis [m^2/gC] z0mr => veg_vp%z0mr , & ! Input: [real(r8) (:) ] ratio of momentum roughness length to canopy top height (-) diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 2cb4e01c8a46..21d7ed671f31 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -143,6 +143,13 @@ module VegetationPropertiesType real(r8), pointer :: nstor(:) => null() !Nitrogen storage pool timescale real(r8), pointer :: br_xr(:) => null() !Base rate for excess respiration real(r8), pointer :: tc_stress => null() !Critial temperature for moisture stress + ! new properties for flexible PFT + real(r8), pointer :: climatezone(:) => null() !climate zone adapted + real(r8), pointer :: nonvascular(:) => null() !nonvascular type or vascular + real(r8), pointer :: graminoid(:) => null() !graminoid or not + real(r8), pointer :: generic_crop(:) => null() !generic crop or not for prognostic crop modules (?) + real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf + real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 contains @@ -181,6 +188,8 @@ subroutine veg_vp_init(this) use pftvarcon , only : fnr, act25, kcha, koha, cpha, vcmaxha, jmaxha, tpuha use pftvarcon , only : lmrha, vcmaxhd, jmaxhd, tpuhd, lmrse, qe, theta_cj use pftvarcon , only : bbbopt, mbbopt, nstor, br_xr, tc_stress, lmrhd + ! new properties for flexible PFT + use pftvarcon , only : climatezone, nonvascular, graminoid, generic_crop,needleleaf, nfixer ! class (vegetation_properties_type) :: this @@ -305,6 +314,14 @@ subroutine veg_vp_init(this) allocate(this%tc_stress ) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! new properties for flexible PFT + allocate( this%climatezone(0:numpft)) ; this%climatezone(:) =spval + allocate( this%nonvascular(0:numpft)) ; this%nonvascular(:) =spval + allocate( this%graminoid(0:numpft)) ; this%graminoid(:) =spval + allocate( this%generic_crop(0:numpft)) ; this%generic_crop(:) =spval + allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval + allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval + ! ----------------------------------------------------------------------------------------------------------- do m = 0,numpft @@ -393,6 +410,13 @@ subroutine veg_vp_init(this) this%mbbopt(m) = mbbopt(m) this%nstor(m) = nstor(m) this%br_xr(m) = br_xr(m) + ! new properties for flexible PFT + this%climatezone(m) = climatezone(m) + this%nonvascular(m) = nonvascular(m) + this%graminoid(m) = graminoid(m) + this%generic_crop(m) = generic_crop(m) + this%needleleaf(m) = needleleaf(m) + this%nfixer(m) = nfixer(m) end do From a0765f2dd4f96669d82fc1cf2b78b2449f086447 Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Tue, 31 Oct 2023 17:07:41 -0400 Subject: [PATCH 573/904] Fix of possible bugs in surfrdMod.F90 --- components/elm/src/main/surfrdMod.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/elm/src/main/surfrdMod.F90 b/components/elm/src/main/surfrdMod.F90 index 9cbcd4b2b8e0..1a957e8cd38a 100755 --- a/components/elm/src/main/surfrdMod.F90 +++ b/components/elm/src/main/surfrdMod.F90 @@ -1105,6 +1105,8 @@ subroutine surfrd_pftformat( begg, endg, ncid ) end if fert_p_cft = 0.0_r8 + wt_nat_patch(begg:endg, :, :) = 0.0_r8 + wt_cft(begg:endg, :, :) = 0.0_r8 if (.not. create_crop_landunit) then call ncd_io(ncid=ncid, varname='PCT_NAT_PFT', flag='read', data=wt_nat_patch, & dim1name=grlnd, readvar=readvar) From c5afb348b306b6bebc60389201a6d50340b2d37e Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Tue, 3 Jan 2023 16:06:50 -0500 Subject: [PATCH 574/904] Make natpft patch and croppft patch are merged correctly, especially when flexible length pfts are user-provided, and/or surfdata 'lsmpft' dimension is less than 'pft' dimension in parameter file. --- components/elm/src/main/elm_initializeMod.F90 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/elm/src/main/elm_initializeMod.F90 b/components/elm/src/main/elm_initializeMod.F90 index 86e72030b946..9fe4959c241d 100755 --- a/components/elm/src/main/elm_initializeMod.F90 +++ b/components/elm/src/main/elm_initializeMod.F90 @@ -59,6 +59,7 @@ subroutine initialize1( ) ! !USES: use elm_varpar , only: elm_varpar_init, natpft_lb, natpft_ub use elm_varpar , only: cft_lb, cft_ub, maxpatch_glcmec + use elm_varpar , only: mxpft, numveg, mxpft_nc, numpft use elm_varpar , only: update_pft_array_bounds use elm_varpar , only: surfpft_lb, surfpft_ub use elm_varcon , only: elm_varcon_init @@ -293,6 +294,17 @@ subroutine initialize1( ) ! Independent of model resolution, Needs to stay before surfrd_get_data call pftconrd() + ! by user-defined PFT (numbers and names), 'numpft/mxpft_nc' changed and other indices + ! a few arrays had been allocated in elm_initializedMod.F90:L266-268 and require redo after this 'pftconrd' call + if ((numpft/=mxpft .or. numpft/=numveg) .or. (mxpft_nc/=24 .or. mxpft_nc/=numveg)) then + if (associated(wt_nat_patch)) deallocate(wt_nat_patch) + allocate (wt_nat_patch (begg:endg,1:max_topounits, surfpft_lb:surfpft_ub )) + if (associated(wt_cft)) deallocate(wt_cft) + allocate (wt_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) + if (associated(fert_cft)) deallocate(fert_cft) + allocate (fert_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) + endif + call soilorder_conrd() ! Read in FATES parameter values early in the call sequence as well From d01128e7e34b798cb175cdacfa057f0d913075c4 Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Tue, 30 Jan 2024 11:58:45 -0500 Subject: [PATCH 575/904] Edit added 'usrpft_codetest_I1850CNPRDCTCBC' test for e3sm_land_developer in effect. --- .../elm/usrpft_codetest_I1850CNPRDCTCBC/user_nl_elm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_codetest_I1850CNPRDCTCBC/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_codetest_I1850CNPRDCTCBC/user_nl_elm index b2fef61a4be1..921f5d7714b2 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_codetest_I1850CNPRDCTCBC/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/usrpft_codetest_I1850CNPRDCTCBC/user_nl_elm @@ -1,5 +1,5 @@ fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/surfdata_42_FLUXNETSITES_simyr1850_c170912.nc' -paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_c211124.nc' +! paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_c211124.nc' ! ! When 'user-pft' codes ready, comment out the above line while let the following in. ! @@ -7,4 +7,4 @@ paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_c211124.nc' ! woody=1 for tree, or =2 for shrub, and defined other pft flags: ! climatezone(0-4), nonvascular(0/1), graminoid(0/1), generic_crop(0/1), nfixer(0/1), needleleaf(0/1) ! for not going by default, changing one PFT name by appending '_unexpected'. -! paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_c211124vpft.nc' + paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_c211124vpft.nc' From 1cd6dff299d2200e983911605dc2561d61ea9d67 Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Wed, 4 Jan 2023 14:42:50 -0500 Subject: [PATCH 576/904] (optional) Vcmax25 at top canopy for each PFT is as an output variable. --- components/elm/src/biogeophys/PhotosynthesisMod.F90 | 4 ++++ components/elm/src/biogeophys/PhotosynthesisType.F90 | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/components/elm/src/biogeophys/PhotosynthesisMod.F90 b/components/elm/src/biogeophys/PhotosynthesisMod.F90 index f080c457fa3c..aee433610da7 100644 --- a/components/elm/src/biogeophys/PhotosynthesisMod.F90 +++ b/components/elm/src/biogeophys/PhotosynthesisMod.F90 @@ -387,6 +387,7 @@ subroutine Photosynthesis ( bounds, fn, filterp, & gb_mol => photosyns_vars%gb_mol_patch , & ! Output: [real(r8) (:) ] leaf boundary layer conductance (umol H2O/m**2/s) gs_mol => photosyns_vars%gs_mol_patch , & ! Output: [real(r8) (:,:) ] leaf stomatal conductance (umol H2O/m**2/s) vcmax_z => photosyns_vars%vcmax_z_patch , & ! Output: [real(r8) (:,:) ] maximum rate of carboxylation (umol co2/m**2/s) + vcmax25_top => photosyns_vars%vcmax25_top_patch , & ! Output: [real(r8) (:) ] maximum rate of carboxylation at top canopy at 25oC (umol co2/m**2/s) cp => photosyns_vars%cp_patch , & ! Output: [real(r8) (:) ] CO2 compensation point (Pa) kc => photosyns_vars%kc_patch , & ! Output: [real(r8) (:) ] Michaelis-Menten constant for CO2 (Pa) ko => photosyns_vars%ko_patch , & ! Output: [real(r8) (:) ] Michaelis-Menten constant for O2 (Pa) @@ -788,6 +789,9 @@ subroutine Photosynthesis ( bounds, fn, filterp, & vcmax_z(p,iv) = vcmax_z(p,iv) * btran(p) lmr_z(p,iv) = lmr_z(p,iv) * btran(p) + + ! output variable + vcmax25_top(p) = vcmax25top end do ! canopy layer loop end do ! patch loop diff --git a/components/elm/src/biogeophys/PhotosynthesisType.F90 b/components/elm/src/biogeophys/PhotosynthesisType.F90 index 59ae928c23a5..5cb94628d8d2 100644 --- a/components/elm/src/biogeophys/PhotosynthesisType.F90 +++ b/components/elm/src/biogeophys/PhotosynthesisType.F90 @@ -26,6 +26,7 @@ module PhotosynthesisType real(r8), pointer :: ag_patch (:,:) => null()! patch co-limited gross leaf photosynthesis (umol CO2/m**2/s) real(r8), pointer :: an_patch (:,:) => null()! patch net leaf photosynthesis (umol CO2/m**2/s) real(r8), pointer :: vcmax_z_patch (:,:) => null()! patch maximum rate of carboxylation (umol co2/m**2/s) + real(r8), pointer :: vcmax25_top_patch (:) => null()! patch maximum rate of carboxylation at top canopy at 25oC (umol co2/m**2/s) real(r8), pointer :: cp_patch (:) => null()! patch CO2 compensation point (Pa) real(r8), pointer :: kc_patch (:) => null()! patch Michaelis-Menten constant for CO2 (Pa) real(r8), pointer :: ko_patch (:) => null()! patch Michaelis-Menten constant for O2 (Pa) @@ -143,6 +144,7 @@ subroutine InitAllocate(this, bounds) allocate(this%ag_patch (begp:endp,1:nlevcan)) ; this%ag_patch (:,:) = spval allocate(this%an_patch (begp:endp,1:nlevcan)) ; this%an_patch (:,:) = spval allocate(this%vcmax_z_patch (begp:endp,1:nlevcan)) ; this%vcmax_z_patch (:,:) = spval + allocate(this%vcmax25_top_patch (begp:endp)) ; this%vcmax25_top_patch (:) = spval allocate(this%cp_patch (begp:endp)) ; this%cp_patch (:) = spval allocate(this%kc_patch (begp:endp)) ; this%kc_patch (:) = spval allocate(this%ko_patch (begp:endp)) ; this%ko_patch (:) = spval @@ -263,6 +265,12 @@ subroutine InitHistory(this, bounds) call hist_addfld1d (fname='PSNSHA', units='umolCO2/m^2/s', & avgflag='A', long_name='shaded leaf photosynthesis', & ptr_patch=this%psnsha_patch) + + this%vcmax25_top_patch(begp:endp) = spval + call hist_addfld1d (fname='VCMAX25TOP', units='umolCO2/m^2/s', & + avgflag='A', long_name='vcmax at top canopy at 25oC', & + ptr_patch=this%vcmax25_top_patch, default='inactive') + end if if ( use_c13 ) then @@ -423,6 +431,8 @@ subroutine TimeStepInit (this, bounds) this%psnsun_wj_patch(p) = 0._r8 this%psnsun_wp_patch(p) = 0._r8 + this%vcmax25_top_patch(p) = 0._r8 + this%psnsha_patch(p) = 0._r8 this%psnsha_wc_patch(p) = 0._r8 this%psnsha_wj_patch(p) = 0._r8 From 92077c1d238d3ebfc7a00aee1422d6e0ccbf601f Mon Sep 17 00:00:00 2001 From: "Yuan, Fengming" Date: Tue, 12 Mar 2024 14:36:02 -0400 Subject: [PATCH 577/904] crop and perenial crop pft flag, 'iscft', is added so that crop pft indexing may be not in need anymore. --- .../elm/src/biogeochem/AllocationMod.F90 | 40 ++++++------- .../src/biogeochem/CNAllocationBetrMod.F90 | 40 ++++++------- .../elm/src/biogeochem/CNCarbonFluxType.F90 | 16 +++--- .../elm/src/biogeochem/CNCarbonStateType.F90 | 8 +-- .../src/biogeochem/CNGapMortalityBeTRMod.F90 | 6 +- .../src/biogeochem/CNNStateUpdate1BeTRMod.F90 | 10 ++-- .../src/biogeochem/CNNStateUpdate2BeTRMod.F90 | 4 +- .../elm/src/biogeochem/CNNitrogenFluxType.F90 | 2 +- .../src/biogeochem/CNNitrogenStateType.F90 | 6 +- .../elm/src/biogeochem/CNPhenologyBeTRMod.F90 | 11 +++- .../src/biogeochem/CarbonStateUpdate1Mod.F90 | 29 ++++------ .../src/biogeochem/CarbonStateUpdate2Mod.F90 | 4 +- components/elm/src/biogeochem/CropType.F90 | 4 +- .../elm/src/biogeochem/DryDepVelocity.F90 | 6 +- components/elm/src/biogeochem/FireMod.F90 | 27 +++------ .../elm/src/biogeochem/GapMortalityMod.F90 | 12 ++-- .../elm/src/biogeochem/GrowthRespMod.F90 | 4 +- .../elm/src/biogeochem/MaintenanceRespMod.F90 | 4 +- .../biogeochem/NitrogenStateUpdate1Mod.F90 | 14 ++--- .../biogeochem/NitrogenStateUpdate2Mod.F90 | 4 +- .../src/biogeochem/PhenologyFluxLimitMod.F90 | 24 ++++---- .../elm/src/biogeochem/PhenologyMod.F90 | 40 +++++++------ .../elm/src/biogeochem/PhosphorusFluxType.F90 | 1 - .../src/biogeochem/PhosphorusStateType.F90 | 4 +- .../biogeochem/PhosphorusStateUpdate1Mod.F90 | 14 ++--- .../biogeochem/PhosphorusStateUpdate2Mod.F90 | 4 +- .../src/biogeochem/PrecisionControlMod.F90 | 10 +--- .../elm/src/biogeochem/RootDynamicsMod.F90 | 4 +- .../elm/src/biogeochem/VOCEmissionMod.F90 | 5 +- .../elm/src/biogeochem/VegStructUpdateMod.F90 | 7 +-- components/elm/src/biogeophys/SedYieldMod.F90 | 10 +--- .../elm/src/data_types/VegetationDataType.F90 | 29 +++++----- .../data_types/VegetationPropertiesType.F90 | 8 +-- components/elm/src/main/filterMod.F90 | 8 +-- components/elm/src/main/pftvarcon.F90 | 57 ++++++++----------- components/elm/src/main/surfrdMod.F90 | 2 + components/elm/src/main/surfrdUtilsMod.F90 | 2 +- 37 files changed, 221 insertions(+), 259 deletions(-) diff --git a/components/elm/src/biogeochem/AllocationMod.F90 b/components/elm/src/biogeochem/AllocationMod.F90 index a31a07cd8f6c..0110367f59ce 100644 --- a/components/elm/src/biogeochem/AllocationMod.F90 +++ b/components/elm/src/biogeochem/AllocationMod.F90 @@ -386,7 +386,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp use elm_varctl , only : carbonphosphorus_only! use pftvarcon , only: npcropmin, declfact, bfact, aleaff, arootf, astemf, noveg use pftvarcon , only: arooti, fleafi, allconsl, allconss, grperc, grpnow, nsoybean - use pftvarcon , only: percrop + use pftvarcon , only: iscft, percrop use elm_varpar , only: nlevdecomp use elm_varcon , only: nitrif_n2o_loss_frac, secspday ! @@ -585,7 +585,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (ivt(p) >= npcropmin) then + else if (iscft(ivt(p)) >= 1) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) end if @@ -672,7 +672,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp f5 = 0._r8 ! continued intializations from above - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops if (croplive(p) .and. percrop(ivt(p)) == 0.0_r8 ) then ! same phases appear in subroutine CropPhenology @@ -822,7 +822,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp p_allometry(p) = 1._r8/cpl + f1/cpfr + (f3*f4*(1._r8+f2))/cplw + & (f3*(1._r8-f4)*(1._r8+f2))/cpdw - else if (ivt(p) >= npcropmin) then ! skip generic crops + else if (iscft(ivt(p)) >= 1) then ! skip generic crops cng = graincn(ivt(p)) cpg = graincp(ivt(p)) c_allometry(p) = (1._r8+g1)*(1._r8+f1+f5+f3*(1._r8+f2)) @@ -852,10 +852,10 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp ! retransn pool has N from leaves, stems, and roots for ! retranslocation - if (ivt(p) >= npcropmin .and. grain_flag(p) == 1._r8) then + if (iscft(ivt(p)) >= 1 .and. grain_flag(p) == 1._r8) then avail_retransn(p) = plant_ndemand(p) avail_retransp(p) = plant_pdemand(p) - else if (ivt(p) < npcropmin .and. annsum_potential_gpp(p) > 0._r8) then + else if (iscft(ivt(p)) < 1 .and. annsum_potential_gpp(p) > 0._r8) then avail_retransn(p) = (annmax_retransn(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt avail_retransp(p) = (annmax_retransp(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt else @@ -1859,7 +1859,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & !$acc routine seq use elm_varctl , only: iulog use pftvarcon , only: noveg - use pftvarcon , only: npcropmin, grperc, grpnow + use pftvarcon , only: iscft, grperc, grpnow use elm_varpar , only: nlevdecomp use elm_varcon , only: nitrif_n2o_loss_frac, secspday ! @@ -2142,7 +2142,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & fcur = fcur2(ivt(p)) - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops if (croplive(p)) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -2328,7 +2328,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & fcur = fcur2(ivt(p)) - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops if (croplive(p)) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -2345,10 +2345,10 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & sminn_to_npool(p) = sminn_to_plant_patch(p) sminp_to_ppool(p) = sminp_to_plant_patch(p) - if (ivt(p) >= npcropmin .and. grain_flag(p) == 1._r8) then + if (iscft(ivt(p)) >= 1 .and. grain_flag(p) == 1._r8) then avail_retransn(p) = retransn(p)/dt avail_retransp(p) = retransp(p)/dt - else if (ivt(p) < npcropmin .and. annsum_potential_gpp(p) > 0._r8) then + else if (iscft(ivt(p)) < 1 .and. annsum_potential_gpp(p) > 0._r8) then avail_retransn(p) = (annmax_retransn(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt avail_retransp(p) = (annmax_retransp(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt else @@ -2374,7 +2374,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (ivt(p) >= npcropmin) then + else if (iscft(ivt(p)) >= 1) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) end if @@ -2473,7 +2473,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & p_allometry(p) = 1._r8/cpl + f1/cpfr + (f3*f4*(1._r8+f2))/cplw + & (f3*(1._r8-f4)*(1._r8+f2))/cpdw - else if (ivt(p) >= npcropmin) then ! skip generic crops + else if (iscft(ivt(p)) >= 1) then ! skip generic crops cng = graincn(ivt(p)) cpg = graincp(ivt(p)) c_allometry(p) = (1._r8+g1)*(1._r8+f1+f5+f3*(1._r8+f2)) @@ -2559,7 +2559,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f2 * f3 * (1._r8 - f4) * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) * (1 + g1) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * (1._r8 - fcur) * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * (1._r8 - f4) * fcur * (1 + g1) @@ -2590,7 +2590,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_deadcrootc(p) = nlc * f2 * f3 * (1._r8 - f4) * fcur cpool_to_deadcrootc_storage(p) = nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -2643,7 +2643,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_deadcrootn(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * fcur npool_to_deadcrootn_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cng = graincn(ivt(p)) npool_to_livestemn(p) = (nlc * f3 * f4 / cnlw) * fcur npool_to_livestemn_storage(p) = (nlc * f3 * f4 / cnlw) * (1._r8 - fcur) @@ -2687,7 +2687,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_deadcrootp(p) = (nlc * f2 * f3 * (1._r8 - f4) / cpdw)* fcur ppool_to_deadcrootp_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cpdw)* (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpg = graincp(ivt(p)) ppool_to_livestemp(p) = (nlc * f3 * f4 / cplw) * fcur ppool_to_livestemp_storage(p) = (nlc * f3 * f4 / cplw) * (1._r8 -fcur) @@ -2717,7 +2717,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) gresp_storage = gresp_storage + cpool_to_deadcrootc_storage(p) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) gresp_storage = gresp_storage + cpool_to_grainc_storage(p) end if @@ -2771,7 +2771,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_deadcrootn(p) = cpool_to_deadcrootc(p) / cndw npool_to_deadcrootn_storage(p) = cpool_to_deadcrootc_storage(p) / cndw end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cng = graincn(ivt(p)) supplement_to_plantn(p) = supplement_to_plantn(p) + cpool_to_livestemc(p) / cnlw - npool_to_livestemn(p) supplement_to_plantn(p) = supplement_to_plantn(p) + cpool_to_livestemc_storage(p) / cnlw & @@ -2843,7 +2843,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_deadcrootp(p) = cpool_to_deadcrootc(p) / cpdw ppool_to_deadcrootp_storage(p) = cpool_to_deadcrootc_storage(p) / cpdw end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpg = graincp(ivt(p)) supplement_to_plantp(p) = supplement_to_plantp(p) + max(cpool_to_livestemc(p) / cplw & - ppool_to_livestemp(p),0._r8) diff --git a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 index 42fad84a9d1d..cc8b72046a17 100644 --- a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 +++ b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 @@ -246,7 +246,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp use shr_sys_mod , only: shr_sys_flush use elm_varctl , only: iulog,cnallocate_carbon_only,cnallocate_carbonnitrogen_only,& cnallocate_carbonphosphorus_only - use pftvarcon , only: npcropmin, declfact, bfact, aleaff, arootf, astemf, noveg + use pftvarcon , only: iscft, declfact, bfact, aleaff, arootf, astemf, noveg use pftvarcon , only: arooti, fleafi, allconsl, allconss, grperc, grpnow, nsoybean use elm_varpar , only: nlevdecomp use elm_varcon , only: nitrif_n2o_loss_frac, secspday @@ -645,7 +645,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (ivt(p) >= npcropmin) then + else if (iscft(ivt(p)) >= 1) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) end if @@ -726,7 +726,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp f5 = 0._r8 ! continued intializations from above - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops if (croplive(p)) then ! same phases appear in subroutine CropPhenology @@ -859,7 +859,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp p_allometry(p) = 1._r8/cpl + f1/cpfr + (f3*f4*(1._r8+f2))/cplw + & (f3*(1._r8-f4)*(1._r8+f2))/cpdw - else if (ivt(p) >= npcropmin) then ! skip generic crops + else if (iscft(ivt(p)) >= 1) then ! skip generic crops cng = graincn(ivt(p)) cpg = graincp(ivt(p)) c_allometry(p) = (1._r8+g1)*(1._r8+f1+f5+f3*(1._r8+f2)) @@ -889,10 +889,10 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp ! retransn pool has N from leaves, stems, and roots for ! retranslocation - if (ivt(p) >= npcropmin .and. grain_flag(p) == 1._r8) then + if (iscft(ivt(p)) >= 1 .and. grain_flag(p) == 1._r8) then avail_retransn(p) = plant_ndemand(p) avail_retransp(p) = plant_pdemand(p) - else if (ivt(p) < npcropmin .and. annsum_potential_gpp(p) > 0._r8) then + else if (iscft(ivt(p)) < 1 .and. annsum_potential_gpp(p) > 0._r8) then avail_retransn(p) = (annmax_retransn(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt avail_retransp(p) = (annmax_retransp(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt else @@ -1105,10 +1105,10 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & use shr_sys_mod , only: shr_sys_flush use elm_varctl , only: iulog,cnallocate_carbon_only,cnallocate_carbonnitrogen_only,& cnallocate_carbonphosphorus_only -! use pftvarcon , only: npcropmin, declfact, bfact, aleaff, arootf, astemf +! use pftvarcon , only: iscft, declfact, bfact, aleaff, arootf, astemf ! use pftvarcon , only: arooti, fleafi, allconsl, allconss, grperc, grpnow, nsoybean use pftvarcon , only: noveg - use pftvarcon , only: npcropmin, grperc, grpnow + use pftvarcon , only: iscft, grperc, grpnow use elm_varpar , only: nlevdecomp !!nlevsoi, use elm_varcon , only: nitrif_n2o_loss_frac, secspday ! use landunit_varcon , only: istsoil, istcrop @@ -1393,7 +1393,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & fcur = fcur2(ivt(p)) - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops if (croplive(p)) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -1412,10 +1412,10 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & plant_n_buffer_patch(p) = plant_n_buffer_patch(p) * (1._r8-dt/taun) plant_p_buffer_patch(p) = plant_p_buffer_patch(p) * (1._r8-dt/taun) - if (ivt(p) >= npcropmin .and. grain_flag(p) == 1._r8) then + if (iscft(ivt(p)) >= 1 .and. grain_flag(p) == 1._r8) then avail_retransn(p) = retransn(p)/dt avail_retransp(p) = retransp(p)/dt - else if (ivt(p) < npcropmin .and. annsum_potential_gpp(p) > 0._r8) then + else if (iscft(ivt(p)) < 1 .and. annsum_potential_gpp(p) > 0._r8) then avail_retransn(p) = (annmax_retransn(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt avail_retransp(p) = (annmax_retransp(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt else @@ -1436,7 +1436,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (ivt(p) >= npcropmin) then + else if (iscft(ivt(p)) >= 1) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) end if ! try to take mr from xsmr storage pool first @@ -1530,7 +1530,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & p_allometry(p) = 1._r8/cpl + f1/cpfr + (f3*f4*(1._r8+f2))/cplw + & (f3*(1._r8-f4)*(1._r8+f2))/cpdw - else if (ivt(p) >= npcropmin) then ! skip generic crops + else if (iscft(ivt(p)) >= 1) then ! skip generic crops cng = graincn(ivt(p)) cpg = graincp(ivt(p)) c_allometry(p) = (1._r8+g1)*(1._r8+f1+f5+f3*(1._r8+f2)) @@ -1614,7 +1614,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f2 * f3 * (1._r8 - f4) * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) * (1 + g1) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * (1._r8 - fcur) * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * (1._r8 - f4) * fcur * (1 + g1) @@ -1645,7 +1645,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_deadcrootc(p) = nlc * f2 * f3 * (1._r8 - f4) * fcur cpool_to_deadcrootc_storage(p) = nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -1698,7 +1698,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_deadcrootn(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * fcur npool_to_deadcrootn_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cng = graincn(ivt(p)) npool_to_livestemn(p) = (nlc * f3 * f4 / cnlw) * fcur npool_to_livestemn_storage(p) = (nlc * f3 * f4 / cnlw) * (1._r8 - fcur) @@ -1742,7 +1742,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_deadcrootp(p) = (nlc * f2 * f3 * (1._r8 - f4) / cpdw)* fcur ppool_to_deadcrootp_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cpdw)* (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpg = graincp(ivt(p)) ppool_to_livestemp(p) = (nlc * f3 * f4 / cplw) * fcur ppool_to_livestemp_storage(p) = (nlc * f3 * f4 / cplw) * (1._r8 -fcur) @@ -1772,7 +1772,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) gresp_storage = gresp_storage + cpool_to_deadcrootc_storage(p) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) gresp_storage = gresp_storage + cpool_to_grainc_storage(p) end if @@ -1820,7 +1820,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_deadcrootn(p) = cpool_to_deadcrootc(p) / cndw npool_to_deadcrootn_storage(p) = cpool_to_deadcrootc_storage(p) / cndw end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cng = graincn(ivt(p)) supplement_to_sminn_vr(c,1) = supplement_to_sminn_vr(c,1) + cpool_to_livestemc(p) / cnlw - npool_to_livestemn(p) supplement_to_sminn_vr(c,1) = supplement_to_sminn_vr(c,1) + cpool_to_livestemc_storage(p) / cnlw & @@ -1891,7 +1891,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_deadcrootp(p) = cpool_to_deadcrootc(p) / cpdw ppool_to_deadcrootp_storage(p) = cpool_to_deadcrootc_storage(p) / cpdw end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpg = graincp(ivt(p)) supplement_to_sminp_vr(c,1) = supplement_to_sminp_vr(c,1) + max(cpool_to_livestemc(p) / cplw & - ppool_to_livestemp(p),0._r8) diff --git a/components/elm/src/biogeochem/CNCarbonFluxType.F90 b/components/elm/src/biogeochem/CNCarbonFluxType.F90 index 092af4fa66bf..61a3f736f621 100644 --- a/components/elm/src/biogeochem/CNCarbonFluxType.F90 +++ b/components/elm/src/biogeochem/CNCarbonFluxType.F90 @@ -10,7 +10,7 @@ module CNCarbonFluxType use landunit_varcon , only : istsoil, istcrop, istdlak use elm_varctl , only : use_c13, use_fates use CH4varcon , only : allowlakeprod - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft use CNDecompCascadeConType , only : decomp_cascade_con use VegetationType , only : veg_pp use ColumnType , only : col_pp @@ -1675,7 +1675,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%cpool_livecroot_storage_gr_patch(p) + & this%cpool_deadcroot_storage_gr_patch(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%mr_patch(p) = & this%mr_patch(p) + & this%grain_mr_patch(p) @@ -1700,7 +1700,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%storage_gr_patch(p) ! autotrophic respiration (AR) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%ar_patch(p) = & this%mr_patch(p) + & this%gr_patch(p) + & @@ -1812,7 +1812,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%wood_harvestc_patch(p) = & this%hrv_deadstemc_to_prod10c_patch(p) + & this%hrv_deadstemc_to_prod100c_patch(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%wood_harvestc_patch(p) = & this%wood_harvestc_patch(p) + & this%hrv_cropc_to_prod1c_patch(p) @@ -1842,7 +1842,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%m_gresp_xfer_to_fire_patch(p) + & this%m_cpool_to_fire_patch(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%litfall_patch(p) = & this%litfall_patch(p) + & @@ -1878,7 +1878,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%hrv_leafc_to_litter_patch(p) + & this%leafc_to_litter_patch(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%leafc_loss_patch(p) = & this%leafc_loss_patch(p) + & this%hrv_leafc_to_prod1c_patch(p) @@ -1920,7 +1920,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%hrv_deadcrootc_storage_to_litter_patch(p) + & this%hrv_deadcrootc_xfer_to_litter_patch(p) ! putting the harvested crop stem and grain in the wood loss bdrewniak - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%woodc_loss_patch(p) = & this%woodc_loss_patch(p) + & this%hrv_grainc_to_prod1c_patch(p) + & @@ -2592,7 +2592,7 @@ subroutine summary_cflux_for_ch4( this, bounds, num_soilp, filter_soilp, num_soi this%cpool_to_deadstemc_patch(p) + & this%deadstemc_xfer_to_deadstemc_patch(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%agnpp_patch(p) = & this%agnpp_patch(p) + & this%cpool_to_grainc_patch(p) + & diff --git a/components/elm/src/biogeochem/CNCarbonStateType.F90 b/components/elm/src/biogeochem/CNCarbonStateType.F90 index 8f258bbb6a9a..46440bee2355 100644 --- a/components/elm/src/biogeochem/CNCarbonStateType.F90 +++ b/components/elm/src/biogeochem/CNCarbonStateType.F90 @@ -12,7 +12,7 @@ module CNCarbonStateType use elm_varctl , only : iulog, use_vertsoilc, spinup_state use decompMod , only : bounds_type use CNStateType , only : cnstate_type - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft use CNDecompCascadeConType , only : decomp_cascade_con use VegetationPropertiesType , only : veg_vp use abortutils , only : endrun @@ -400,7 +400,7 @@ subroutine InitCold(this, bounds, ratio, c12_carbonstate_vars) ! ! !USES: use landunit_varcon , only: istsoil - use pftvarcon , only: noveg, npcropmin + use pftvarcon , only: noveg ! ! !ARGUMENTS: class(carbonstate_type) :: this @@ -459,7 +459,7 @@ subroutine InitCold(this, bounds, ratio, c12_carbonstate_vars) if (veg_vp%evergreen(veg_pp%itype(p)) == 1._r8) then this%leafc_patch(p) = 1._r8 * ratio this%leafc_storage_patch(p) = 0._r8 - else if (veg_pp%itype(p) >= npcropmin) then ! prognostic crop types + else if (iscft(veg_pp%itype(p)) >= 1) then ! prognostic crop types this%leafc_patch(p) = 0._r8 this%leafc_storage_patch(p) = 0._r8 else @@ -866,7 +866,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%gresp_storage_patch(p) + & this%gresp_xfer_patch(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%storvegc_patch(p) = & this%storvegc_patch(p) + & this%grainc_storage_patch(p) + & diff --git a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 index e3965d6eea3a..370e82ab7ff5 100644 --- a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 @@ -91,7 +91,7 @@ subroutine CNGapMortality (& ! !USES: use elm_time_manager , only: get_days_per_year use elm_varcon , only: secspday - use pftvarcon , only: npcropmin + use pftvarcon , only: iscft use elm_varctl , only: spinup_state, spinup_mortality_factor ! ! !ARGUMENTS: @@ -188,7 +188,7 @@ subroutine CNGapMortality (& * gap_indicator(gid_m_deadcrootn_to_litter) - if (ivt(p) < npcropmin) then + if (iscft(veg_pp%itype(p)) < 1) then veg_nf%m_retransn_to_litter(p) = veg_ns%retransn(p) * m & * gap_indicator(gid_m_retransn_to_litter) end if @@ -239,7 +239,7 @@ subroutine CNGapMortality (& veg_pf%m_deadstemp_to_litter(p) = veg_ps%deadstemp(p) * m veg_pf%m_livecrootp_to_litter(p) = veg_ps%livecrootp(p) * m veg_pf%m_deadcrootp_to_litter(p) = veg_ps%deadcrootp(p) * m - if (ivt(p) < npcropmin) then + if (iscft(veg_pp%itype(p)) < 1) then veg_pf%m_retransp_to_litter(p) = veg_ps%retransp(p) * m end if diff --git a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 index 4c7fd83d2108..f2a431e7283c 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 @@ -10,7 +10,7 @@ module CNNStateUpdate1BeTRMod use elm_varpar , only : crop_prog, i_met_lit, i_cel_lit, i_lig_lit, i_cwd use elm_varctl , only : iulog use elm_varcon , only : nitrif_n2o_loss_frac - use pftvarcon , only : crop, generic_crop + use pftvarcon , only : iscft use VegetationPropertiesType , only : veg_vp use CNDecompCascadeConType , only : decomp_cascade_con use CNStateType , only : cnstate_type @@ -101,7 +101,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) - veg_nf%deadcrootn_xfer_to_deadcrootn(p)*dt end if - if (crop(ivt(p)) >= 1 .and. generic_crop(ivt(p)) < 1) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) - veg_nf%livestemn_xfer_to_livestemn(p)*dt @@ -126,7 +126,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%livecrootn(p) = veg_ns%livecrootn(p) - veg_nf%livecrootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%livecrootn_to_retransn(p)*dt end if - if (crop(ivt(p)) >= 1 .and. generic_crop(ivt(p)) < 1) then ! Beth adds retrans from froot + if (iscft(ivt(p)) >= 1) then ! Beth adds retrans from froot veg_ns%frootn(p) = veg_ns%frootn(p) - veg_nf%frootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%frootn_to_retransn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_litter(p)*dt @@ -172,7 +172,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_storage(p) = veg_ns%deadcrootn_storage(p) + veg_nf%npool_to_deadcrootn_storage(p)*dt end if - if (crop(ivt(p)) >= 1 .and. generic_crop(ivt(p)) < 1) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%npool_to_livestemn(p)*dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn_storage(p)*dt @@ -200,7 +200,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) + veg_nf%deadcrootn_storage_to_xfer(p)*dt end if - if (crop(ivt(p)) >= 1 .and. generic_crop(ivt(p)) < 1) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn_storage(p) = veg_ns%livestemn_storage(p) - veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) + veg_nf%livestemn_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/CNNStateUpdate2BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate2BeTRMod.F90 index 60100b657aef..fbfa6110d77f 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate2BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate2BeTRMod.F90 @@ -14,7 +14,7 @@ module CNNStateUpdate2BeTRMod use CNNitrogenFLuxType , only : nitrogenflux_type use VegetationType , only : veg_pp use VegetationDataType , only : veg_ns, veg_nf - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft !! bgc interface & pflotran: use elm_varctl , only : use_pflotran, pf_cmode ! @@ -147,7 +147,7 @@ subroutine NStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn(p) = veg_ns%deadcrootn(p) - veg_nf%hrv_deadcrootn_to_litter(p) * dt veg_ns%retransn(p) = veg_ns%retransn(p) - veg_nf%hrv_retransn_to_litter(p) * dt - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >=1) then ! skip 2 generic crops veg_ns%livestemn(p)= veg_ns%livestemn(p) - veg_nf%hrv_livestemn_to_prod1n(p) * dt veg_ns%leafn(p) = veg_ns%leafn(p) - veg_nf%hrv_leafn_to_prod1n(p) * dt veg_ns%grainn(p) = veg_ns%grainn(p) - veg_nf%hrv_grainn_to_prod1n(p) * dt diff --git a/components/elm/src/biogeochem/CNNitrogenFluxType.F90 b/components/elm/src/biogeochem/CNNitrogenFluxType.F90 index d0b76c195afa..b6f02ce91f82 100644 --- a/components/elm/src/biogeochem/CNNitrogenFluxType.F90 +++ b/components/elm/src/biogeochem/CNNitrogenFluxType.F90 @@ -1078,7 +1078,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil ! !USES: use elm_varpar , only: nlevdecomp,ndecomp_cascade_transitions,ndecomp_pools use subgridAveMod , only: p2c - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft use tracer_varcon , only: is_active_betr_bgc use elm_varpar , only: nlevdecomp_full ! diff --git a/components/elm/src/biogeochem/CNNitrogenStateType.F90 b/components/elm/src/biogeochem/CNNitrogenStateType.F90 index dda6477899a7..041c757ae598 100644 --- a/components/elm/src/biogeochem/CNNitrogenStateType.F90 +++ b/components/elm/src/biogeochem/CNNitrogenStateType.F90 @@ -12,7 +12,7 @@ module CNNitrogenStateType use elm_varctl , only : use_vertsoilc, use_century_decomp, use_fan use elm_varctl , only : iulog, override_bgc_restart_mismatch_dump, spinup_state use decompMod , only : bounds_type - use pftvarcon , only : npcropmin, nstor + use pftvarcon , only : iscft, nstor use CNDecompCascadeConType , only : decomp_cascade_con use VegetationPropertiesType , only : veg_vp use abortutils , only : endrun @@ -502,7 +502,7 @@ subroutine InitCold(this, bounds, & ! !USES: use elm_varpar , only : crop_prog use decompMod , only : bounds_type - use pftvarcon , only : noveg, npcropmin + use pftvarcon , only : noveg, iscft ! ! !ARGUMENTS: class(nitrogenstate_type) :: this @@ -1007,7 +1007,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%npool_patch(p) + & this%retransn_patch(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%dispvegn_patch(p) = & this%dispvegn_patch(p) + & this%grainn_patch(p) diff --git a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 index 23f461ad3055..aaeb58d9ee91 100644 --- a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 @@ -1868,7 +1868,8 @@ subroutine CropPhenologyInit(bounds) ! initialized, and after ecophyscon file is read in. ! ! !USES: - use pftvarcon , only: npcropmin, npcropmax, mnNHplantdate + use elm_varpar , only : mxpft + use pftvarcon , only: iscft, crop, mnNHplantdate use pftvarcon , only: mnSHplantdate, mxNHplantdate use pftvarcon , only: mxSHplantdate use elm_time_manager, only: get_calday @@ -1893,13 +1894,17 @@ subroutine CropPhenologyInit(bounds) ! Convert planting dates into julian day minplantjday(:,:) = huge(1) maxplantjday(:,:) = huge(1) - do n = npcropmin, npcropmax + do n = 0, mxpft + if (iscft(n)>=1) then minplantjday(n,inNH) = int( get_calday( mnNHplantdate(n), 0 ) ) maxplantjday(n,inNH) = int( get_calday( mxNHplantdate(n), 0 ) ) + end if end do - do n = npcropmin, npcropmax + do n = 0, mxpft + if (iscft(n)>=1) then minplantjday(n,inSH) = int( get_calday( mnSHplantdate(n), 0 ) ) maxplantjday(n,inSH) = int( get_calday( mxSHplantdate(n), 0 ) ) + end if end do ! Figure out what hemisphere each PFT is in diff --git a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 index 25455dbbb20e..7513662f4670 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 @@ -11,7 +11,7 @@ module CarbonStateUpdate1Mod use elm_varcon , only : dzsoi_decomp use elm_varctl , only : nu_com, use_c13, use_c14 use elm_varctl , only : use_pflotran, pf_cmode, use_fates - use pftvarcon , only : crop, generic_crop, percrop + use pftvarcon , only : iscft use CNDecompCascadeConType , only : decomp_cascade_type use CNStateType , only : cnstate_type use CNDecompCascadeConType , only : decomp_cascade_con @@ -284,8 +284,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%deadcrootc(p) = veg_cs%deadcrootc(p) + veg_cf%deadcrootc_xfer_to_deadcrootc(p)*dt veg_cs%deadcrootc_xfer(p) = veg_cs%deadcrootc_xfer(p) - veg_cf%deadcrootc_xfer_to_deadcrootc(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%livestemc_xfer_to_livestemc(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) - veg_cf%livestemc_xfer_to_livestemc(p)*dt @@ -304,8 +303,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%livecrootc(p) = veg_cs%livecrootc(p) - veg_cf%livecrootc_to_deadcrootc(p)*dt veg_cs%deadcrootc(p) = veg_cs%deadcrootc(p) + veg_cf%livecrootc_to_deadcrootc(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_cs%livestemc(p) = veg_cs%livestemc(p) - veg_cf%livestemc_to_litter(p)*dt veg_cs%grainc(p) = veg_cs%grainc(p) - veg_cf%grainc_to_food(p)*dt @@ -321,8 +319,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livestem_curmr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livecroot_curmr(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livestem_curmr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%grain_curmr(p)*dt end if @@ -340,8 +337,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livestem_xsmr(p)*dt veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livecroot_xsmr(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livestem_xsmr(p)*dt veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%grain_xsmr(p)*dt if (harvdate(p) < 999) then ! beginning at harvest, send to atm @@ -377,8 +373,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_deadcrootc_storage(p)*dt veg_cs%deadcrootc_storage(p) = veg_cs%deadcrootc_storage(p) + veg_cf%cpool_to_deadcrootc_storage(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if ( iscft(ivt(p)) >= 1 ) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_livestemc(p)*dt veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%cpool_to_livestemc(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_livestemc_storage(p)*dt @@ -398,8 +393,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livecroot_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_deadcroot_gr(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livestem_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_grain_gr(p)*dt end if @@ -413,8 +407,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_livecroot_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_deadcroot_gr(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_livestem_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_grain_gr(p)*dt end if @@ -428,8 +421,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livecroot_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_deadcroot_storage_gr(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if ( iscft(ivt(p)) >= 1 ) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livestem_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_grain_storage_gr(p)*dt end if @@ -455,8 +447,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%gresp_storage(p) = veg_cs%gresp_storage(p) - veg_cf%gresp_storage_to_xfer(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) + veg_cf%gresp_storage_to_xfer(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if ( iscft(ivt(p)) >= 1 ) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_cs%livestemc_storage(p) = veg_cs%livestemc_storage(p) - veg_cf%livestemc_storage_to_xfer(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) + veg_cf%livestemc_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/CarbonStateUpdate2Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate2Mod.F90 index 48174fb8ddfc..985c544d29ff 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate2Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate2Mod.F90 @@ -9,7 +9,7 @@ module CarbonStateUpdate2Mod use shr_log_mod , only : errMsg => shr_log_errMsg use abortutils , only : endrun use elm_varpar , only : nlevdecomp, i_met_lit, i_cel_lit, i_lig_lit, i_cwd - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft use elm_varctl , only : use_pflotran, pf_cmode use VegetationType , only : veg_pp use tracer_varcon , only : is_active_betr_bgc @@ -186,7 +186,7 @@ subroutine CarbonStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soilp, veg_cs%deadcrootc(p) = veg_cs%deadcrootc(p) - veg_cf%hrv_deadcrootc_to_litter(p) * dt ! crops - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_cs%livestemc(p) = veg_cs%livestemc(p) - veg_cf%hrv_livestemc_to_prod1c(p) *dt veg_cs%leafc(p) = veg_cs%leafc(p) - veg_cf%hrv_leafc_to_prod1c(p) *dt veg_cs%grainc(p) = veg_cs%grainc(p) - veg_cf%hrv_grainc_to_prod1c(p) *dt diff --git a/components/elm/src/biogeochem/CropType.F90 b/components/elm/src/biogeochem/CropType.F90 index bc3d8c44ff41..58f7757e5fcb 100644 --- a/components/elm/src/biogeochem/CropType.F90 +++ b/components/elm/src/biogeochem/CropType.F90 @@ -417,7 +417,7 @@ subroutine Restart(this, bounds, ncid, flag) use restUtilMod use ncdio_pio use VegetationType , only : veg_pp - use pftvarcon , only : npcropmin, npcropmax + use pftvarcon , only : iscft ! ! !ARGUMENTS: class(crop_type), intent(inout) :: this @@ -454,7 +454,7 @@ subroutine Restart(this, bounds, ncid, flag) interpinic_flag='copy', readvar=readvar, data=restyear) if (readvar) then do p = bounds%begp, bounds%endp - if (veg_pp%itype(p) >= npcropmin .and. veg_pp%itype(p) <= npcropmax .and. & + if (iscft(veg_pp%itype(p)) >= 1 .and. & veg_pp%active(p)) then this%nyrs_crop_active_patch(p) = restyear end if diff --git a/components/elm/src/biogeochem/DryDepVelocity.F90 b/components/elm/src/biogeochem/DryDepVelocity.F90 index 293713da54a7..a79792b4e858 100644 --- a/components/elm/src/biogeochem/DryDepVelocity.F90 +++ b/components/elm/src/biogeochem/DryDepVelocity.F90 @@ -146,7 +146,7 @@ subroutine depvel_compute( bounds, & use landunit_varcon, only : istsoil, istice, istice_mec, istdlak, istwet use elm_varctl , only : iulog use pftvarcon , only : noveg, nonvascular - use pftvarcon , only : woody, graminoid, generic_crop, crop, percrop + use pftvarcon , only : woody, graminoid, iscft, crop use pftvarcon , only : needleleaf ! ! !ARGUMENTS: @@ -293,10 +293,8 @@ subroutine depvel_compute( bounds, & if (nonvascular(elmveg) == 1.0_r8 ) wesveg = 3 ! assuming moss like grass !if (elmveg == nc3crop ) wesveg = 2 !if (elmveg == nc3irrig ) wesveg = 2 - if (generic_crop(elmveg) == 1.0_r8 ) wesveg = 2 - if (crop(elmveg) == 1.0_r8 ) wesveg = 2 - if (percrop(elmveg) == 1.0_r8 ) wesveg = 2 !if (elmveg >= npcropmin .and. elmveg <= npcropmax ) wesveg = 2 + if (crop(elmveg) == 1.0_r8 .or. iscft(elmveg) == 1.0_r8 ) wesveg = 2 #ifndef _OPENACC if (wesveg == wveg_unset )then write(iulog,*) 'elmveg = ', elmveg, 'lun_pp%itype = ', lun_pp%itype(l) diff --git a/components/elm/src/biogeochem/FireMod.F90 b/components/elm/src/biogeochem/FireMod.F90 index 24e34e37c555..7aa4e6a50f9e 100644 --- a/components/elm/src/biogeochem/FireMod.F90 +++ b/components/elm/src/biogeochem/FireMod.F90 @@ -126,7 +126,7 @@ subroutine FireArea (bounds, & use elm_varcon , only: secspday, spval use elm_varctl , only: use_nofire, spinup_state, spinup_mortality_factor use dynSubgridControlMod , only: run_has_transient_landcover - use pftvarcon , only: noveg, woody, graminoid, generic_crop, crop, percrop + use pftvarcon , only: noveg, woody, graminoid, iscft, crop use pftvarcon , only: climatezone, needleleaf, evergreen ! ! !ARGUMENTS: @@ -305,16 +305,12 @@ subroutine FireArea (bounds, & if (pi <= col_pp%npfts(c)) then p = col_pp%pfti(c) + pi - 1 ! For crop veg types - if( generic_crop(veg_pp%itype(p)) == 1 .or. & - crop(veg_pp%itype(p)) == 1 .or. & - percrop(veg_pp%itype(p)) == 1 )then + if( crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p)) == 1 )then cropf_col(c) = cropf_col(c) + veg_pp%wtcol(p) end if ! For natural vegetation (non-crop and non-bare-soil) if( veg_pp%itype(p) /= noveg .and. & - (generic_crop(veg_pp%itype(p)) == 0 .and. & - crop(veg_pp%itype(p)) == 0 .and. & - percrop(veg_pp%itype(p)) == 0) )then + (crop(veg_pp%itype(p)) == 0 .and. iscft(veg_pp%itype(p)) == 0) )then lfwt(c) = lfwt(c) + veg_pp%wtcol(p) end if end if @@ -336,9 +332,7 @@ subroutine FireArea (bounds, & ! column-level litter carbon ! is available, so we use leaf carbon to estimate the ! litter carbon for crop PFTs - if( (generic_crop(veg_pp%itype(p)) == 1 .or. & - crop(veg_pp%itype(p)) == 1 .or. & - percrop(veg_pp%itype(p)) == 1) .and. & + if( (crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p)) == 1) .and. & veg_pp%wtcol(p) > 0._r8 .and. leafc_col(c) > 0._r8 )then fuelc_crop(c)=fuelc_crop(c) + (leafc(p) + leafc_storage(p) + & leafc_xfer(p))*veg_pp%wtcol(p)/cropf_col(c) + & @@ -376,9 +370,7 @@ subroutine FireArea (bounds, & p = col_pp%pfti(c) + pi - 1 ! For non-crop -- natural vegetation and bare-soil - if( (generic_crop(veg_pp%itype(p)) == 0 .and. & - crop(veg_pp%itype(p)) == 0 .and. & - percrop(veg_pp%itype(p)) == 0) .and. & + if( (crop(veg_pp%itype(p)) == 0 .and. iscft(veg_pp%itype(p)) == 0) .and. & cropf_col(c) < 1.0_r8 ) then if( btran2(p) .ne. spval) then if (btran2(p) <= 1._r8 ) then @@ -517,9 +509,7 @@ subroutine FireArea (bounds, & p = col_pp%pfti(c) + pi - 1 ! For crop if( forc_t(t) >= SHR_CONST_TKFRZ .and. & - (generic_crop(veg_pp%itype(p))>=1 .or. & - crop(veg_pp%itype(p))>=1 .or. & - percrop(veg_pp%itype(p))>=1) .and. & + (crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p)) == 1) .and. & kmo == abm_lf(c) .and. forc_rain(t)+forc_snow(t) == 0._r8 .and. & burndate(p) >= 999 .and. veg_pp%wtcol(p) > 0._r8 )then ! catch crop burn time @@ -690,7 +680,7 @@ subroutine FireFluxes (num_soilc, filter_soilc, num_soilp, filter_soilp, & !$acc routine seq use pftvarcon , only: cc_leaf,cc_lstem,cc_dstem,cc_other,fm_leaf,fm_lstem,fm_other,fm_root,fm_lroot,fm_droot use pftvarcon , only: lf_flab,lf_fcel,lf_flig,fr_flab,fr_fcel,fr_flig - use pftvarcon , only: generic_crop, crop, percrop + use pftvarcon , only: iscft, crop use elm_varpar , only: max_patch_per_col use elm_varctl , only: spinup_state, spinup_mortality_factor use dynSubgridControlMod , only: get_flanduse_timeseries @@ -992,8 +982,7 @@ subroutine FireFluxes (num_soilc, filter_soilc, num_soilp, filter_soilp, & c = veg_pp%column(p) itype = veg_pp%itype(p) - if( (generic_crop(veg_pp%itype(p)) == 0 .and. crop(veg_pp%itype(p)) == 0 .and. & - percrop(veg_pp%itype(p)) == 0 ) .and. & + if( (crop(veg_pp%itype(p)) == 0 .and. iscft(veg_pp%itype(p)) == 0) .and. & cropf_col(c) < 1.0_r8)then ! For non-crop (bare-soil and natural vegetation) if (transient_landcover) then !true when landuse data is used diff --git a/components/elm/src/biogeochem/GapMortalityMod.F90 b/components/elm/src/biogeochem/GapMortalityMod.F90 index 3257a6365b65..734be187805f 100644 --- a/components/elm/src/biogeochem/GapMortalityMod.F90 +++ b/components/elm/src/biogeochem/GapMortalityMod.F90 @@ -87,7 +87,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & ! !USES: !$acc routine seq use elm_varcon , only: secspday - use pftvarcon , only: npcropmin + use pftvarcon , only: iscft use elm_varctl , only: spinup_state, spinup_mortality_factor ! ! !ARGUMENTS: @@ -142,7 +142,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & ! displayed pools veg_cf%m_leafc_to_litter(p) = 0._r8 veg_cf%m_livestemc_to_litter(p) = 0._r8 - if(ivt(p) < npcropmin .or. (ivt(p) >= npcropmin .and. croplive(p))) then + if(iscft(ivt(p)) < 1 .or. (iscft(ivt(p)) >= 1 .and. croplive(p))) then veg_cf%m_leafc_to_litter(p) = veg_cs%leafc(p) * m veg_cf%m_livestemc_to_litter(p) = veg_cs%livestemc(p) * m end if @@ -181,7 +181,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & ! displayed pools veg_nf%m_leafn_to_litter(p) = 0._r8 veg_nf%m_livestemn_to_litter(p) = 0._r8 - if(ivt(p) < npcropmin .or. (ivt(p) >= npcropmin .and. croplive(p))) then + if(iscft(ivt(p)) < 1 .or. (iscft(ivt(p)) >= 1 .and. croplive(p))) then veg_nf%m_leafn_to_litter(p) = veg_ns%leafn(p) * m veg_nf%m_livestemn_to_litter(p) = veg_ns%livestemn(p) * m end if @@ -190,7 +190,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_nf%m_livecrootn_to_litter(p) = veg_ns%livecrootn(p) * m veg_nf%m_deadcrootn_to_litter(p) = veg_ns%deadcrootn(p) * m veg_nf%m_retransn_to_litter(p) = 0._r8 - if (ivt(p) < npcropmin) then + if (iscft(ivt(p)) < 1) then veg_nf%m_retransn_to_litter(p) = veg_ns%retransn(p) * m end if veg_nf%m_npool_to_litter(p) = veg_ns%npool(p) * m @@ -225,7 +225,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & ! displayed pools veg_pf%m_leafp_to_litter(p) = 0._r8 veg_pf%m_livestemp_to_litter(p) = 0._r8 - if(ivt(p) < npcropmin .or. (ivt(p) >= npcropmin .and. croplive(p))) then + if(iscft(ivt(p)) < 1 .or. (iscft(ivt(p)) >= 1 .and. croplive(p))) then veg_pf%m_leafp_to_litter(p) = veg_ps%leafp(p) * m veg_pf%m_livestemp_to_litter(p) = veg_ps%livestemp(p) * m endif @@ -235,7 +235,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_pf%m_deadcrootp_to_litter(p) = veg_ps%deadcrootp(p) * m veg_pf%m_retransp_to_litter(p) = 0._r8 - if (ivt(p) < npcropmin) then + if (iscft(ivt(p)) < 1) then veg_pf%m_retransp_to_litter(p) = veg_ps%retransp(p) * m end if veg_pf%m_ppool_to_litter(p) = veg_ps%ppool(p) * m diff --git a/components/elm/src/biogeochem/GrowthRespMod.F90 b/components/elm/src/biogeochem/GrowthRespMod.F90 index a5002d108e51..34a9a64c56c9 100644 --- a/components/elm/src/biogeochem/GrowthRespMod.F90 +++ b/components/elm/src/biogeochem/GrowthRespMod.F90 @@ -7,7 +7,7 @@ module GrowthRespMod ! ! !USES: use shr_kind_mod , only : r8 => shr_kind_r8 - use pftvarcon , only : grperc, grpnow, npcropmin + use pftvarcon , only : grperc, grpnow, iscft use VegetationPropertiesType , only : veg_vp use VegetationType , only : veg_pp use VegetationDataType , only : veg_cf @@ -102,7 +102,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) do fp = 1,num_soilp p = filter_soilp(fp) - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops cpool_livestem_gr(p) = cpool_to_livestemc(p) * grperc(ivt(p)) diff --git a/components/elm/src/biogeochem/MaintenanceRespMod.F90 b/components/elm/src/biogeochem/MaintenanceRespMod.F90 index eab60926ab60..48d179f97d1f 100644 --- a/components/elm/src/biogeochem/MaintenanceRespMod.F90 +++ b/components/elm/src/biogeochem/MaintenanceRespMod.F90 @@ -12,7 +12,7 @@ module MaintenanceRespMod use decompMod , only : bounds_type use abortutils , only : endrun use shr_log_mod , only : errMsg => shr_log_errMsg - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft use SharedParamsMod , only : ParamsShareInst use VegetationPropertiesType , only : veg_vp use SoilStateType , only : soilstate_type @@ -185,7 +185,7 @@ subroutine MaintenanceResp(bounds, & if (woody(ivt(p)) >= 1.0_r8) then livestem_mr(p) = livestemn(p)*br_mr*tc livecroot_mr(p) = livecrootn(p)*br_mr*tc - else if (ivt(p) >= npcropmin .and. livestemn(p) .gt. 0._r8) then + else if (iscft(ivt(p)) >= 1 .and. livestemn(p) .gt. 0._r8) then livestem_mr(p) = livestemn(p)*br_mr*tc grain_mr(p) = grainn(p)*br_mr*tc end if diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 index d72dc4dc598e..80fc34e6d0ba 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 @@ -9,7 +9,7 @@ module NitrogenStateUpdate1Mod use elm_varpar , only : crop_prog, i_met_lit, i_cel_lit, i_lig_lit, i_cwd use elm_varctl , only : iulog use elm_varcon , only : nitrif_n2o_loss_frac - use pftvarcon , only : crop, generic_crop, percrop + use pftvarcon , only : iscft use VegetationPropertiesType , only : veg_vp use CNDecompCascadeConType , only : decomp_cascade_con use CNStateType , only : cnstate_type @@ -297,8 +297,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) - veg_nf%deadcrootn_xfer_to_deadcrootn(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) - veg_nf%livestemn_xfer_to_livestemn(p)*dt @@ -323,8 +322,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%livecrootn(p) = veg_ns%livecrootn(p) - veg_nf%livecrootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%livecrootn_to_retransn(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! Beth adds retrans from froot + if (iscft(ivt(p)) >= 1) then ! Beth adds retrans from froot veg_ns%frootn(p) = veg_ns%frootn(p) - veg_nf%frootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%frootn_to_retransn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_litter(p)*dt @@ -375,8 +373,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_storage(p) = veg_ns%deadcrootn_storage(p) + veg_nf%npool_to_deadcrootn_storage(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%npool_to_livestemn(p)*dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn_storage(p)*dt @@ -404,8 +401,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) + veg_nf%deadcrootn_storage_to_xfer(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn_storage(p) = veg_ns%livestemn_storage(p) - veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) + veg_nf%livestemn_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate2Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate2Mod.F90 index 9121954e6363..5d897c5905a9 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate2Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate2Mod.F90 @@ -14,7 +14,7 @@ module NitrogenStateUpdate2Mod use ColumnDataType , only : col_ns, col_nf use VegetationType , only : veg_pp use VegetationDataType , only : veg_ns, veg_nf - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft ! bgc interface & pflotran: use elm_varctl , only : use_pflotran, pf_cmode ! @@ -172,7 +172,7 @@ subroutine NitrogenStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soil veg_ns%retransn(p) = veg_ns%retransn(p) - veg_nf%hrv_retransn_to_litter(p) * dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%hrv_npool_to_litter(p) * dt - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_ns%livestemn(p)= veg_ns%livestemn(p) - veg_nf%hrv_livestemn_to_prod1n(p) * dt veg_ns%leafn(p) = veg_ns%leafn(p) - veg_nf%hrv_leafn_to_prod1n(p) * dt veg_ns%grainn(p) = veg_ns%grainn(p) - veg_nf%hrv_grainn_to_prod1n(p) * dt diff --git a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 index 786efbe6323a..49909f54929a 100644 --- a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 +++ b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 @@ -10,7 +10,7 @@ module PhenologyFLuxLimitMod use VegetationType , only : veg_pp use VegetationPropertiesType , only : veg_vp use elm_time_manager , only : get_step_size - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft use elm_varctl , only : iulog use abortutils , only : endrun implicit none @@ -644,7 +644,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_deadcrootc_xfer) = veg_cs%deadcrootc_xfer(p) ystates(s_deadcrootc_storage) = veg_cs%deadcrootc_storage(p) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then ystates(s_livestemc) = veg_cs%livestemc(p) ystates(s_livestemc_xfer) = veg_cs%livestemc_xfer(p) ystates(s_livestemc_storage) = veg_cs%livestemc_storage(p) @@ -675,7 +675,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& + veg_cf%cpool_livecroot_storage_gr(p) & + veg_cf%cpool_deadcroot_storage_gr(p) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then ar_p= ar_p & + veg_cf%livestem_curmr(p) & + veg_cf%grain_curmr(p) & @@ -713,7 +713,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_deadcrootc_storage_to_xfer) = veg_cf%deadcrootc_storage_to_xfer(p) rfluxes(f_gresp_storage_to_xfer) = veg_cf%gresp_storage_to_xfer(p) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then rfluxes(f_cpool_to_livestemc) = veg_cf%cpool_to_livestemc(p) rfluxes(f_cpool_to_livestemc_storage) = veg_cf%cpool_to_livestemc_storage(p) rfluxes(f_cpool_to_grainc) = veg_cf%cpool_to_grainc(p) @@ -767,7 +767,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_deadcrootc_storage_to_xfer) , veg_cf%deadcrootc_storage_to_xfer(p)) call fpmax(rfluxes(f_gresp_storage_to_xfer) , veg_cf%gresp_storage_to_xfer(p)) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then call fpmax(rfluxes(f_cpool_to_livestemc) , veg_cf%cpool_to_livestemc(p)) call fpmax(rfluxes(f_cpool_to_livestemc_storage) , veg_cf%cpool_to_livestemc_storage(p)) call fpmax(rfluxes(f_cpool_to_grainc) , veg_cf%cpool_to_grainc(p)) @@ -806,7 +806,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& call ascal(veg_cf%cpool_livecroot_storage_gr(p), rscal) call ascal(veg_cf%cpool_deadcroot_storage_gr(p), rscal) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then call ascal(veg_cf%livestem_curmr(p) , rscal) call ascal(veg_cf%grain_curmr(p) , rscal) call ascal(veg_cf%cpool_livestem_gr(p) , rscal) @@ -878,7 +878,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_deadcrootn_xfer) = veg_ns%deadcrootn_xfer(p) ystates(s_deadcrootn_storage) = veg_ns%deadcrootn_storage(p) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then ystates(s_grainn) = veg_ns%grainn(p) ystates(s_grainn_xfer) = veg_ns%grainn_xfer(p) ystates(s_grainn_storage) = veg_ns%grainn_storage(p) @@ -917,7 +917,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_deadcrootn_xfer_to_deadcrootn) = veg_nf%deadcrootn_xfer_to_deadcrootn(p) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then rfluxes(f_npool_to_livestemn) = veg_nf%npool_to_livestemn(p) rfluxes(f_npool_to_livestemn_storage) = veg_nf%npool_to_livestemn_storage(p) rfluxes(f_npool_to_grainn) = veg_nf%npool_to_grainn(p) @@ -973,7 +973,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_deadcrootn_xfer_to_deadcrootn) , veg_nf%deadcrootn_xfer_to_deadcrootn(p)) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then call fpmax(rfluxes(f_npool_to_livestemn) , veg_nf%npool_to_livestemn(p)) call fpmax(rfluxes(f_npool_to_livestemn_storage) , veg_nf%npool_to_livestemn_storage(p)) call fpmax(rfluxes(f_npool_to_grainn) , veg_nf%npool_to_grainn(p)) @@ -1063,7 +1063,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_deadcrootn_xfer) = veg_ps%deadcrootp_xfer(p) ystates(s_deadcrootn_storage) = veg_ps%deadcrootp_storage(p) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then ystates(s_grainn) = veg_ps%grainp(p) ystates(s_grainn_xfer) = veg_ps%grainp_xfer(p) ystates(s_grainn_storage) = veg_ps%grainp_storage(p) @@ -1102,7 +1102,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_deadcrootn_xfer_to_deadcrootn) = veg_pf%deadcrootp_xfer_to_deadcrootp(p) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then rfluxes(f_npool_to_livestemn) = veg_pf%ppool_to_livestemp(p) rfluxes(f_npool_to_livestemn_storage) = veg_pf%ppool_to_livestemp_storage(p) rfluxes(f_npool_to_grainn) = veg_pf%ppool_to_grainp(p) @@ -1158,7 +1158,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_deadcrootn_xfer_to_deadcrootn) , veg_pf%deadcrootp_xfer_to_deadcrootp(p)) endif - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then call fpmax(rfluxes(f_npool_to_livestemn) , veg_pf%ppool_to_livestemp(p)) call fpmax(rfluxes(f_npool_to_livestemn_storage) , veg_pf%ppool_to_livestemp_storage(p)) call fpmax(rfluxes(f_npool_to_grainn) , veg_pf%ppool_to_grainp(p)) diff --git a/components/elm/src/biogeochem/PhenologyMod.F90 b/components/elm/src/biogeochem/PhenologyMod.F90 index d5ddfc2f44ed..2a52cb928837 100644 --- a/components/elm/src/biogeochem/PhenologyMod.F90 +++ b/components/elm/src/biogeochem/PhenologyMod.F90 @@ -2177,7 +2177,8 @@ subroutine CropPhenologyInit(bounds) ! initialized, and after ecophyscon file is read in. ! ! !USES: - use pftvarcon , only: npcropmin, npcropmax, nppercropmin, nppercropmax, mnNHplantdate + use elm_varpar , only : mxpft + use pftvarcon , only: iscft, mnNHplantdate use pftvarcon , only: mnSHplantdate, mxNHplantdate use pftvarcon , only: mxSHplantdate use elm_time_manager, only: get_calday @@ -2203,23 +2204,28 @@ subroutine CropPhenologyInit(bounds) ! Convert planting dates into julian day minplantjday(:,:) = huge(1) maxplantjday(:,:) = huge(1) - do n = npcropmin, npcropmax + do n = 0, mxpft + if (iscft(n)>=1) then minplantjday(n,inNH) = int( get_calday( mnNHplantdate(n), 0 ) ) maxplantjday(n,inNH) = int( get_calday( mxNHplantdate(n), 0 ) ) + end if end do - do n = npcropmin, npcropmax + do n = 0, mxpft + if (iscft(n)>=1) then minplantjday(n,inSH) = int( get_calday( mnSHplantdate(n), 0 ) ) maxplantjday(n,inSH) = int( get_calday( mxSHplantdate(n), 0 ) ) + end if end do - do n = nppercropmin, nppercropmax - minplantjday(n,inNH) = int( get_calday( mnNHplantdate(n), 0 ) ) - maxplantjday(n,inNH) = int( get_calday( mxNHplantdate(n), 0 ) ) - end do - do n = nppercropmin, nppercropmax - minplantjday(n,inSH) = int( get_calday( mnSHplantdate(n), 0 ) ) - maxplantjday(n,inSH) = int( get_calday( mxSHplantdate(n), 0 ) ) - end do + ! with flag 'iscft', the following is not needed anymore + !do n = nppercropmin, nppercropmax + ! minplantjday(n,inNH) = int( get_calday( mnNHplantdate(n), 0 ) ) + ! maxplantjday(n,inNH) = int( get_calday( mxNHplantdate(n), 0 ) ) + !end do + !do n = nppercropmin, nppercropmax + ! minplantjday(n,inSH) = int( get_calday( mnSHplantdate(n), 0 ) ) + ! maxplantjday(n,inSH) = int( get_calday( mxSHplantdate(n), 0 ) ) + !end do ! Figure out what hemisphere each PFT is in do p = bounds%begp, bounds%endp @@ -2965,7 +2971,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & ! ! !USES: !$acc routine seq - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft ! ! !ARGUMENTS: integer , intent(in) :: num_soilp ! number of soil patches in filter @@ -3062,7 +3068,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & if (offset_counter(p) == dt) then t1 = 1.0_r8 / dt - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then ! this assumes that offset_counter == dt for crops ! if this were ever changed, we'd need to add code to the "else" leafc_to_litter(p) = (1.0_r8 - presharv(ivt(p))) * ((t1 * leafc(p)) + cpool_to_leafc(p)) @@ -3079,7 +3085,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & end if if ( nu_com .eq. 'RD') then - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then if (offset_counter(p) == dt) then t1 = 1.0_r8 / dt @@ -3113,7 +3119,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & else if (offset_counter(p) == dt) then t1 = 1.0_r8 / dt - if (ivt(p) >= npcropmin) then + if (iscft(ivt(p)) >= 1) then ! this assumes that offset_counter == dt for crops ! if this were ever changed, we'd need to add code to the "else" leafn_to_litter(p) = (1.0_r8 - presharv(ivt(p))) * ((t1 * leafn(p)) + npool_to_leafn(p)) @@ -3387,7 +3393,7 @@ subroutine CNLitterToColumn (num_soilp, filter_soilp, & ! !USES: !$acc routine seq use elm_varpar , only : max_patch_per_col, nlevdecomp - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft ! ! !ARGUMENTS: integer , intent(in) :: num_soilp ! number of soil columns in filter @@ -3497,7 +3503,7 @@ subroutine CNLitterToColumn (num_soilp, filter_soilp, & ! new ones for now (slevis) ! The food is now directed to the product pools (BDrewniak) - if (ivt(p) >= npcropmin) then ! add livestemc to litter + if (iscft(ivt(p)) >= 1) then ! add livestemc to litter ! stem litter carbon fluxes phenology_c_to_litr_met_c(c,j) = phenology_c_to_litr_met_c(c,j) & + livestemc_to_litter(p) * lf_flab(ivt(p)) * wt_col * leaf_prof(p,j) diff --git a/components/elm/src/biogeochem/PhosphorusFluxType.F90 b/components/elm/src/biogeochem/PhosphorusFluxType.F90 index f34d7556f696..52bda3118fd2 100644 --- a/components/elm/src/biogeochem/PhosphorusFluxType.F90 +++ b/components/elm/src/biogeochem/PhosphorusFluxType.F90 @@ -561,7 +561,6 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil ! !USES: use elm_varpar , only: nlevdecomp,ndecomp_cascade_transitions,ndecomp_pools use subgridAveMod , only: p2c - use pftvarcon , only: npcropmin ! pflotran ! use elm_varctl , only: use_pflotran, pf_cmode ! diff --git a/components/elm/src/biogeochem/PhosphorusStateType.F90 b/components/elm/src/biogeochem/PhosphorusStateType.F90 index 0be7dd7f533f..58dbe630c2ef 100644 --- a/components/elm/src/biogeochem/PhosphorusStateType.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateType.F90 @@ -12,7 +12,7 @@ module PhosphorusStateType use elm_varctl , only : use_vertsoilc, use_century_decomp use elm_varctl , only : iulog, override_bgc_restart_mismatch_dump, spinup_state use decompMod , only : bounds_type - use pftvarcon , only : npcropmin, nstor + use pftvarcon , only : iscft, nstor use CNDecompCascadeConType , only : decomp_cascade_con use VegetationPropertiesType , only : veg_vp use abortutils , only : endrun @@ -254,7 +254,7 @@ subroutine InitCold(this, bounds, & ! !USES: use elm_varpar , only : crop_prog use decompMod , only : bounds_type - use pftvarcon , only : noveg, npcropmin + use pftvarcon , only : noveg, iscft ! ! !ARGUMENTS: class(phosphorusstate_type) :: this diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 index e29074eb332f..01369ed83fa7 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 @@ -8,7 +8,7 @@ module PhosphorusStateUpdate1Mod use elm_varpar , only : nlevdecomp, ndecomp_pools, ndecomp_cascade_transitions use elm_varpar , only : crop_prog, i_met_lit, i_cel_lit, i_lig_lit, i_cwd use elm_varctl , only : iulog - use pftvarcon , only : crop, generic_crop, percrop + use pftvarcon , only : iscft use soilorder_varcon , only : smax,ks_sorption use VegetationPropertiesType , only : veg_vp use CNDecompCascadeConType , only : decomp_cascade_con @@ -255,8 +255,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_xfer(p) = veg_ps%deadcrootp_xfer(p) - veg_pf%deadcrootp_xfer_to_deadcrootp(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ps%livestemp(p) = veg_ps%livestemp(p) + veg_pf%livestemp_xfer_to_livestemp(p)*dt veg_ps%livestemp_xfer(p) = veg_ps%livestemp_xfer(p) - veg_pf%livestemp_xfer_to_livestemp(p)*dt @@ -281,8 +280,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%livecrootp(p) = veg_ps%livecrootp(p) - veg_pf%livecrootp_to_retransp(p)*dt veg_ps%retransp(p) = veg_ps%retransp(p) + veg_pf%livecrootp_to_retransp(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! Beth adds retrans from froot + if (iscft(ivt(p)) >= 1) then ! Beth adds retrans from froot veg_ps%frootp(p) = veg_ps%frootp(p) - veg_pf%frootp_to_retransp(p)*dt veg_ps%retransp(p) = veg_ps%retransp(p) + veg_pf%frootp_to_retransp(p)*dt veg_ps%livestemp(p) = veg_ps%livestemp(p) - veg_pf%livestemp_to_litter(p)*dt @@ -333,8 +331,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_storage(p) = veg_ps%deadcrootp_storage(p) + veg_pf%ppool_to_deadcrootp_storage(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_livestemp(p)*dt veg_ps%livestemp(p) = veg_ps%livestemp(p) + veg_pf%ppool_to_livestemp(p)*dt veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_livestemp_storage(p)*dt @@ -362,8 +359,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_xfer(p) = veg_ps%deadcrootp_xfer(p) + veg_pf%deadcrootp_storage_to_xfer(p)*dt end if - if ( (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) .and. & - generic_crop(ivt(p)) == 0 ) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ps%livestemp_storage(p) = veg_ps%livestemp_storage(p) - veg_pf%livestemp_storage_to_xfer(p)*dt veg_ps%livestemp_xfer(p) = veg_ps%livestemp_xfer(p) + veg_pf%livestemp_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate2Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate2Mod.F90 index 2385083451cc..4aa7563c70f7 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate2Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate2Mod.F90 @@ -12,7 +12,7 @@ module PhosphorusStateUpdate2Mod !use PhosphorusStateType , only : phosphorusstate_type !use PhosphorusFLuxType , only : phosphorusflux_type use VegetationType , only : veg_pp - use pftvarcon , only : npcropmin + use pftvarcon , only : iscft use tracer_varcon , only : is_active_betr_bgc ! bgc interface & pflotran: use elm_varctl , only : use_pflotran, pf_cmode @@ -182,7 +182,7 @@ subroutine PhosphorusStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_so veg_ps%retransp(p) = veg_ps%retransp(p) - veg_pf%hrv_retransp_to_litter(p) * dt veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%hrv_ppool_to_litter(p) * dt - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops veg_ps%livestemp(p)= veg_ps%livestemp(p) - veg_pf%hrv_livestemp_to_prod1p(p) * dt veg_ps%leafp(p) = veg_ps%leafp(p) - veg_pf%hrv_leafp_to_prod1p(p) * dt veg_ps%grainp(p) = veg_ps%grainp(p) - veg_pf%hrv_grainp_to_prod1p(p) * dt diff --git a/components/elm/src/biogeochem/PrecisionControlMod.F90 b/components/elm/src/biogeochem/PrecisionControlMod.F90 index 8767d300b244..2423592c05c2 100644 --- a/components/elm/src/biogeochem/PrecisionControlMod.F90 +++ b/components/elm/src/biogeochem/PrecisionControlMod.F90 @@ -39,7 +39,7 @@ subroutine PrecisionControl(num_soilc, filter_soilc, num_soilp, filter_soilp) !$acc routine seq use elm_varctl , only : iulog, use_c13, use_c14, use_fates use elm_varpar , only : nlevdecomp_full, crop_prog - use pftvarcon , only : crop, generic_crop, percrop + use pftvarcon , only : iscft use tracer_varcon , only : is_active_betr_bgc use CNDecompCascadeConType , only : decomp_cascade_con ! @@ -211,9 +211,7 @@ subroutine PrecisionControl(num_soilc, filter_soilc, num_soilp, filter_soilp) veg_ps%frootp_xfer(p) = 0._r8 end if - if ( crop_prog .and. & - (crop(veg_pp%itype(p)) >= 1 .or. & - percrop(veg_pp%itype(p)) >= 1) )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then ! grain C and N if (abs(veg_cs%grainc(p)) < ccrit) then pc = pc + veg_cs%grainc(p) @@ -515,9 +513,7 @@ subroutine PrecisionControl(num_soilc, filter_soilc, num_soilp, filter_soilp) endif end if - if ( crop_prog .and. & - (crop(veg_pp%itype(p)) >= 1 .or. & - percrop(veg_pp%itype(p)) >= 1) )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then ! xsmrpool (C only) if (abs(veg_cs%xsmrpool(p)) < ccrit) then pc = pc + veg_cs%xsmrpool(p) diff --git a/components/elm/src/biogeochem/RootDynamicsMod.F90 b/components/elm/src/biogeochem/RootDynamicsMod.F90 index 3c78043aa6ac..f5ffc4c67844 100644 --- a/components/elm/src/biogeochem/RootDynamicsMod.F90 +++ b/components/elm/src/biogeochem/RootDynamicsMod.F90 @@ -10,7 +10,7 @@ module RootDynamicsMod use elm_varpar , only : nlevsoi, nlevgrnd use elm_varctl , only : use_vertsoilc use decompMod , only : bounds_type - use pftvarcon , only : noveg, npcropmin, roota_par, rootb_par, root_dmx, evergreen + use pftvarcon , only : noveg, iscft, roota_par, rootb_par, root_dmx, evergreen use CanopyStateType , only: canopystate_type use CNStateType , only : cnstate_type use CNCarbonStateType , only : carbonstate_type @@ -136,7 +136,7 @@ subroutine RootDynamics(bounds, num_soilc, filter_soilc, num_soilp, filter_soilp p = filter_soilp(f) c = pcolumn(p) if (ivt(p) /= noveg) then - if ((ivt(p)) >= npcropmin) then !skip generic crop types + if (iscft(ivt(p)) >= 1) then !skip generic crop types if (huigrain(p) > 0._r8) then root_depth(p) = max(zi(c,2), min(hui(p)/huigrain(p)* root_dmx(ivt(p)), root_dmx(ivt(p)))) end if diff --git a/components/elm/src/biogeochem/VOCEmissionMod.F90 b/components/elm/src/biogeochem/VOCEmissionMod.F90 index 64a72fe78b8e..fb7bd66958db 100644 --- a/components/elm/src/biogeochem/VOCEmissionMod.F90 +++ b/components/elm/src/biogeochem/VOCEmissionMod.F90 @@ -10,7 +10,7 @@ module VOCEmissionMod use elm_varctl , only : iulog use elm_varpar , only : numpft, nlevcan use elm_varpar , only : numveg ! fixed as 16, while numpft above may be variable - use pftvarcon , only : woody, graminoid, generic_crop, crop, percrop + use pftvarcon , only : woody, graminoid, iscft, crop use pftvarcon , only : needleleaf, evergreen use shr_megan_mod , only : shr_megan_megcomps_n, shr_megan_megcomp_t, shr_megan_linkedlist use shr_megan_mod , only : shr_megan_mechcomps_n, shr_megan_mechcomps, shr_megan_mapped_emisfctrs @@ -703,8 +703,7 @@ function get_map_EF(ivt_in, g_in, ti_in, vocemis_vars) get_map_EF = vocemis_vars%efisop_grc(4,g_in, ti_in) else if (graminoid(ivt_in) == 1) then !grass get_map_EF = vocemis_vars%efisop_grc(5,g_in, ti_in) - else if (generic_crop(ivt_in) == 1 .or. crop(ivt_in) == 1 & - .or. percrop(ivt_in) == 1) then !crops + else if (crop(ivt_in) == 1 .or. iscft(ivt_in) == 1) then !crops get_map_EF = vocemis_vars%efisop_grc(6,g_in, ti_in) end if diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index de7062b810d5..c40e0fc0ea0d 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -37,7 +37,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! vegetation structure (LAI, SAI, height) ! ! !USES: - use pftvarcon , only : noveg, woody, generic_crop, crop, percrop + use pftvarcon , only : noveg, woody, iscft, crop use pftvarcon , only : ncorn, ncornirrig, ztopmx, laimx use pftvarcon , only : nmiscanthus, nmiscanthusirrig, nswitchgrass, nswitchgrassirrig use elm_time_manager , only : get_rad_step_size @@ -147,7 +147,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! alpha are set by PFT, and alpha is scaled to CLM time step by multiplying by ! dt and dividing by dtsmonth (seconds in average 30 day month) ! tsai_min scaled by 0.5 to match MODIS satellite derived values - if (generic_crop(ivt(p)) == 1) then ! generic crops + if (crop(ivt(p)) == 1 .and. iscft(ivt(p)) == 0) then ! generic crops tsai_alpha = 1.0_r8-1.0_r8*dt/dtsmonth tsai_min = 0.1_r8 @@ -193,8 +193,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & hbot(p) = max(0._r8, min(3._r8, htop(p)-1._r8)) - else if ( generic_crop(ivt(p)) <1 .and. & - (crop(ivt(p)) >= 1 .or. percrop(ivt(p)) >= 1) ) then ! prognostic crops + else if (iscft(ivt(p)) >= 1) then ! prognostic crops if (tlai(p) >= laimx(ivt(p))) peaklai(p) = 1 ! used in CNAllocation diff --git a/components/elm/src/biogeophys/SedYieldMod.F90 b/components/elm/src/biogeophys/SedYieldMod.F90 index e50544edde6c..e0d63ed063c9 100644 --- a/components/elm/src/biogeophys/SedYieldMod.F90 +++ b/components/elm/src/biogeophys/SedYieldMod.F90 @@ -63,7 +63,7 @@ subroutine SoilErosion (bounds, num_soilc, filter_soilc, & use elm_time_manager, only : get_step_size use landunit_varcon , only : istcrop, istsoil, istice use pftvarcon , only : gcbc_p, gcbc_q, gcbr_p, gcbr_q - use pftvarcon , only : generic_crop, crop, percrop + use pftvarcon , only : crop, iscft ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -218,9 +218,7 @@ subroutine SoilErosion (bounds, num_soilc, filter_soilc, & (veg_cs%livecrootc(p)+veg_cs%deadcrootc(p))*croot_prof(p,1) ) fgndcov = exp( -gcbc_p(veg_pp%itype(p))*PCT_gnd - & gcbr_p(veg_pp%itype(p))*Broot ) - if( generic_crop(veg_pp%itype(p)) >= 1 .or. & - crop(veg_pp%itype(p)) >= 1 .or. & - percrop(veg_pp%itype(p)) >=1 )then + if( crop(veg_pp%itype(p)) >= 1 .or. iscft(veg_pp%itype(p)) >=1 )then Es_Pcrp = Es_Pcrp + pfactor(c) * ftillage * flitho * & fgndcov * veg_pp%wtcol(p) * K * (KE_DT+KE_LD) @@ -268,9 +266,7 @@ subroutine SoilErosion (bounds, num_soilc, filter_soilc, & nh = 0.03_r8 + 0.05_r8*max(Crsd,Clai) fsr = fsr + veg_pp%wtcol(p) * (0.03_r8/nh)**0.6_r8 - if ( generic_crop(veg_pp%itype(p)) >= 1 .or. & - crop(veg_pp%itype(p)) >= 1 .or. & - percrop(veg_pp%itype(p)) >=1 ) then + if ( crop(veg_pp%itype(p)) >= 1 .or. iscft(veg_pp%itype(p)) >= 1 ) then ftillage_tc = ftillage_tc + ftillage * veg_pp%wtcol(p) Es_Q = Es_Q + 19.1_r8 * qfactor(c) * 2._r8/COH * flitho * fslp * & diff --git a/components/elm/src/data_types/VegetationDataType.F90 b/components/elm/src/data_types/VegetationDataType.F90 index eea30fc55fef..ecbdca64df33 100644 --- a/components/elm/src/data_types/VegetationDataType.F90 +++ b/components/elm/src/data_types/VegetationDataType.F90 @@ -17,7 +17,7 @@ module VegetationDataType use elm_varcon , only : spval, ispval, sb use elm_varcon , only : c13ratio, c14ratio use landunit_varcon , only : istsoil, istcrop - use pftvarcon , only : npcropmin, noveg, nstor + use pftvarcon , only : iscft, noveg, nstor use elm_varctl , only : iulog, use_cn, spinup_state, spinup_mortality_factor, use_fates use elm_varctl , only : nu_com, use_crop, use_c13 use elm_varctl , only : use_lch4, use_betr @@ -2447,7 +2447,7 @@ subroutine veg_cs_init(this, begp, endp, carbon_type, ratio) if (veg_vp%evergreen(veg_pp%itype(p)) == 1._r8) then this%leafc(p) = 1._r8 * ratio this%leafc_storage(p) = 0._r8 - else if (veg_pp%itype(p) >= npcropmin) then ! prognostic crop types + else if (iscft(veg_pp%itype(p)) >= 1) then ! prognostic crop types this%leafc(p) = 0._r8 this%leafc_storage(p) = 0._r8 else @@ -3572,7 +3572,7 @@ subroutine veg_cs_summary(this, bounds, num_soilc, filter_soilc, num_soilp, filt this%gresp_storage(p) + & this%gresp_xfer(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%storvegc(p) = & this%storvegc(p) + & this%grainc_storage(p) + & @@ -4249,7 +4249,7 @@ subroutine veg_ns_summary(this, bounds, num_soilc, filter_soilc, num_soilp, filt this%npool(p) + & this%retransn(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%dispvegn(p) = & this%dispvegn(p) + & this%grainn(p) @@ -4997,7 +4997,7 @@ subroutine veg_ps_summary (this, bounds, num_soilc, filter_soilc, num_soilp, fil this%ppool(p) + & this%retransp(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%dispvegp(p) = & this%dispvegp(p) + & this%grainp(p) @@ -8189,7 +8189,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%cpool_livecroot_storage_gr(p) + & this%cpool_deadcroot_storage_gr(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%mr(p) = & this%mr(p) + & this%grain_mr(p) @@ -8214,7 +8214,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%storage_gr(p) ! autotrophic respiration (AR) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%ar(p) = & this%mr(p) + & this%gr(p) + & @@ -8324,8 +8324,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%wood_harvestc(p) = & this%hrv_deadstemc_to_prod10c(p) + & this%hrv_deadstemc_to_prod100c(p) - - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%wood_harvestc(p) = & this%wood_harvestc(p) + & this%hrv_cropc_to_prod1c(p) @@ -8355,7 +8354,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%m_gresp_xfer_to_fire(p) + & this%m_cpool_to_fire(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%litfall(p) = & this%litfall(p) + & this%livestemc_to_litter(p) + & @@ -8394,7 +8393,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%leafc_loss(p) = this%leafc_loss(p) + & this%hrv_leafc_to_litter(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%leafc_loss(p) = & this%leafc_loss(p) + & this%hrv_leafc_to_prod1c(p) @@ -8438,7 +8437,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%hrv_deadcrootc_storage_to_litter(p) + & this%hrv_deadcrootc_xfer_to_litter(p) ! putting the harvested crop stem and grain in the wood loss bdrewniak - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%woodc_loss(p) = & this%woodc_loss(p) + & this%hrv_grainc_to_prod1c(p) + & @@ -8563,7 +8562,7 @@ subroutine veg_cf_summary_for_ch4( this, bounds, num_soilp, filter_soilp) this%cpool_to_deadstemc(p) + & this%deadstemc_xfer_to_deadstemc(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%agnpp(p) = & this%agnpp(p) + & this%cpool_to_grainc(p) + & @@ -9721,7 +9720,7 @@ subroutine veg_nf_summary(this, bounds, num_soilc, filter_soilc, num_soilp, filt this%wood_harvestn(p) = & this%hrv_deadstemn_to_prod10n(p) + & this%hrv_deadstemn_to_prod100n(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%wood_harvestn(p) = & this%wood_harvestn(p) + & this%hrv_cropn_to_prod1n(p) @@ -10777,7 +10776,7 @@ subroutine veg_pf_summary(this, bounds, num_soilc, filter_soilc, num_soilp, filt this%wood_harvestp(p) = & this%hrv_deadstemp_to_prod10p(p) + & this%hrv_deadstemp_to_prod100p(p) - if ( crop_prog .and. veg_pp%itype(p) >= npcropmin )then + if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then this%wood_harvestp(p) = & this%wood_harvestp(p) + & this%hrv_cropp_to_prod1p(p) diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 21d7ed671f31..16f251377a43 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -147,7 +147,7 @@ module VegetationPropertiesType real(r8), pointer :: climatezone(:) => null() !climate zone adapted real(r8), pointer :: nonvascular(:) => null() !nonvascular type or vascular real(r8), pointer :: graminoid(:) => null() !graminoid or not - real(r8), pointer :: generic_crop(:) => null() !generic crop or not for prognostic crop modules (?) + real(r8), pointer :: iscft(:) => null() !generic crop (0) or (1) crop for prognostic crop modules real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 @@ -189,7 +189,7 @@ subroutine veg_vp_init(this) use pftvarcon , only : lmrha, vcmaxhd, jmaxhd, tpuhd, lmrse, qe, theta_cj use pftvarcon , only : bbbopt, mbbopt, nstor, br_xr, tc_stress, lmrhd ! new properties for flexible PFT - use pftvarcon , only : climatezone, nonvascular, graminoid, generic_crop,needleleaf, nfixer + use pftvarcon , only : climatezone, nonvascular, graminoid, iscft,needleleaf, nfixer ! class (vegetation_properties_type) :: this @@ -318,7 +318,7 @@ subroutine veg_vp_init(this) allocate( this%climatezone(0:numpft)) ; this%climatezone(:) =spval allocate( this%nonvascular(0:numpft)) ; this%nonvascular(:) =spval allocate( this%graminoid(0:numpft)) ; this%graminoid(:) =spval - allocate( this%generic_crop(0:numpft)) ; this%generic_crop(:) =spval + allocate( this%iscft(0:numpft)) ; this%iscft(:) =spval allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval ! ----------------------------------------------------------------------------------------------------------- @@ -414,7 +414,7 @@ subroutine veg_vp_init(this) this%climatezone(m) = climatezone(m) this%nonvascular(m) = nonvascular(m) this%graminoid(m) = graminoid(m) - this%generic_crop(m) = generic_crop(m) + this%iscft(m) = iscft(m) this%needleleaf(m) = needleleaf(m) this%nfixer(m) = nfixer(m) diff --git a/components/elm/src/main/filterMod.F90 b/components/elm/src/main/filterMod.F90 index 768235bea40a..eb3f2f2ef1d0 100644 --- a/components/elm/src/main/filterMod.F90 +++ b/components/elm/src/main/filterMod.F90 @@ -272,7 +272,7 @@ subroutine setFiltersOneGroup(bounds, this_filter, include_inactive, icemask_grc ! ! !USES: use decompMod , only : BOUNDS_LEVEL_CLUMP - use pftvarcon , only : npcropmin, nppercropmin + use pftvarcon , only : iscft, crop, percrop use landunit_varcon, only : istsoil, istcrop, istice_mec use column_varcon, only : icol_road_perv ! @@ -420,17 +420,17 @@ subroutine setFiltersOneGroup(bounds, this_filter, include_inactive, icemask_grc t =veg_pp%topounit(p) if (top_pp%active(t)) then if (veg_pp%active(p) .or. include_inactive) then - if (veg_pp%itype(p) < npcropmin) then + if (iscft(veg_pp%itype(p)) < 1) then l =veg_pp%landunit(p) if (lun_pp%itype(l) == istsoil .or. lun_pp%itype(l) == istcrop) then fnc = fnc + 1 this_filter(nc)%soilnopcropp(fnc) = p end if else - if (veg_pp%itype(p) < nppercropmin) then + if (percrop(veg_pp%itype(p)) < 1) then fc = fc + 1 this_filter(nc)%pcropp(fc) = p - else if (veg_pp%itype(p) >= nppercropmin) then + else if (percrop(veg_pp%itype(p)) >= 1) then fpc = fpc + 1 this_filter(nc)%ppercropp(fpc) = p end if diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index 1b06a49cf2f6..dc5d837472ae 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -318,7 +318,7 @@ module pftvarcon real(r8), allocatable :: nonvascular(:) ! nonvascular lifeform flag (0 = vascular, 1 = moss, 2 = lichen) real(r8), allocatable :: needleleaf(:) ! needleleaf lifeform flag (0 = broadleaf, 1 = needleleaf) real(r8), allocatable :: graminoid(:) ! graminoid lifeform flag (0 = nonvascular+woody+crop+percrop, 1 = graminoid) - real(r8), allocatable :: generic_crop(:) ! generic_crop (0 = non_crop or prognostic crop, 1 = generic crop, i.e. crop when use_crop=false) + real(r8), allocatable :: iscft(:) ! crop function type flag (0 = non_crop or generic crop, i.e. crop when use_crop=false, 1 = prognostic crop with cft created) real(r8), allocatable :: nfixer(:) ! nitrogen fixer flag (0 = inable, 1 = able to nitrogen fixation from atm. N2) @@ -641,7 +641,7 @@ subroutine pftconrd allocate( climatezone (0:mxpft) ) allocate( nonvascular (0:mxpft) ) allocate( graminoid (0:mxpft) ) - allocate( generic_crop (0:mxpft) ) + allocate( iscft (0:mxpft) ) allocate( needleleaf (0:mxpft) ) allocate( nfixer (0:mxpft) ) @@ -1127,23 +1127,13 @@ subroutine pftconrd end if end if - call ncd_io('iscft', generic_crop, 'read', ncid, readvar=readv) ! read-in is 'CFT' or not for crop type + call ncd_io('iscft', iscft, 'read', ncid, readvar=readv) ! read-in is 'CFT' or not for crop type if ( .not. readv ) then if (PFT_DEFAULT) then - generic_crop(:) = 0 ! will assign a value below + iscft(:) = 0 ! will assign a value below else call endrun(msg='ERROR: error in reading in user-defined pft data'//errMsg(__FILE__,__LINE__)) end if - else - do i = 0, npft-1 - if ((crop(i)>=1 .or. percrop(i)>=1) .and. generic_crop(i)==0) then - ! at this moment, read-in 'generic_crop' is actually 'iscft' (i.e. crop as CFT or not) - ! So must re-assign - generic_crop(i) = 1 - else - generic_crop(i) = 0 - end if - end do end if call ncd_pio_closefile(ncid) @@ -1243,7 +1233,7 @@ subroutine pftconrd ! when not using those indexing of PFT orders anymore in other codes than here. needleleaf(noveg+1:ndllf_dcd_brl_tree) = 1 graminoid(nc3_arctic_grass:nc4_grass) = 1 - generic_crop(nc3crop:nc3irrig) = 1 + iscft(npcropmin:max(npcropmax,nppercropmax)) = 1 nfixer(nsoybean) = 1 nfixer(nsoybeanirrig) = 1 @@ -1280,13 +1270,16 @@ subroutine pftconrd ncft0 = -1 noncropmax = 0 do i = 0, npft-1 - if (crop(i)>=1 .or. percrop(i)>=1 .or. generic_crop(i)>=1) then - numcft = numcft + 1 + if (crop(i)>=1 .or. percrop(i)>=1 .or. iscft(i)>=1) then + numcft = numcft + 1 ! includes generic_crop, while cft_size NOT (???? todo checking) if(use_crop) then - ! if 'generic_crop' specifically defined, + + ! the following assumes that all crop pfts are in a block + + ! if 'generic crop' (crop=1) specifically flagged by iscft=0 ! 'crop' will not be counted into prognostic - if (crop(i)>=1 .and. generic_crop(i)==0) then + if (crop(i)>=1 .and. iscft(i)==1) then npcropmax = i if(npcropmin<=0) npcropmin = i end if @@ -1299,11 +1292,9 @@ subroutine pftconrd end if else - if(crop(i)>=1 .or. generic_crop(i)>=1) then - ! in case either 'crop' or 'generic_crop' or both defined when not use_crop=.true. - generic_crop(i) = 1 - else - generic_crop(i) = 0 + if(crop(i)>=1 .or. percrop(i)>=1) then + ! in case either 'crop' or 'generic crop' or both defined, it must be generic, when not use_crop=.true. + iscft(i) = 0 end if end if @@ -1322,7 +1313,7 @@ subroutine pftconrd ! need to check 'noveg' if ( woody(i)<=0 .and. graminoid(i)<=0 .and. nonvascular(i)<=0 .and. & - generic_crop(i)<=0 .and. crop(i)<=0 .and. percrop(i)<=0) then + iscft(i)<=0 .and. crop(i)<=0 .and. percrop(i)<=0) then if (noveg>=0) then ! not yet support multiple non-vegetated PFT ! this also will catch error of no actual PFT if npft>1 @@ -1375,23 +1366,23 @@ subroutine pftconrd if ( .not. use_fates ) then do i = 0, mxpft if (i == noveg) then - if ( (nonvascular(i)+woody(i)+graminoid(i)+generic_crop(i)+crop(i)+percrop(i)) >= 1 .or. & + if ( (nonvascular(i)+woody(i)+graminoid(i)+max(iscft(i),crop(i)+percrop(i))) >= 1 .or. & (needleleaf(i)+evergreen(i)+stress_decid(i)+season_decid(i)+nfixer(i)) >= 1 ) then print *, 'ERROR: Incorrect not-vegetated PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: not_vegetated has at least one positive PFT flag '//errMsg(__FILE__, __LINE__)) end if - else if ( (nonvascular(i)+woody(i)+graminoid(i)+generic_crop(i)+crop(i)+percrop(i)) >= 1) then - if (nonvascular(i) >= 1 .and. (woody(i)+graminoid(i)+generic_crop(i)+crop(i)+percrop(i)) >= 1) then + else if ( (nonvascular(i)+woody(i)+graminoid(i)+max(iscft(i),crop(i)+percrop(i))) >= 1) then + if (nonvascular(i) >= 1 .and. (woody(i)+graminoid(i)+max(iscft(i),crop(i)+percrop(i))) >= 1) then print *, 'ERROR: Incorrect nonvasculr PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: nonvascular PFT cannot be any of woody/graminoid/crop type '//errMsg(__FILE__, __LINE__)) - else if (woody(i) >= 1 .and. (nonvascular(i)+graminoid(i)+generic_crop(i)+crop(i)+percrop(i)) >= 1) then + else if (woody(i) >= 1 .and. (nonvascular(i)+graminoid(i)+max(iscft(i),crop(i)+percrop(i))) >= 1) then print *, 'ERROR: Incorrect woody PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: woody PFT cannot be any of nonvascular/graminoid/crop type - '//errMsg(__FILE__, __LINE__)) - else if (graminoid(i) >= 1 .and. (nonvascular(i)+woody(i)+generic_crop(i)+crop(i)+percrop(i)) >=1 ) then + else if (graminoid(i) >= 1 .and. (nonvascular(i)+woody(i)+max(iscft(i),crop(i)+percrop(i))) >=1 ) then print *, 'ERROR: Incorrect graminoid PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: graminoid PFT cannot be any of nonvascular/woody/crop type - '//errMsg(__FILE__, __LINE__)) - else if ( (generic_crop(i)+crop(i)+percrop(i)) >= 1 .and. (nonvascular(i)+woody(i)+graminoid(i)) >= 1) then + else if ( (max(iscft(i),crop(i)+percrop(i))) >= 1 .and. (nonvascular(i)+woody(i)+graminoid(i)) >= 1) then print *, 'ERROR: Incorrect crop PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: crop PFT cannot be any of nonvascular/woody/graminoid type - '//errMsg(__FILE__, __LINE__)) end if @@ -1417,11 +1408,11 @@ subroutine pftconrd if (masterproc) then write(iulog,*) write(iulog,*) 'Using PFT physiological parameters from: ', paramfile - write(iulog,*) ' -- index -- name -- climate zone -- -- woody -- -- needleleaf -- -- evergreen -- -- stress_decid -- -- season_decid -- -- graminoid-- -- generic_crop -- -- crop -- -- perennial crop -- -- nfixer --' + write(iulog,*) ' -- index -- name -- climate zone -- -- woody -- -- needleleaf -- -- evergreen -- -- stress_decid -- -- season_decid -- -- graminoid-- -- iscft -- -- crop -- -- perennial crop -- -- nfixer --' do i = 0, npft-1 write(iulog,*) i, pftname(i), int(climatezone(i)), int(woody(i)), int(needleleaf(i)), & int(evergreen(i)), int(stress_decid(i)), int(season_decid(i)), & - int(graminoid(i)), int(generic_crop(i)), int(crop(i)), int(percrop(i)), int(nfixer(i)) + int(graminoid(i)), int(iscft(i)), int(crop(i)), int(percrop(i)), int(nfixer(i)) end do write(iulog,*) end if diff --git a/components/elm/src/main/surfrdMod.F90 b/components/elm/src/main/surfrdMod.F90 index 1a957e8cd38a..d61ac41e38e4 100755 --- a/components/elm/src/main/surfrdMod.F90 +++ b/components/elm/src/main/surfrdMod.F90 @@ -1265,6 +1265,8 @@ subroutine surfrd_veg_all(begg, endg, ncid, ns,ntpu) do nl = begg,endg do t = 1, max_topounits + ! (TODO) the following assumes that rainfed/irrigated crop are ordered side by side + ! indexing is fixed wt_cft(nl,t,nc3crop) = wt_cft(nl,t,nc3crop) + wt_cft(nl,t,nc3irrig) wt_cft(nl,t,nc3irrig) = 0._r8 wt_cft(nl,t,ncorn) = wt_cft(nl,t,ncorn) + wt_cft(nl,t,ncornirrig) diff --git a/components/elm/src/main/surfrdUtilsMod.F90 b/components/elm/src/main/surfrdUtilsMod.F90 index 1e58b5174afa..c581f968aaf6 100644 --- a/components/elm/src/main/surfrdUtilsMod.F90 +++ b/components/elm/src/main/surfrdUtilsMod.F90 @@ -310,7 +310,7 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose use elm_varctl , only : irrigate use elm_varpar , only : cft_lb, cft_ub, cft_size use pftvarcon , only: is_pft_known_to_model - use pftvarcon , only : npcropmax, mergetoelmpft, npcropmin + use pftvarcon , only : npcropmax, mergetoelmpft use topounit_varcon , only : max_topounits ! TKT use GridcellType , only : grc_pp ! TKT ! From 0dcb1658da7ecc13c83e3a0b09026151b69ba77e Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Sun, 19 May 2024 13:04:00 -0400 Subject: [PATCH 578/904] Bug fix to remove topi, topf and use max_topounits, plus cleanup --- components/elm/src/main/surfrdUtilsMod.F90 | 33 ++++++++-------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/components/elm/src/main/surfrdUtilsMod.F90 b/components/elm/src/main/surfrdUtilsMod.F90 index c581f968aaf6..2c1e42559dac 100644 --- a/components/elm/src/main/surfrdUtilsMod.F90 +++ b/components/elm/src/main/surfrdUtilsMod.F90 @@ -36,7 +36,7 @@ subroutine check_sums_equal_1_3d(arr, lb, name, caller) ! Confirm that sum(arr(n,:)) == 1 for all n. If this isn't true for any n, abort with a message. ! ! Uses - use topounit_varcon, only : max_topounits, has_topounit + use topounit_varcon, only : max_topounits ! !ARGUMENTS: integer , intent(in) :: lb ! lower bound of the first dimension of arr @@ -123,7 +123,7 @@ subroutine convert_cft_to_pft( begg, endg, cftsize, wt_cft ) ! a crop landunit, and put them on the vegetated landunit. ! !USES: use elm_varsur , only : wt_lunit, wt_nat_patch - use elm_varpar , only : cft_size, surfpft_size + use elm_varpar , only : cft_size use elm_varpar , only : natpft_size use landunit_varcon , only : istsoil, istcrop use topounit_varcon , only : max_topounits @@ -170,7 +170,6 @@ subroutine convert_pft_to_cft( begg, endg ) ! the new crop landunit ! !USES: use elm_varsur , only : wt_lunit, wt_nat_patch, wt_cft - use elm_varpar , only : cft_size, surfpft_size use elm_varpar , only : cft_size, cft_lb, cft_ub, surfpft_lb, surfpft_ub use landunit_varcon , only : istsoil, istcrop use topounit_varcon , only : max_topounits @@ -261,7 +260,6 @@ subroutine collapse_crop_var(crop_var, begg, endg) use elm_varpar, only: cft_lb, cft_ub, cft_size use pftvarcon , only: is_pft_known_to_model use topounit_varcon , only : max_topounits ! TKT - use GridcellType, only : grc_pp ! ! !ARGUMENTS: ! Use begg and endg rather than 'bounds', because bounds may not be @@ -311,8 +309,7 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose use elm_varpar , only : cft_lb, cft_ub, cft_size use pftvarcon , only: is_pft_known_to_model use pftvarcon , only : npcropmax, mergetoelmpft - use topounit_varcon , only : max_topounits ! TKT - use GridcellType , only : grc_pp ! TKT + use topounit_varcon , only : max_topounits ! ! !ARGUMENTS: @@ -323,16 +320,15 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose ! Weight and fertilizer of each CFT in each grid cell; dimensioned [g, cft_lb:cft_ub] ! This array is modified in-place - real(r8), intent(inout) :: wt_cft(begg:,1:, cft_lb:) !TKT - real(r8), intent(inout) :: fert_cft(begg:,1:, cft_lb:) !TKT + real(r8), intent(inout) :: wt_cft(begg:,1:, cft_lb:) + real(r8), intent(inout) :: fert_cft(begg:,1:, cft_lb:) real(r8), intent(inout) :: fert_p_cft(begg:, 1:, cft_lb:) logical, intent(in) :: verbose ! If true, print some extra information ! ! !LOCAL VARIABLES: - integer :: g, t,t2 ! TKT + integer :: g, t,t2 integer :: m - !integer, allocatable :: ntpu(:) ! To store number of topounits per grid TKT real(r8) :: wt_cft_to real(r8) :: wt_cft_from real(r8) :: wt_cft_merge @@ -340,7 +336,7 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose character(len=*), parameter :: subname = 'collapse_crop_types' !----------------------------------------------------------------------- - SHR_ASSERT_ALL((ubound(wt_cft) == (/endg,max_topounits, cft_ub/)), errMsg(__FILE__, __LINE__)) ! TKT + SHR_ASSERT_ALL((ubound(wt_cft) == (/endg,max_topounits, cft_ub/)), errMsg(__FILE__, __LINE__)) SHR_ASSERT_ALL((ubound(fert_cft) == (/endg,max_topounits, cft_ub/)), errMsg(__FILE__, __LINE__)) SHR_ASSERT_ALL((ubound(fert_p_cft) == (/endg,max_topounits, cft_ub/)), errMsg(__FILE__, __LINE__)) @@ -353,14 +349,12 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose ! ------------------------------------------------------------------------ ! If not using irrigation, merge irrigated CFTs into rainfed CFTs ! ------------------------------------------------------------------------ - !allocate(ntpu(begg:endg)) if (.not. irrigate) then if (verbose .and. masterproc) then write(iulog,*) trim(subname)//' crop=.T. and irrigate=.F., so merging irrigated pfts with rainfed' end if do g = begg, endg - !ntpu(g) = grc_pp%ntopounits(g) ! Left Hand Side: merged rainfed+irrigated crop pfts from nc3crop to ! npcropmax-1, stride 2 ! Right Hand Side: rainfed crop pfts from nc3crop to npcropmax-1, @@ -369,16 +363,14 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose ! stride 2 ! where stride 2 means "every other" - do t = grc_pp%topi(g), grc_pp%topf(g) ! TKT - t2 = t - grc_pp%topi(g) + 1 - + do t2 = 1, max_topounits wt_cft(g,t2, cft_lb:cft_ub-1:2) = & - wt_cft(g,t2, cft_lb:cft_ub-1:2) + wt_cft(g,t2, cft_lb+1:cft_ub:2) ! TKT + wt_cft(g,t2, cft_lb:cft_ub-1:2) + wt_cft(g,t2, cft_lb+1:cft_ub:2) wt_cft(g,t2, cft_lb+1:cft_ub:2) = 0._r8 - end do ! TKT + end do end do - call check_sums_equal_1_3d(wt_cft, begg, 'wt_cft', subname//': irrigation') ! TKT + call check_sums_equal_1_3d(wt_cft, begg, 'wt_cft', subname//': irrigation') end if ! ------------------------------------------------------------------------ @@ -412,8 +404,7 @@ subroutine collapse_crop_types(wt_cft, fert_cft, fert_p_cft, begg, endg, verbose end do end do - call check_sums_equal_1_3d(wt_cft, begg, 'wt_cft', subname//': mergetoelmpft') ! TKT - !deallocate(ntpu) + call check_sums_equal_1_3d(wt_cft, begg, 'wt_cft', subname//': mergetoelmpft') end subroutine collapse_crop_types end module surfrdUtilsMod From ee819167f2d14b08f6046050902ce30a299e7518 Mon Sep 17 00:00:00 2001 From: Fengming Yuan Date: Sun, 21 Jul 2024 13:02:28 -0400 Subject: [PATCH 579/904] Move a few array allocations to after pftconrd(). (#6) --- components/elm/src/main/elm_initializeMod.F90 | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/components/elm/src/main/elm_initializeMod.F90 b/components/elm/src/main/elm_initializeMod.F90 index 9fe4959c241d..b7fe36f24ed9 100755 --- a/components/elm/src/main/elm_initializeMod.F90 +++ b/components/elm/src/main/elm_initializeMod.F90 @@ -269,10 +269,10 @@ subroutine initialize1( ) allocate (wt_lunit (begg:endg,1:max_topounits, max_lunit )) allocate (urban_valid (begg:endg,1:max_topounits )) - allocate (wt_nat_patch (begg:endg,1:max_topounits, surfpft_lb:surfpft_ub )) - allocate (wt_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) - allocate (fert_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) - allocate (fert_p_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) + !allocate (wt_nat_patch (begg:endg,1:max_topounits, surfpft_lb:surfpft_ub )) + !allocate (wt_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) + !allocate (fert_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) + !allocate (fert_p_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) if (create_glacier_mec_landunit) then allocate (wt_glc_mec (begg:endg,1:max_topounits, maxpatch_glcmec)) allocate (topo_glc_mec(begg:endg,1:max_topounits, maxpatch_glcmec)) @@ -294,16 +294,13 @@ subroutine initialize1( ) ! Independent of model resolution, Needs to stay before surfrd_get_data call pftconrd() - ! by user-defined PFT (numbers and names), 'numpft/mxpft_nc' changed and other indices - ! a few arrays had been allocated in elm_initializedMod.F90:L266-268 and require redo after this 'pftconrd' call - if ((numpft/=mxpft .or. numpft/=numveg) .or. (mxpft_nc/=24 .or. mxpft_nc/=numveg)) then - if (associated(wt_nat_patch)) deallocate(wt_nat_patch) - allocate (wt_nat_patch (begg:endg,1:max_topounits, surfpft_lb:surfpft_ub )) - if (associated(wt_cft)) deallocate(wt_cft) - allocate (wt_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) - if (associated(fert_cft)) deallocate(fert_cft) - allocate (fert_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) - endif + ! if by user-defined PFT (numbers and names), 'numpft/mxpft_nc' may be changed including other derived indices + ! + ! a few arrays allocation previously done above is moved here i.e. after this 'pftconrd' call + allocate (wt_nat_patch (begg:endg,1:max_topounits, surfpft_lb:surfpft_ub )) + allocate (wt_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) + allocate (fert_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) + allocate (fert_p_cft (begg:endg,1:max_topounits, cft_lb:cft_ub )) call soilorder_conrd() From d2164819c243e6760e60d1d3d11961a4215a7e9e Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Sun, 21 Jul 2024 22:07:12 -0400 Subject: [PATCH 580/904] Convert iscft() from real(r8) to logical. This is a demonstration of how the same approach could be used for other flags that are used as logicals but allocated as reals or integers. [BFB] --- .../elm/src/biogeochem/AllocationMod.F90 | 36 +++++++-------- .../src/biogeochem/CNAllocationBetrMod.F90 | 38 +++++++--------- .../elm/src/biogeochem/CNCarbonFluxType.F90 | 14 +++--- .../elm/src/biogeochem/CNCarbonStateType.F90 | 4 +- .../src/biogeochem/CNGapMortalityBeTRMod.F90 | 4 +- .../src/biogeochem/CNNStateUpdate1BeTRMod.F90 | 8 ++-- .../src/biogeochem/CNNStateUpdate2BeTRMod.F90 | 2 +- .../elm/src/biogeochem/CNNitrogenFluxType.F90 | 1 - .../src/biogeochem/CNNitrogenStateType.F90 | 2 +- .../elm/src/biogeochem/CNPhenologyBeTRMod.F90 | 4 +- .../src/biogeochem/CarbonStateUpdate1Mod.F90 | 18 ++++---- .../src/biogeochem/CarbonStateUpdate2Mod.F90 | 2 +- components/elm/src/biogeochem/CropType.F90 | 2 +- .../elm/src/biogeochem/DryDepVelocity.F90 | 2 +- components/elm/src/biogeochem/FireMod.F90 | 12 ++--- .../elm/src/biogeochem/GapMortalityMod.F90 | 10 ++--- .../elm/src/biogeochem/GrowthRespMod.F90 | 2 +- .../elm/src/biogeochem/MaintenanceRespMod.F90 | 2 +- .../biogeochem/NitrogenStateUpdate1Mod.F90 | 8 ++-- .../biogeochem/NitrogenStateUpdate2Mod.F90 | 2 +- .../src/biogeochem/PhenologyFluxLimitMod.F90 | 22 ++++----- .../elm/src/biogeochem/PhenologyMod.F90 | 22 +++------ .../biogeochem/PhosphorusStateUpdate1Mod.F90 | 8 ++-- .../biogeochem/PhosphorusStateUpdate2Mod.F90 | 2 +- .../src/biogeochem/PrecisionControlMod.F90 | 4 +- .../elm/src/biogeochem/RootDynamicsMod.F90 | 2 +- .../elm/src/biogeochem/VOCEmissionMod.F90 | 2 +- .../elm/src/biogeochem/VegStructUpdateMod.F90 | 4 +- components/elm/src/biogeophys/SedYieldMod.F90 | 4 +- .../elm/src/data_types/VegetationDataType.F90 | 26 +++++------ .../data_types/VegetationPropertiesType.F90 | 4 +- components/elm/src/main/filterMod.F90 | 2 +- components/elm/src/main/pftvarcon.F90 | 45 +++++++++++-------- 33 files changed, 157 insertions(+), 163 deletions(-) diff --git a/components/elm/src/biogeochem/AllocationMod.F90 b/components/elm/src/biogeochem/AllocationMod.F90 index 0110367f59ce..141194397fb0 100644 --- a/components/elm/src/biogeochem/AllocationMod.F90 +++ b/components/elm/src/biogeochem/AllocationMod.F90 @@ -585,7 +585,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (iscft(ivt(p)) >= 1) then + else if (iscft(ivt(p))) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) end if @@ -672,7 +672,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp f5 = 0._r8 ! continued intializations from above - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops if (croplive(p) .and. percrop(ivt(p)) == 0.0_r8 ) then ! same phases appear in subroutine CropPhenology @@ -822,7 +822,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp p_allometry(p) = 1._r8/cpl + f1/cpfr + (f3*f4*(1._r8+f2))/cplw + & (f3*(1._r8-f4)*(1._r8+f2))/cpdw - else if (iscft(ivt(p)) >= 1) then ! skip generic crops + else if (iscft(ivt(p))) then ! skip generic crops cng = graincn(ivt(p)) cpg = graincp(ivt(p)) c_allometry(p) = (1._r8+g1)*(1._r8+f1+f5+f3*(1._r8+f2)) @@ -852,10 +852,10 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp ! retransn pool has N from leaves, stems, and roots for ! retranslocation - if (iscft(ivt(p)) >= 1 .and. grain_flag(p) == 1._r8) then + if (iscft(ivt(p)) .and. grain_flag(p) == 1._r8) then avail_retransn(p) = plant_ndemand(p) avail_retransp(p) = plant_pdemand(p) - else if (iscft(ivt(p)) < 1 .and. annsum_potential_gpp(p) > 0._r8) then + else if ((.not. iscft(ivt(p))) .and. annsum_potential_gpp(p) > 0._r8) then avail_retransn(p) = (annmax_retransn(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt avail_retransp(p) = (annmax_retransp(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt else @@ -2142,7 +2142,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & fcur = fcur2(ivt(p)) - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops if (croplive(p)) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -2328,7 +2328,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & fcur = fcur2(ivt(p)) - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops if (croplive(p)) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -2345,10 +2345,10 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & sminn_to_npool(p) = sminn_to_plant_patch(p) sminp_to_ppool(p) = sminp_to_plant_patch(p) - if (iscft(ivt(p)) >= 1 .and. grain_flag(p) == 1._r8) then + if (iscft(ivt(p)) .and. grain_flag(p) == 1._r8) then avail_retransn(p) = retransn(p)/dt avail_retransp(p) = retransp(p)/dt - else if (iscft(ivt(p)) < 1 .and. annsum_potential_gpp(p) > 0._r8) then + else if ((.not. iscft(ivt(p))) .and. annsum_potential_gpp(p) > 0._r8) then avail_retransn(p) = (annmax_retransn(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt avail_retransp(p) = (annmax_retransp(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt else @@ -2374,7 +2374,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (iscft(ivt(p)) >= 1) then + else if (iscft(ivt(p))) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) end if @@ -2473,7 +2473,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & p_allometry(p) = 1._r8/cpl + f1/cpfr + (f3*f4*(1._r8+f2))/cplw + & (f3*(1._r8-f4)*(1._r8+f2))/cpdw - else if (iscft(ivt(p)) >= 1) then ! skip generic crops + else if (iscft(ivt(p))) then ! skip generic crops cng = graincn(ivt(p)) cpg = graincp(ivt(p)) c_allometry(p) = (1._r8+g1)*(1._r8+f1+f5+f3*(1._r8+f2)) @@ -2559,7 +2559,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f2 * f3 * (1._r8 - f4) * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) * (1 + g1) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * (1._r8 - fcur) * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * (1._r8 - f4) * fcur * (1 + g1) @@ -2590,7 +2590,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_deadcrootc(p) = nlc * f2 * f3 * (1._r8 - f4) * fcur cpool_to_deadcrootc_storage(p) = nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -2643,7 +2643,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_deadcrootn(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * fcur npool_to_deadcrootn_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * (1._r8 - fcur) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cng = graincn(ivt(p)) npool_to_livestemn(p) = (nlc * f3 * f4 / cnlw) * fcur npool_to_livestemn_storage(p) = (nlc * f3 * f4 / cnlw) * (1._r8 - fcur) @@ -2687,7 +2687,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_deadcrootp(p) = (nlc * f2 * f3 * (1._r8 - f4) / cpdw)* fcur ppool_to_deadcrootp_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cpdw)* (1._r8 - fcur) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpg = graincp(ivt(p)) ppool_to_livestemp(p) = (nlc * f3 * f4 / cplw) * fcur ppool_to_livestemp_storage(p) = (nlc * f3 * f4 / cplw) * (1._r8 -fcur) @@ -2717,7 +2717,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) gresp_storage = gresp_storage + cpool_to_deadcrootc_storage(p) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) gresp_storage = gresp_storage + cpool_to_grainc_storage(p) end if @@ -2771,7 +2771,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_deadcrootn(p) = cpool_to_deadcrootc(p) / cndw npool_to_deadcrootn_storage(p) = cpool_to_deadcrootc_storage(p) / cndw end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cng = graincn(ivt(p)) supplement_to_plantn(p) = supplement_to_plantn(p) + cpool_to_livestemc(p) / cnlw - npool_to_livestemn(p) supplement_to_plantn(p) = supplement_to_plantn(p) + cpool_to_livestemc_storage(p) / cnlw & @@ -2843,7 +2843,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_deadcrootp(p) = cpool_to_deadcrootc(p) / cpdw ppool_to_deadcrootp_storage(p) = cpool_to_deadcrootc_storage(p) / cpdw end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpg = graincp(ivt(p)) supplement_to_plantp(p) = supplement_to_plantp(p) + max(cpool_to_livestemc(p) / cplw & - ppool_to_livestemp(p),0._r8) diff --git a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 index cc8b72046a17..8cfc342bb404 100644 --- a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 +++ b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 @@ -645,7 +645,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (iscft(ivt(p)) >= 1) then + else if (iscft(ivt(p))) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) end if @@ -726,7 +726,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp f5 = 0._r8 ! continued intializations from above - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops if (croplive(p)) then ! same phases appear in subroutine CropPhenology @@ -859,7 +859,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp p_allometry(p) = 1._r8/cpl + f1/cpfr + (f3*f4*(1._r8+f2))/cplw + & (f3*(1._r8-f4)*(1._r8+f2))/cpdw - else if (iscft(ivt(p)) >= 1) then ! skip generic crops + else if (iscft(ivt(p))) then ! skip generic crops cng = graincn(ivt(p)) cpg = graincp(ivt(p)) c_allometry(p) = (1._r8+g1)*(1._r8+f1+f5+f3*(1._r8+f2)) @@ -889,10 +889,10 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp ! retransn pool has N from leaves, stems, and roots for ! retranslocation - if (iscft(ivt(p)) >= 1 .and. grain_flag(p) == 1._r8) then + if (iscft(ivt(p)) .and. grain_flag(p) == 1._r8) then avail_retransn(p) = plant_ndemand(p) avail_retransp(p) = plant_pdemand(p) - else if (iscft(ivt(p)) < 1 .and. annsum_potential_gpp(p) > 0._r8) then + else if ((.not. iscft(ivt(p))) .and. annsum_potential_gpp(p) > 0._r8) then avail_retransn(p) = (annmax_retransn(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt avail_retransp(p) = (annmax_retransp(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt else @@ -1105,14 +1105,10 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & use shr_sys_mod , only: shr_sys_flush use elm_varctl , only: iulog,cnallocate_carbon_only,cnallocate_carbonnitrogen_only,& cnallocate_carbonphosphorus_only -! use pftvarcon , only: iscft, declfact, bfact, aleaff, arootf, astemf -! use pftvarcon , only: arooti, fleafi, allconsl, allconss, grperc, grpnow, nsoybean use pftvarcon , only: noveg use pftvarcon , only: iscft, grperc, grpnow use elm_varpar , only: nlevdecomp !!nlevsoi, use elm_varcon , only: nitrif_n2o_loss_frac, secspday -! use landunit_varcon , only: istsoil, istcrop -! use elm_time_manager , only: get_step_size ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -1393,7 +1389,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & fcur = fcur2(ivt(p)) - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops if (croplive(p)) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -1412,10 +1408,10 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & plant_n_buffer_patch(p) = plant_n_buffer_patch(p) * (1._r8-dt/taun) plant_p_buffer_patch(p) = plant_p_buffer_patch(p) * (1._r8-dt/taun) - if (iscft(ivt(p)) >= 1 .and. grain_flag(p) == 1._r8) then + if (iscft(ivt(p)) .and. grain_flag(p) == 1._r8) then avail_retransn(p) = retransn(p)/dt avail_retransp(p) = retransp(p)/dt - else if (iscft(ivt(p)) < 1 .and. annsum_potential_gpp(p) > 0._r8) then + else if ((.not. iscft(ivt(p))) .and. annsum_potential_gpp(p) > 0._r8) then avail_retransn(p) = (annmax_retransn(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt avail_retransp(p) = (annmax_retransp(p)/2._r8)*(gpp(p)/annsum_potential_gpp(p))/dt else @@ -1436,7 +1432,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) >= 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (iscft(ivt(p)) >= 1) then + else if (iscft(ivt(p))) then if (croplive(p)) mr = mr + livestem_mr(p) + grain_mr(p) end if ! try to take mr from xsmr storage pool first @@ -1530,7 +1526,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & p_allometry(p) = 1._r8/cpl + f1/cpfr + (f3*f4*(1._r8+f2))/cplw + & (f3*(1._r8-f4)*(1._r8+f2))/cpdw - else if (iscft(ivt(p)) >= 1) then ! skip generic crops + else if (iscft(ivt(p))) then ! skip generic crops cng = graincn(ivt(p)) cpg = graincp(ivt(p)) c_allometry(p) = (1._r8+g1)*(1._r8+f1+f5+f3*(1._r8+f2)) @@ -1614,7 +1610,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f2 * f3 * (1._r8 - f4) * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) * (1 + g1) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * fcur * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * f4 * (1._r8 - fcur) * (1 + g1) cpool_to_xsmrpool(p) = cpool_to_xsmrpool(p) + nlc * f3 * (1._r8 - f4) * fcur * (1 + g1) @@ -1645,7 +1641,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & cpool_to_deadcrootc(p) = nlc * f2 * f3 * (1._r8 - f4) * fcur cpool_to_deadcrootc_storage(p) = nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -1698,7 +1694,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_deadcrootn(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * fcur npool_to_deadcrootn_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * (1._r8 - fcur) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cng = graincn(ivt(p)) npool_to_livestemn(p) = (nlc * f3 * f4 / cnlw) * fcur npool_to_livestemn_storage(p) = (nlc * f3 * f4 / cnlw) * (1._r8 - fcur) @@ -1742,7 +1738,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_deadcrootp(p) = (nlc * f2 * f3 * (1._r8 - f4) / cpdw)* fcur ppool_to_deadcrootp_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cpdw)* (1._r8 - fcur) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpg = graincp(ivt(p)) ppool_to_livestemp(p) = (nlc * f3 * f4 / cplw) * fcur ppool_to_livestemp_storage(p) = (nlc * f3 * f4 / cplw) * (1._r8 -fcur) @@ -1772,7 +1768,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) gresp_storage = gresp_storage + cpool_to_deadcrootc_storage(p) end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) gresp_storage = gresp_storage + cpool_to_grainc_storage(p) end if @@ -1820,7 +1816,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & npool_to_deadcrootn(p) = cpool_to_deadcrootc(p) / cndw npool_to_deadcrootn_storage(p) = cpool_to_deadcrootc_storage(p) / cndw end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cng = graincn(ivt(p)) supplement_to_sminn_vr(c,1) = supplement_to_sminn_vr(c,1) + cpool_to_livestemc(p) / cnlw - npool_to_livestemn(p) supplement_to_sminn_vr(c,1) = supplement_to_sminn_vr(c,1) + cpool_to_livestemc_storage(p) / cnlw & @@ -1891,7 +1887,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & ppool_to_deadcrootp(p) = cpool_to_deadcrootc(p) / cpdw ppool_to_deadcrootp_storage(p) = cpool_to_deadcrootc_storage(p) / cpdw end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpg = graincp(ivt(p)) supplement_to_sminp_vr(c,1) = supplement_to_sminp_vr(c,1) + max(cpool_to_livestemc(p) / cplw & - ppool_to_livestemp(p),0._r8) diff --git a/components/elm/src/biogeochem/CNCarbonFluxType.F90 b/components/elm/src/biogeochem/CNCarbonFluxType.F90 index 61a3f736f621..a6c56a935f05 100644 --- a/components/elm/src/biogeochem/CNCarbonFluxType.F90 +++ b/components/elm/src/biogeochem/CNCarbonFluxType.F90 @@ -1675,7 +1675,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%cpool_livecroot_storage_gr_patch(p) + & this%cpool_deadcroot_storage_gr_patch(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%mr_patch(p) = & this%mr_patch(p) + & this%grain_mr_patch(p) @@ -1700,7 +1700,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%storage_gr_patch(p) ! autotrophic respiration (AR) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%ar_patch(p) = & this%mr_patch(p) + & this%gr_patch(p) + & @@ -1812,7 +1812,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%wood_harvestc_patch(p) = & this%hrv_deadstemc_to_prod10c_patch(p) + & this%hrv_deadstemc_to_prod100c_patch(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%wood_harvestc_patch(p) = & this%wood_harvestc_patch(p) + & this%hrv_cropc_to_prod1c_patch(p) @@ -1842,7 +1842,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%m_gresp_xfer_to_fire_patch(p) + & this%m_cpool_to_fire_patch(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%litfall_patch(p) = & this%litfall_patch(p) + & @@ -1878,7 +1878,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%hrv_leafc_to_litter_patch(p) + & this%leafc_to_litter_patch(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%leafc_loss_patch(p) = & this%leafc_loss_patch(p) + & this%hrv_leafc_to_prod1c_patch(p) @@ -1920,7 +1920,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%hrv_deadcrootc_storage_to_litter_patch(p) + & this%hrv_deadcrootc_xfer_to_litter_patch(p) ! putting the harvested crop stem and grain in the wood loss bdrewniak - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%woodc_loss_patch(p) = & this%woodc_loss_patch(p) + & this%hrv_grainc_to_prod1c_patch(p) + & @@ -2592,7 +2592,7 @@ subroutine summary_cflux_for_ch4( this, bounds, num_soilp, filter_soilp, num_soi this%cpool_to_deadstemc_patch(p) + & this%deadstemc_xfer_to_deadstemc_patch(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%agnpp_patch(p) = & this%agnpp_patch(p) + & this%cpool_to_grainc_patch(p) + & diff --git a/components/elm/src/biogeochem/CNCarbonStateType.F90 b/components/elm/src/biogeochem/CNCarbonStateType.F90 index 46440bee2355..b733773fb20d 100644 --- a/components/elm/src/biogeochem/CNCarbonStateType.F90 +++ b/components/elm/src/biogeochem/CNCarbonStateType.F90 @@ -459,7 +459,7 @@ subroutine InitCold(this, bounds, ratio, c12_carbonstate_vars) if (veg_vp%evergreen(veg_pp%itype(p)) == 1._r8) then this%leafc_patch(p) = 1._r8 * ratio this%leafc_storage_patch(p) = 0._r8 - else if (iscft(veg_pp%itype(p)) >= 1) then ! prognostic crop types + else if (iscft(veg_pp%itype(p))) then ! prognostic crop types this%leafc_patch(p) = 0._r8 this%leafc_storage_patch(p) = 0._r8 else @@ -866,7 +866,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%gresp_storage_patch(p) + & this%gresp_xfer_patch(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%storvegc_patch(p) = & this%storvegc_patch(p) + & this%grainc_storage_patch(p) + & diff --git a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 index 370e82ab7ff5..3bbbf1db1f60 100644 --- a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 @@ -188,7 +188,7 @@ subroutine CNGapMortality (& * gap_indicator(gid_m_deadcrootn_to_litter) - if (iscft(veg_pp%itype(p)) < 1) then + if (.not. iscft(veg_pp%itype(p))) then veg_nf%m_retransn_to_litter(p) = veg_ns%retransn(p) * m & * gap_indicator(gid_m_retransn_to_litter) end if @@ -239,7 +239,7 @@ subroutine CNGapMortality (& veg_pf%m_deadstemp_to_litter(p) = veg_ps%deadstemp(p) * m veg_pf%m_livecrootp_to_litter(p) = veg_ps%livecrootp(p) * m veg_pf%m_deadcrootp_to_litter(p) = veg_ps%deadcrootp(p) * m - if (iscft(veg_pp%itype(p)) < 1) then + if (.not. iscft(veg_pp%itype(p))) then veg_pf%m_retransp_to_litter(p) = veg_ps%retransp(p) * m end if diff --git a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 index f2a431e7283c..465c92f82b4c 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 @@ -101,7 +101,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) - veg_nf%deadcrootn_xfer_to_deadcrootn(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) - veg_nf%livestemn_xfer_to_livestemn(p)*dt @@ -126,7 +126,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%livecrootn(p) = veg_ns%livecrootn(p) - veg_nf%livecrootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%livecrootn_to_retransn(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! Beth adds retrans from froot + if (iscft(ivt(p))) then ! Beth adds retrans from froot veg_ns%frootn(p) = veg_ns%frootn(p) - veg_nf%frootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%frootn_to_retransn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_litter(p)*dt @@ -172,7 +172,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_storage(p) = veg_ns%deadcrootn_storage(p) + veg_nf%npool_to_deadcrootn_storage(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%npool_to_livestemn(p)*dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn_storage(p)*dt @@ -200,7 +200,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) + veg_nf%deadcrootn_storage_to_xfer(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn_storage(p) = veg_ns%livestemn_storage(p) - veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) + veg_nf%livestemn_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/CNNStateUpdate2BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate2BeTRMod.F90 index fbfa6110d77f..70b56142c882 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate2BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate2BeTRMod.F90 @@ -147,7 +147,7 @@ subroutine NStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_ns%deadcrootn(p) = veg_ns%deadcrootn(p) - veg_nf%hrv_deadcrootn_to_litter(p) * dt veg_ns%retransn(p) = veg_ns%retransn(p) - veg_nf%hrv_retransn_to_litter(p) * dt - if (iscft(ivt(p)) >=1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_ns%livestemn(p)= veg_ns%livestemn(p) - veg_nf%hrv_livestemn_to_prod1n(p) * dt veg_ns%leafn(p) = veg_ns%leafn(p) - veg_nf%hrv_leafn_to_prod1n(p) * dt veg_ns%grainn(p) = veg_ns%grainn(p) - veg_nf%hrv_grainn_to_prod1n(p) * dt diff --git a/components/elm/src/biogeochem/CNNitrogenFluxType.F90 b/components/elm/src/biogeochem/CNNitrogenFluxType.F90 index b6f02ce91f82..5394e0d05fa1 100644 --- a/components/elm/src/biogeochem/CNNitrogenFluxType.F90 +++ b/components/elm/src/biogeochem/CNNitrogenFluxType.F90 @@ -1078,7 +1078,6 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil ! !USES: use elm_varpar , only: nlevdecomp,ndecomp_cascade_transitions,ndecomp_pools use subgridAveMod , only: p2c - use pftvarcon , only : iscft use tracer_varcon , only: is_active_betr_bgc use elm_varpar , only: nlevdecomp_full ! diff --git a/components/elm/src/biogeochem/CNNitrogenStateType.F90 b/components/elm/src/biogeochem/CNNitrogenStateType.F90 index 041c757ae598..a7f7315c6c83 100644 --- a/components/elm/src/biogeochem/CNNitrogenStateType.F90 +++ b/components/elm/src/biogeochem/CNNitrogenStateType.F90 @@ -1007,7 +1007,7 @@ subroutine Summary(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soil this%npool_patch(p) + & this%retransn_patch(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%dispvegn_patch(p) = & this%dispvegn_patch(p) + & this%grainn_patch(p) diff --git a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 index aaeb58d9ee91..ac3b58538e8f 100644 --- a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 @@ -1895,13 +1895,13 @@ subroutine CropPhenologyInit(bounds) minplantjday(:,:) = huge(1) maxplantjday(:,:) = huge(1) do n = 0, mxpft - if (iscft(n)>=1) then + if (iscft(n)) then minplantjday(n,inNH) = int( get_calday( mnNHplantdate(n), 0 ) ) maxplantjday(n,inNH) = int( get_calday( mxNHplantdate(n), 0 ) ) end if end do do n = 0, mxpft - if (iscft(n)>=1) then + if (iscft(n)) then minplantjday(n,inSH) = int( get_calday( mnSHplantdate(n), 0 ) ) maxplantjday(n,inSH) = int( get_calday( mxSHplantdate(n), 0 ) ) end if diff --git a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 index 7513662f4670..1eb7f64d025a 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 @@ -284,7 +284,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%deadcrootc(p) = veg_cs%deadcrootc(p) + veg_cf%deadcrootc_xfer_to_deadcrootc(p)*dt veg_cs%deadcrootc_xfer(p) = veg_cs%deadcrootc_xfer(p) - veg_cf%deadcrootc_xfer_to_deadcrootc(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%livestemc_xfer_to_livestemc(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) - veg_cf%livestemc_xfer_to_livestemc(p)*dt @@ -303,7 +303,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%livecrootc(p) = veg_cs%livecrootc(p) - veg_cf%livecrootc_to_deadcrootc(p)*dt veg_cs%deadcrootc(p) = veg_cs%deadcrootc(p) + veg_cf%livecrootc_to_deadcrootc(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_cs%livestemc(p) = veg_cs%livestemc(p) - veg_cf%livestemc_to_litter(p)*dt veg_cs%grainc(p) = veg_cs%grainc(p) - veg_cf%grainc_to_food(p)*dt @@ -319,7 +319,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livestem_curmr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livecroot_curmr(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%livestem_curmr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%grain_curmr(p)*dt end if @@ -337,7 +337,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livestem_xsmr(p)*dt veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livecroot_xsmr(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%livestem_xsmr(p)*dt veg_cs%xsmrpool(p) = veg_cs%xsmrpool(p) - veg_cf%grain_xsmr(p)*dt if (harvdate(p) < 999) then ! beginning at harvest, send to atm @@ -373,7 +373,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_deadcrootc_storage(p)*dt veg_cs%deadcrootc_storage(p) = veg_cs%deadcrootc_storage(p) + veg_cf%cpool_to_deadcrootc_storage(p)*dt end if - if ( iscft(ivt(p)) >= 1 ) then ! skip 2 generic crops + if ( iscft(ivt(p))) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_livestemc(p)*dt veg_cs%livestemc(p) = veg_cs%livestemc(p) + veg_cf%cpool_to_livestemc(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_to_livestemc_storage(p)*dt @@ -393,7 +393,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livecroot_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_deadcroot_gr(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livestem_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_grain_gr(p)*dt end if @@ -407,7 +407,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_livecroot_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_deadcroot_gr(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_livestem_gr(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) - veg_cf%transfer_grain_gr(p)*dt end if @@ -421,7 +421,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livecroot_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_deadcroot_storage_gr(p)*dt end if - if ( iscft(ivt(p)) >= 1 ) then ! skip 2 generic crops + if ( iscft(ivt(p))) then ! skip 2 generic crops veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_livestem_storage_gr(p)*dt veg_cs%cpool(p) = veg_cs%cpool(p) - veg_cf%cpool_grain_storage_gr(p)*dt end if @@ -447,7 +447,7 @@ subroutine CarbonStateUpdate1(bounds, & veg_cs%gresp_storage(p) = veg_cs%gresp_storage(p) - veg_cf%gresp_storage_to_xfer(p)*dt veg_cs%gresp_xfer(p) = veg_cs%gresp_xfer(p) + veg_cf%gresp_storage_to_xfer(p)*dt end if - if ( iscft(ivt(p)) >= 1 ) then ! skip 2 generic crops + if ( iscft(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_cs%livestemc_storage(p) = veg_cs%livestemc_storage(p) - veg_cf%livestemc_storage_to_xfer(p)*dt veg_cs%livestemc_xfer(p) = veg_cs%livestemc_xfer(p) + veg_cf%livestemc_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/CarbonStateUpdate2Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate2Mod.F90 index 985c544d29ff..4676a1e351d1 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate2Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate2Mod.F90 @@ -186,7 +186,7 @@ subroutine CarbonStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soilp, veg_cs%deadcrootc(p) = veg_cs%deadcrootc(p) - veg_cf%hrv_deadcrootc_to_litter(p) * dt ! crops - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_cs%livestemc(p) = veg_cs%livestemc(p) - veg_cf%hrv_livestemc_to_prod1c(p) *dt veg_cs%leafc(p) = veg_cs%leafc(p) - veg_cf%hrv_leafc_to_prod1c(p) *dt veg_cs%grainc(p) = veg_cs%grainc(p) - veg_cf%hrv_grainc_to_prod1c(p) *dt diff --git a/components/elm/src/biogeochem/CropType.F90 b/components/elm/src/biogeochem/CropType.F90 index 58f7757e5fcb..8394d21ca348 100644 --- a/components/elm/src/biogeochem/CropType.F90 +++ b/components/elm/src/biogeochem/CropType.F90 @@ -454,7 +454,7 @@ subroutine Restart(this, bounds, ncid, flag) interpinic_flag='copy', readvar=readvar, data=restyear) if (readvar) then do p = bounds%begp, bounds%endp - if (iscft(veg_pp%itype(p)) >= 1 .and. & + if (iscft(veg_pp%itype(p)) .and. & veg_pp%active(p)) then this%nyrs_crop_active_patch(p) = restyear end if diff --git a/components/elm/src/biogeochem/DryDepVelocity.F90 b/components/elm/src/biogeochem/DryDepVelocity.F90 index a79792b4e858..b6ef82a632ef 100644 --- a/components/elm/src/biogeochem/DryDepVelocity.F90 +++ b/components/elm/src/biogeochem/DryDepVelocity.F90 @@ -294,7 +294,7 @@ subroutine depvel_compute( bounds, & !if (elmveg == nc3crop ) wesveg = 2 !if (elmveg == nc3irrig ) wesveg = 2 !if (elmveg >= npcropmin .and. elmveg <= npcropmax ) wesveg = 2 - if (crop(elmveg) == 1.0_r8 .or. iscft(elmveg) == 1.0_r8 ) wesveg = 2 + if (crop(elmveg) == 1.0_r8 .or. iscft(elmveg)) wesveg = 2 #ifndef _OPENACC if (wesveg == wveg_unset )then write(iulog,*) 'elmveg = ', elmveg, 'lun_pp%itype = ', lun_pp%itype(l) diff --git a/components/elm/src/biogeochem/FireMod.F90 b/components/elm/src/biogeochem/FireMod.F90 index 7aa4e6a50f9e..b84aa2c3ab6e 100644 --- a/components/elm/src/biogeochem/FireMod.F90 +++ b/components/elm/src/biogeochem/FireMod.F90 @@ -305,12 +305,12 @@ subroutine FireArea (bounds, & if (pi <= col_pp%npfts(c)) then p = col_pp%pfti(c) + pi - 1 ! For crop veg types - if( crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p)) == 1 )then + if( crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p)))then cropf_col(c) = cropf_col(c) + veg_pp%wtcol(p) end if ! For natural vegetation (non-crop and non-bare-soil) if( veg_pp%itype(p) /= noveg .and. & - (crop(veg_pp%itype(p)) == 0 .and. iscft(veg_pp%itype(p)) == 0) )then + (crop(veg_pp%itype(p)) == 0 .and. .not. iscft(veg_pp%itype(p))) )then lfwt(c) = lfwt(c) + veg_pp%wtcol(p) end if end if @@ -332,7 +332,7 @@ subroutine FireArea (bounds, & ! column-level litter carbon ! is available, so we use leaf carbon to estimate the ! litter carbon for crop PFTs - if( (crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p)) == 1) .and. & + if( (crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p))) .and. & veg_pp%wtcol(p) > 0._r8 .and. leafc_col(c) > 0._r8 )then fuelc_crop(c)=fuelc_crop(c) + (leafc(p) + leafc_storage(p) + & leafc_xfer(p))*veg_pp%wtcol(p)/cropf_col(c) + & @@ -370,7 +370,7 @@ subroutine FireArea (bounds, & p = col_pp%pfti(c) + pi - 1 ! For non-crop -- natural vegetation and bare-soil - if( (crop(veg_pp%itype(p)) == 0 .and. iscft(veg_pp%itype(p)) == 0) .and. & + if( (crop(veg_pp%itype(p)) == 0 .and. .not. iscft(veg_pp%itype(p))) .and. & cropf_col(c) < 1.0_r8 ) then if( btran2(p) .ne. spval) then if (btran2(p) <= 1._r8 ) then @@ -509,7 +509,7 @@ subroutine FireArea (bounds, & p = col_pp%pfti(c) + pi - 1 ! For crop if( forc_t(t) >= SHR_CONST_TKFRZ .and. & - (crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p)) == 1) .and. & + (crop(veg_pp%itype(p)) == 1 .or. iscft(veg_pp%itype(p))) .and. & kmo == abm_lf(c) .and. forc_rain(t)+forc_snow(t) == 0._r8 .and. & burndate(p) >= 999 .and. veg_pp%wtcol(p) > 0._r8 )then ! catch crop burn time @@ -982,7 +982,7 @@ subroutine FireFluxes (num_soilc, filter_soilc, num_soilp, filter_soilp, & c = veg_pp%column(p) itype = veg_pp%itype(p) - if( (crop(veg_pp%itype(p)) == 0 .and. iscft(veg_pp%itype(p)) == 0) .and. & + if( (crop(veg_pp%itype(p)) == 0 .and. .not. iscft(veg_pp%itype(p))) .and. & cropf_col(c) < 1.0_r8)then ! For non-crop (bare-soil and natural vegetation) if (transient_landcover) then !true when landuse data is used diff --git a/components/elm/src/biogeochem/GapMortalityMod.F90 b/components/elm/src/biogeochem/GapMortalityMod.F90 index 734be187805f..3a58551ae076 100644 --- a/components/elm/src/biogeochem/GapMortalityMod.F90 +++ b/components/elm/src/biogeochem/GapMortalityMod.F90 @@ -142,7 +142,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & ! displayed pools veg_cf%m_leafc_to_litter(p) = 0._r8 veg_cf%m_livestemc_to_litter(p) = 0._r8 - if(iscft(ivt(p)) < 1 .or. (iscft(ivt(p)) >= 1 .and. croplive(p))) then + if((.not. iscft(ivt(p))) .or. (iscft(ivt(p)) .and. croplive(p))) then veg_cf%m_leafc_to_litter(p) = veg_cs%leafc(p) * m veg_cf%m_livestemc_to_litter(p) = veg_cs%livestemc(p) * m end if @@ -181,7 +181,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & ! displayed pools veg_nf%m_leafn_to_litter(p) = 0._r8 veg_nf%m_livestemn_to_litter(p) = 0._r8 - if(iscft(ivt(p)) < 1 .or. (iscft(ivt(p)) >= 1 .and. croplive(p))) then + if((.not. iscft(ivt(p))) .or. (iscft(ivt(p)) .and. croplive(p))) then veg_nf%m_leafn_to_litter(p) = veg_ns%leafn(p) * m veg_nf%m_livestemn_to_litter(p) = veg_ns%livestemn(p) * m end if @@ -190,7 +190,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_nf%m_livecrootn_to_litter(p) = veg_ns%livecrootn(p) * m veg_nf%m_deadcrootn_to_litter(p) = veg_ns%deadcrootn(p) * m veg_nf%m_retransn_to_litter(p) = 0._r8 - if (iscft(ivt(p)) < 1) then + if (.not. iscft(ivt(p))) then veg_nf%m_retransn_to_litter(p) = veg_ns%retransn(p) * m end if veg_nf%m_npool_to_litter(p) = veg_ns%npool(p) * m @@ -225,7 +225,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & ! displayed pools veg_pf%m_leafp_to_litter(p) = 0._r8 veg_pf%m_livestemp_to_litter(p) = 0._r8 - if(iscft(ivt(p)) < 1 .or. (iscft(ivt(p)) >= 1 .and. croplive(p))) then + if((.not. iscft(ivt(p))) .or. (iscft(ivt(p)) .and. croplive(p))) then veg_pf%m_leafp_to_litter(p) = veg_ps%leafp(p) * m veg_pf%m_livestemp_to_litter(p) = veg_ps%livestemp(p) * m endif @@ -235,7 +235,7 @@ subroutine GapMortality (num_soilc, filter_soilc, num_soilp, filter_soilp, & veg_pf%m_deadcrootp_to_litter(p) = veg_ps%deadcrootp(p) * m veg_pf%m_retransp_to_litter(p) = 0._r8 - if (iscft(ivt(p)) < 1) then + if (.not. iscft(ivt(p))) then veg_pf%m_retransp_to_litter(p) = veg_ps%retransp(p) * m end if veg_pf%m_ppool_to_litter(p) = veg_ps%ppool(p) * m diff --git a/components/elm/src/biogeochem/GrowthRespMod.F90 b/components/elm/src/biogeochem/GrowthRespMod.F90 index 34a9a64c56c9..cb7ad522265f 100644 --- a/components/elm/src/biogeochem/GrowthRespMod.F90 +++ b/components/elm/src/biogeochem/GrowthRespMod.F90 @@ -102,7 +102,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) do fp = 1,num_soilp p = filter_soilp(fp) - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops cpool_livestem_gr(p) = cpool_to_livestemc(p) * grperc(ivt(p)) diff --git a/components/elm/src/biogeochem/MaintenanceRespMod.F90 b/components/elm/src/biogeochem/MaintenanceRespMod.F90 index 48d179f97d1f..54005275b296 100644 --- a/components/elm/src/biogeochem/MaintenanceRespMod.F90 +++ b/components/elm/src/biogeochem/MaintenanceRespMod.F90 @@ -185,7 +185,7 @@ subroutine MaintenanceResp(bounds, & if (woody(ivt(p)) >= 1.0_r8) then livestem_mr(p) = livestemn(p)*br_mr*tc livecroot_mr(p) = livecrootn(p)*br_mr*tc - else if (iscft(ivt(p)) >= 1 .and. livestemn(p) .gt. 0._r8) then + else if (iscft(ivt(p)) .and. livestemn(p) .gt. 0._r8) then livestem_mr(p) = livestemn(p)*br_mr*tc grain_mr(p) = grainn(p)*br_mr*tc end if diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 index 80fc34e6d0ba..c1b2bfc5d65a 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 @@ -297,7 +297,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) - veg_nf%deadcrootn_xfer_to_deadcrootn(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%livestemn_xfer_to_livestemn(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) - veg_nf%livestemn_xfer_to_livestemn(p)*dt @@ -322,7 +322,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%livecrootn(p) = veg_ns%livecrootn(p) - veg_nf%livecrootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%livecrootn_to_retransn(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! Beth adds retrans from froot + if (iscft(ivt(p))) then ! Beth adds retrans from froot veg_ns%frootn(p) = veg_ns%frootn(p) - veg_nf%frootn_to_retransn(p)*dt veg_ns%retransn(p) = veg_ns%retransn(p) + veg_nf%frootn_to_retransn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) - veg_nf%livestemn_to_litter(p)*dt @@ -373,7 +373,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_storage(p) = veg_ns%deadcrootn_storage(p) + veg_nf%npool_to_deadcrootn_storage(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn(p)*dt veg_ns%livestemn(p) = veg_ns%livestemn(p) + veg_nf%npool_to_livestemn(p)*dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%npool_to_livestemn_storage(p)*dt @@ -401,7 +401,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp veg_ns%deadcrootn_xfer(p) = veg_ns%deadcrootn_xfer(p) + veg_nf%deadcrootn_storage_to_xfer(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ns%livestemn_storage(p) = veg_ns%livestemn_storage(p) - veg_nf%livestemn_storage_to_xfer(p)*dt veg_ns%livestemn_xfer(p) = veg_ns%livestemn_xfer(p) + veg_nf%livestemn_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate2Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate2Mod.F90 index 5d897c5905a9..d8e6785e52cb 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate2Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate2Mod.F90 @@ -172,7 +172,7 @@ subroutine NitrogenStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soil veg_ns%retransn(p) = veg_ns%retransn(p) - veg_nf%hrv_retransn_to_litter(p) * dt veg_ns%npool(p) = veg_ns%npool(p) - veg_nf%hrv_npool_to_litter(p) * dt - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_ns%livestemn(p)= veg_ns%livestemn(p) - veg_nf%hrv_livestemn_to_prod1n(p) * dt veg_ns%leafn(p) = veg_ns%leafn(p) - veg_nf%hrv_leafn_to_prod1n(p) * dt veg_ns%grainn(p) = veg_ns%grainn(p) - veg_nf%hrv_grainn_to_prod1n(p) * dt diff --git a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 index 49909f54929a..87b9637d5512 100644 --- a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 +++ b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 @@ -644,7 +644,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_deadcrootc_xfer) = veg_cs%deadcrootc_xfer(p) ystates(s_deadcrootc_storage) = veg_cs%deadcrootc_storage(p) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then ystates(s_livestemc) = veg_cs%livestemc(p) ystates(s_livestemc_xfer) = veg_cs%livestemc_xfer(p) ystates(s_livestemc_storage) = veg_cs%livestemc_storage(p) @@ -675,7 +675,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& + veg_cf%cpool_livecroot_storage_gr(p) & + veg_cf%cpool_deadcroot_storage_gr(p) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then ar_p= ar_p & + veg_cf%livestem_curmr(p) & + veg_cf%grain_curmr(p) & @@ -713,7 +713,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_deadcrootc_storage_to_xfer) = veg_cf%deadcrootc_storage_to_xfer(p) rfluxes(f_gresp_storage_to_xfer) = veg_cf%gresp_storage_to_xfer(p) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then rfluxes(f_cpool_to_livestemc) = veg_cf%cpool_to_livestemc(p) rfluxes(f_cpool_to_livestemc_storage) = veg_cf%cpool_to_livestemc_storage(p) rfluxes(f_cpool_to_grainc) = veg_cf%cpool_to_grainc(p) @@ -767,7 +767,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_deadcrootc_storage_to_xfer) , veg_cf%deadcrootc_storage_to_xfer(p)) call fpmax(rfluxes(f_gresp_storage_to_xfer) , veg_cf%gresp_storage_to_xfer(p)) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then call fpmax(rfluxes(f_cpool_to_livestemc) , veg_cf%cpool_to_livestemc(p)) call fpmax(rfluxes(f_cpool_to_livestemc_storage) , veg_cf%cpool_to_livestemc_storage(p)) call fpmax(rfluxes(f_cpool_to_grainc) , veg_cf%cpool_to_grainc(p)) @@ -806,7 +806,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& call ascal(veg_cf%cpool_livecroot_storage_gr(p), rscal) call ascal(veg_cf%cpool_deadcroot_storage_gr(p), rscal) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then call ascal(veg_cf%livestem_curmr(p) , rscal) call ascal(veg_cf%grain_curmr(p) , rscal) call ascal(veg_cf%cpool_livestem_gr(p) , rscal) @@ -878,7 +878,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_deadcrootn_xfer) = veg_ns%deadcrootn_xfer(p) ystates(s_deadcrootn_storage) = veg_ns%deadcrootn_storage(p) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then ystates(s_grainn) = veg_ns%grainn(p) ystates(s_grainn_xfer) = veg_ns%grainn_xfer(p) ystates(s_grainn_storage) = veg_ns%grainn_storage(p) @@ -917,7 +917,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_deadcrootn_xfer_to_deadcrootn) = veg_nf%deadcrootn_xfer_to_deadcrootn(p) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then rfluxes(f_npool_to_livestemn) = veg_nf%npool_to_livestemn(p) rfluxes(f_npool_to_livestemn_storage) = veg_nf%npool_to_livestemn_storage(p) rfluxes(f_npool_to_grainn) = veg_nf%npool_to_grainn(p) @@ -973,7 +973,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_deadcrootn_xfer_to_deadcrootn) , veg_nf%deadcrootn_xfer_to_deadcrootn(p)) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then call fpmax(rfluxes(f_npool_to_livestemn) , veg_nf%npool_to_livestemn(p)) call fpmax(rfluxes(f_npool_to_livestemn_storage) , veg_nf%npool_to_livestemn_storage(p)) call fpmax(rfluxes(f_npool_to_grainn) , veg_nf%npool_to_grainn(p)) @@ -1063,7 +1063,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& ystates(s_deadcrootn_xfer) = veg_ps%deadcrootp_xfer(p) ystates(s_deadcrootn_storage) = veg_ps%deadcrootp_storage(p) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then ystates(s_grainn) = veg_ps%grainp(p) ystates(s_grainn_xfer) = veg_ps%grainp_xfer(p) ystates(s_grainn_storage) = veg_ps%grainp_storage(p) @@ -1102,7 +1102,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& rfluxes(f_deadcrootn_xfer_to_deadcrootn) = veg_pf%deadcrootp_xfer_to_deadcrootp(p) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then rfluxes(f_npool_to_livestemn) = veg_pf%ppool_to_livestemp(p) rfluxes(f_npool_to_livestemn_storage) = veg_pf%ppool_to_livestemp_storage(p) rfluxes(f_npool_to_grainn) = veg_pf%ppool_to_grainp(p) @@ -1158,7 +1158,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& call fpmax(rfluxes(f_deadcrootn_xfer_to_deadcrootn) , veg_pf%deadcrootp_xfer_to_deadcrootp(p)) endif - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then call fpmax(rfluxes(f_npool_to_livestemn) , veg_pf%ppool_to_livestemp(p)) call fpmax(rfluxes(f_npool_to_livestemn_storage) , veg_pf%ppool_to_livestemp_storage(p)) call fpmax(rfluxes(f_npool_to_grainn) , veg_pf%ppool_to_grainp(p)) diff --git a/components/elm/src/biogeochem/PhenologyMod.F90 b/components/elm/src/biogeochem/PhenologyMod.F90 index 2a52cb928837..08e9b94da524 100644 --- a/components/elm/src/biogeochem/PhenologyMod.F90 +++ b/components/elm/src/biogeochem/PhenologyMod.F90 @@ -2205,28 +2205,18 @@ subroutine CropPhenologyInit(bounds) minplantjday(:,:) = huge(1) maxplantjday(:,:) = huge(1) do n = 0, mxpft - if (iscft(n)>=1) then + if (iscft(n)) then minplantjday(n,inNH) = int( get_calday( mnNHplantdate(n), 0 ) ) maxplantjday(n,inNH) = int( get_calday( mxNHplantdate(n), 0 ) ) end if end do do n = 0, mxpft - if (iscft(n)>=1) then + if (iscft(n)) then minplantjday(n,inSH) = int( get_calday( mnSHplantdate(n), 0 ) ) maxplantjday(n,inSH) = int( get_calday( mxSHplantdate(n), 0 ) ) end if end do - ! with flag 'iscft', the following is not needed anymore - !do n = nppercropmin, nppercropmax - ! minplantjday(n,inNH) = int( get_calday( mnNHplantdate(n), 0 ) ) - ! maxplantjday(n,inNH) = int( get_calday( mxNHplantdate(n), 0 ) ) - !end do - !do n = nppercropmin, nppercropmax - ! minplantjday(n,inSH) = int( get_calday( mnSHplantdate(n), 0 ) ) - ! maxplantjday(n,inSH) = int( get_calday( mxSHplantdate(n), 0 ) ) - !end do - ! Figure out what hemisphere each PFT is in do p = bounds%begp, bounds%endp g = veg_pp%gridcell(p) @@ -3068,7 +3058,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & if (offset_counter(p) == dt) then t1 = 1.0_r8 / dt - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then ! this assumes that offset_counter == dt for crops ! if this were ever changed, we'd need to add code to the "else" leafc_to_litter(p) = (1.0_r8 - presharv(ivt(p))) * ((t1 * leafc(p)) + cpool_to_leafc(p)) @@ -3085,7 +3075,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & end if if ( nu_com .eq. 'RD') then - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then if (offset_counter(p) == dt) then t1 = 1.0_r8 / dt @@ -3119,7 +3109,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & else if (offset_counter(p) == dt) then t1 = 1.0_r8 / dt - if (iscft(ivt(p)) >= 1) then + if (iscft(ivt(p))) then ! this assumes that offset_counter == dt for crops ! if this were ever changed, we'd need to add code to the "else" leafn_to_litter(p) = (1.0_r8 - presharv(ivt(p))) * ((t1 * leafn(p)) + npool_to_leafn(p)) @@ -3503,7 +3493,7 @@ subroutine CNLitterToColumn (num_soilp, filter_soilp, & ! new ones for now (slevis) ! The food is now directed to the product pools (BDrewniak) - if (iscft(ivt(p)) >= 1) then ! add livestemc to litter + if (iscft(ivt(p))) then ! add livestemc to litter ! stem litter carbon fluxes phenology_c_to_litr_met_c(c,j) = phenology_c_to_litr_met_c(c,j) & + livestemc_to_litter(p) * lf_flab(ivt(p)) * wt_col * leaf_prof(p,j) diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 index 01369ed83fa7..6e1111169221 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 @@ -255,7 +255,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_xfer(p) = veg_ps%deadcrootp_xfer(p) - veg_pf%deadcrootp_xfer_to_deadcrootp(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ps%livestemp(p) = veg_ps%livestemp(p) + veg_pf%livestemp_xfer_to_livestemp(p)*dt veg_ps%livestemp_xfer(p) = veg_ps%livestemp_xfer(p) - veg_pf%livestemp_xfer_to_livestemp(p)*dt @@ -280,7 +280,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%livecrootp(p) = veg_ps%livecrootp(p) - veg_pf%livecrootp_to_retransp(p)*dt veg_ps%retransp(p) = veg_ps%retransp(p) + veg_pf%livecrootp_to_retransp(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! Beth adds retrans from froot + if (iscft(ivt(p))) then ! Beth adds retrans from froot veg_ps%frootp(p) = veg_ps%frootp(p) - veg_pf%frootp_to_retransp(p)*dt veg_ps%retransp(p) = veg_ps%retransp(p) + veg_pf%frootp_to_retransp(p)*dt veg_ps%livestemp(p) = veg_ps%livestemp(p) - veg_pf%livestemp_to_litter(p)*dt @@ -331,7 +331,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_storage(p) = veg_ps%deadcrootp_storage(p) + veg_pf%ppool_to_deadcrootp_storage(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_livestemp(p)*dt veg_ps%livestemp(p) = veg_ps%livestemp(p) + veg_pf%ppool_to_livestemp(p)*dt veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%ppool_to_livestemp_storage(p)*dt @@ -359,7 +359,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi veg_ps%deadcrootp_xfer(p) = veg_ps%deadcrootp_xfer(p) + veg_pf%deadcrootp_storage_to_xfer(p)*dt end if - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero veg_ps%livestemp_storage(p) = veg_ps%livestemp_storage(p) - veg_pf%livestemp_storage_to_xfer(p)*dt veg_ps%livestemp_xfer(p) = veg_ps%livestemp_xfer(p) + veg_pf%livestemp_storage_to_xfer(p)*dt diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate2Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate2Mod.F90 index 4aa7563c70f7..c6b6cd8ee6d3 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate2Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate2Mod.F90 @@ -182,7 +182,7 @@ subroutine PhosphorusStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_so veg_ps%retransp(p) = veg_ps%retransp(p) - veg_pf%hrv_retransp_to_litter(p) * dt veg_ps%ppool(p) = veg_ps%ppool(p) - veg_pf%hrv_ppool_to_litter(p) * dt - if (iscft(ivt(p)) >= 1) then ! skip 2 generic crops + if (iscft(ivt(p))) then ! skip 2 generic crops veg_ps%livestemp(p)= veg_ps%livestemp(p) - veg_pf%hrv_livestemp_to_prod1p(p) * dt veg_ps%leafp(p) = veg_ps%leafp(p) - veg_pf%hrv_leafp_to_prod1p(p) * dt veg_ps%grainp(p) = veg_ps%grainp(p) - veg_pf%hrv_grainp_to_prod1p(p) * dt diff --git a/components/elm/src/biogeochem/PrecisionControlMod.F90 b/components/elm/src/biogeochem/PrecisionControlMod.F90 index 2423592c05c2..10f601487cda 100644 --- a/components/elm/src/biogeochem/PrecisionControlMod.F90 +++ b/components/elm/src/biogeochem/PrecisionControlMod.F90 @@ -211,7 +211,7 @@ subroutine PrecisionControl(num_soilc, filter_soilc, num_soilp, filter_soilp) veg_ps%frootp_xfer(p) = 0._r8 end if - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then ! grain C and N if (abs(veg_cs%grainc(p)) < ccrit) then pc = pc + veg_cs%grainc(p) @@ -513,7 +513,7 @@ subroutine PrecisionControl(num_soilc, filter_soilc, num_soilp, filter_soilp) endif end if - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then ! xsmrpool (C only) if (abs(veg_cs%xsmrpool(p)) < ccrit) then pc = pc + veg_cs%xsmrpool(p) diff --git a/components/elm/src/biogeochem/RootDynamicsMod.F90 b/components/elm/src/biogeochem/RootDynamicsMod.F90 index f5ffc4c67844..9ea6a9812b0a 100644 --- a/components/elm/src/biogeochem/RootDynamicsMod.F90 +++ b/components/elm/src/biogeochem/RootDynamicsMod.F90 @@ -136,7 +136,7 @@ subroutine RootDynamics(bounds, num_soilc, filter_soilc, num_soilp, filter_soilp p = filter_soilp(f) c = pcolumn(p) if (ivt(p) /= noveg) then - if (iscft(ivt(p)) >= 1) then !skip generic crop types + if (iscft(ivt(p))) then !skip generic crop types if (huigrain(p) > 0._r8) then root_depth(p) = max(zi(c,2), min(hui(p)/huigrain(p)* root_dmx(ivt(p)), root_dmx(ivt(p)))) end if diff --git a/components/elm/src/biogeochem/VOCEmissionMod.F90 b/components/elm/src/biogeochem/VOCEmissionMod.F90 index fb7bd66958db..0a17f8803e73 100644 --- a/components/elm/src/biogeochem/VOCEmissionMod.F90 +++ b/components/elm/src/biogeochem/VOCEmissionMod.F90 @@ -703,7 +703,7 @@ function get_map_EF(ivt_in, g_in, ti_in, vocemis_vars) get_map_EF = vocemis_vars%efisop_grc(4,g_in, ti_in) else if (graminoid(ivt_in) == 1) then !grass get_map_EF = vocemis_vars%efisop_grc(5,g_in, ti_in) - else if (crop(ivt_in) == 1 .or. iscft(ivt_in) == 1) then !crops + else if (crop(ivt_in) == 1 .or. iscft(ivt_in)) then !crops get_map_EF = vocemis_vars%efisop_grc(6,g_in, ti_in) end if diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index c40e0fc0ea0d..5943da355a04 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -147,7 +147,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! alpha are set by PFT, and alpha is scaled to CLM time step by multiplying by ! dt and dividing by dtsmonth (seconds in average 30 day month) ! tsai_min scaled by 0.5 to match MODIS satellite derived values - if (crop(ivt(p)) == 1 .and. iscft(ivt(p)) == 0) then ! generic crops + if (crop(ivt(p)) == 1 .and. .not. iscft(ivt(p))) then ! generic crops tsai_alpha = 1.0_r8-1.0_r8*dt/dtsmonth tsai_min = 0.1_r8 @@ -193,7 +193,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & hbot(p) = max(0._r8, min(3._r8, htop(p)-1._r8)) - else if (iscft(ivt(p)) >= 1) then ! prognostic crops + else if (iscft(ivt(p))) then ! prognostic crops if (tlai(p) >= laimx(ivt(p))) peaklai(p) = 1 ! used in CNAllocation diff --git a/components/elm/src/biogeophys/SedYieldMod.F90 b/components/elm/src/biogeophys/SedYieldMod.F90 index e0d63ed063c9..250929f4a423 100644 --- a/components/elm/src/biogeophys/SedYieldMod.F90 +++ b/components/elm/src/biogeophys/SedYieldMod.F90 @@ -218,7 +218,7 @@ subroutine SoilErosion (bounds, num_soilc, filter_soilc, & (veg_cs%livecrootc(p)+veg_cs%deadcrootc(p))*croot_prof(p,1) ) fgndcov = exp( -gcbc_p(veg_pp%itype(p))*PCT_gnd - & gcbr_p(veg_pp%itype(p))*Broot ) - if( crop(veg_pp%itype(p)) >= 1 .or. iscft(veg_pp%itype(p)) >=1 )then + if( crop(veg_pp%itype(p)) >= 1 .or. iscft(veg_pp%itype(p)))then Es_Pcrp = Es_Pcrp + pfactor(c) * ftillage * flitho * & fgndcov * veg_pp%wtcol(p) * K * (KE_DT+KE_LD) @@ -266,7 +266,7 @@ subroutine SoilErosion (bounds, num_soilc, filter_soilc, & nh = 0.03_r8 + 0.05_r8*max(Crsd,Clai) fsr = fsr + veg_pp%wtcol(p) * (0.03_r8/nh)**0.6_r8 - if ( crop(veg_pp%itype(p)) >= 1 .or. iscft(veg_pp%itype(p)) >= 1 ) then + if ( crop(veg_pp%itype(p)) >= 1 .or. iscft(veg_pp%itype(p))) then ftillage_tc = ftillage_tc + ftillage * veg_pp%wtcol(p) Es_Q = Es_Q + 19.1_r8 * qfactor(c) * 2._r8/COH * flitho * fslp * & diff --git a/components/elm/src/data_types/VegetationDataType.F90 b/components/elm/src/data_types/VegetationDataType.F90 index ecbdca64df33..3bc7021f1509 100644 --- a/components/elm/src/data_types/VegetationDataType.F90 +++ b/components/elm/src/data_types/VegetationDataType.F90 @@ -2447,7 +2447,7 @@ subroutine veg_cs_init(this, begp, endp, carbon_type, ratio) if (veg_vp%evergreen(veg_pp%itype(p)) == 1._r8) then this%leafc(p) = 1._r8 * ratio this%leafc_storage(p) = 0._r8 - else if (iscft(veg_pp%itype(p)) >= 1) then ! prognostic crop types + else if (iscft(veg_pp%itype(p))) then ! prognostic crop types this%leafc(p) = 0._r8 this%leafc_storage(p) = 0._r8 else @@ -3572,7 +3572,7 @@ subroutine veg_cs_summary(this, bounds, num_soilc, filter_soilc, num_soilp, filt this%gresp_storage(p) + & this%gresp_xfer(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%storvegc(p) = & this%storvegc(p) + & this%grainc_storage(p) + & @@ -4249,7 +4249,7 @@ subroutine veg_ns_summary(this, bounds, num_soilc, filter_soilc, num_soilp, filt this%npool(p) + & this%retransn(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%dispvegn(p) = & this%dispvegn(p) + & this%grainn(p) @@ -4997,7 +4997,7 @@ subroutine veg_ps_summary (this, bounds, num_soilc, filter_soilc, num_soilp, fil this%ppool(p) + & this%retransp(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%dispvegp(p) = & this%dispvegp(p) + & this%grainp(p) @@ -8189,7 +8189,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%cpool_livecroot_storage_gr(p) + & this%cpool_deadcroot_storage_gr(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%mr(p) = & this%mr(p) + & this%grain_mr(p) @@ -8214,7 +8214,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%storage_gr(p) ! autotrophic respiration (AR) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%ar(p) = & this%mr(p) + & this%gr(p) + & @@ -8324,7 +8324,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%wood_harvestc(p) = & this%hrv_deadstemc_to_prod10c(p) + & this%hrv_deadstemc_to_prod100c(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%wood_harvestc(p) = & this%wood_harvestc(p) + & this%hrv_cropc_to_prod1c(p) @@ -8354,7 +8354,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%m_gresp_xfer_to_fire(p) + & this%m_cpool_to_fire(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%litfall(p) = & this%litfall(p) + & this%livestemc_to_litter(p) + & @@ -8393,7 +8393,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%leafc_loss(p) = this%leafc_loss(p) + & this%hrv_leafc_to_litter(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%leafc_loss(p) = & this%leafc_loss(p) + & this%hrv_leafc_to_prod1c(p) @@ -8437,7 +8437,7 @@ subroutine veg_cf_summary(this, bounds, num_soilp, filter_soilp, num_soilc, filt this%hrv_deadcrootc_storage_to_litter(p) + & this%hrv_deadcrootc_xfer_to_litter(p) ! putting the harvested crop stem and grain in the wood loss bdrewniak - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%woodc_loss(p) = & this%woodc_loss(p) + & this%hrv_grainc_to_prod1c(p) + & @@ -8562,7 +8562,7 @@ subroutine veg_cf_summary_for_ch4( this, bounds, num_soilp, filter_soilp) this%cpool_to_deadstemc(p) + & this%deadstemc_xfer_to_deadstemc(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%agnpp(p) = & this%agnpp(p) + & this%cpool_to_grainc(p) + & @@ -9720,7 +9720,7 @@ subroutine veg_nf_summary(this, bounds, num_soilc, filter_soilc, num_soilp, filt this%wood_harvestn(p) = & this%hrv_deadstemn_to_prod10n(p) + & this%hrv_deadstemn_to_prod100n(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%wood_harvestn(p) = & this%wood_harvestn(p) + & this%hrv_cropn_to_prod1n(p) @@ -10776,7 +10776,7 @@ subroutine veg_pf_summary(this, bounds, num_soilc, filter_soilc, num_soilp, filt this%wood_harvestp(p) = & this%hrv_deadstemp_to_prod10p(p) + & this%hrv_deadstemp_to_prod100p(p) - if ( crop_prog .and. iscft(veg_pp%itype(p)) >= 1 )then + if ( crop_prog .and. iscft(veg_pp%itype(p)))then this%wood_harvestp(p) = & this%wood_harvestp(p) + & this%hrv_cropp_to_prod1p(p) diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 16f251377a43..12f9a7b22ffa 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -147,7 +147,7 @@ module VegetationPropertiesType real(r8), pointer :: climatezone(:) => null() !climate zone adapted real(r8), pointer :: nonvascular(:) => null() !nonvascular type or vascular real(r8), pointer :: graminoid(:) => null() !graminoid or not - real(r8), pointer :: iscft(:) => null() !generic crop (0) or (1) crop for prognostic crop modules + logical, pointer :: iscft(:) => null() !.false. = generic crop, .true. = prognostic crop real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 @@ -318,7 +318,7 @@ subroutine veg_vp_init(this) allocate( this%climatezone(0:numpft)) ; this%climatezone(:) =spval allocate( this%nonvascular(0:numpft)) ; this%nonvascular(:) =spval allocate( this%graminoid(0:numpft)) ; this%graminoid(:) =spval - allocate( this%iscft(0:numpft)) ; this%iscft(:) =spval + allocate( this%iscft(0:numpft)) ; this%iscft(:) =.false. allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval ! ----------------------------------------------------------------------------------------------------------- diff --git a/components/elm/src/main/filterMod.F90 b/components/elm/src/main/filterMod.F90 index eb3f2f2ef1d0..e1dde9445598 100644 --- a/components/elm/src/main/filterMod.F90 +++ b/components/elm/src/main/filterMod.F90 @@ -420,7 +420,7 @@ subroutine setFiltersOneGroup(bounds, this_filter, include_inactive, icemask_grc t =veg_pp%topounit(p) if (top_pp%active(t)) then if (veg_pp%active(p) .or. include_inactive) then - if (iscft(veg_pp%itype(p)) < 1) then + if (.not. iscft(veg_pp%itype(p))) then l =veg_pp%landunit(p) if (lun_pp%itype(l) == istsoil .or. lun_pp%itype(l) == istcrop) then fnc = fnc + 1 diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index dc5d837472ae..9dcbb3cd6da6 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -318,7 +318,8 @@ module pftvarcon real(r8), allocatable :: nonvascular(:) ! nonvascular lifeform flag (0 = vascular, 1 = moss, 2 = lichen) real(r8), allocatable :: needleleaf(:) ! needleleaf lifeform flag (0 = broadleaf, 1 = needleleaf) real(r8), allocatable :: graminoid(:) ! graminoid lifeform flag (0 = nonvascular+woody+crop+percrop, 1 = graminoid) - real(r8), allocatable :: iscft(:) ! crop function type flag (0 = non_crop or generic crop, i.e. crop when use_crop=false, 1 = prognostic crop with cft created) + logical , allocatable :: iscft(:) ! crop function type flag (.false. = non_crop or generic crop, i.e. crop when use_crop=false, .true. = prognostic crop with cft created) + real(r8), allocatable :: temp_iscft(:) ! for file read, translated to logical afterwards real(r8), allocatable :: nfixer(:) ! nitrogen fixer flag (0 = inable, 1 = able to nitrogen fixation from atm. N2) @@ -369,6 +370,7 @@ subroutine pftconrd logical :: PFT_DEFAULT ! pft names are default, i.e. NOT user-defined integer :: ncft0, ncft ! crop pft index of first/last when 'create_crop_landunit' is true integer :: noncropmax ! max non-crop pft index (to check when 'create_crop_landunit' is true) + real(r8) :: local_iscft ! a local transfer of iscft from logical to integer for use in error checks character(len=32) :: subname = 'pftconrd' ! subroutine name ! ! Expected PFT names: The names expected on the paramfile file and the order they are expected to be in. @@ -642,6 +644,7 @@ subroutine pftconrd allocate( nonvascular (0:mxpft) ) allocate( graminoid (0:mxpft) ) allocate( iscft (0:mxpft) ) + allocate( temp_iscft (0:mxpft) ) allocate( needleleaf (0:mxpft) ) allocate( nfixer (0:mxpft) ) @@ -1127,17 +1130,22 @@ subroutine pftconrd end if end if - call ncd_io('iscft', iscft, 'read', ncid, readvar=readv) ! read-in is 'CFT' or not for crop type + call ncd_io('iscft', temp_iscft, 'read', ncid, readvar=readv) ! read-in is 'CFT' or not for crop type if ( .not. readv ) then if (PFT_DEFAULT) then - iscft(:) = 0 ! will assign a value below + temp_iscft(:) = 0._r8 ! will assign a value below else call endrun(msg='ERROR: error in reading in user-defined pft data'//errMsg(__FILE__,__LINE__)) end if end if call ncd_pio_closefile(ncid) - + + ! transfer the temporary real to logical + do i=0, mxpft + if (temp_iscft(i) == 1._r8) iscft(i) = .true. + if (temp_iscft(i) == 0._r8) iscft(i) = .false. + end do if ( PFT_DEFAULT ) then ! if still reading in default PFT physiology file, @@ -1233,7 +1241,7 @@ subroutine pftconrd ! when not using those indexing of PFT orders anymore in other codes than here. needleleaf(noveg+1:ndllf_dcd_brl_tree) = 1 graminoid(nc3_arctic_grass:nc4_grass) = 1 - iscft(npcropmin:max(npcropmax,nppercropmax)) = 1 + iscft(npcropmin:max(npcropmax,nppercropmax)) = .true. nfixer(nsoybean) = 1 nfixer(nsoybeanirrig) = 1 @@ -1251,7 +1259,6 @@ subroutine pftconrd !------------------------------------------------------------------------------------------- - ! NOT default PFT file else @@ -1270,16 +1277,16 @@ subroutine pftconrd ncft0 = -1 noncropmax = 0 do i = 0, npft-1 - if (crop(i)>=1 .or. percrop(i)>=1 .or. iscft(i)>=1) then + if (crop(i)>=1 .or. percrop(i)>=1 .or. iscft(i)) then numcft = numcft + 1 ! includes generic_crop, while cft_size NOT (???? todo checking) if(use_crop) then ! the following assumes that all crop pfts are in a block - ! if 'generic crop' (crop=1) specifically flagged by iscft=0 + ! if 'generic crop' (crop=1) specifically flagged by iscft=.false. ! 'crop' will not be counted into prognostic - if (crop(i)>=1 .and. iscft(i)==1) then + if (crop(i)>=1 .and. iscft(i)) then npcropmax = i if(npcropmin<=0) npcropmin = i end if @@ -1294,7 +1301,7 @@ subroutine pftconrd else if(crop(i)>=1 .or. percrop(i)>=1) then ! in case either 'crop' or 'generic crop' or both defined, it must be generic, when not use_crop=.true. - iscft(i) = 0 + iscft(i) = .false. end if end if @@ -1313,7 +1320,7 @@ subroutine pftconrd ! need to check 'noveg' if ( woody(i)<=0 .and. graminoid(i)<=0 .and. nonvascular(i)<=0 .and. & - iscft(i)<=0 .and. crop(i)<=0 .and. percrop(i)<=0) then + .not. iscft(i) .and. crop(i)<=0 .and. percrop(i)<=0) then if (noveg>=0) then ! not yet support multiple non-vegetated PFT ! this also will catch error of no actual PFT if npft>1 @@ -1365,24 +1372,26 @@ subroutine pftconrd ! checking of pft flags' conflict if ( .not. use_fates ) then do i = 0, mxpft + if (iscft(i)) local_iscft = 1._r8 + if (.not. iscft(i)) local_iscft = 0._r8 if (i == noveg) then - if ( (nonvascular(i)+woody(i)+graminoid(i)+max(iscft(i),crop(i)+percrop(i))) >= 1 .or. & + if ( (nonvascular(i)+woody(i)+graminoid(i)+max(local_iscft,crop(i)+percrop(i))) >= 1 .or. & (needleleaf(i)+evergreen(i)+stress_decid(i)+season_decid(i)+nfixer(i)) >= 1 ) then print *, 'ERROR: Incorrect not-vegetated PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: not_vegetated has at least one positive PFT flag '//errMsg(__FILE__, __LINE__)) end if - else if ( (nonvascular(i)+woody(i)+graminoid(i)+max(iscft(i),crop(i)+percrop(i))) >= 1) then - if (nonvascular(i) >= 1 .and. (woody(i)+graminoid(i)+max(iscft(i),crop(i)+percrop(i))) >= 1) then + else if ( (nonvascular(i)+woody(i)+graminoid(i)+max(local_iscft,crop(i)+percrop(i))) >= 1) then + if (nonvascular(i) >= 1 .and. (woody(i)+graminoid(i)+max(local_iscft,crop(i)+percrop(i))) >= 1) then print *, 'ERROR: Incorrect nonvasculr PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: nonvascular PFT cannot be any of woody/graminoid/crop type '//errMsg(__FILE__, __LINE__)) - else if (woody(i) >= 1 .and. (nonvascular(i)+graminoid(i)+max(iscft(i),crop(i)+percrop(i))) >= 1) then + else if (woody(i) >= 1 .and. (nonvascular(i)+graminoid(i)+max(local_iscft,crop(i)+percrop(i))) >= 1) then print *, 'ERROR: Incorrect woody PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: woody PFT cannot be any of nonvascular/graminoid/crop type - '//errMsg(__FILE__, __LINE__)) - else if (graminoid(i) >= 1 .and. (nonvascular(i)+woody(i)+max(iscft(i),crop(i)+percrop(i))) >=1 ) then + else if (graminoid(i) >= 1 .and. (nonvascular(i)+woody(i)+max(local_iscft,crop(i)+percrop(i))) >=1 ) then print *, 'ERROR: Incorrect graminoid PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: graminoid PFT cannot be any of nonvascular/woody/crop type - '//errMsg(__FILE__, __LINE__)) - else if ( (max(iscft(i),crop(i)+percrop(i))) >= 1 .and. (nonvascular(i)+woody(i)+graminoid(i)) >= 1) then + else if ( (max(local_iscft,crop(i)+percrop(i))) >= 1 .and. (nonvascular(i)+woody(i)+graminoid(i)) >= 1) then print *, 'ERROR: Incorrect crop PFT flags: ', i, ' ', trim(pftname(i)) call endrun(msg=' ERROR: crop PFT cannot be any of nonvascular/woody/graminoid type - '//errMsg(__FILE__, __LINE__)) end if @@ -1412,7 +1421,7 @@ subroutine pftconrd do i = 0, npft-1 write(iulog,*) i, pftname(i), int(climatezone(i)), int(woody(i)), int(needleleaf(i)), & int(evergreen(i)), int(stress_decid(i)), int(season_decid(i)), & - int(graminoid(i)), int(iscft(i)), int(crop(i)), int(percrop(i)), int(nfixer(i)) + int(graminoid(i)), int(local_iscft), int(crop(i)), int(percrop(i)), int(nfixer(i)) end do write(iulog,*) end if From 0f841e67a2a152cd35c85043783459057b891c78 Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Mon, 26 Aug 2024 22:26:27 -0400 Subject: [PATCH 581/904] Fixes a bug, uninitialized variable in pftvarcon --- components/elm/src/main/pftvarcon.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index 9dcbb3cd6da6..fc5bfd54e7d3 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -1421,7 +1421,7 @@ subroutine pftconrd do i = 0, npft-1 write(iulog,*) i, pftname(i), int(climatezone(i)), int(woody(i)), int(needleleaf(i)), & int(evergreen(i)), int(stress_decid(i)), int(season_decid(i)), & - int(graminoid(i)), int(local_iscft), int(crop(i)), int(percrop(i)), int(nfixer(i)) + int(graminoid(i)), int(temp_iscft(i)), int(crop(i)), int(percrop(i)), int(nfixer(i)) end do write(iulog,*) end if From 1848615f0968fa5fa35b2ab5317845e9ecc33326 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 27 Aug 2024 09:34:09 -0700 Subject: [PATCH 582/904] Pm-cpu must still use gcc-11.2 There are several libraries we use that were built with gcc-11. --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index d0e015349b3a..baa622fb513f 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -218,7 +218,7 @@ PrgEnv-gnu/8.5.0 - gcc/12.2.0 + gcc/11.2.0 cray-libsci/23.02.1.1 From 485ca5850eb05cade08a12ccd3506b76a176ecbb Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Tue, 27 Aug 2024 13:44:42 -0500 Subject: [PATCH 583/904] Correct linoz_data_type for hist-nat --- .../use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml index 8f3a2644527d..94cdfc6d000d 100644 --- a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml @@ -107,7 +107,7 @@ 1850 linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc atm/cam/chem/trop_mozart/ub -INTERP_MISSING_MONTHS +CYCLICAL SERIAL From ffb3b6103bc182b235a0c9392e56be41c801dc85 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 27 Aug 2024 15:10:10 -0500 Subject: [PATCH 584/904] initialize factors in area correction computations to 1.0 only intel on chrysalis complained about it It could be a cause for trouble later on --- driver-moab/main/component_mod.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/driver-moab/main/component_mod.F90 b/driver-moab/main/component_mod.F90 index 88d152a95685..3e8ba9042a58 100644 --- a/driver-moab/main/component_mod.F90 +++ b/driver-moab/main/component_mod.F90 @@ -748,6 +748,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe lsize = comp(1)%mblsize allocate(areas (lsize, 3)) ! lsize is along grid; read mask too allocate(factors (lsize, 2)) + factors = 1.0 ! initialize with 1.0 all factors; then maybe correct them ! get areas tagname='area:aream:mask'//C_NULL_CHAR arrsize = 3 * lsize From 30efb49b035afdbf84cc06a2d8a61faaf505b9b6 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 27 Aug 2024 14:40:26 -0700 Subject: [PATCH 585/904] Undo gcc downgrade --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index baa622fb513f..d0e015349b3a 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -218,7 +218,7 @@ PrgEnv-gnu/8.5.0 - gcc/11.2.0 + gcc/12.2.0 cray-libsci/23.02.1.1 From e9c56ec7fcf812f3e92fc9bac1341fc68716deba Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Tue, 27 Aug 2024 17:04:21 -0500 Subject: [PATCH 586/904] Turn on depth-int solver for 1km GIS; add more output vars --- .../bld/namelist_files/albany_input.gis_1to10km_r01.yaml | 3 +++ .../bld/namelist_files/albany_input.gis_1to10km_r02.yaml | 2 ++ components/mpas-albany-landice/cime_config/buildnml | 2 ++ 3 files changed, 7 insertions(+) diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r01.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r01.yaml index 589e163bba7f..c1b2e310f323 100644 --- a/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r01.yaml +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r01.yaml @@ -2,6 +2,9 @@ --- ANONYMOUS: + Problem: + Depth Integrated Model: true + # Discretization Description Discretization: #Exodus Output File Name: albany_output.exo diff --git a/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r02.yaml b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r02.yaml index 2b70fd9b30fa..45539411756a 100644 --- a/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r02.yaml +++ b/components/mpas-albany-landice/bld/namelist_files/albany_input.gis_1to10km_r02.yaml @@ -1,7 +1,9 @@ %YAML 1.1 --- ANONYMOUS: + Problem: + Depth Integrated Model: true Basal Cubature Degree: 4 LandIce Field Norm: sliding_velocity_basalside: diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index a42011e54073..9489b6dfa8fd 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -238,8 +238,10 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') From 383f137c1e79ea37664cc72ac41a1956fbd0dbd7 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Tue, 27 Aug 2024 22:00:20 -0500 Subject: [PATCH 587/904] Add compset hist-all-xaer --- cime_config/allactive/config_compsets.xml | 10 ++ ..._eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml | 117 ++++++++++++++++++ .../eam/cime_config/config_component.xml | 1 + .../elm/cime_config/config_component.xml | 1 + 4 files changed, 129 insertions(+) create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index 63883bff15d9..28eefc45b765 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -158,6 +158,16 @@ 20TRSOI_EAM%CMIP6-VOLC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + WCYCL20TR-xaer + 20TRSOI_EAM%CMIP6-xAER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + + + hist-all-xaer + 20TRSOI_EAM%CMIP6-xAER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV + + diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml new file mode 100644 index 000000000000..359a292e9e12 --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml @@ -0,0 +1,117 @@ + + + + +.true. + + +.true. +.true. +.true. + + +atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc +SERIAL + + +atm/cam/ggas/GHG_CMIP-1-2-0_Annual_Global_0000-2014_c20180105.nc +RAMPED + + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1850-2014_c180205_1850-aero_hist-volcano.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_elev_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc + + +CYCLICAL +1850 +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H4_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C2H6_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_C3H8_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH2O_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3CHO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CH3COCH3_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_CO_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_ISOP_surface_1850-2014_1.9x2.5_c20210323.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_MTERP_surface_1850-2014_1.9x2.5_c20230126.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_e3sm_NO_surface_1850-2014_1.9x2.5_c20220425.nc +atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160416.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/emissions-cmip6_e3sm_SOAG0_surf_1850-2014_1.9x2.5_c20230201.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1850-2014_c180205.nc +atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions_E90_surface_1750-2015_1.9x2.5_c20210408.nc + + + + +CYCLICAL +1849 +oxid_1.9x2.5_L26_1850-2015_c20181106.nc +'' +atm/cam/chem/trop_mozart_aero/oxid +'prsd_O3:O3','prsd_NO3:NO3','prsd_OH:OH' + + +ch4_oxid_1.9x2.5_L26_1990-1999clim.c090804.nc +atm/cam/chem/methane +CYCLICAL +1995 +'' +'prsd_ch4:CH4' + + + + 'A:H2OLNZ:H2O', 'N:O2:O2', 'N:CO2:CO2', + 'A:O3:O3', 'A:N2OLNZ:N2O', 'A:CH4LNZ:CH4', + 'N:CFC11:CFC11', 'N:CFC12:CFC12', + 'M:mam5_mode1:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode2:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode2_rrtmg_c130628.nc', + 'M:mam5_mode3:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_c141106.nc', + 'M:mam5_mode4:$INPUTDATA_ROOT/atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc', + 'M:mam5_mode5:$INPUTDATA_ROOT/atm/cam/physprops/mam5_mode5_rrtmg_sig1.2_dgnl.40_c03072023.nc' + + + +3 +1 +'atm/cam/chem/trop_mam/marine_BGC/' +'CYCLICAL' +'monthly_macromolecules_0.1deg_bilinear_latlon_year01_merge_date.nc' +0 +0 +'chla:CHL1','mpoly:TRUEPOLYC','mprot:TRUEPROTC','mlip:TRUELIPC' + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +SERIAL +linv3_1849-2017_CMIP6_Hist_10deg_58km_c20230705.nc +atm/cam/chem/trop_mozart/ub +INTERP_MISSING_MONTHS + + +SERIAL + + +'xactive_lnd' +'O3','H2O2','CH2O','CH3OOH','NO','NO2','HNO3','HO2NO2','PAN','CO','CH3COCH3','C2H5OOH','CH3CHO','H2SO4','SO2','NO3','N2O5','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'NEU' +'C2H5OOH','CH2O','CH3CHO','CH3OOH','H2O2','H2SO4','HNO3','HO2NO2','SO2','SOAG0','SOAG15','SOAG24','SOAG35','SOAG34','SOAG33','SOAG32','SOAG31' +'CH2O', 'CH3O2', 'CH3OOH', 'PAN', 'CO', 'C2H6', 'C3H8', 'C2H4', 'ROHO2', 'CH3COCH3', 'C2H5O2', 'C2H5OOH', 'CH3CHO', 'CH3CO3', 'ISOP', 'ISOPO2', 'MVKMACR', 'MVKO2' +'' + diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 8f1e68161cb7..655d7be5503c 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -131,6 +131,7 @@ 20TR_eam_CMIP6-aer_chemUCI-Linoz-mam5-vbs 20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs 20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs + 20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs 20TR_eam_CMIP6-ozone_chemUCI-Linoz-mam5-vbs 20TR_eam_CMIP6-lulc_chemUCI-Linoz-mam5-vbs 20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs diff --git a/components/elm/cime_config/config_component.xml b/components/elm/cime_config/config_component.xml index b191a0a2248d..00518b5dc780 100755 --- a/components/elm/cime_config/config_component.xml +++ b/components/elm/cime_config/config_component.xml @@ -79,6 +79,7 @@ 20thC_CMIP6xLULC_transient 20thC_CMIP6xLULC_transient 20thC_CMIP6_transient + 20thC_CMIP6_transient 20thC_CMIP6_transient 20thC_CMIP6bgc_transient 20thC_CMIP6bgc_transient From ac671bd8d34c9e532978e4aded129955400ee307 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 28 Aug 2024 09:32:44 -0700 Subject: [PATCH 588/904] Do static libc++ linking --- cime_config/machines/cmake_macros/gnu_pm-cpu.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake b/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake index 226d07350a78..c2093f7f7fbb 100644 --- a/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake @@ -10,3 +10,6 @@ set(MPIFC "ftn") set(SCC "gcc") set(SCXX "g++") set(SFC "gfortran") + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -static-libstdc++") + From 58b4220130cd4c5eea4908dee0c56bbef97f7347 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 28 Aug 2024 13:54:27 -0600 Subject: [PATCH 589/904] Revert ers change --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index eacb1b7f5ed9..3dfad8399b80 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit eacb1b7f5ed984f0c124076c78fd1e80dbd9b7a1 +Subproject commit 3dfad8399b803d724365c559c5b84e5546049e0c From bd79ca9ce10bafa8ec2d2f170bec5863d73b860e Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Wed, 28 Aug 2024 20:35:07 +0000 Subject: [PATCH 590/904] Run in 4-CCS mode --- cime_config/machines/config_machines.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index b6ac815f58f6..fed0110bc32b 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3385,7 +3385,7 @@ 104 104 104 - 12 + 48 FALSE mpiexec @@ -3444,6 +3444,7 @@ 20 $ENV{KOKKOS_ROOT} 1 + 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4 0 From 6ae05889a8664d227995f661cdbeae13e3b9136d Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Wed, 28 Aug 2024 20:43:22 +0000 Subject: [PATCH 591/904] Disable openmp-offload --- cime_config/machines/Depends.oneapi-ifxgpu.cmake | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cime_config/machines/Depends.oneapi-ifxgpu.cmake b/cime_config/machines/Depends.oneapi-ifxgpu.cmake index 0dd35e56bcc9..5a958df26eba 100644 --- a/cime_config/machines/Depends.oneapi-ifxgpu.cmake +++ b/cime_config/machines/Depends.oneapi-ifxgpu.cmake @@ -1,14 +1,5 @@ -set(CPPDEFS "${CPPDEFS} -DMPAS_OPENMP_OFFLOAD") -list(APPEND MPAS_ADD_ACC_FLAGS - ${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_mesh_pool.f90 - ${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_velocity_solver_variational.f90 - ${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_velocity_solver.f90 -) - -foreach(ITEM IN LISTS MPAS_ADD_ACC_FLAGS) - e3sm_add_flags("${ITEM}" "-fiopenmp -fopenmp-targets=spir64") -endforeach() - # compile mpas_seaice_core_interface.f90 with ifort, not ifx -e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort") +if (NOT MPILIB STREQUAL "openmpi") + e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort") +endif() From 004c1870aa962eab2287fc99b6831c000a4a0bab Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 28 Aug 2024 15:18:23 -0700 Subject: [PATCH 592/904] add several SHOC variables to be output for diagnostic purposes --- .../shoc/eamxx_shoc_process_interface.cpp | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 657f9487611c..f07912796002 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -91,6 +91,20 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("ustar", scalar2d, m/s, grid_name, ps); add_field("obklen", scalar2d, m, grid_name, ps); + // Diagnostic output - mid point grid + add_field("brunt", scalar3d_mid, pow(s,-1), grid_name, ps); + add_field("shoc_mix", scalar3d_mid, m, grid_name, ps); + add_field("isotropy", scalar3d_mid, s, grid_name, ps); + + // Diagnostic output - interface grid + add_field("wthl_sec", scalar3d_int, K*(m/s), grid_name, ps); + add_field("thl_sec", scalar3d_int, pow(K,2), grid_name, ps); + add_field("wqw_sec", scalar3d_int, (kg/kg)*(m/s), grid_name, ps); + add_field("qw_sec", scalar3d_int, pow(kg/kg,2), grid_name, ps); + add_field("uw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("vw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("w3", scalar3d_int, pow(m/s,3), grid_name, ps); + // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -495,6 +509,38 @@ void SHOCMacrophysics::run_impl (const double dt) default_policy, shoc_postprocess); Kokkos::fence(); + + // SHOC output diagnostics + const auto& shoc_mix = get_field_out("shoc_mix").get_view(); + Kokkos::deep_copy(shoc_mix,history_output.shoc_mix); + + const auto& brunt = get_field_out("brunt").get_view(); + Kokkos::deep_copy(brunt,history_output.brunt); + + const auto& w3 = get_field_out("w3").get_view(); + Kokkos::deep_copy(w3,history_output.w3); + + const auto& isotropy = get_field_out("isotropy").get_view(); + Kokkos::deep_copy(isotropy,history_output.isotropy); + + const auto& wthl_sec = get_field_out("wthl_sec").get_view(); + Kokkos::deep_copy(wthl_sec,history_output.wthl_sec); + + const auto& wqw_sec = get_field_out("wqw_sec").get_view(); + Kokkos::deep_copy(wqw_sec,history_output.wqw_sec); + + const auto& uw_sec = get_field_out("uw_sec").get_view(); + Kokkos::deep_copy(uw_sec,history_output.uw_sec); + + const auto& vw_sec = get_field_out("vw_sec").get_view(); + Kokkos::deep_copy(vw_sec,history_output.vw_sec); + + const auto& qw_sec = get_field_out("qw_sec").get_view(); + Kokkos::deep_copy(qw_sec,history_output.qw_sec); + + const auto& thl_sec = get_field_out("thl_sec").get_view(); + Kokkos::deep_copy(thl_sec,history_output.thl_sec); + } // ========================================================================================= void SHOCMacrophysics::finalize_impl() From 050d14990f5420c51fa55a03be332d49854d4a7e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 28 Aug 2024 16:12:23 -0700 Subject: [PATCH 593/904] Fixes namelist to enable fine grid simulations --- .../cime_config/namelist_defaults_scream.xml | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index a785fb322912..dadb6dbba35b 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,7 +269,7 @@ be lost if SCREAM_HACK_XML is not enabled. - + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc @@ -279,20 +279,19 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc - - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120np4_mono_20220502.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc From 78c4a4848a04a21fffc6100d5fb148f927775085 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 28 Aug 2024 22:01:41 -0500 Subject: [PATCH 594/904] remove some white spaces --- .../atmosphere_surface_coupling_importer.cpp | 4 +-- .../homme/interface/phys_grid_mod.F90 | 6 ++-- .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 36 +++++++++---------- .../mct_coupling/scream_cxx_f90_interface.cpp | 12 +++---- .../eamxx/src/mct_coupling/scream_f2c_mod.F90 | 2 +- .../src/share/atm_process/SCDataManager.hpp | 6 ++-- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 7686319ad3d8..ee3e21e7461b 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -80,7 +80,7 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr 0) then appname="HM_COARSE"//C_NULL_CHAR - ATM_ID1 = 120 ! + ATM_ID1 = 120 ! ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, MHID) if (ierr > 0 ) & call abortmp('Error: cannot register moab app') @@ -641,7 +641,7 @@ subroutine phys_grid_init (pgN) endif appname="HM_PGX"//C_NULL_CHAR ATM_ID1 = ATMID(1) ! this number should not conflict with other components IDs; how do we know? - ! + ! ! in this case, we reuse the main atm id, mhid will not be used for intersection anymore ! still, need to be careful ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, mhpgid) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index b5513afb4cca..9f976d6f0c27 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -19,7 +19,7 @@ module atm_comp_mct #ifdef HAVE_MOAB use seq_comm_mct , only: mphaid ! atm physics grid id in MOAB, on atm pes - use iso_c_binding + use iso_c_binding #ifdef MOABCOMP use seq_comm_mct, only: seq_comm_compare_mb_mct #endif @@ -55,7 +55,7 @@ module atm_comp_mct #ifdef HAVE_MOAB - ! to store all fields to be set in moab + ! to store all fields to be set in moab integer , private :: mblsize, totalmbls, nsend, totalmbls_r, nrecv real(r8) , allocatable, target, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes real(r8) , allocatable, target, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes @@ -63,7 +63,7 @@ module atm_comp_mct integer :: mpicom_atm_moab ! used just for mpi-reducing the difference between moab tags and mct avs integer :: rank2 #endif -#endif +#endif !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -128,7 +128,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) character(CL) :: calendar ! calendar string #ifdef HAVE_MOAB integer :: ent_type - character(CXX) :: tagname ! will store all seq_flds_a2x_fields + character(CXX) :: tagname ! will store all seq_flds_a2x_fields #ifdef MOABDEBUG character*100 outfile, wopts #endif @@ -210,7 +210,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call atm_domain_mct (lsize, gsMap_atm, dom_atm) #ifdef HAVE_MOAB - call moab_atm_phys_scream() + call moab_atm_phys_scream() #ifdef MOABCOMP mpicom_atm_moab = mpicom_atm ! just to store it to be used later in MOABCOMP rank2 = my_task ! again, just to use it for MOABCOMP output @@ -242,7 +242,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) a2x_am = 0. ! initialize everything with 0 nrecv = mct_avect_nRattr(x2a) - totalmbls_r = mblsize * nrecv ! size of the double array used to receive + totalmbls_r = mblsize * nrecv ! size of the double array used to receive allocate (x2a_am(mblsize, nrecv) ) ! these will be received by moab tags, then used to set cam in surf data x2a_am = 0. ! initialize everything with 0 #endif @@ -268,9 +268,9 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(trim(username),username_c) call string_f2c(trim(hostname),hostname_c) call scream_init_atm (caseid_c,hostname_c,username_c) -#ifdef HAVE_MOAB +#ifdef HAVE_MOAB ! data should be set now inside moab from import and export fields - ! do we import and export or just export at init stage ? + ! do we import and export or just export at init stage ? tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR ent_type = 0 ! vertices, point cloud ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, totalmbls , ent_type, a2x_am ) @@ -293,7 +293,7 @@ end subroutine atm_init_mct subroutine atm_run_mct(EClock, cdata, x2a, a2x) use iso_c_binding, only: c_double use scream_f2c_mod, only: scream_run -#ifdef HAVE_MOAB +#ifdef HAVE_MOAB use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields use shr_kind_mod , only: cxx=>shr_kind_cxx use iMOAB, only: iMOAB_SetDoubleTagStorage, iMOAB_GetDoubleTagStorage @@ -309,7 +309,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! character(CXX) :: mct_field, modelStr -#endif +#endif #endif ! !INPUT/OUTPUT PARAMETERS: @@ -325,7 +325,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) type(mct_gGrid) , pointer :: ggrid real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream -#ifdef HAVE_MOAB +#ifdef HAVE_MOAB integer :: ierr character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields #ifdef MOABDEBUG @@ -365,12 +365,12 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) tagname=trim(seq_flds_x2a_fields)//C_NULL_CHAR ent_type = 0 ! vertices, point cloud ierr = iMOAB_GetDoubleTagStorage ( mphaid, tagname, totalmbls_r , ent_type, x2a_am ) - + #endif ! Run scream call scream_run( dt_scream ) -#ifdef HAVE_MOAB +#ifdef HAVE_MOAB ! export data to moab data structures tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR ent_type = 0 ! vertices, point cloud @@ -517,9 +517,9 @@ subroutine moab_atm_phys_scream() #ifdef MOABDEBUG character*100 outfile, wopts #endif - character(CXX) :: tagname ! will store all seq_flds_a2x_fields + character(CXX) :: tagname ! will store all seq_flds_a2x_fields character*32 appname - integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) + integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) !------------------------------------------------------------------- ! ! Local Variables @@ -543,7 +543,7 @@ subroutine moab_atm_phys_scream() if (my_task == master_task) then print *, " moab_atm_phys_scream:: register MOAB app:", trim(appname), " mphaid=", mphaid endif - ! find global ids + ! find global ids num_local_cols = scream_get_num_local_cols() num_global_cols = scream_get_num_global_cols() @@ -644,7 +644,7 @@ subroutine moab_atm_phys_scream() call mpi_abort(mpicom_atm,ierr,mpi_ierr) endif #endif - ! define fields seq_flds_a2x_fields + ! define fields seq_flds_a2x_fields tagtype = 1 ! dense, double numco = 1 ! one value per vertex / entity tagname = trim(seq_flds_a2x_fields)//C_NULL_CHAR @@ -653,7 +653,7 @@ subroutine moab_atm_phys_scream() print *, "Error: fail to define seq_flds_a2x_fields " call mpi_abort(mpicom_atm,ierr,mpi_ierr) endif - ! make sure this is defined too; it could have the same fields, but in different order, or really different + ! make sure this is defined too; it could have the same fields, but in different order, or really different ! fields; need to make sure we have them tagname = trim(seq_flds_x2a_fields)//C_NULL_CHAR ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 52e0d692e0a5..4d70b3109f9e 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -165,18 +165,18 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, } void scream_setup_surface_coupling (const char*& import_field_names, int*& import_cpl_indices, - double*& x2a_ptr, + double*& x2a_ptr, #ifdef HAVE_MOAB double*& x2a_moab_ptr, -#endif +#endif int*& import_vector_components, double*& import_constant_multiple, bool*& do_import_during_init, const int& num_cpl_imports, const int& num_scream_imports, const int& import_field_size, char*& export_field_names, int*& export_cpl_indices, - double*& a2x_ptr, + double*& a2x_ptr, #ifdef HAVE_MOAB double*& a2x_moab_ptr, -#endif +#endif int*& export_vector_components, double*& export_constant_multiple, bool*& do_export_during_init, const int& num_cpl_exports, const int& num_scream_exports, const int& export_field_size) @@ -207,14 +207,14 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor num_cpl_imports, num_scream_imports, import_field_size, x2a_ptr, #ifdef HAVE_MOAB x2a_moab_ptr, -#endif +#endif names_in[0], import_cpl_indices, import_vector_components, import_constant_multiple, do_import_during_init); ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Export, num_cpl_exports, num_scream_exports, export_field_size, a2x_ptr, #ifdef HAVE_MOAB a2x_moab_ptr, -#endif +#endif names_out[0], export_cpl_indices, export_vector_components, export_constant_multiple, do_export_during_init); }); diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 5e09def98df0..74b45f8330b9 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -58,7 +58,7 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices a2x_ptr, & #ifdef HAVE_MOAB a2x_moab_ptr, & -#endif +#endif export_vector_components, & export_constant_multiple, do_export_during_init, & num_cpl_exports, num_scream_exports, export_field_size) bind(c) diff --git a/components/eamxx/src/share/atm_process/SCDataManager.hpp b/components/eamxx/src/share/atm_process/SCDataManager.hpp index 2dcfd3c5016c..76bc9e5bd843 100644 --- a/components/eamxx/src/share/atm_process/SCDataManager.hpp +++ b/components/eamxx/src/share/atm_process/SCDataManager.hpp @@ -22,10 +22,10 @@ struct SCDataManager { ~SCDataManager() = default; void setup_internals (const int num_cpl_fields, const int num_scream_fields, const int field_size, - Real* field_data_ptr, + Real* field_data_ptr, #ifdef HAVE_MOAB Real* field_data_moab_ptr, -#endif +#endif char* field_names, int* field_cpl_indices_ptr, int* field_vector_components_ptr, Real* field_constant_multiple_ptr, bool* transfer_during_init_ptr) @@ -117,7 +117,7 @@ struct SCDataManager { view_2d m_field_data; #ifdef HAVE_MOAB - view_2d m_field_data_moab; + view_2d m_field_data_moab; #endif name_t* m_field_names; view_1d m_field_cpl_indices; From c9d94bbdfdf1a622e74bc001dbd126f78fc1d11c Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Wed, 28 Aug 2024 22:25:14 -0500 Subject: [PATCH 595/904] Put modified tracer_data.F90 on the correct path --- components/eam/src/chemistry/utils/tracer_data.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/chemistry/utils/tracer_data.F90 b/components/eam/src/chemistry/utils/tracer_data.F90 index 073aacff6e47..62ccbcef8a25 100644 --- a/components/eam/src/chemistry/utils/tracer_data.F90 +++ b/components/eam/src/chemistry/utils/tracer_data.F90 @@ -660,7 +660,7 @@ subroutine advance_trcdata( flds, file, state, pbuf2d ) call t_startf('read_next_trcdata') call read_next_trcdata(state, flds, file ) call t_stopf('read_next_trcdata') - if(masterproc) write(iulog,*) 'READ_NEXT_TRCDATA ', flds%fldnam + if(masterproc) write(iulog,*) 'READ_NEXT_TRCDATA ', flds%fldnam,data_time end if endif From c5eb35f37c838964d732226c8a5b659b6387a5a8 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Wed, 28 Aug 2024 22:27:03 -0500 Subject: [PATCH 596/904] Remove the mis-placed tracer_data.F90 --- .../eam/src/chemistry/mozart/tracer_data.F90 | 2884 ----------------- 1 file changed, 2884 deletions(-) delete mode 100644 components/eam/src/chemistry/mozart/tracer_data.F90 diff --git a/components/eam/src/chemistry/mozart/tracer_data.F90 b/components/eam/src/chemistry/mozart/tracer_data.F90 deleted file mode 100644 index 62ccbcef8a25..000000000000 --- a/components/eam/src/chemistry/mozart/tracer_data.F90 +++ /dev/null @@ -1,2884 +0,0 @@ -module tracer_data -!----------------------------------------------------------------------- -! module used to read (and interpolate) offline tracer data (sources and -! mixing ratios) -! Created by: Francis Vitt -- 2 May 2006 -! Modified by : Jim Edwards -- 10 March 2009 -! Modified by : Cheryl Craig and Chih-Chieh (Jack) Chen -- February 2010 -! Modified by : Jinbo Xie --March 2023 -! Added a new option in interpolate_trcdata to work on linoz -! inputdata. A new UCI interpolation that better conserves -! mass in implemented and added a new function. It is called -! when linoz data are used in interpolate_trcdata. -! -!----------------------------------------------------------------------- - - use perf_mod, only : t_startf, t_stopf - use shr_kind_mod, only : r8 => shr_kind_r8,r4 => shr_kind_r4, shr_kind_cl, SHR_KIND_CS - use time_manager, only : get_curr_date, get_step_size, get_curr_calday - use spmd_utils, only : masterproc - use ppgrid, only : pcols, pver, pverp, begchunk, endchunk - use cam_abortutils, only : endrun - use cam_logfile, only : iulog - - use physics_buffer, only : physics_buffer_desc, pbuf_get_field, pbuf_get_index - use time_manager, only : set_time_float_from_date, set_date_from_time_float - use pio, only : file_desc_t, var_desc_t, & - pio_seterrorhandling, pio_internal_error, pio_bcast_error, & - pio_setdebuglevel, & - pio_char, pio_noerr, & - pio_inq_dimid, pio_inq_varid, & - pio_def_dim, pio_def_var, & - pio_put_att, pio_put_var, & - pio_get_var, pio_get_att, pio_nowrite, pio_inq_dimlen, & - pio_inq_vardimid, pio_inq_dimlen, pio_closefile, & - pio_inquire_variable - - implicit none - - private ! all unless made public - save - - public :: trfld, input3d, input2d, trfile - public :: trcdata_init - public :: advance_trcdata - public :: get_fld_data - public :: put_fld_data - public :: get_fld_ndx - public :: write_trc_restart - public :: read_trc_restart - public :: init_trc_restart - public :: incr_filename - !added public for linoz new function diagnostic - public :: read_next_trcdata - public :: interpolate_trcdata - public :: get_model_time - !!PUBLIC MEMBERS - - type input3d - real(r8), dimension(:,:,:), pointer :: data => null() - endtype input3d - - type input2d - real(r8), dimension(:,:), pointer :: data => null() - endtype input2d - - type trfld - real(r8), dimension(:,:,:), pointer :: data => null() - type(input3d), dimension(4) :: input - character(len=32) :: srcnam - character(len=32) :: fldnam - character(len=32) :: units - type(var_desc_t) :: var_id - integer :: coords(4) ! LATDIM | LONDIM | LEVDIM | TIMDIM - integer :: order(4) ! LATDIM | LONDIM | LEVDIM | TIMDIM - logical :: srf_fld = .false. - integer :: pbuf_ndx = -1 - endtype trfld - - type trfile - type(input2d), dimension(4) :: ps_in - character(len=shr_kind_cl) :: pathname = ' ' - character(len=shr_kind_cl) :: curr_filename = ' ' - character(len=shr_kind_cl) :: next_filename = ' ' - type(file_desc_t) :: curr_fileid - type(file_desc_t) :: next_fileid - - type(var_desc_t), pointer :: currfnameid => null() ! pio restart file var id - type(var_desc_t), pointer :: nextfnameid => null() ! pio restart file var id - - character(len=shr_kind_cl) :: filenames_list = '' - real(r8) :: datatimem = -1.e36_r8 ! time of prv. values read in - real(r8) :: datatimep = -1.e36_r8 ! time of nxt. values read in - real(r8) :: datatimes(4) - integer :: interp_recs - real(r8), pointer, dimension(:) :: curr_data_times => null() - real(r8), pointer, dimension(:) :: next_data_times => null() - logical :: remove_trc_file = .false. ! delete file when finished with it - real(r8) :: offset_time - integer :: cyc_ndx_beg - integer :: cyc_ndx_end - integer :: cyc_yr = 0 - real(r8) :: one_yr = 0 - real(r8) :: curr_mod_time ! model time - calendar day - real(r8) :: next_mod_time ! model time - calendar day - next time step - integer :: nlon - integer :: nlat - integer :: nlev - integer :: nilev - integer :: ps_coords(3) ! LATDIM | LONDIM | TIMDIM - integer :: ps_order(3) ! LATDIM | LONDIM | TIMDIM - real(r8), pointer, dimension(:) :: lons => null() - real(r8), pointer, dimension(:) :: lats => null() - real(r8), pointer, dimension(:) :: levs => null() - real(r8), pointer, dimension(:) :: ilevs => null() - real(r8), pointer, dimension(:) :: hyam => null() - real(r8), pointer, dimension(:) :: hybm => null() - real(r8), pointer, dimension(:,:) :: ps => null() - real(r8), pointer, dimension(:) :: hyai => null() - real(r8), pointer, dimension(:) :: hybi => null() - real(r8), pointer, dimension(:,:) :: weight_x => null(), weight_y => null() - integer, pointer, dimension(:) :: count_x => null(), count_y => null() - integer, pointer, dimension(:,:) :: index_x => null(), index_y => null() - real(r8) :: p0 - type(var_desc_t) :: ps_id - logical, allocatable, dimension(:) :: in_pbuf - logical :: has_ps = .false. - logical :: zonal_ave = .false. - logical :: alt_data = .false. - logical :: cyclical = .false. - logical :: cyclical_list = .false. - logical :: weight_by_lat = .false. - logical :: conserve_column = .false. - logical :: fill_in_months = .false. - logical :: fixed = .false. - logical :: initialized = .false. - logical :: top_bndry = .false. - logical :: stepTime = .false. ! Do not interpolate in time, but use stepwise times - logical :: linoz_v3 = .false. !set for linoz_v3 interpolation only - logical :: linoz_v2 = .false. !set for linoz_v2 interpolation only - endtype trfile - - integer, public, parameter :: MAXTRCRS = 100 - - integer, parameter :: LONDIM = 1 - integer, parameter :: LATDIM = 2 - integer, parameter :: LEVDIM = 3 - integer, parameter :: TIMDIM = 4 - - integer, parameter :: PS_TIMDIM = 3 - - integer, parameter :: ZA_LATDIM = 1 - integer, parameter :: ZA_LEVDIM = 2 - integer, parameter :: ZA_TIMDIM = 3 - - integer, parameter :: nm=1 ! array index for previous (minus) data - integer, parameter :: np=2 ! array index for next (plus) data - - integer :: plon, plat - -contains - -!-------------------------------------------------------------------------- -!-------------------------------------------------------------------------- - subroutine trcdata_init( specifier, filename, filelist, datapath, flds, file, & - rmv_file, data_cycle_yr, data_fixed_ymd, data_fixed_tod, data_type ) - - use mo_constants, only : d2r - use cam_control_mod, only : nsrest - use dyn_grid, only : get_dyn_grid_parm - use string_utils, only : to_upper - use horizontal_interpolate, only : xy_interp_init -#if ( defined SPMD ) - use mpishorthand, only: mpicom, mpir8, mpiint -#endif - - implicit none - - character(len=*), intent(in) :: specifier(:) - character(len=*), intent(in) :: filename - character(len=*), intent(in) :: filelist - character(len=*), intent(in) :: datapath - type(trfld), dimension(:), pointer :: flds - type(trfile), intent(inout) :: file - logical, intent(in) :: rmv_file - integer, intent(in) :: data_cycle_yr - integer, intent(in) :: data_fixed_ymd - integer, intent(in) :: data_fixed_tod - character(len=*), intent(in) :: data_type - - integer :: f, mxnflds, astat - integer :: str_yr, str_mon, str_day - integer :: lon_dimid, lat_dimid, lev_dimid, tim_dimid, old_dimid - integer :: dimids(4), did - type(var_desc_t) :: varid - integer :: idx - integer :: ierr - integer :: errcode - real(r8) :: start_time, time1, time2 - integer :: i1,i2,j1,j2 - integer :: nvardims, vardimids(4) - - character(len=80) :: data_units - - call specify_fields( specifier, flds ) - - file%datatimep=-1.e36_r8 - file%datatimem=-1.e36_r8 - - mxnflds = 0 - if (associated(flds)) mxnflds = size( flds ) - - if (mxnflds < 1) return - - file%remove_trc_file = rmv_file - file%pathname = trim(datapath) - file%filenames_list = trim(filelist) - - file%fill_in_months = .false. - file%cyclical = .false. - file%cyclical_list = .false. - -! does not work when compiled with pathf90 -! select case ( to_upper(data_type) ) - select case ( data_type ) - case( 'FIXED' ) - file%fixed = .true. - case( 'INTERP_MISSING_MONTHS' ) - file%fill_in_months = .true. - case( 'CYCLICAL' ) - file%cyclical = .true. - file%cyc_yr = data_cycle_yr - case( 'CYCLICAL_LIST' ) - file%cyclical_list = .true. - file%cyc_yr = data_cycle_yr - case( 'SERIAL' ) - case default - write(iulog,*) 'trcdata_init: invalid data type: '//trim(data_type)//' file: '//trim(filename) - write(iulog,*) 'trcdata_init: valid data types: SERIAL | CYCLICAL | CYCLICAL_LIST | FIXED | INTERP_MISSING_MONTHS ' - call endrun('trcdata_init: invalid data type: '//trim(data_type)//' file: '//trim(filename)) - endselect - - if ( (.not.file%fixed) .and. ((data_fixed_ymd>0._r8) .or.(data_fixed_tod>0._r8))) then - call endrun('trcdata_init: Cannot specify data_fixed_ymd or data_fixed_tod if data type is not FIXED') - endif - if ( (.not.file%cyclical) .and. (data_cycle_yr>0._r8) ) then - call endrun('trcdata_init: Cannot specify data_cycle_yr if data type is not CYCLICAL') - endif - - if (masterproc) then - write(iulog,*) 'trcdata_init: data type: '//trim(data_type)//' file: '//trim(filename) - endif - - ! if there is no list of files (len_trim(file%filenames_list)<1) then - ! -> set curr_filename from namelist rather from restart data - if ( len_trim(file%curr_filename)<1 .or. len_trim(file%filenames_list)<1 .or. file%fixed ) then ! initial run - file%curr_filename = trim(filename) - - call get_model_time(file) - - if ( file%fixed ) then - str_yr = data_fixed_ymd/10000 - str_mon = (data_fixed_ymd - str_yr*10000)/100 - str_day = data_fixed_ymd - str_yr*10000 - str_mon*100 - call set_time_float_from_date( start_time, str_yr, str_mon, str_day, data_fixed_tod ) - file%offset_time = start_time - file%curr_mod_time - else - file%offset_time = 0 - endif - endif - - call set_time_float_from_date( time2, 2, 1, 1, 0 ) - call set_time_float_from_date( time1, 1, 1, 1, 0 ) - file%one_yr = time2-time1 - - if ( file%cyclical .or. file%cyclical_list) then - file%cyc_ndx_beg = -1 - file%cyc_ndx_end = -1 - if ( file%cyc_yr /= 0 ) then - call set_time_float_from_date( time1, file%cyc_yr , 1, 1, 0 ) - call set_time_float_from_date( time2, file%cyc_yr+1, 1, 1, 0 ) - file%one_yr = time2-time1 - endif - - call open_trc_datafile( file%curr_filename, file%pathname, file%curr_fileid, file%curr_data_times, & - cyc_ndx_beg=file%cyc_ndx_beg, cyc_ndx_end=file%cyc_ndx_end, cyc_yr=file%cyc_yr ) - else - call open_trc_datafile( file%curr_filename, file%pathname, file%curr_fileid, file%curr_data_times ) - file%curr_data_times = file%curr_data_times - file%offset_time - endif - - call pio_seterrorhandling(File%curr_fileid, PIO_BCAST_ERROR) - ierr = pio_inq_dimid( file%curr_fileid, 'lon', idx ) - call pio_seterrorhandling(File%curr_fileid, PIO_INTERNAL_ERROR) - - file%zonal_ave = (ierr/=PIO_NOERR) - - plon = get_dyn_grid_parm('plon') - plat = get_dyn_grid_parm('plat') - - if ( .not. file%zonal_ave ) then - - call get_dimension( file%curr_fileid, 'lon', file%nlon, dimid=old_dimid, data=file%lons ) - - file%lons = file%lons * d2r - - lon_dimid = old_dimid - - endif - - ierr = pio_inq_dimid( file%curr_fileid, 'time', old_dimid) - - ! Hack to work with weird netCDF and old gcc or NAG bug. - tim_dimid = old_dimid - - call get_dimension( file%curr_fileid, 'lat', file%nlat, dimid=old_dimid, data=file%lats ) - file%lats = file%lats * d2r - - lat_dimid = old_dimid - - allocate( file%ps(file%nlon,file%nlat), stat=astat ) - if( astat /= 0 ) then - write(iulog,*) 'trcdata_init: file%ps allocation error = ',astat - call endrun('trcdata_init: failed to allocate x array') - end if - - call pio_seterrorhandling(File%curr_fileid, PIO_BCAST_ERROR) - ierr = pio_inq_varid( file%curr_fileid, 'PS', file%ps_id ) - file%has_ps = (ierr==PIO_NOERR) - ierr = pio_inq_dimid( file%curr_fileid, 'altitude', idx ) - file%alt_data = (ierr==PIO_NOERR) - - call pio_seterrorhandling(File%curr_fileid, PIO_INTERNAL_ERROR) - - if ( file%has_ps) then - ierr = pio_inq_vardimid (file%curr_fileid, file%ps_id, dimids(1:3)) - do did = 1,3 - if ( dimids(did) == lon_dimid ) then - file%ps_coords(LONDIM) = did - file%ps_order(did) = LONDIM - else if ( dimids(did) == lat_dimid ) then - file%ps_coords(LATDIM) = did - file%ps_order(did) = LATDIM - else if ( dimids(did) == tim_dimid ) then - file%ps_coords(PS_TIMDIM) = did - file%ps_order(did) = PS_TIMDIM - endif - enddo - endif - - if (masterproc) then - write(iulog,*) 'trcdata_init: file%has_ps = ' , file%has_ps - endif ! masterproc - - if (file%alt_data) then - call get_dimension( file%curr_fileid, 'altitude_int', file%nilev, data=file%ilevs ) - call get_dimension( file%curr_fileid, 'altitude', file%nlev, dimid=old_dimid, data=file%levs ) - else - call get_dimension( file%curr_fileid, 'lev', file%nlev, dimid=old_dimid, data=file%levs ) - !!added for Linoz_v3 - call get_dimension( file%curr_fileid, 'ilev', file%nilev, data=file%ilevs ) - if (old_dimid>0) then - file%levs = file%levs*100._r8 ! mbar->pascals - endif - endif - - ! For some bizarre reason, netCDF with older gcc is keeping a pointer to the dimid, and overwriting it later! - ! Hackish workaround is to make a copy... - lev_dimid = old_dimid - - if (file%has_ps) then - - allocate( file%hyam(file%nlev), file%hybm(file%nlev), stat=astat ) - if( astat /= 0 ) then - write(iulog,*) 'trcdata_init: file%hyam,file%hybm allocation error = ',astat - call endrun('trcdata_init: failed to allocate file%hyam and file%hybm arrays') - end if - - allocate( file%hyai(file%nlev+1), file%hybi(file%nlev+1), stat=astat ) - if( astat /= 0 ) then - write(iulog,*) 'trcdata_init: file%hyai,file%hybi allocation error = ',astat - call endrun('trcdata_init: failed to allocate file%hyai and file%hybi arrays') - end if - - call pio_seterrorhandling(File%curr_fileid, PIO_BCAST_ERROR) - ierr = pio_inq_varid( file%curr_fileid, 'P0', varid) - call pio_seterrorhandling(File%curr_fileid, PIO_INTERNAL_ERROR) - - if ( ierr == PIO_NOERR ) then - ierr = pio_get_var( file%curr_fileid, varid, file%p0 ) - else - file%p0 = 100000._r8 - endif - ierr = pio_inq_varid( file%curr_fileid, 'hyam', varid ) - ierr = pio_get_var( file%curr_fileid, varid, file%hyam ) - ierr = pio_inq_varid( file%curr_fileid, 'hybm', varid ) - ierr = pio_get_var( file%curr_fileid, varid, file%hybm ) - if (file%conserve_column) then - ierr = pio_inq_varid( file%curr_fileid, 'hyai', varid ) - ierr = pio_get_var( file%curr_fileid, varid, file%hyai ) - ierr = pio_inq_varid( file%curr_fileid, 'hybi', varid ) - ierr = pio_get_var( file%curr_fileid, varid, file%hybi ) - endif - - allocate( file %ps (pcols,begchunk:endchunk), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate file%ps array; error = ',astat - call endrun - end if - allocate( file%ps_in(1)%data(pcols,begchunk:endchunk), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate file%ps_in(1)%data array; error = ',astat - call endrun - end if - allocate( file%ps_in(2)%data(pcols,begchunk:endchunk), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate file%ps_in(2)%data array; error = ',astat - call endrun - end if - if( file%fill_in_months ) then - allocate( file%ps_in(3)%data(pcols,begchunk:endchunk), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate file%ps_in(3)%data array; error = ',astat - call endrun - end if - allocate( file%ps_in(4)%data(pcols,begchunk:endchunk), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate file%ps_in(4)%data array; error = ',astat - call endrun - end if - end if - endif - - flds_loop: do f = 1,mxnflds - - ! initialize the coordinate values to -1, - ! to defend against fields that do not have certain dimension, e.g., zonal average surface fields - ! and check against it when assigning value to cnt3 for that dimension - - do did = 1,4 - flds(f)%coords(did) = -1 - end do - - ! get netcdf variable id for the field - ierr = pio_inq_varid( file%curr_fileid, flds(f)%srcnam, flds(f)%var_id ) - - ! determine if the field has a vertical dimension - if (lev_dimid>0) then - ierr = pio_inquire_variable( file%curr_fileid, flds(f)%var_id, ndims=nvardims ) - ierr = pio_inquire_variable( file%curr_fileid, flds(f)%var_id, dimids=vardimids(:nvardims) ) - flds(f)%srf_fld = .not.any(vardimids(:nvardims)==lev_dimid) - else - flds(f)%srf_fld = .true. - endif - - ! allocate memory only if not already in pbuf2d - - if ( .not. file%in_pbuf(f) ) then - if ( flds(f)%srf_fld .or. file%top_bndry ) then - allocate( flds(f) %data(pcols,1,begchunk:endchunk), stat=astat ) - else - allocate( flds(f) %data(pcols,pver,begchunk:endchunk), stat=astat ) - endif - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate flds(f)%data array; error = ',astat - call endrun - end if - else - flds(f)%pbuf_ndx = pbuf_get_index(flds(f)%fldnam,errcode) - endif - - if (flds(f)%srf_fld) then - allocate( flds(f)%input(1)%data(pcols,1,begchunk:endchunk), stat=astat ) - else - allocate( flds(f)%input(1)%data(pcols,file%nlev,begchunk:endchunk), stat=astat ) - endif - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate flds(f)%input(1)%data array; error = ',astat - call endrun - end if - if (flds(f)%srf_fld) then - allocate( flds(f)%input(2)%data(pcols,1,begchunk:endchunk), stat=astat ) - else - allocate( flds(f)%input(2)%data(pcols,file%nlev,begchunk:endchunk), stat=astat ) - endif - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate flds(f)%input(2)%data array; error = ',astat - call endrun - end if - - if( file%fill_in_months ) then - if (flds(f)%srf_fld) then - allocate( flds(f)%input(3)%data(pcols,1,begchunk:endchunk), stat=astat ) - else - allocate( flds(f)%input(3)%data(pcols,file%nlev,begchunk:endchunk), stat=astat ) - endif - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate flds(f)%input(3)%data array; error = ',astat - call endrun - end if - if (flds(f)%srf_fld) then - allocate( flds(f)%input(4)%data(pcols,1,begchunk:endchunk), stat=astat ) - else - allocate( flds(f)%input(4)%data(pcols,file%nlev,begchunk:endchunk), stat=astat ) - endif - if( astat/= 0 ) then - write(iulog,*) 'trcdata_init: failed to allocate flds(f)%input(4)%data array; error = ',astat - call endrun - end if - endif - - if ( file%zonal_ave ) then - ierr = pio_inq_vardimid (file%curr_fileid, flds(f)%var_id, dimids(1:3)) - do did = 1,3 - if ( dimids(did) == lat_dimid ) then - flds(f)%coords(ZA_LATDIM) = did - flds(f)%order(did) = ZA_LATDIM - else if ( dimids(did) == lev_dimid ) then - flds(f)%coords(ZA_LEVDIM) = did - flds(f)%order(did) = ZA_LEVDIM - else if ( dimids(did) == tim_dimid ) then - flds(f)%coords(ZA_TIMDIM) = did - flds(f)%order(did) = ZA_TIMDIM - endif - enddo - else if ( flds(f)%srf_fld ) then - ierr = pio_inq_vardimid (file%curr_fileid, flds(f)%var_id, dimids(1:3)) - do did = 1,3 - if ( dimids(did) == lon_dimid ) then - flds(f)%coords(LONDIM) = did - flds(f)%order(did) = LONDIM - else if ( dimids(did) == lat_dimid ) then - flds(f)%coords(LATDIM) = did - flds(f)%order(did) = LATDIM - else if ( dimids(did) == tim_dimid ) then - flds(f)%coords(PS_TIMDIM) = did - flds(f)%order(did) = PS_TIMDIM - endif - enddo - else - ierr = pio_inq_vardimid (file%curr_fileid, flds(f)%var_id, dimids) - do did = 1,4 - if ( dimids(did) == lon_dimid ) then - flds(f)%coords(LONDIM) = did - flds(f)%order(did) = LONDIM - else if ( dimids(did) == lat_dimid ) then - flds(f)%coords(LATDIM) = did - flds(f)%order(did) = LATDIM - else if ( dimids(did) == lev_dimid ) then - flds(f)%coords(LEVDIM) = did - flds(f)%order(did) = LEVDIM - else if ( dimids(did) == tim_dimid ) then - flds(f)%coords(TIMDIM) = did - flds(f)%order(did) = TIMDIM - endif - enddo - endif - - ierr = pio_get_att( file%curr_fileid, flds(f)%var_id, 'units', data_units) - data_units = trim(data_units) - flds(f)%units = data_units(1:32) - - enddo flds_loop - -! if weighting by latitude, compute weighting for horizontal interpolation - if( file%weight_by_lat ) then -! get dimensions of CAM resolution - plon = get_dyn_grid_parm('plon') - plat = get_dyn_grid_parm('plat') - -! weight_x & weight_y are weighting function for x & y interpolation - allocate(file%weight_x(plon,file%nlon)) - allocate(file%weight_y(plat,file%nlat)) - allocate(file%count_x(plon)) - allocate(file%count_y(plat)) - allocate(file%index_x(plon,file%nlon)) - allocate(file%index_y(plat,file%nlat)) - file%weight_x(:,:) = 0.0_r8 - file%weight_y(:,:) = 0.0_r8 - file%count_x(:) = 0 - file%count_y(:) = 0 - file%index_x(:,:) = 0 - file%index_y(:,:) = 0 - - if(masterproc) then -! compute weighting - call xy_interp_init(file%nlon,file%nlat,file%lons,file%lats,plon,plat,file%weight_x,file%weight_y) - - do i2=1,plon - file%count_x(i2) = 0 - do i1=1,file%nlon - if(file%weight_x(i2,i1).gt.0.0_r8 ) then - file%count_x(i2) = file%count_x(i2) + 1 - file%index_x(i2,file%count_x(i2)) = i1 - endif - enddo - enddo - - do j2=1,plat - file%count_y(j2) = 0 - do j1=1,file%nlat - if(file%weight_y(j2,j1).gt.0.0_r8 ) then - file%count_y(j2) = file%count_y(j2) + 1 - file%index_y(j2,file%count_y(j2)) = j1 - endif - enddo - enddo - endif - -#if ( defined SPMD) - call mpibcast(file%weight_x, plon*file%nlon, mpir8 , 0, mpicom) - call mpibcast(file%weight_y, plat*file%nlat, mpir8 , 0, mpicom) - call mpibcast(file%count_x, plon, mpiint , 0, mpicom) - call mpibcast(file%count_y, plat, mpiint , 0, mpicom) - call mpibcast(file%index_x, plon*file%nlon, mpiint , 0, mpicom) - call mpibcast(file%index_y, plat*file%nlat, mpiint , 0, mpicom) -#endif - endif - - end subroutine trcdata_init - -!----------------------------------------------------------------------- -! Reads more data if needed and interpolates data to current model time -!----------------------------------------------------------------------- - subroutine advance_trcdata( flds, file, state, pbuf2d ) - use physics_types,only : physics_state - use physics_buffer, only : physics_buffer_desc - use ppgrid, only : pver,pcols - - implicit none - - type(trfile), intent(inout) :: file - type(trfld), intent(inout) :: flds(:) - type(physics_state), intent(in) :: state(begchunk:endchunk) - - type(physics_buffer_desc), optional, pointer :: pbuf2d(:,:) - integer :: ncol - real(r8) :: data_time - real(r8) :: t(pcols,pver) ! input temperature (K) - real(r8) :: rho(pcols,pver) ! input temperature (K) - real(r8) :: pmid(pcols,pver) ! pressure at layer midpoints (pa) -!--------------------------------------BLH----------------------------------- - - call t_startf('advance_trcdata') - if ( .not.( file%fixed .and. file%initialized ) ) then - - call get_model_time(file) - - data_time = file%datatimep - - if ( file%cyclical .or. file%cyclical_list ) then - ! wrap around - if ( (file%datatimepfile%datatimem) ) then - data_time = data_time + file%one_yr - endif - endif - - ! For stepTime need to advance if the times are equal - ! Should not impact other runs? - if ( file%curr_mod_time >= data_time ) then - call t_startf('read_next_trcdata') - call read_next_trcdata(state, flds, file ) - call t_stopf('read_next_trcdata') - if(masterproc) write(iulog,*) 'READ_NEXT_TRCDATA ', flds%fldnam,data_time - end if - - endif - - ! need to interpolate the data, regardless - ! each mpi task needs to interpolate - call t_startf('interpolate_trcdata') - if(present(pbuf2d)) then - call interpolate_trcdata( state, flds, file, pbuf2d ) - else - call interpolate_trcdata( state, flds, file ) - endif - call t_stopf('interpolate_trcdata') - - file%initialized = .true. - - call t_stopf('advance_trcdata') - - end subroutine advance_trcdata - -!------------------------------------------------------------------- -!------------------------------------------------------------------- - subroutine get_fld_data( flds, field_name, data, ncol, lchnk, pbuf ) - - use physics_buffer, only : physics_buffer_desc, pbuf_get_field - - implicit none - - type(trfld), intent(inout) :: flds(:) - character(len=*), intent(in) :: field_name - real(r8), intent(out) :: data(:,:) - integer, intent(in) :: lchnk - integer, intent(in) :: ncol - type(physics_buffer_desc), pointer :: pbuf(:) - - - integer :: f, nflds - real(r8),pointer :: tmpptr(:,:) - - data(:,:) = 0._r8 - nflds = size(flds) - - do f = 1, nflds - if ( trim(flds(f)%fldnam) == trim(field_name) ) then - if ( flds(f)%pbuf_ndx>0 ) then - call pbuf_get_field(pbuf, flds(f)%pbuf_ndx, tmpptr) - data(:ncol,:) = tmpptr(:ncol,:) - else - data(:ncol,:) = flds(f)%data(:ncol,:,lchnk) - endif - endif - enddo - - end subroutine get_fld_data - -!------------------------------------------------------------------- -!------------------------------------------------------------------- - subroutine put_fld_data( flds, field_name, data, ncol, lchnk, pbuf ) - - use physics_buffer, only : physics_buffer_desc, pbuf_get_field - - implicit none - - type(trfld), intent(inout) :: flds(:) - character(len=*), intent(in) :: field_name - real(r8), intent(in) :: data(:,:) - integer, intent(in) :: lchnk - integer, intent(in) :: ncol - type(physics_buffer_desc), pointer :: pbuf(:) - - - integer :: f, nflds - real(r8),pointer :: tmpptr(:,:) - - nflds = size(flds) - - do f = 1, nflds - if ( trim(flds(f)%fldnam) == trim(field_name) ) then - if ( flds(f)%pbuf_ndx>0 ) then - call pbuf_get_field(pbuf, flds(f)%pbuf_ndx, tmpptr) - tmpptr(:ncol,:) = data(:ncol,:) - else - flds(f)%data(:ncol,:,lchnk) = data(:ncol,:) - endif - endif - enddo - - end subroutine put_fld_data - -!------------------------------------------------------------------- -!------------------------------------------------------------------- - subroutine get_fld_ndx( flds, field_name, idx ) - - implicit none - - type(trfld), intent(in) :: flds(:) - character(len=*), intent(in) :: field_name - integer, intent(out) :: idx - integer :: f, nflds - - idx = -1 - nflds = size(flds) - - do f = 1, nflds - if ( trim(flds(f)%fldnam) == trim(field_name) ) then - idx = f - return - endif - enddo - - end subroutine get_fld_ndx - -!------------------------------------------------------------------------------ -!------------------------------------------------------------------------------ - subroutine get_model_time(file) - implicit none - type(trfile), intent(inout) :: file - - integer yr, mon, day, ncsec ! components of a date - - call get_curr_date(yr, mon, day, ncsec) - - if ( file%cyclical .or. file%cyclical_list) yr = file%cyc_yr - call set_time_float_from_date( file%curr_mod_time, yr, mon, day, ncsec ) - file%next_mod_time = file%curr_mod_time + get_step_size()/86400._r8 - - end subroutine get_model_time - -!------------------------------------------------------------------------------ -!------------------------------------------------------------------------------ - subroutine check_files( file, fids, itms, times_found) - - implicit none - - type(trfile), intent(inout) :: file - type(file_desc_t), intent(out) :: fids(2) ! ids of files that contains these recs - integer, optional, intent(out) :: itms(2) - logical, optional, intent(inout) :: times_found - - !----------------------------------------------------------------------- - ! ... local variables - !----------------------------------------------------------------------- - logical :: list_cycled - - list_cycled = .false. - - !----------------------------------------------------------------------- - ! If next time beyond the end of the time list, - ! then increment the filename and move on to the next file - !----------------------------------------------------------------------- - if ((file%next_mod_time > file%curr_data_times(size(file%curr_data_times))).or.file%cyclical_list) then - if (file%cyclical_list) then - if ( associated(file%next_data_times) ) then - if ((file%curr_mod_time > file%datatimep)) then - - call advance_file(file) - - endif - endif - - endif - if ( .not. associated(file%next_data_times) ) then - ! open next file if not already opened... - if (file%cyclical_list) then - file%next_filename = incr_filename( file%curr_filename, filenames_list=file%filenames_list, datapath=file%pathname ,& - cyclical_list=file%cyclical_list, list_cycled=list_cycled) - else - file%next_filename = incr_filename( file%curr_filename, filenames_list=file%filenames_list, datapath=file%pathname) - endif - call open_trc_datafile( file%next_filename, file%pathname, file%next_fileid, file%next_data_times ) - file%next_data_times = file%next_data_times - file%offset_time - endif - endif - - !----------------------------------------------------------------------- - ! If using next_data_times and the current is greater than or equal to the next, then - ! close the current file, and set up for next file. - !----------------------------------------------------------------------- - if ( associated(file%next_data_times) ) then - if (file%cyclical_list .and. list_cycled) then ! special case - list cycled - - file%datatimem = file%curr_data_times(size(file%curr_data_times)) - itms(1)=size(file%curr_data_times) - fids(1)=file%curr_fileid - - file%datatimep = file%next_data_times(1) - itms(2)=1 - fids(2) = file%next_fileid - - times_found = .true. - - else if (file%curr_mod_time >= file%next_data_times(1)) then - - call advance_file(file) - - endif - endif - - end subroutine check_files - -!----------------------------------------------------------------------- -!----------------------------------------------------------------------- - function incr_filename( filename, filenames_list, datapath, cyclical_list, list_cycled ) - - !----------------------------------------------------------------------- - ! ... Increment or decrement a date string withing a filename - ! the filename date section is assumed to be of the form - ! yyyy-dd-mm - !----------------------------------------------------------------------- - - use string_utils, only : incstr - use shr_file_mod, only : shr_file_getunit, shr_file_freeunit - - implicit none - - - character(len=*), intent(in) :: filename ! present dynamical dataset filename - character(len=*), optional, intent(in) :: filenames_list - character(len=*), optional, intent(in) :: datapath - logical , optional, intent(in) :: cyclical_list ! If true, allow list to cycle - logical , optional, intent(out) :: list_cycled - character(len=shr_kind_cl) :: incr_filename ! next filename in the sequence - - - ! set new next_filename ... - - !----------------------------------------------------------------------- - ! ... local variables - !----------------------------------------------------------------------- - integer :: pos, pos1, istat - character(len=shr_kind_cl) :: fn_new, line, filepath - character(len=6) :: seconds - character(len=5) :: num - integer :: ios,unitnumber - - if (present(list_cycled)) list_cycled = .false. - - if (( .not. present(filenames_list)) .or.(len_trim(filenames_list) == 0)) then - !----------------------------------------------------------------------- - ! ... ccm type filename - !----------------------------------------------------------------------- - pos = len_trim( filename ) - fn_new = filename(:pos) - if ( masterproc ) write(iulog,*) 'incr_flnm: old filename = ',trim(fn_new) - if( fn_new(pos-2:) == '.nc' ) then - pos = pos - 3 - end if - istat = incstr( fn_new(:pos), 1 ) - if( istat /= 0 ) then - write(iulog,*) 'incr_flnm: incstr returned ', istat - write(iulog,*) ' while trying to decrement ',trim( fn_new ) - call endrun - end if - - else - - !------------------------------------------------------------------- - ! ... open filenames_list - !------------------------------------------------------------------- - if ( masterproc ) write(iulog,*) 'incr_flnm: old filename = ',trim(filename) - if ( masterproc ) write(iulog,*) 'incr_flnm: open filenames_list : ',trim(filenames_list) - unitnumber = shr_file_getUnit() - if ( present(datapath) ) then - filepath = trim(datapath) //'/'// trim(filenames_list) - else - filepath = trim(datapath) - endif - - open( unit=unitnumber, file=filepath, iostat=ios, status="OLD") - if (ios /= 0) then - call endrun('not able to open filenames_list file: '//trim(filepath)) - endif - - !------------------------------------------------------------------- - ! ... read file names - !------------------------------------------------------------------- - read( unit=unitnumber, fmt='(A)', iostat=ios ) line - if (ios /= 0) then - call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) - endif - - !------------------------------------------------------------------- - ! If current filename is '', then initialize with the first filename read in - ! and skip this section. - !------------------------------------------------------------------- - if (filename /= '') then - - !------------------------------------------------------------------- - ! otherwise read until find current filename - !------------------------------------------------------------------- - do while( trim(line) /= trim(filename) ) - read( unit=unitnumber, fmt='(A)', iostat=ios ) line - if (ios /= 0) then - call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) - endif - enddo - - !------------------------------------------------------------------- - ! Read next filename - !------------------------------------------------------------------- - read( unit=unitnumber, fmt='(A)', iostat=ios ) line - - !--------------------------------------------------------------------------------- - ! If cyclical_list, then an end of file is not an error, but rather - ! a signal to rewind and start over - !--------------------------------------------------------------------------------- - - if (ios /= 0) then - if (present(cyclical_list)) then - if (cyclical_list) then - list_cycled=.true. - rewind(unitnumber) - read( unit=unitnumber, fmt='(A)', iostat=ios ) line - ! Error here should never happen, but check just in case - if (ios /= 0) then - call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) - endif - else - call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) - endif - else - call endrun('not able to increment file name from filenames_list file: '//trim(filenames_list)) - endif - endif - - endif - - !--------------------------------------------------------------------------------- - ! Assign the current filename and close the filelist - !--------------------------------------------------------------------------------- - fn_new = trim(line) - - close(unit=unitnumber) - call shr_file_freeUnit(unitnumber) - endif - - !--------------------------------------------------------------------------------- - ! return the current filename - !--------------------------------------------------------------------------------- - incr_filename = trim(fn_new) - if ( masterproc ) write(iulog,*) 'incr_flnm: new filename = ',trim(incr_filename) - - end function incr_filename - -!------------------------------------------------------------------------------ -!------------------------------------------------------------------------------ - subroutine find_times( itms, fids, time, file, datatimem, datatimep, times_found ) - - implicit none - - type(trfile), intent(in) :: file - real(r8), intent(out) :: datatimem, datatimep - - integer, intent(out) :: itms(2) ! record numbers that bracket time - type(file_desc_t), intent(out) :: fids(2) ! ids of files that contains these recs - - real(r8), intent(in) :: time ! time of interest - logical, intent(inout) :: times_found - - integer :: np1 ! current forward time index of dataset - integer :: n,i ! - integer :: curr_tsize, next_tsize, all_tsize - integer :: astat - integer :: cyc_tsize - - real(r8), allocatable, dimension(:):: all_data_times - - curr_tsize = size(file%curr_data_times) - next_tsize = 0 - if ( associated(file%next_data_times)) next_tsize = size(file%next_data_times) - - all_tsize = curr_tsize + next_tsize - - allocate( all_data_times( all_tsize ), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'find_times: failed to allocate all_data_times array; error = ',astat - call endrun - end if - - all_data_times(:curr_tsize) = file%curr_data_times(:) - if (next_tsize > 0) all_data_times(curr_tsize+1:all_tsize) = file%next_data_times(:) - - if ( .not. file%cyclical ) then - if ( all( all_data_times(:) > time ) ) then - write(iulog,*) 'FIND_TIMES: ALL data times are after ', time - write(iulog,*) 'FIND_TIMES: data times: ',all_data_times(:) - write(iulog,*) 'FIND_TIMES: time: ',time - call endrun('find_times: all(all_data_times(:) > time) '// trim(file%curr_filename) ) - endif - - ! find bracketing times - find_times_loop : do n=1, all_tsize-1 - np1 = n + 1 - datatimem = all_data_times(n) !+ file%offset_time - datatimep = all_data_times(np1) !+ file%offset_time - ! When stepTime, datatimep may not equal the time (as only datatimem is used) - ! Should not break other runs? - if ( (time .ge. datatimem) .and. (time .lt. datatimep) ) then - times_found = .true. - exit find_times_loop - endif - enddo find_times_loop - - else ! file%cyclical - - cyc_tsize = file%cyc_ndx_end - file%cyc_ndx_beg + 1 - - if ( cyc_tsize > 1 ) then - - call findplb(all_data_times(file%cyc_ndx_beg:file%cyc_ndx_end),cyc_tsize, time, n ) - - if (n == cyc_tsize) then - np1 = 1 - else - np1 = n+1 - endif - - datatimem = all_data_times(n +file%cyc_ndx_beg-1) - datatimep = all_data_times(np1+file%cyc_ndx_beg-1) - times_found = .true. - - endif - endif - - if ( .not. times_found ) then - if (masterproc) then - write(iulog,*)'FIND_TIMES: Failed to find dates bracketing desired time =', time - write(iulog,*)' datatimem = ',file%datatimem - write(iulog,*)' datatimep = ',file%datatimep - write(iulog,*)' all_data_times = ',all_data_times - !call endrun() - return - endif - endif - - deallocate( all_data_times, stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'find_times: failed to deallocate all_data_times array; error = ',astat - call endrun - end if - - if ( .not. file%cyclical ) then - itms(1) = n - itms(2) = np1 - else - itms(1) = n +file%cyc_ndx_beg-1 - itms(2) = np1 +file%cyc_ndx_beg-1 - endif - - fids(:) = file%curr_fileid - - do i=1,2 - if ( itms(i) > curr_tsize ) then - itms(i) = itms(i) - curr_tsize - fids(i) = file%next_fileid - endif - enddo - - end subroutine find_times - -!------------------------------------------------------------------------ -!------------------------------------------------------------------------ - subroutine read_next_trcdata(state, flds, file ) - - use shr_const_mod, only:pi => shr_const_pi - use physics_types,only : physics_state - use ppgrid, only: pcols, pver, pverp,begchunk,endchunk - use physconst, only: rair - use iop_data_mod - use rad_constituents, only: rad_cnst_get_info, rad_cnst_get_aer_props, & - rad_cnst_get_mode_props, rad_cnst_get_mode_num - - implicit none - - type (trfile), intent(inout) :: file - type (trfld),intent(inout) :: flds(:) - type(physics_state), intent(in) :: state(begchunk:endchunk) - - integer :: recnos(4),i,f,nflds ! - integer :: cnt4(4) ! array of counts for each dimension - integer :: strt4(4) ! array of starting indices - integer :: cnt3(3) ! array of counts for each dimension - integer :: strt3(3) ! array of starting indices - type(file_desc_t) :: fids(4) - logical :: times_found - - integer :: cur_yr, cur_mon, cur_day, cur_sec, yr1, yr2, mon, date, sec - real(r8) :: series1_time, series2_time - type(file_desc_t) :: fid1, fid2 - - integer :: nspec,nmodes,n,ii,kk,l1,k,lchnk - real(r8) :: profile_p(pver),pp(pver),meanP,meanO,sumii,volfrac,specdens,aero_den(6) - real(r8) :: q_a(3,6) - real(r8) :: rho(pcols,pver) ! air density (kg m-3) - character(len=20) :: aername - character(len=3) :: arnam(7) = (/'so4','pom','soa','bc ','dst','ncl','num'/) - - nflds = size(flds) - times_found = .false. - - if(single_column .and. scm_observed_aero) then - - call ver_profile_aero(pp) - ! The following do loop gets the species properties and calculates the aerosol - ! mass mixing ratio of each species from observed total number and size distribution - ! properties in the unit kg/m^3 for the 3 modes. Data is read from the forcing file. - ! For mode 1 (accumulation mode) q_a(1,1)=q(so4),q_a(1,2)=q(pom),q_a(1,3)=q(soa), - ! q_a(1,4)=q(bc),q_a(1,5)=q(dst),q_a(1,6)=q(ncl). - ! For mode 2 (aitken mode) q_a(2,1)=q(so4),q_a(2,2)=q(soa),q_a(2,3)=q(ncl). - ! For mode 3 (coarse mode) q_a(3,1)=q(dst),q_a(3,2)=q(ncl),q_a(3,3)=q(so4). - - call rad_cnst_get_info(0, nmodes=nmodes) - do n=1, nmodes - call rad_cnst_get_info(0, n, nspec=nspec) - do l1 = 1, nspec - call rad_cnst_get_aer_props(0, n,l1,density_aer=specdens) - call rad_cnst_get_aer_props(0, n, l1, aername=aername) - aero_den(l1)=specdens - q_a(n,l1) = specdens*scm_div(n,l1)*scm_num(n)*((pi/6.0_r8)*( & - scm_dgnum(n)**3)*exp(4.5_r8*(log(scm_std(n))**2) )) - enddo - enddo - - do k = 1, pver - do ii = 1, state(begchunk)%ncol - rho(ii,k) = state(begchunk)%pmid(ii,k)/(rair*state(begchunk)%t(ii,k)) - enddo - enddo - end if - - do while( .not. times_found ) - call find_times( recnos, fids, file%curr_mod_time,file,file%datatimem,file%datatimep, times_found ) - if ( .not. times_found ) then - call check_files( file, fids, recnos, times_found ) - endif - enddo - - ! If single column do not interpolate aerosol data, just use the step function. - ! The exception is if we are trying to "replay" a column from the full model - if(single_column .and. .not. use_replay) then - file%stepTime = .true. - endif - - if (file%stepTime) then - file%interp_recs = 1 - else - file%interp_recs = 2 - end if - - if ( file%fill_in_months ) then - - if( file%datatimep-file%datatimem > file%one_yr ) then - - call get_curr_date(cur_yr, cur_mon, cur_day, cur_sec) - - call set_date_from_time_float(file%datatimem, yr1, mon, date, sec ) - call set_date_from_time_float(file%datatimep, yr2, mon, date, sec ) - - call set_time_float_from_date( series1_time, yr1, cur_mon, cur_day, cur_sec ) - call set_time_float_from_date( series2_time, yr2, cur_mon, cur_day, cur_sec ) - - fid1 = fids(1) - fid2 = fids(2) - file%cyclical = .true. - call set_cycle_indices( fid1, file%cyc_ndx_beg, file%cyc_ndx_end, yr1) - call find_times( recnos(1:2), fids(1:2), series1_time, file, file%datatimes(1), file%datatimes(2), times_found ) - - if ( .not. times_found ) then - call endrun('read_next_trcdata: time not found for series1_time') - endif - call set_cycle_indices( fid2, file%cyc_ndx_beg, file%cyc_ndx_end, yr2) - - if ( fid1%fh /= fid2%fh ) then - file%cyc_ndx_beg = file%cyc_ndx_beg + size(file%curr_data_times) - file%cyc_ndx_end = file%cyc_ndx_end + size(file%curr_data_times) - endif - call find_times( recnos(3:4), fids(3:4), series2_time, file, file%datatimes(3), file%datatimes(4), times_found ) - if ( .not. times_found ) then - call endrun('read_next_trcdata: time not found for series2_time') - endif - file%cyclical = .false. - file%interp_recs = 4 - - call set_date_from_time_float( file%datatimes(1), yr1, mon, date, sec ) - call set_time_float_from_date( file%datatimem, cur_yr, mon, date, sec ) - if (file%datatimes(1) > file%datatimes(2) ) then ! wrap around - if ( cur_mon == 1 ) then - call set_time_float_from_date( file%datatimem, cur_yr-1, mon, date, sec ) - endif - endif - - call set_date_from_time_float( file%datatimes(2), yr1, mon, date, sec ) - call set_time_float_from_date( file%datatimep, cur_yr, mon, date, sec ) - if (file%datatimes(1) > file%datatimes(2) ) then ! wrap around - if ( cur_mon == 12 ) then - call set_time_float_from_date( file%datatimep, cur_yr+1, mon, date, sec ) - endif - endif - - endif - - endif - - ! - ! Set up hyperslab corners - ! - strt4(:) = 1 - strt3(:) = 1 - - do i=1,file%interp_recs - - do f = 1,nflds - if ( file%zonal_ave ) then - ! Defend against zonal mean surface fields that do not set the value via dimension match - if (flds(f)%coords(ZA_LATDIM) .gt. 0) cnt3(flds(f)%coords(ZA_LATDIM)) = file%nlat - if (flds(f)%srf_fld) then - ! Defend against zonal mean surface fields that do not set the value via dimension match - if (flds(f)%coords(ZA_LEVDIM) .gt. 0) cnt3(flds(f)%coords(ZA_LEVDIM)) = 1 - else - cnt3(flds(f)%coords(ZA_LEVDIM)) = file%nlev - endif - cnt3(flds(f)%coords(ZA_TIMDIM)) = 1 - strt3(flds(f)%coords(ZA_TIMDIM)) = recnos(i) - !! - if (file%linoz_v3 .or. file%linoz_v2) then - !!check if these are the surface variables - !!no need to do interpolate since only used - !!in preprocessing, - !!clim +57 is the correspondent srf variable - if (index(flds(f)%fldnam,"_clim") .gt.0.and.& - index(flds(f)%fldnam,"P_clim") .le.0.and.& - index(flds(f)%fldnam,"L_clim") .le.0.and.& - index(flds(f)%fldnam,"_srf") .le.0)then - call read_za_trc_linoz( fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt3, cnt3, file, & - (/ flds(f)%order(ZA_LATDIM),flds(f)%order(ZA_LEVDIM)/), & - vid_srf=flds(f+57)%var_id ) - elseif (index(flds(f)%fldnam,"_srf").gt.0) then - if (index(flds(f)%fldnam,"ch4_avg_srf").gt.0) then - cnt3(1)=1!set 1st dim since no ZA_LATDIM - endif - call read_zasrf_trc_linoz(fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt3, cnt3, file) - else - call read_za_trc_linoz( fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt3, cnt3, file, & - (/ flds(f)%order(ZA_LATDIM),flds(f)%order(ZA_LEVDIM) /) ) - endif - else - call read_za_trc( fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt3, cnt3, file, & - (/ flds(f)%order(ZA_LATDIM),flds(f)%order(ZA_LEVDIM) /) ) - endif - else if ( flds(f)%srf_fld ) then - cnt3( flds(f)%coords(LONDIM)) = file%nlon - cnt3( flds(f)%coords(LATDIM)) = file%nlat - cnt3( flds(f)%coords(PS_TIMDIM)) = 1 - strt3(flds(f)%coords(PS_TIMDIM)) = recnos(i) - call read_2d_trc( fids(i), flds(f)%var_id, flds(f)%input(i)%data(:,1,:), strt3, cnt3, file, & - (/ flds(f)%order(LONDIM),flds(f)%order(LATDIM) /) ) - else - cnt4(flds(f)%coords(LONDIM)) = file%nlon - cnt4(flds(f)%coords(LATDIM)) = file%nlat - cnt4(flds(f)%coords(LEVDIM)) = file%nlev - cnt4(flds(f)%coords(TIMDIM)) = 1 - strt4(flds(f)%coords(TIMDIM)) = recnos(i) - call read_3d_trc( fids(i), flds(f)%var_id, flds(f)%input(i)%data, strt4, cnt4, file, & - (/ flds(f)%order(LONDIM),flds(f)%order(LATDIM),flds(f)%order(LEVDIM) /)) - - ! - ! This section sets the observed aersol mass and number mixing ratios in the - ! appropriate variables. The observed aerosol inforamtion is read from the - ! forcing file as total number and size distribution parameters. Then the total - ! volume is calculated.Using the desnsity of each species, the mass is calculated. - ! Finally the number is partition among the each species using the species fraction - ! data read from the forcing file. - ! - if(single_column .and. scm_observed_aero) then - kk=index(trim(flds(f)%fldnam),'_')-1 - if(index(trim(flds(f)%fldnam),'1') > 0 .and.index(trim(flds(f)%fldnam),'log') < 1) then - if(flds(f)%fldnam(1:kk).eq.arnam(1).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(1),flds(f)%input(i)%data, & - rho,pp,q_a(1,1),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(2).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(2),flds(f)%input(i)%data, & - rho,pp,q_a(1,2),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(3).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(3),flds(f)%input(i)%data, & - rho,pp,q_a(1,3),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(4).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(4),flds(f)%input(i)%data, & - rho,pp,q_a(1,4),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(5).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(5),flds(f)%input(i)%data, & - rho,pp,q_a(1,5),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(6).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(6),flds(f)%input(i)%data, & - rho,pp,q_a(1,6),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(7).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(7),flds(f)%input(i)%data, & - rho,pp,scm_num(1),state(begchunk)%ncol) - endif - elseif(index(trim(flds(f)%fldnam),'2') > 0 .and.index(trim(flds(f)%fldnam),'log') < 1) then - if(flds(f)%fldnam(1:kk).eq.arnam(1).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(1),flds(f)%input(i)%data, & - rho,pp,q_a(2,1),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(3).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(3),flds(f)%input(i)%data, & - rho,pp,q_a(2,2),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(6).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(6),flds(f)%input(i)%data, & - rho,pp,q_a(2,3),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(7).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(7),flds(f)%input(i)%data, & - rho,pp,scm_num(2),state(begchunk)%ncol) - endif - elseif(index(trim(flds(f)%fldnam),'3') > 0 .and.index(trim(flds(f)%fldnam),'log') < 1) then - if(flds(f)%fldnam(1:kk).eq.arnam(1).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(1),flds(f)%input(i)%data, & - rho,pp,q_a(3,3),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(5).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(5),flds(f)%input(i)%data, & - rho,pp,q_a(3,1),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(6).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(6),flds(f)%input(i)%data, & - rho,pp,q_a(3,2),state(begchunk)%ncol) - elseif(flds(f)%fldnam(1:kk).eq.arnam(7).and.index(trim(flds(f)%fldnam),'log') < 1) then - call replace_aero_data(flds(f)%fldnam,arnam(7),flds(f)%input(i)%data, & - rho,pp,scm_num(3),state(begchunk)%ncol) - endif - endif - endif !scm_observed_aero - endif - - enddo - - if ( file%has_ps ) then - cnt3(file%ps_coords(LONDIM)) = file%nlon - cnt3(file%ps_coords(LATDIM)) = file%nlat - cnt3(file%ps_coords(PS_TIMDIM)) = 1 - strt3(file%ps_coords(PS_TIMDIM)) = recnos(i) - call read_2d_trc( fids(i), file%ps_id, file%ps_in(i)%data, strt3, cnt3, file, & - (/ file%ps_order(LONDIM),file%ps_order(LATDIM) /) ) - endif - - enddo - - end subroutine read_next_trcdata - -!-------------------------------------------------------------------------------- -!This subroutine replaces the climatological aerosol information by the observed -!once after they are read -! - subroutine replace_aero_data(aerofulnam,spnam,aero_q_data,rho,pp,q_mix,ncoli) - use ppgrid, only: pcols,pver,begchunk,endchunk - - implicit none - real(r8), intent(inout) :: aero_q_data(pcols,pver,begchunk:endchunk) - real(r8), intent(in) :: rho(pcols,pver),pp(pver) - real(r8) :: sumii,meanO - real(r8), intent(in) :: q_mix - character(len=32),intent(in) ::aerofulnam - character(len=3),intent(in) :: spnam - character(len=32) ::aerosubnam - integer, intent(in) :: ncoli - integer :: ii,k,countj - - if(trim(aerofulnam(1:2)).eq.'bc') then - aerosubnam=aerofulnam(1:4) - else - aerosubnam=aerofulnam(1:5) - endif - - if((trim(aerosubnam).eq.(trim(spnam)//'_a').or.trim(aerosubnam).eq.(trim(spnam)//'_c')) & - .and.index(trim(aerofulnam),'log') < 1) then - - aero_q_data=q_mix - sumii=0._r8 - countj =0 - do ii = 1, ncoli - do k = 1, pver - if(pp(k).gt.0._r8) then - countj=countj+1 - endif - if(trim(spnam).ne.'num') then - aero_q_data(ii,k,begchunk)=(aero_q_data(ii,k,begchunk)*pp(k)) /rho(ii,k) - endif - sumii=sumii + aero_q_data(ii,k,begchunk) - enddo - enddo - meanO=sumii/countj - do k = 1, pver - if(meanO.ne.0.) then - aero_q_data(1,k,begchunk)=pp(k)*meanO - else - aero_q_data(1,k,begchunk)=0._r8 - endif - enddo - endif - - end subroutine replace_aero_data - -!--------------------------------------------------------------------------- -!This subroutine generates a heavyside type profiles for the observed aerosol. -!This setting is constant profile in the lower atmosphere and then exponentially -!decreasing to zero at the top of the atmosphere. The level of initial decay is -!controlled by "initial_val". Larger than -3.5 pushes the decay point up and smaller -!brings it closer to the surface. -! - subroutine ver_profile_aero(vertprof_aero) - use mo_constants, only : pi - use ppgrid, only: pcols,pver - - implicit none - real(r8), intent(inout) :: vertprof_aero(pver) - real(r8) :: initial_val = -3.5_r8 - integer :: k - - do k=1,pver - if(k==1) then - vertprof_aero(k)=0._r8 - elseif(k==2) then - vertprof_aero(k)=1._r8/(1._r8 + exp(-2._r8*initial_val * pi)) - else - vertprof_aero(k)=1._r8/(1._r8 + exp(-2._r8*(initial_val * pi + pi/4._r8*(k-2)))) - endif - enddo - end subroutine ver_profile_aero - -!------------------------------------------------------------------------ - - - subroutine read_2d_trc( fid, vid, loc_arr, strt, cnt, file, order ) - use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish - - use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p, get_lon_all_p, get_lat_all_p - use mo_constants, only : pi - use dycore, only: dycore_is - use polar_avg, only: polar_average - use horizontal_interpolate, only : xy_interp - - implicit none - type(file_desc_t), intent(in) :: fid - type(var_desc_t), intent(in) :: vid - integer, intent(in) :: strt(:), cnt(:), order(2) - real(r8),intent(out) :: loc_arr(:,:) - type (trfile), intent(in) :: file - - real(r8) :: to_lats(pcols), to_lons(pcols), wrk(pcols) - real(r8), allocatable, target :: wrk2d(:,:) - real(r8), pointer :: wrk2d_in(:,:) - - integer :: tsize, c, i, j, ierr, ncols - real(r8), parameter :: zero=0_r8, twopi=2_r8*pi - type(interp_type) :: lon_wgts, lat_wgts - integer :: lons(pcols), lats(pcols) - - nullify(wrk2d_in) - allocate( wrk2d(cnt(1),cnt(2)), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'read_2d_trc: wrk2d allocation error = ',ierr - call endrun - end if - - if(order(1)/=1 .or. order(2)/=2 .or. cnt(1)/=file%nlon .or. cnt(2)/=file%nlat) then - allocate( wrk2d_in(file%nlon, file%nlat), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'read_2d_trc: wrk2d_in allocation error = ',ierr - call endrun - end if - end if - - - - ierr = pio_get_var( fid, vid, strt, cnt, wrk2d ) - if(associated(wrk2d_in)) then - wrk2d_in = reshape( wrk2d(:,:),(/file%nlon,file%nlat/), order=order ) - deallocate(wrk2d) - else - wrk2d_in => wrk2d - end if - - j=1 - -! if weighting by latitude, the perform horizontal interpolation by using weight_x, weight_y - - if(file%weight_by_lat) then - - call t_startf('xy_interp') - - do c = begchunk,endchunk - ncols = get_ncols_p(c) - call get_lon_all_p(c,ncols,lons) - call get_lat_all_p(c,ncols,lats) - - call xy_interp(file%nlon,file%nlat,1,plon,plat,pcols,ncols,file%weight_x,file%weight_y,wrk2d_in,loc_arr(:,c-begchunk+1), & - lons,lats,file%count_x,file%count_y,file%index_x,file%index_y) - enddo - - call t_stopf('xy_interp') - - else - do c=begchunk,endchunk - ncols = get_ncols_p(c) - call get_rlat_all_p(c, pcols, to_lats) - call get_rlon_all_p(c, pcols, to_lons) - - call lininterp_init(file%lons, file%nlon, to_lons, ncols, 2, lon_wgts, zero, twopi) - call lininterp_init(file%lats, file%nlat, to_lats, ncols, 1, lat_wgts) - - call lininterp(wrk2d_in, file%nlon, file%nlat, loc_arr(1:ncols,c-begchunk+1), ncols, lon_wgts, lat_wgts) - - call lininterp_finish(lon_wgts) - call lininterp_finish(lat_wgts) - end do - endif - - if(allocated(wrk2d)) then - deallocate(wrk2d) - else - deallocate(wrk2d_in) - end if - if(dycore_is('LR')) call polar_average(loc_arr) - end subroutine read_2d_trc - -!------------------------------------------------------------------------ - - subroutine read_za_trc( fid, vid, loc_arr, strt, cnt, file, order ) - use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish - use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p - use mo_constants, only : pi - use dycore, only : dycore_is - use polar_avg, only : polar_average - - implicit none - type(file_desc_t), intent(in) :: fid - type(var_desc_t), intent(in) :: vid - integer, intent(in) :: strt(:), cnt(:) - integer, intent(in) :: order(2) - real(r8), intent(out):: loc_arr(:,:,:) - type (trfile), intent(in) :: file - - type(interp_type) :: lat_wgts - real(r8) :: to_lats(pcols), to_lons(pcols), wrk(pcols) - real(r8), allocatable, target :: wrk2d(:,:) - real(r8), pointer :: wrk2d_in(:,:) - integer :: c, k, ierr, ncols - - nullify(wrk2d_in) - allocate( wrk2d(cnt(1),cnt(2)), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'read_2d_trc: wrk2d allocation error = ',ierr - call endrun - end if - - if(order(1)/=1 .or. order(2)/=2 .or. cnt(1)/=file%nlat .or. cnt(2)/=file%nlev) then - allocate( wrk2d_in(file%nlat, file%nlev), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'read_2d_trc: wrk2d_in allocation error = ',ierr - call endrun - end if - end if - - - ierr = pio_get_var( fid, vid, strt, cnt, wrk2d ) - if(associated(wrk2d_in)) then - wrk2d_in = reshape( wrk2d(:,:),(/file%nlat,file%nlev/), order=order ) - deallocate(wrk2d) - else - wrk2d_in => wrk2d - end if - - - - do c=begchunk,endchunk - ncols = get_ncols_p(c) - call get_rlat_all_p(c, pcols, to_lats) - - call lininterp_init(file%lats, file%nlat, to_lats, ncols, 1, lat_wgts) - do k=1,file%nlev - call lininterp(wrk2d_in(:,k), file%nlat, wrk(1:ncols), ncols, lat_wgts) - loc_arr(1:ncols,k,c-begchunk+1) = wrk(1:ncols) - end do - call lininterp_finish(lat_wgts) - end do - - if(allocated(wrk2d)) then - deallocate(wrk2d) - else - deallocate(wrk2d_in) - end if -! if(dycore_is('LR')) call polar_average(loc_arr) - end subroutine read_za_trc -!------------------------------------------------------------------------ - subroutine read_za_trc_linoz( fid, vid, loc_arr, strt, cnt, file, order ,vid_srf) - use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish - use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p - use mo_constants, only : pi - use dycore, only : dycore_is - use polar_avg, only : polar_average - - implicit none - type(file_desc_t), intent(in) :: fid - type(var_desc_t), intent(in) :: vid - integer, intent(in) :: strt(:), cnt(:) - integer, intent(in) :: order(2) - real(r8), intent(out):: loc_arr(:,:,:) - type (trfile), intent(in) :: file - !! - type(var_desc_t), intent(in), optional :: vid_srf - integer :: cnt_srf(2) - integer :: strt_srf(2) - !! - type(interp_type) :: lat_wgts - real(r8) :: to_lats(pcols), to_lons(pcols), wrk(pcols) - real(r8), allocatable, target :: wrk2d(:,:) - real(r8), allocatable, target :: wrksrf(:) - real(r8), pointer :: wrk2d_in(:,:) - integer :: c, k, ierr, ncols - - nullify(wrk2d_in) - allocate( wrk2d(cnt(1),cnt(2)), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'read_2d_trc: wrk2d allocation error = ',ierr - call endrun - end if - - if(order(1)/=1 .or. order(2)/=2 .or. cnt(1)/=file%nlat .or. cnt(2)/=file%nlev) then - allocate( wrk2d_in(file%nlat, file%nlev), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'read_2d_trc: wrk2d_in allocation error = ',ierr - call endrun - end if - end if - !! - ierr = pio_get_var( fid, vid, strt, cnt, wrk2d ) - !! - if (file%linoz_v3) then - !!since io reads in global data for every thread - !!and interpolate to local chunk, we can do surface, - !!polar, and surface padding preprocessfor every variable - !!before interpolation - !!it is put here rather than the data formation process - !!to prevent forget when producing forcing data - !read in order of 0.1hPa~985hPa, pad top with 2nd layer - wrk2d(:,1)=wrk2d(:,2) - !both N/S poles need to pad by nearest data - wrk2d(1,:)=wrk2d(2,:) - wrk2d(file%nlat,:)=wrk2d(file%nlat-1,:) - !!read in srf data to pad surface - !!they are inputs from CMIP forcing and other references - cnt_srf(1)=cnt(1) - cnt_srf(2)=cnt(3) - strt_srf(1)=strt(1) - strt_srf(2)=strt(3) - !!check if vid_srf is present to determine clim variables - if (present(vid_srf)) then - !!padding for clim terms - allocate(wrksrf(cnt(1)), stat=ierr ) - ierr = pio_get_var( fid, vid_srf, strt_srf, cnt_srf, wrksrf ) - !!surface padding - wrk2d(:,file%nlev)=wrksrf - deallocate(wrksrf) - endif - !! - endif - !! - if(associated(wrk2d_in)) then - wrk2d_in = reshape( wrk2d(:,:),(/file%nlat,file%nlev/), order=order ) - deallocate(wrk2d) - else - wrk2d_in => wrk2d - end if - - do c=begchunk,endchunk - ncols = get_ncols_p(c) - call get_rlat_all_p(c, pcols, to_lats) - call lininterp_init(file%lats, file%nlat, to_lats, ncols, 1, lat_wgts) - do k=1,file%nlev - call lininterp(wrk2d_in(:,k), file%nlat, wrk(1:ncols), ncols, lat_wgts) - loc_arr(1:ncols,k,c-begchunk+1) = wrk(1:ncols) - end do - call lininterp_finish(lat_wgts) - end do - - if(allocated(wrk2d)) then - deallocate(wrk2d) - else - deallocate(wrk2d_in) - end if - !! - if(allocated(wrksrf)) then - deallocate(wrksrf) - end if -! if(dycore_is('LR')) call polar_average(loc_arr) - end subroutine read_za_trc_linoz - -!------------------------------------------------------------------------ - subroutine read_zasrf_trc_linoz( fid, vid, loc_arr, strt, cnt, file) - use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish - use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p - !! - implicit none - type(file_desc_t), intent(in) :: fid - type(var_desc_t), intent(in) :: vid - integer, intent(in) :: strt(:), cnt(:) - real(r8), intent(out):: loc_arr(:,:,:) - type (trfile), intent(in) :: file - !! - real(r8), allocatable, target :: wrk(:) - real(r8), pointer :: wrk_in(:) - real(r8) :: wrk_out(pcols) - type(interp_type) :: lat_wgts - real(r8) :: to_lats(pcols), to_lons(pcols) - integer :: c, k, ierr, ncols - integer :: cnt_srf(2) - integer :: strt_srf(2) - !! - nullify(wrk_in) - allocate( wrk(cnt(1)), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'read_zasrf_trc_linoz: wrk allocation error = ',ierr - call endrun - end if - !! - cnt_srf(1)=cnt(1) - cnt_srf(2)=cnt(3) - strt_srf(1)=strt(1) - strt_srf(2)=strt(3) - !! - !for surface variable with the dimension of (time,lat) or (time,1) - !for (time) it is also set like (time,1) - ierr = pio_get_var( fid, vid, strt_srf, cnt_srf, wrk ) - !! - if(associated(wrk_in)) then - wrk_in = reshape( wrk(:),(/file%nlat/)) - deallocate(wrk) - else - wrk_in => wrk - end if - !! - do c=begchunk,endchunk - ncols = get_ncols_p(c) - call get_rlat_all_p(c, pcols, to_lats) - call lininterp_init(file%lats, file%nlat, to_lats, ncols, 1, lat_wgts) - !! - if (cnt(1).eq.1) then!!for single timeseries - do k=1,1 - loc_arr(1:ncols,k,c-begchunk+1) = wrk_in(1) - end do - else - do k=1,1 - call lininterp(wrk_in(:), file%nlat, wrk_out(1:ncols), ncols, lat_wgts) - loc_arr(1:ncols,k,c-begchunk+1) = wrk_out(1:ncols) - end do - end if - !! - call lininterp_finish(lat_wgts) - end do - !! - if(allocated(wrk)) then - deallocate(wrk) - else - deallocate(wrk_in) - end if - !! - end subroutine read_zasrf_trc_linoz - -!------------------------------------------------------------------------ - - subroutine read_3d_trc( fid, vid, loc_arr, strt, cnt, file, order) - use interpolate_data, only : lininterp_init, lininterp, interp_type, lininterp_finish - use phys_grid, only : pcols, begchunk, endchunk, get_ncols_p, get_rlat_all_p, get_rlon_all_p, get_lon_all_p,& - get_lat_all_p - use mo_constants, only : pi - use dycore, only : dycore_is - use polar_avg, only : polar_average - use dycore, only : dycore_is - use horizontal_interpolate, only : xy_interp - - implicit none - - type(file_desc_t), intent(in) :: fid - type(var_desc_t), intent(in) :: vid - integer, intent(in) :: strt(:), cnt(:), order(3) - real(r8),intent(out) :: loc_arr(:,:,:) - - type (trfile), intent(in) :: file - - integer :: i,j,k, astat, c, ncols - integer :: lons(pcols), lats(pcols) - - integer :: jlim(2), jl, ju, ierr - integer :: gndx - - real(r8), allocatable, target :: wrk3d(:,:,:) - real(r8), pointer :: wrk3d_in(:,:,:) - real(r8) :: to_lons(pcols), to_lats(pcols) - real(r8), parameter :: zero=0_r8, twopi=2_r8*pi - type(interp_type) :: lon_wgts, lat_wgts - - loc_arr(:,:,:) = 0._r8 - nullify(wrk3d_in) - allocate(wrk3d(cnt(1),cnt(2),cnt(3)), stat=ierr) - if( ierr /= 0 ) then - write(iulog,*) 'read_3d_trc: wrk3d allocation error = ',ierr - call endrun - end if - - ierr = pio_get_var( fid, vid, strt, cnt, wrk3d ) - - if(order(1)/=1 .or. order(2)/=2 .or. order(3)/=3 .or. & - cnt(1)/=file%nlon.or.cnt(2)/=file%nlat.or.cnt(3)/=file%nlev) then - allocate(wrk3d_in(file%nlon,file%nlat,file%nlev),stat=ierr) - if( ierr /= 0 ) then - write(iulog,*) 'read_3d_trc: wrk3d allocation error = ',ierr - call endrun - end if - wrk3d_in = reshape( wrk3d(:,:,:),(/file%nlon,file%nlat,file%nlev/), order=order ) - deallocate(wrk3d) - else - wrk3d_in => wrk3d - end if - - j=1 - -! If weighting by latitude, then perform horizontal interpolation by using weight_x, weight_y - - if(file%weight_by_lat) then - - call t_startf('xy_interp') - - do c = begchunk,endchunk - ncols = get_ncols_p(c) - call get_lon_all_p(c,ncols,lons) - call get_lat_all_p(c,ncols,lats) - - call xy_interp(file%nlon,file%nlat,file%nlev,plon,plat,pcols,ncols,file%weight_x,file%weight_y,wrk3d_in, & - loc_arr(:,:,c-begchunk+1), lons,lats,file%count_x,file%count_y,file%index_x,file%index_y) - enddo - - call t_stopf('xy_interp') - - else - do c=begchunk,endchunk - ncols = get_ncols_p(c) - call get_rlat_all_p(c, pcols, to_lats) - call get_rlon_all_p(c, pcols, to_lons) - - call lininterp_init(file%lons, file%nlon, to_lons(1:ncols), ncols, 2, lon_wgts, zero, twopi) - call lininterp_init(file%lats, file%nlat, to_lats(1:ncols), ncols, 1, lat_wgts) - - - call lininterp(wrk3d_in, file%nlon, file%nlat, file%nlev, loc_arr(:,:,c-begchunk+1), ncols, pcols, lon_wgts, lat_wgts) - - - call lininterp_finish(lon_wgts) - call lininterp_finish(lat_wgts) - end do - endif - - if(allocated(wrk3d)) then - deallocate( wrk3d, stat=astat ) - else - deallocate( wrk3d_in, stat=astat ) - end if - if( astat/= 0 ) then - write(iulog,*) 'read_3d_trc: failed to deallocate wrk3d array; error = ',astat - call endrun - endif - if(dycore_is('LR')) call polar_average(file%nlev, loc_arr) - end subroutine read_3d_trc - -!------------------------------------------------------------------------------ - - subroutine interpolate_trcdata( state, flds, file, pbuf2d ) - use mo_util, only : rebin - use physics_types,only : physics_state - use physconst, only : cday - use physics_buffer, only : physics_buffer_desc, pbuf_get_field - - implicit none - - type(physics_state), intent(in) :: state(begchunk:endchunk) - type (trfld), intent(inout) :: flds(:) - type (trfile), intent(inout) :: file - - type(physics_buffer_desc), optional, pointer :: pbuf2d(:,:) - - - real(r8) :: fact1, fact2 - real(r8) :: deltat - integer :: f,nflds,c,ncol, i,k - real(r8) :: ps(pcols) - real(r8) :: datain(pcols,file%nlev) - real(r8) :: pin(pcols,file%nlev) - real(r8) :: pint(pcols,file%nilev) - - real(r8) :: model_z(pverp) - real(r8), parameter :: m2km = 1.e-3_r8 - real(r8), pointer :: data_out3d(:,:,:) - real(r8), pointer :: data_out(:,:) - integer :: chnk_offset - - nflds = size(flds) - - if ( file%interp_recs == 4 ) then - deltat = file%datatimes(3) - file%datatimes(1) - fact1 = (file%datatimes(3) - file%datatimem)/deltat - fact2 = 1._r8-fact1 -!$OMP PARALLEL DO PRIVATE (C, NCOL, F) - do c = begchunk,endchunk - ncol = state(c)%ncol - if ( file%has_ps ) then - file%ps_in(1)%data(:ncol,c) = fact1*file%ps_in(1)%data(:ncol,c) + fact2*file%ps_in(3)%data(:ncol,c) - endif - do f = 1,nflds - flds(f)%input(1)%data(:ncol,:,c) = fact1*flds(f)%input(1)%data(:ncol,:,c) + fact2*flds(f)%input(3)%data(:ncol,:,c) - enddo - enddo - - deltat = file%datatimes(4) - file%datatimes(2) - fact1 = (file%datatimes(4) - file%datatimep)/deltat - fact2 = 1._r8-fact1 - -!$OMP PARALLEL DO PRIVATE (C, NCOL, F) - do c = begchunk,endchunk - ncol = state(c)%ncol - if ( file%has_ps ) then - file%ps_in(2)%data(:ncol,c) = fact1*file%ps_in(2)%data(:ncol,c) + fact2*file%ps_in(4)%data(:ncol,c) - endif - do f = 1,nflds - flds(f)%input(2)%data(:ncol,:,c) = fact1*flds(f)%input(2)%data(:ncol,:,c) + fact2*flds(f)%input(4)%data(:ncol,:,c) - enddo - enddo - - endif - !------------------------------------------------------------------------- - ! If file%interp_recs=1 then no time interpolation -- set - ! fact1=1 and fact2=0 and will just use first value unmodified - !------------------------------------------------------------------------- - - if (file%interp_recs == 1) then - fact1=1._r8 - fact2=0._r8 - else - file%interp_recs = 2 - - deltat = file%datatimep - file%datatimem - - if ( file%cyclical .and. (deltat < 0._r8) ) then - deltat = deltat+file%one_yr - if ( file%datatimep >= file%curr_mod_time ) then - fact1 = (file%datatimep - file%curr_mod_time)/deltat - else - fact1 = (file%datatimep+file%one_yr - file%curr_mod_time)/deltat - endif - else - fact1 = (file%datatimep - file%curr_mod_time)/deltat - endif - - ! this assures that FIXED data are b4b on restarts - if ( file%fixed ) then - fact1 = dble(int(fact1*cday+.5_r8))/dble(cday) - endif - fact2 = 1._r8-fact1 - endif - - chnk_offset=-begchunk+1 - - fld_loop: do f = 1,nflds - - if (flds(f)%pbuf_ndx<=0) then - data_out3d => flds(f)%data(:,:,:) - endif - -!$OMP PARALLEL DO PRIVATE (C, NCOL, PS, I, K, PIN, DATAIN, MODEL_Z, DATA_OUT) - do c = begchunk,endchunk - if (flds(f)%pbuf_ndx>0) then - if(.not.present(pbuf2d)) then - call endrun ('tracer_data.F90(subr interpolate_trcdata):' // & - 'pbuf2d must be passed as an argument for pbuf_get_field subr call') - endif - call pbuf_get_field(pbuf2d, c, flds(f)%pbuf_ndx, data_out) - else - data_out => data_out3d(:,:,c+chnk_offset) - endif - ncol = state(c)%ncol - if (file%alt_data) then - - if (fact2 == 0) then ! This needed as %data is not set if fact2=0 (and lahey compiler core dumps) - datain(:ncol,:) = fact1*flds(f)%input(nm)%data(:ncol,:,c) - else - datain(:ncol,:) = fact1*flds(f)%input(nm)%data(:ncol,:,c) + fact2*flds(f)%input(np)%data(:ncol,:,c) - end if - do i = 1,ncol - model_z(1:pverp) = m2km * state(c)%zi(i,pverp:1:-1) - call rebin( file%nlev, pver, file%ilevs, model_z, datain(i,:), data_out(i,:) ) - enddo - - else - - if ( file%nlev>1 ) then - if ( file%has_ps ) then - if (fact2 == 0) then ! This needed as %data is not set if fact2=0 (and lahey compiler core dumps) - ps(:ncol) = fact1*file%ps_in(nm)%data(:ncol,c) - else - ps(:ncol) = fact1*file%ps_in(nm)%data(:ncol,c) + fact2*file%ps_in(np)%data(:ncol,c) - end if - do i = 1,ncol - do k = 1,file%nlev - pin(i,k) = file%p0*file%hyam(k) + ps(i)*file%hybm(k) - enddo - enddo - else - do k = 1,file%nlev - pin(:,k) = file%levs(k) - enddo - !!Currently designed for linoz_v2/v3 use - if (file%linoz_v3 .or. file%linoz_v2) then - do k = 1,file%nilev - pint(:,k) = file%ilevs(k) - enddo - endif - endif - endif - - if (flds(f)%srf_fld) then - do i = 1,ncol - if (fact2 == 0) then ! This needed as %data is not set if fact2=0 (and lahey compiler core dumps) - data_out(i,1) = & - fact1*flds(f)%input(nm)%data(i,1,c) - else - data_out(i,1) = & - fact1*flds(f)%input(nm)%data(i,1,c) + fact2*flds(f)%input(np)%data(i,1,c) - endif - enddo - else - if (fact2 == 0) then ! This needed as %data is not set if fact2=0 (and lahey compiler core dumps) - datain(:ncol,:) = fact1*flds(f)%input(nm)%data(:ncol,:,c) - else - datain(:ncol,:) = fact1*flds(f)%input(nm)%data(:ncol,:,c) + fact2*flds(f)%input(np)%data(:ncol,:,c) - end if - if ( file%top_bndry ) then - call vert_interp_ub(ncol, file%nlev, file%levs, datain(:ncol,:), data_out(:ncol,:) ) - else if(file%conserve_column) then - call vert_interp_mixrat(ncol,file%nlev,pver,state(c)%pint, & - datain, data_out(:,:), & - file%p0,ps,file%hyai,file%hybi) - else if(file%linoz_v3 .or. file%linoz_v2) then - !!uci chemistry for linoz that better conserves mass - !!uci interpolation for 55 out of 57 variables in linoz_v3 - !!excluding t_clim, o3col_clim - !!for linoz_v2 it is simiar with less variables - if (flds(f)%fldnam.ne.'t_clim ' & - .and.flds(f)%fldnam.ne.'o3col_clim ') then - !!file ilevs is in hPa, while model level in Pa, so times 100 - call vert_interp_uci(ncol, file%nlev, 100*file%ilevs, state(c)%pint, datain, data_out(:,:) ) - else - call vert_interp(ncol, file%nlev, pin, state(c)%pmid, datain, data_out(:,:) ) - endif - !! - else - call vert_interp(ncol, file%nlev, pin, state(c)%pmid, datain, data_out(:,:) ) - endif - endif - - endif - enddo - - enddo fld_loop - - end subroutine interpolate_trcdata - -!----------------------------------------------------------------------- -!----------------------------------------------------------------------- - subroutine get_dimension( fid, dname, dsize, dimid, data ) - implicit none - type(file_desc_t), intent(inout) :: fid - character(*), intent(in) :: dname - integer, intent(out) :: dsize - - integer, optional, intent(out) :: dimid - real(r8), optional, pointer, dimension(:) :: data - - integer :: vid, ierr, id - - call pio_seterrorhandling( fid, PIO_BCAST_ERROR) - ierr = pio_inq_dimid( fid, dname, id ) - call pio_seterrorhandling( fid, PIO_INTERNAL_ERROR) - - if ( ierr==PIO_NOERR ) then - - ierr = pio_inq_dimlen( fid, id, dsize ) - - if ( present(dimid) ) then - dimid = id - endif - - if ( present(data) ) then - if ( associated(data) ) then - deallocate(data, stat=ierr) - if( ierr /= 0 ) then - write(iulog,*) 'get_dimension: data deallocation error = ',ierr - call endrun('get_dimension: failed to deallocate data array') - end if - endif - allocate( data(dsize), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'get_dimension: data allocation error = ',ierr - call endrun('get_dimension: failed to allocate data array') - end if - - ierr = pio_inq_varid( fid, dname, vid ) - ierr = pio_get_var( fid, vid, data ) - endif - else - dsize = 1 - if ( present(dimid) ) then - dimid = -1 - endif - endif - - end subroutine get_dimension - -!----------------------------------------------------------------------- -!----------------------------------------------------------------------- - subroutine set_cycle_indices( fileid, cyc_ndx_beg, cyc_ndx_end, cyc_yr ) - - implicit none - - type(file_desc_t), intent(inout) :: fileid - integer, intent(out) :: cyc_ndx_beg - integer, intent(out) :: cyc_ndx_end - integer, intent(in) :: cyc_yr - - integer, allocatable , dimension(:) :: dates, datesecs - integer :: timesize, i, astat, year, ierr - type(var_desc_T) :: dateid - call get_dimension( fileid, 'time', timesize ) - cyc_ndx_beg=-1 - - allocate( dates(timesize), stat=astat ) - if( astat/= 0 ) then - write(*,*) 'set_cycle_indices: failed to allocate dates array; error = ',astat - call endrun - end if - - ierr = pio_inq_varid( fileid, 'date', dateid ) - ierr = pio_get_var( fileid, dateid, dates ) - - do i=1,timesize - year = dates(i) / 10000 - if ( year == cyc_yr ) then - if (cyc_ndx_beg < 0) then - cyc_ndx_beg = i - endif - cyc_ndx_end = i - endif - enddo - deallocate( dates, stat=astat ) - if( astat/= 0 ) then - write(*,*) 'set_cycle_indices: failed to deallocate dates array; error = ',astat - call endrun - end if - if (cyc_ndx_beg < 0) then - write(*,*) 'set_cycle_indices: cycle year not found : ' , cyc_yr - call endrun('set_cycle_indices: cycle year not found') - endif - - end subroutine set_cycle_indices -!----------------------------------------------------------------------- - -!----------------------------------------------------------------------- - subroutine open_trc_datafile( fname, path, piofile, times, cyc_ndx_beg, cyc_ndx_end, cyc_yr ) - - use ioFileMod, only: getfil - use cam_pio_utils, only: cam_pio_openfile - - implicit none - - character(*), intent(in) :: fname - character(*), intent(in) :: path - type(file_desc_t), intent(inout) :: piofile - real(r8), pointer :: times(:) - - integer, optional, intent(out) :: cyc_ndx_beg - integer, optional, intent(out) :: cyc_ndx_end - integer, optional, intent(in) :: cyc_yr - - character(len=shr_kind_cl) :: filen, filepath - integer :: year, month, day, dsize, i, timesize - integer :: dateid,secid - integer, allocatable , dimension(:) :: dates, datesecs - integer :: astat, ierr - logical :: need_first_ndx - - if (len_trim(path) == 0) then - filepath = trim(fname) - else - filepath = trim(path) // '/' // trim(fname) - end if - ! - ! open file and get fileid - ! - call getfil( filepath, filen, 0 ) - call cam_pio_openfile( piofile, filen, PIO_NOWRITE) - if(masterproc) write(iulog,*)'open_trc_datafile: ',trim(filen) - - call get_dimension(piofile, 'time', timesize) - - if ( associated(times) ) then - deallocate(times, stat=ierr) - if( ierr /= 0 ) then - write(iulog,*) 'open_trc_datafile: data deallocation error = ',ierr - call endrun('open_trc_datafile: failed to deallocate data array') - end if - endif - allocate( times(timesize), stat=ierr ) - if( ierr /= 0 ) then - write(iulog,*) 'open_trc_datafile: data allocation error = ',ierr - call endrun('open_trc_datafile: failed to allocate data array') - end if - - allocate( dates(timesize), stat=astat ) - if( astat/= 0 ) then - if(masterproc) write(iulog,*) 'open_trc_datafile: failed to allocate dates array; error = ',astat - call endrun - end if - allocate( datesecs(timesize), stat=astat ) - if( astat/= 0 ) then - if(masterproc) write(iulog,*) 'open_trc_datafile: failed to allocate datesec array; error = ',astat - call endrun - end if - - ierr = pio_inq_varid( piofile, 'date', dateid ) - call pio_seterrorhandling( piofile, PIO_BCAST_ERROR) - ierr = pio_inq_varid( piofile, 'datesec', secid ) - call pio_seterrorhandling( piofile, PIO_INTERNAL_ERROR) - - if(ierr==PIO_NOERR) then - ierr = pio_get_var( piofile, secid, datesecs ) - else - datesecs=0 - end if - - ierr = pio_get_var( piofile, dateid, dates ) - need_first_ndx=.true. - - do i=1,timesize - year = dates(i) / 10000 - month = mod(dates(i),10000)/100 - day = mod(dates(i),100) - call set_time_float_from_date( times(i), year, month, day, datesecs(i) ) - if ( present(cyc_yr) ) then - if ( year == cyc_yr ) then - if ( present(cyc_ndx_beg) .and. need_first_ndx ) then - cyc_ndx_beg = i - need_first_ndx = .false. - endif - if ( present(cyc_ndx_end) ) then - cyc_ndx_end = i - endif - endif - endif - enddo - - deallocate( dates, stat=astat ) - if( astat/= 0 ) then - if(masterproc) write(iulog,*) 'open_trc_datafile: failed to deallocate dates array; error = ',astat - call endrun - end if - deallocate( datesecs, stat=astat ) - if( astat/= 0 ) then - if(masterproc) write(iulog,*) 'open_trc_datafile: failed to deallocate datesec array; error = ',astat - call endrun - end if - - if ( present(cyc_yr) .and. present(cyc_ndx_beg) ) then - if (cyc_ndx_beg < 0) then - write(iulog,*) 'open_trc_datafile: cycle year not found : ' , cyc_yr - call endrun('open_trc_datafile: cycle year not found') - endif - endif - - end subroutine open_trc_datafile - -!-------------------------------------------------------------------------- -!-------------------------------------------------------------------------- - subroutine specify_fields( specifier, fields ) - - implicit none - - character(len=*), intent(in) :: specifier(:) - type(trfld), pointer, dimension(:) :: fields - - integer :: fld_cnt, astat - integer :: i,j - character(len=shr_kind_cl) :: str1, str2 - character(len=32), allocatable, dimension(:) :: fld_name, src_name - integer :: nflds - - nflds = size(specifier) - - allocate(fld_name(nflds), src_name(nflds), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'specify_fields: failed to allocate fld_name, src_name arrays; error = ',astat - call endrun - end if - - fld_cnt = 0 - - count_cnst: do i = 1, nflds - - if ( len_trim( specifier(i) ) == 0 ) then - exit count_cnst - endif - - j = scan( specifier(i),':') - - if (j > 0) then - str1 = trim(adjustl( specifier(i)(:j-1) )) - str2 = trim(adjustl( specifier(i)(j+1:) )) - fld_name(i) = trim(adjustl( str1 )) - src_name(i) = trim(adjustl( str2 )) - else - fld_name(i) = trim(adjustl( specifier(i) )) - src_name(i) = trim(adjustl( specifier(i) )) - endif - - fld_cnt = fld_cnt + 1 - - enddo count_cnst - - if( fld_cnt < 1 ) then - nullify(fields) - return - end if - - !----------------------------------------------------------------------- - ! ... allocate field type array - !----------------------------------------------------------------------- - allocate( fields(fld_cnt), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'specify_fields: failed to allocate fields array; error = ',astat - call endrun - end if - - do i = 1,fld_cnt - fields(i)%fldnam = fld_name(i) - fields(i)%srcnam = src_name(i) - enddo - - deallocate(fld_name, src_name) - - end subroutine specify_fields - -!------------------------------------------------------------------------------ - - subroutine init_trc_restart( whence, piofile, tr_file ) - - implicit none - character(len=*), intent(in) :: whence - type(file_desc_t), intent(inout) :: piofile - type(trfile), intent(inout) :: tr_file - - character(len=32) :: name - integer :: ioerr, mcdimid, maxlen - - - ! Dimension should already be defined in restart file - call pio_seterrorhandling(pioFile, PIO_BCAST_ERROR) - ioerr = pio_inq_dimid(pioFile,'max_chars', mcdimid) - call pio_seterrorhandling(pioFile, PIO_INTERNAL_ERROR) - ! but define it if nessasary - if(ioerr/= PIO_NOERR) then - ioerr = pio_def_dim(pioFile, 'max_chars', SHR_KIND_CL, mcdimid) - end if - - if(len_trim(tr_file%curr_filename)>1) then - allocate(tr_file%currfnameid) - name = trim(whence)//'_curr_fname' - ioerr = pio_def_var(pioFile, name,pio_char, (/mcdimid/), tr_file%currfnameid) - ioerr = pio_put_att(pioFile, tr_file%currfnameid, 'offset_time', tr_file%offset_time) - maxlen = len_trim(tr_file%curr_filename) - ioerr = pio_put_att(pioFile, tr_file%currfnameid, 'actual_len', maxlen) - else - nullify(tr_file%currfnameid) - end if - - if(len_trim(tr_file%next_filename)>1) then - allocate(tr_file%nextfnameid) - name = trim(whence)//'_next_fname' - ioerr = pio_def_var(pioFile, name,pio_char, (/mcdimid/), tr_file%nextfnameid) - maxlen = len_trim(tr_file%next_filename) - ioerr = pio_put_att(pioFile, tr_file%nextfnameid, 'actual_len', maxlen) - else - nullify(tr_file%nextfnameid) - end if - end subroutine init_trc_restart -!------------------------------------------------------------------------- -! writes file names to restart file -!------------------------------------------------------------------------- - subroutine write_trc_restart( piofile, tr_file ) - - implicit none - - type(file_desc_t), intent(inout) :: piofile - type(trfile), intent(inout) :: tr_file - - integer :: ioerr, slen ! error status - if(associated(tr_file%currfnameid)) then - ioerr = pio_put_var(pioFile, tr_file%currfnameid, tr_file%curr_filename) - deallocate(tr_file%currfnameid) - nullify(tr_file%currfnameid) - end if - if(associated(tr_file%nextfnameid)) then - ioerr = pio_put_var(pioFile, tr_file%nextfnameid, tr_file%next_filename) - deallocate(tr_file%nextfnameid) - nullify(tr_file%nextfnameid) - end if - end subroutine write_trc_restart - -!------------------------------------------------------------------------- -! reads file names from restart file -!------------------------------------------------------------------------- - subroutine read_trc_restart( whence, piofile, tr_file ) - - implicit none - - character(len=*), intent(in) :: whence - type(file_desc_t), intent(inout) :: piofile - type(trfile), intent(inout) :: tr_file - type(var_desc_t) :: vdesc - character(len=64) :: name - integer :: ioerr ! error status - integer :: slen - - call PIO_SetErrorHandling(piofile, PIO_BCAST_ERROR) - name = trim(whence)//'_curr_fname' - ioerr = pio_inq_varid(piofile, name, vdesc) - if(ioerr==PIO_NOERR) then - tr_file%curr_filename=' ' - ioerr = pio_get_att(piofile, vdesc, 'offset_time', tr_file%offset_time) - ioerr = pio_get_att(piofile, vdesc, 'actual_len', slen) - ioerr = pio_get_var(piofile, vdesc, tr_file%curr_filename) - if(slen P2 (decreasing up) -!--- integrates (p-avg) the value F0 from F on the grid P -!--- assumes model pressure increases from top to bottom. -!---NOTE reverse order in P's -!---Assume that the quantity is constant over range halfway to layer above/below -!--- and calculate box edges from model top to model bottom -! -!---For a model level between pressure range P1 > P2 (decreasing up) -!---calculate the SOM Z-moments of the loss freq at std z* (log-p) intervals -!-------- the pressure levels BETWEEN z* values are: -! P(i) < P(i+1) bounds z*(i) -!-------- The MOMENTS for a square-wave or 'bar': F(x)=F0 b<=x<=c, =0.0 else -!----- S0 = f0 (x) [from x=b to x=c] -!----------------------------------------------------------------------- - implicit none - integer, intent(in) :: NL - real(r8), intent(in) :: P1,P2,P(NL+1),F(NL) - real(r8), intent(out):: F0 - integer I - real(r8) XB,XC,PC,PB,SGNF0,PF1,PF2 -!----------------------------------------------------------------------- - F0 = 0._r8 - ! - do I = 1,NL - PF1=P(I) - PF2=P(I+1) - ! - PC = min(P1,PF2) - PB = max(P2,PF1) - ! - if (PC .gt. PB) then -!--- have condition: P1 .ge. PC .gt. PB .ge. P2 -!--- and 0 .le. XB .lt. XC .le. 1 - XC = (PC-P2)/(P1-P2) - XB = (PB-P2)/(P1-P2) -!-------- assume that the quantity, F, is constant over interval [XLO,XUP], -!-------- F0: (c-b), -!-------- calculate its contribution to the moments in the interval [0,1] - F0 = F0 +F(I) *(XC -XB) - endif - enddo -!---limiter on Z-moments: force monotonicity (tables can be + or -) - SGNF0 = sign(1._r8, F0) - F0 = abs(F0) - F0 = SGNF0 * F0 - END SUBROUTINE vert_interp_uci_single -!------------------------------------------------------------------------------ - subroutine vert_interp_ub( ncol, nlevs, plevs, datain, dataout ) - use ref_pres, only : ptop_ref - - - !----------------------------------------------------------------------- - ! - ! Interpolate data from current time-interpolated values to top interface pressure - ! -- from mo_tgcm_ubc.F90 - !-------------------------------------------------------------------------- - implicit none - ! Arguments - ! - integer, intent(in) :: ncol - integer, intent(in) :: nlevs - real(r8), intent(in) :: plevs(nlevs) - real(r8), intent(in) :: datain(ncol,nlevs) - real(r8), intent(out) :: dataout(ncol) - - ! - ! local variables - ! - integer :: i,ku,kl,kk - real(r8) :: pinterp, delp - - pinterp = ptop_ref - - if( pinterp <= plevs(1) ) then - kl = 1 - ku = 1 - delp = 0._r8 - else if( pinterp >= plevs(nlevs) ) then - kl = nlevs - ku = nlevs - delp = 0._r8 - else - - do kk = 2,nlevs - if( pinterp <= plevs(kk) ) then - ku = kk - kl = kk - 1 - delp = log( pinterp/plevs(kk) ) / log( plevs(kk-1)/plevs(kk) ) - exit - end if - end do - - end if - - do i = 1,ncol - dataout(i) = datain(i,kl) + delp * (datain(i,ku) - datain(i,kl)) - end do - - end subroutine vert_interp_ub -!------------------------------------------------------------------------------ - -!------------------------------------------------------------------------------ -!------------------------------------------------------------------------------ - subroutine advance_file(file) - - !------------------------------------------------------------------------------ - ! This routine advances to the next file - !------------------------------------------------------------------------------ - - use shr_sys_mod, only: shr_sys_system - use ioFileMod, only: getfil - - implicit none - - type(trfile), intent(inout) :: file - - !----------------------------------------------------------------------- - ! local variables - !----------------------------------------------------------------------- - character(len=shr_kind_cl) :: ctmp - character(len=shr_kind_cl) :: loc_fname - integer :: istat, astat - - !----------------------------------------------------------------------- - ! close current file ... - !----------------------------------------------------------------------- - call pio_closefile( file%curr_fileid ) - - !----------------------------------------------------------------------- - ! remove if requested - !----------------------------------------------------------------------- - if( file%remove_trc_file ) then - call getfil( file%curr_filename, loc_fname, 0 ) - write(iulog,*) 'advance_file: removing file = ',trim(loc_fname) - ctmp = 'rm -f ' // trim(loc_fname) - write(iulog,*) 'advance_file: fsystem issuing command - ' - write(iulog,*) trim(ctmp) - call shr_sys_system( ctmp, istat ) - end if - - !----------------------------------------------------------------------- - ! Advance the filename and file id - !----------------------------------------------------------------------- - file%curr_filename = file%next_filename - file%curr_fileid = file%next_fileid - - !----------------------------------------------------------------------- - ! Advance the curr_data_times - !----------------------------------------------------------------------- - deallocate( file%curr_data_times, stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'advance_file: failed to deallocate file%curr_data_times array; error = ',astat - call endrun - end if - allocate( file%curr_data_times( size( file%next_data_times ) ), stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'advance_file: failed to allocate file%curr_data_times array; error = ',astat - call endrun - end if - file%curr_data_times(:) = file%next_data_times(:) - - !----------------------------------------------------------------------- - ! delete information about next file (as was just assigned to current) - !----------------------------------------------------------------------- - file%next_filename = '' - - deallocate( file%next_data_times, stat=astat ) - if( astat/= 0 ) then - write(iulog,*) 'advance_file: failed to deallocate file%next_data_times array; error = ',astat - call endrun - end if - nullify( file%next_data_times ) - - end subroutine advance_file - -end module tracer_data From 9bb98d766f8cb6e44e40805ad7c856ad6c0861b2 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Wed, 28 Aug 2024 22:45:09 -0500 Subject: [PATCH 597/904] Add ext_frc_volc_type to use_cases having only volcanic SO2 varying over years --- .../use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml | 1 + .../use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml | 1 + .../20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml | 1 + .../use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml | 1 + 4 files changed, 4 insertions(+) diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml index 94cdfc6d000d..493affd96414 100644 --- a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-nat_chemUCI-Linoz-mam5-vbs.xml @@ -22,6 +22,7 @@ +INTERP_MISSING_MONTHS CYCLICAL 1850 atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml index 6f0754423592..83d4c4d92717 100644 --- a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-volc_chemUCI-Linoz-mam5-vbs.xml @@ -23,6 +23,7 @@ +INTERP_MISSING_MONTHS CYCLICAL 1850 atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml index f70010a82c44..671eb826ef14 100644 --- a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xGHG-xaer_chemUCI-Linoz-mam5-vbs.xml @@ -22,6 +22,7 @@ +INTERP_MISSING_MONTHS CYCLICAL 1850 atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc diff --git a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml index 359a292e9e12..8dd525d7e5d3 100644 --- a/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml +++ b/components/eam/bld/namelist_files/use_cases/20TR_eam_CMIP6-xaer_chemUCI-Linoz-mam5-vbs.xml @@ -19,6 +19,7 @@ +INTERP_MISSING_MONTHS CYCLICAL 1850 atm/cam/chem/trop_mozart_aero/emis/chem_gases/2degrees/emissions-cmip6_NO2_aircraft_vertical_1750-2015_1.9x2.5_c20170608.nc From d8d89e69a472ffb5e8f005bfdd20908e97c7a636 Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Thu, 29 Aug 2024 11:48:28 -0700 Subject: [PATCH 598/904] Update shell_commands to remove spa --- .../mam4xx/srf_online_emiss_constituent_fluxes/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands index 05b0b4954116..f47a1729ed74 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -11,7 +11,7 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/u #Update IC file and add the processes #------------------------------------------------------ $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,rrtmgp,mam4_srf_online_emiss" -b From 86b6965b6ca6aaea3f74c5e6ce07fe86e5dde736 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Aug 2024 13:44:57 -0600 Subject: [PATCH 599/904] Update rrtmgp branch --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 02c2e03df33e..8ff525eeed1d 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 02c2e03df33edca6101670168ca187cb94c0768e +Subproject commit 8ff525eeed1d87a2ca6f251c4d16b46222c5554d From 5afe73dcde08ea9c68fc3e8c4486bce49a1c5251 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Aug 2024 13:45:23 -0600 Subject: [PATCH 600/904] Set kokkos to be the default rrtmgp backend --- components/eamxx/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 398efa3bc2df..ac2737f2fb6e 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -210,8 +210,8 @@ endif() # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) -option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE) -option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE) if (SCREAM_RRTMGP_ENABLE_YAKL) add_definitions("-DRRTMGP_ENABLE_YAKL") endif() From 4b573528ff76ff23b3613792db02ee0dea9a4dbc Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Thu, 29 Aug 2024 12:50:13 -0700 Subject: [PATCH 601/904] Adds units for mass and number ofr constituent fluxes --- .../src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 96d26ca22f9e..3a1a1d39b63e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -86,6 +86,7 @@ void MAMConstituentFluxes::set_grids( add_field("phis", scalar2d, m2 / s2, grid_name); // Constituent fluxes at the surface (gasses and aerosols) + //[units: kg/m2/s (mass) or #/m2/s (number)] add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); From ead3cb7f5b1be6a86d01fcdaaf1ec5e230a22dd3 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 29 Aug 2024 13:55:07 -0700 Subject: [PATCH 602/904] add runtime guard around added diagnostics --- .../cime_config/namelist_defaults_scream.xml | 1 + .../shoc/eamxx_shoc_process_interface.cpp | 76 ++++++++++--------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index f09d10b0b320..ef11b40431ab 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -232,6 +232,7 @@ be lost if SCREAM_HACK_XML is not enabled. 7.0 0.1 0.1 + false diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index f07912796002..9a99d0f15992 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -91,19 +91,24 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("ustar", scalar2d, m/s, grid_name, ps); add_field("obklen", scalar2d, m, grid_name, ps); - // Diagnostic output - mid point grid - add_field("brunt", scalar3d_mid, pow(s,-1), grid_name, ps); - add_field("shoc_mix", scalar3d_mid, m, grid_name, ps); - add_field("isotropy", scalar3d_mid, s, grid_name, ps); - - // Diagnostic output - interface grid - add_field("wthl_sec", scalar3d_int, K*(m/s), grid_name, ps); - add_field("thl_sec", scalar3d_int, pow(K,2), grid_name, ps); - add_field("wqw_sec", scalar3d_int, (kg/kg)*(m/s), grid_name, ps); - add_field("qw_sec", scalar3d_int, pow(kg/kg,2), grid_name, ps); - add_field("uw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); - add_field("vw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); - add_field("w3", scalar3d_int, pow(m/s,3), grid_name, ps); + // Extra SHOC output diagnostics + if (m_params.get("extra_shoc_diags", false)) { + + // Diagnostic output - mid point grid + add_field("brunt", scalar3d_mid, pow(s,-1), grid_name, ps); + add_field("shoc_mix", scalar3d_mid, m, grid_name, ps); + add_field("isotropy", scalar3d_mid, s, grid_name, ps); + + // Diagnostic output - interface grid + add_field("wthl_sec", scalar3d_int, K*(m/s), grid_name, ps); + add_field("thl_sec", scalar3d_int, pow(K,2), grid_name, ps); + add_field("wqw_sec", scalar3d_int, (kg/kg)*(m/s), grid_name, ps); + add_field("qw_sec", scalar3d_int, pow(kg/kg,2), grid_name, ps); + add_field("uw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("vw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("w3", scalar3d_int, pow(m/s,3), grid_name, ps); + + } // Extra SHOC output diagnostics // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -510,37 +515,40 @@ void SHOCMacrophysics::run_impl (const double dt) shoc_postprocess); Kokkos::fence(); - // SHOC output diagnostics - const auto& shoc_mix = get_field_out("shoc_mix").get_view(); - Kokkos::deep_copy(shoc_mix,history_output.shoc_mix); + // Extra SHOC output diagnostics + if (m_params.get("extra_shoc_diags", false)) { + + const auto& shoc_mix = get_field_out("shoc_mix").get_view(); + Kokkos::deep_copy(shoc_mix,history_output.shoc_mix); - const auto& brunt = get_field_out("brunt").get_view(); - Kokkos::deep_copy(brunt,history_output.brunt); + const auto& brunt = get_field_out("brunt").get_view(); + Kokkos::deep_copy(brunt,history_output.brunt); - const auto& w3 = get_field_out("w3").get_view(); - Kokkos::deep_copy(w3,history_output.w3); + const auto& w3 = get_field_out("w3").get_view(); + Kokkos::deep_copy(w3,history_output.w3); - const auto& isotropy = get_field_out("isotropy").get_view(); - Kokkos::deep_copy(isotropy,history_output.isotropy); + const auto& isotropy = get_field_out("isotropy").get_view(); + Kokkos::deep_copy(isotropy,history_output.isotropy); - const auto& wthl_sec = get_field_out("wthl_sec").get_view(); - Kokkos::deep_copy(wthl_sec,history_output.wthl_sec); + const auto& wthl_sec = get_field_out("wthl_sec").get_view(); + Kokkos::deep_copy(wthl_sec,history_output.wthl_sec); - const auto& wqw_sec = get_field_out("wqw_sec").get_view(); - Kokkos::deep_copy(wqw_sec,history_output.wqw_sec); + const auto& wqw_sec = get_field_out("wqw_sec").get_view(); + Kokkos::deep_copy(wqw_sec,history_output.wqw_sec); - const auto& uw_sec = get_field_out("uw_sec").get_view(); - Kokkos::deep_copy(uw_sec,history_output.uw_sec); + const auto& uw_sec = get_field_out("uw_sec").get_view(); + Kokkos::deep_copy(uw_sec,history_output.uw_sec); - const auto& vw_sec = get_field_out("vw_sec").get_view(); - Kokkos::deep_copy(vw_sec,history_output.vw_sec); + const auto& vw_sec = get_field_out("vw_sec").get_view(); + Kokkos::deep_copy(vw_sec,history_output.vw_sec); - const auto& qw_sec = get_field_out("qw_sec").get_view(); - Kokkos::deep_copy(qw_sec,history_output.qw_sec); + const auto& qw_sec = get_field_out("qw_sec").get_view(); + Kokkos::deep_copy(qw_sec,history_output.qw_sec); - const auto& thl_sec = get_field_out("thl_sec").get_view(); - Kokkos::deep_copy(thl_sec,history_output.thl_sec); + const auto& thl_sec = get_field_out("thl_sec").get_view(); + Kokkos::deep_copy(thl_sec,history_output.thl_sec); + } // Extra SHOC output diagnostics } // ========================================================================================= void SHOCMacrophysics::finalize_impl() From 4c3e57509382eb889ac070036e5778918f304e6b Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Aug 2024 17:12:20 -0600 Subject: [PATCH 603/904] Allow option to force rrtmgp inputs to be layoutleft --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 260 ++++++++++-------- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 30 +- 2 files changed, 167 insertions(+), 123 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index e89e02fe301b..67536538b48e 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -23,6 +23,48 @@ using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; +namespace { + +struct ConvertToRrtmgpSubview +{ + int beg; + int ncol; + + template + View subview1d(const View& v) const { + return View(v, std::make_pair(beg, beg+ncol)); + } + + template + View subview2d_impl(const View& v, const int inner_dim) const { + return View(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); + } + +#ifdef RRTMGP_LAYOUT_LEFT + template + BufferView subview2d(const FieldView&, const BufferView& buffer_view, const int inner_dim) const { + return BufferView(buffer_view, std::make_pair(0, ncol), Kokkos::ALL); + } +#else + // Be sure to trim the excess + // items from the field manager views due to simd packs. If we don't trim, then + // check_range_k will fail due to looking at unused values. Once rrtmgp can handle + // packs, this won't be necessary. + template + FieldView subview2d(const FieldView& field_view, const BufferView&, const int inner_dim) const { + return subview2d_impl(field_view, inner_dim); + } +#endif + + template + View subview3d(const View& v) const { + // The range assumes these are buffer views, not fields + return View(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + } +}; + +} + RRTMGPRadiation:: RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) @@ -255,7 +297,12 @@ size_t RRTMGPRadiation::requested_buffer_size_in_bytes() const Buffer::num_3d_nlay_nswbands*m_col_chunk_size*(m_nlay)*m_nswbands + Buffer::num_3d_nlay_nlwbands*m_col_chunk_size*(m_nlay)*m_nlwbands + Buffer::num_3d_nlay_nswgpts*m_col_chunk_size*(m_nlay)*m_nswgpts + - Buffer::num_3d_nlay_nlwgpts*m_col_chunk_size*(m_nlay)*m_nlwgpts; + Buffer::num_3d_nlay_nlwgpts*m_col_chunk_size*(m_nlay)*m_nlwgpts * +#if defined(RRTMGP_ENABLE_YAKL) && defined(RRTMGP_ENABLE_KOKKOS) + 2; +#else + 1; +#endif return interface_request * sizeof(Real); } // RRTMGPRadiation::requested_buffer_size @@ -409,13 +456,7 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.cld_tau_lw_bnd.totElems(); #endif - // During the transition to kokkos, the buffer views/arrays will point to the same memory, - // so care is needed to avoid repeating calculations in such a way that answers change. - // Example: buff_view(x) += foo; - // Stuff like this cannot be done twice when both kokkos and yakl are enabled #ifdef RRTMGP_ENABLE_KOKKOS - mem = reinterpret_cast(buffer_manager.get_memory()); - // 1d arrays m_buffer.mu0_k = decltype(m_buffer.mu0_k)(mem, m_col_chunk_size); mem += m_buffer.mu0_k.size(); @@ -847,7 +888,11 @@ void RRTMGPRadiation::run_impl (const double dt) { this->log(LogLevel::debug, "[RRTMGP::run_impl] Col chunk beg,end: " + std::to_string(beg) + ", " + std::to_string(beg+ncol) + "\n"); - + // d_tint and d_dz are used in eamxx calls and therefore + // must be layout right + ulrreal2dk d_tint = ulrreal2dk(m_buffer.d_tint.data(), m_col_chunk_size, m_nlay+1); + ulrreal2dk d_dz = ulrreal2dk(m_buffer.d_dz.data(), m_col_chunk_size, m_nlay); + auto d_mu0 = m_buffer.cosine_zenith; #ifdef RRTMGP_ENABLE_YAKL // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. @@ -920,123 +965,68 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cld_tau_lw_gpt = subview_3d(m_buffer.cld_tau_lw_gpt); #endif #ifdef RRTMGP_ENABLE_KOKKOS - // If YAKL is on, we don't want aliased memory in both the yakl and kokos - // subviews, so make new views and deep_copy. Also, be sure to trim the excess - // items from the field manager views due to simd packs. If we don't trim, then - // check_range_k will fail due to looking at unused values. Once rrtmgp can handle - // packs, this won't be necessary. - auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { - ureal1dk subv(v, std::make_pair(beg, beg+ncol)); -#ifdef RRTMGP_ENABLE_YAKL - real1dk rv(v.label(), ncol); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { - cureal1dk subv(v, std::make_pair(beg, beg+ncol)); -#ifdef RRTMGP_ENABLE_YAKL - creal1dk rv(v.label(), ncol); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); -#ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_2dkc = [&](const cureal2dk& v, const int inner_dim) -> cureal2dk { - cureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); -#ifdef RRTMGP_ENABLE_YAKL - creal2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { - ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); // The range assumes these are buffer views, not fields -#ifdef RRTMGP_ENABLE_YAKL - real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; + ConvertToRrtmgpSubview conv = {beg, ncol}; // Note, ncol will not necessary be m_col_chunk_size because the number of cols // will not always be evenly divided by m_col_chunk_size. In most cases, the // extra space will not cause any problems, but it does sometimes. - auto p_lay_k = subview_2dkc(d_pmid, m_nlay); - auto t_lay_k = subview_2dkc(d_tmid, m_nlay); - auto p_lev_k = subview_2dkc(d_pint, m_nlay+1); - auto p_del_k = subview_2dkc(d_pdel, m_nlay); + auto p_lay_k = conv.subview2d(d_pmid, m_buffer.p_lay_k, m_nlay); + auto t_lay_k = conv.subview2d(d_tmid, m_buffer.t_lay_k, m_nlay); + auto p_lev_k = conv.subview2d(d_pint, m_buffer.p_lev_k, m_nlay+1); + auto z_del_k = conv.subview2d(d_dz, m_buffer.z_del_k, m_nlay); + auto p_del_k = conv.subview2d(d_pdel, m_buffer.p_del_k, m_nlay); + auto t_lev_k = conv.subview2d(d_tint, m_buffer.t_lev_k, m_nlay+1); auto sfc_alb_dir_k = m_buffer.sfc_alb_dir_k; auto sfc_alb_dif_k = m_buffer.sfc_alb_dif_k; - auto sfc_alb_dir_vis_k = subview_1dkc(d_sfc_alb_dir_vis); - auto sfc_alb_dir_nir_k = subview_1dkc(d_sfc_alb_dir_nir); - auto sfc_alb_dif_vis_k = subview_1dkc(d_sfc_alb_dif_vis); - auto sfc_alb_dif_nir_k = subview_1dkc(d_sfc_alb_dif_nir); - auto qc_k = subview_2dkc(d_qc, m_nlay); - auto nc_k = subview_2dkc(d_nc, m_nlay); - auto qi_k = subview_2dkc(d_qi, m_nlay); + auto sfc_alb_dir_vis_k = conv.subview1d(d_sfc_alb_dir_vis); + auto sfc_alb_dir_nir_k = conv.subview1d(d_sfc_alb_dir_nir); + auto sfc_alb_dif_vis_k = conv.subview1d(d_sfc_alb_dif_vis); + auto sfc_alb_dif_nir_k = conv.subview1d(d_sfc_alb_dif_nir); + auto qc_k = conv.subview2d(d_qc, m_buffer.qc_k, m_nlay); + auto nc_k = conv.subview2d(d_nc, m_buffer.nc_k, m_nlay); + auto qi_k = conv.subview2d(d_qi, m_buffer.qi_k, m_nlay); auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; - auto rel_k = subview_2dkc(d_rel, m_nlay); - auto rei_k = subview_2dkc(d_rei, m_nlay); - auto sw_flux_up_k = subview_2dk(d_sw_flux_up, m_nlay+1); - auto sw_flux_dn_k = subview_2dk(d_sw_flux_dn, m_nlay+1); - auto sw_flux_dn_dir_k = subview_2dk(d_sw_flux_dn_dir, m_nlay+1); - auto lw_flux_up_k = subview_2dk(d_lw_flux_up, m_nlay+1); - auto lw_flux_dn_k = subview_2dk(d_lw_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_up_k = subview_2dk(d_sw_clnclrsky_flux_up, m_nlay+1); - auto sw_clnclrsky_flux_dn_k = subview_2dk(d_sw_clnclrsky_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); - auto sw_clrsky_flux_up_k = subview_2dk(d_sw_clrsky_flux_up, m_nlay+1); - auto sw_clrsky_flux_dn_k = subview_2dk(d_sw_clrsky_flux_dn, m_nlay+1); - auto sw_clrsky_flux_dn_dir_k = subview_2dk(d_sw_clrsky_flux_dn_dir, m_nlay+1); - auto sw_clnsky_flux_up_k = subview_2dk(d_sw_clnsky_flux_up, m_nlay+1); - auto sw_clnsky_flux_dn_k = subview_2dk(d_sw_clnsky_flux_dn, m_nlay+1); - auto sw_clnsky_flux_dn_dir_k = subview_2dk(d_sw_clnsky_flux_dn_dir, m_nlay+1); - auto lw_clnclrsky_flux_up_k = subview_2dk(d_lw_clnclrsky_flux_up, m_nlay+1); - auto lw_clnclrsky_flux_dn_k = subview_2dk(d_lw_clnclrsky_flux_dn, m_nlay+1); - auto lw_clrsky_flux_up_k = subview_2dk(d_lw_clrsky_flux_up, m_nlay+1); - auto lw_clrsky_flux_dn_k = subview_2dk(d_lw_clrsky_flux_dn, m_nlay+1); - auto lw_clnsky_flux_up_k = subview_2dk(d_lw_clnsky_flux_up, m_nlay+1); - auto lw_clnsky_flux_dn_k = subview_2dk(d_lw_clnsky_flux_dn, m_nlay+1); + auto rel_k = conv.subview2d(d_rel, m_buffer.eff_radius_qc_k, m_nlay); + auto rei_k = conv.subview2d(d_rei, m_buffer.eff_radius_qi_k, m_nlay); + auto sw_flux_up_k = conv.subview2d(d_sw_flux_up, m_buffer.sw_flux_up_k, m_nlay+1); + auto sw_flux_dn_k = conv.subview2d(d_sw_flux_dn, m_buffer.sw_flux_dn_k, m_nlay+1); + auto sw_flux_dn_dir_k = conv.subview2d(d_sw_flux_dn_dir, m_buffer.sw_flux_dn_dir_k, m_nlay+1); + auto lw_flux_up_k = conv.subview2d(d_lw_flux_up, m_buffer.lw_flux_up_k, m_nlay+1); + auto lw_flux_dn_k = conv.subview2d(d_lw_flux_dn, m_buffer.lw_flux_dn_k, m_nlay+1); + auto sw_clnclrsky_flux_up_k = conv.subview2d(d_sw_clnclrsky_flux_up, m_buffer.sw_clnclrsky_flux_up_k, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = conv.subview2d(d_sw_clnclrsky_flux_dn, m_buffer.sw_clnclrsky_flux_dn_k, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = conv.subview2d(d_sw_clnclrsky_flux_dn_dir, m_buffer.sw_clnclrsky_flux_dn_dir_k, m_nlay+1); + auto sw_clrsky_flux_up_k = conv.subview2d(d_sw_clrsky_flux_up, m_buffer.sw_clrsky_flux_up_k, m_nlay+1); + auto sw_clrsky_flux_dn_k = conv.subview2d(d_sw_clrsky_flux_dn, m_buffer.sw_clrsky_flux_dn_k, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = conv.subview2d(d_sw_clrsky_flux_dn_dir, m_buffer.sw_clrsky_flux_dn_dir_k, m_nlay+1); + auto sw_clnsky_flux_up_k = conv.subview2d(d_sw_clnsky_flux_up, m_buffer.sw_clnsky_flux_up_k, m_nlay+1); + auto sw_clnsky_flux_dn_k = conv.subview2d(d_sw_clnsky_flux_dn, m_buffer.sw_clnsky_flux_dn_k, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = conv.subview2d(d_sw_clnsky_flux_dn_dir, m_buffer.sw_clnsky_flux_dn_dir_k, m_nlay+1); + auto lw_clnclrsky_flux_up_k = conv.subview2d(d_lw_clnclrsky_flux_up, m_buffer.lw_clnclrsky_flux_up_k, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = conv.subview2d(d_lw_clnclrsky_flux_dn, m_buffer.lw_clnclrsky_flux_dn_k, m_nlay+1); + auto lw_clrsky_flux_up_k = conv.subview2d(d_lw_clrsky_flux_up, m_buffer.lw_clrsky_flux_up_k, m_nlay+1); + auto lw_clrsky_flux_dn_k = conv.subview2d(d_lw_clrsky_flux_dn, m_buffer.lw_clrsky_flux_dn_k, m_nlay+1); + auto lw_clnsky_flux_up_k = conv.subview2d(d_lw_clnsky_flux_up, m_buffer.lw_clnsky_flux_up_k, m_nlay+1); + auto lw_clnsky_flux_dn_k = conv.subview2d(d_lw_clnsky_flux_dn, m_buffer.lw_clnsky_flux_dn_k, m_nlay+1); auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; auto sw_bnd_flux_dif_k = m_buffer.sw_bnd_flux_dif_k; auto lw_bnd_flux_up_k = m_buffer.lw_bnd_flux_up_k; auto lw_bnd_flux_dn_k = m_buffer.lw_bnd_flux_dn_k; - auto sfc_flux_dir_vis_k = subview_1dk(d_sfc_flux_dir_vis); - auto sfc_flux_dir_nir_k = subview_1dk(d_sfc_flux_dir_nir); - auto sfc_flux_dif_vis_k = subview_1dk(d_sfc_flux_dif_vis); - auto sfc_flux_dif_nir_k = subview_1dk(d_sfc_flux_dif_nir); + auto sfc_flux_dir_vis_k = conv.subview1d(d_sfc_flux_dir_vis); + auto sfc_flux_dir_nir_k = conv.subview1d(d_sfc_flux_dir_nir); + auto sfc_flux_dif_vis_k = conv.subview1d(d_sfc_flux_dif_vis); + auto sfc_flux_dif_nir_k = conv.subview1d(d_sfc_flux_dif_nir); auto aero_tau_sw_k = m_buffer.aero_tau_sw_k; auto aero_ssa_sw_k = m_buffer.aero_ssa_sw_k; auto aero_g_sw_k = m_buffer.aero_g_sw_k; auto aero_tau_lw_k = m_buffer.aero_tau_lw_k; - auto cld_tau_sw_bnd_k = subview_3dk(m_buffer.cld_tau_sw_bnd_k); - auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); - auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); - auto cld_tau_lw_gpt_k = subview_3dk(m_buffer.cld_tau_lw_gpt_k); + auto cld_tau_sw_bnd_k = conv.subview3d(m_buffer.cld_tau_sw_bnd_k); + auto cld_tau_lw_bnd_k = conv.subview3d(m_buffer.cld_tau_lw_bnd_k); + auto cld_tau_sw_gpt_k = conv.subview3d(m_buffer.cld_tau_sw_gpt_k); + auto cld_tau_lw_gpt_k = conv.subview3d(m_buffer.cld_tau_lw_gpt_k); #endif - auto d_tint = m_buffer.d_tint; - auto d_dz = m_buffer.d_dz; - auto d_mu0 = m_buffer.cosine_zenith; // Set gas concs to "view" only the first ncol columns #ifdef RRTMGP_ENABLE_YAKL @@ -1045,7 +1035,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.ncol = ncol; - m_gas_concs_k.concs = subview_3dk(gas_concs_k); + m_gas_concs_k.concs = conv.subview3d(gas_concs_k); #endif // Copy data from the FieldManager to the YAKL arrays @@ -1156,6 +1146,26 @@ void RRTMGPRadiation::run_impl (const double dt) { } #endif #ifdef RRTMGP_ENABLE_KOKKOS +#ifdef RRTMGP_LAYOUT_LEFT + // Copy to layout left buffer views + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + p_lay_k(i,k) = d_pmid(icol,k); + t_lay_k(i,k) = d_tmid(icol,k); + z_del_k(i,k) = d_dz(i,k); + p_del_k(i,k) = d_pdel(icol,k); + qc_k(i,k) = d_qc(icol,k); + nc_k(i,k) = d_nc(icol,k); + qi_k(i,k) = d_qi(icol,k); + rel_k(i,k) = d_rel(icol,k); + rei_k(i,k) = d_rei(icol,k); + p_lev_k(i,k) = d_pint(icol,k); + t_lev_k(i,k) = d_tint(i,k); + }); + + p_lev_k(i,nlay) = d_pint(icol,nlay); + t_lev_k(i,nlay) = d_tint(i,nlay); +#endif + // Note that RRTMGP expects ordering (col,lay,bnd) but the FM keeps things in (col,bnd,lay) order if (do_aerosol_rad) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { @@ -1215,7 +1225,7 @@ void RRTMGPRadiation::run_impl (const double dt) { auto f = name=="o3" ? get_field_in(full_name) : get_field_out(full_name); auto d_vmr = f.get_view(); #ifdef RRTMGP_ENABLE_KOKKOS - auto tmp2d_k = subview_2dkc(d_vmr, m_nlay); + auto tmp2d_k = conv.subview2d_impl(d_vmr, m_nlay); #endif #ifdef RRTMGP_ENABLE_YAKL @@ -1328,11 +1338,8 @@ void RRTMGPRadiation::run_impl (const double dt) { iwp(i+1,k+1) *= 1e3; #endif #ifdef RRTMGP_ENABLE_KOKKOS -#ifndef RRTMGP_ENABLE_YAKL - // lwp and lwp_k point to the same memory lwp_k(i,k) *= 1e3; iwp_k(i,k) *= 1e3; -#endif #endif }); }); @@ -1385,7 +1392,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS interface_t::rrtmgp_main( ncol, m_nlay, - p_lay_k, t_lay_k, p_lev_k, d_tint, + p_lay_k, t_lay_k, p_lev_k, t_lev_k, m_gas_concs_k, sfc_alb_dir_k, sfc_alb_dif_k, d_mu0, lwp_k, iwp_k, rel_k, rei_k, cldfrac_tot_k, @@ -1592,7 +1599,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); interface_t::compute_aerocom_cloudtop( - ncol, nlay, t_lay_k, p_lay_k, p_del_k, d_dz, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, + ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k); @@ -1658,6 +1665,31 @@ void RRTMGPRadiation::run_impl (const double dt) { const int icol = i + beg; d_sfc_flux_sw_net(icol) = sw_flux_dn_k(i,kbot_k) - sw_flux_up_k(i,kbot_k); d_sfc_flux_lw_dn(icol) = lw_flux_dn_k(i,kbot_k); +#ifdef RRTMGP_LAYOUT_LEFT + // Copy from layout left buffer views back to layout right fields + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { + d_sw_flux_up(icol,k) = sw_flux_up_k(i,k); + d_sw_flux_dn(icol,k) = sw_flux_dn_k(i,k); + d_sw_flux_dn_dir(icol,k) = sw_flux_dn_dir_k(i,k); + d_lw_flux_up(icol,k) = lw_flux_up_k(i,k); + d_lw_flux_dn(icol,k) = lw_flux_dn_k(i,k); + d_sw_clnclrsky_flux_up(icol,k) = sw_clnclrsky_flux_up_k(i,k); + d_sw_clnclrsky_flux_dn(icol,k) = sw_clnclrsky_flux_dn_k(i,k); + d_sw_clnclrsky_flux_dn_dir(icol,k) = sw_clnclrsky_flux_dn_dir_k(i,k); + d_sw_clrsky_flux_up(icol,k) = sw_clrsky_flux_up_k(i,k); + d_sw_clrsky_flux_dn(icol,k) = sw_clrsky_flux_dn_k(i,k); + d_sw_clrsky_flux_dn_dir(icol,k) = sw_clrsky_flux_dn_dir_k(i,k); + d_sw_clnsky_flux_up(icol,k) = sw_clnsky_flux_up_k(i,k); + d_sw_clnsky_flux_dn(icol,k) = sw_clnsky_flux_dn_k(i,k); + d_sw_clnsky_flux_dn_dir(icol,k) = sw_clnsky_flux_dn_dir_k(i,k); + d_lw_clnclrsky_flux_up(icol,k) = lw_clnclrsky_flux_up_k(i,k); + d_lw_clnclrsky_flux_dn(icol,k) = lw_clnclrsky_flux_dn_k(i,k); + d_lw_clrsky_flux_up(icol,k) = lw_clrsky_flux_up_k(i,k); + d_lw_clrsky_flux_dn(icol,k) = lw_clrsky_flux_dn_k(i,k); + d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up_k(i,k); + d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn_k(i,k); + }); +#endif // Extract optical properties for COSP Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067_k); diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index e036e7ac08c0..329c7eeaba8d 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -14,15 +14,26 @@ namespace scream { * exactly ONE instance of this class in its list of subcomponents. */ +// rrtmgp is performance tuned for layout left views but will accept any +// view. We probably want to stick with layout left views for performance +// reasons even though this requires us to make copies of our fields (they +// are layout right). +#define RRTMGP_LAYOUT_LEFT + class RRTMGPRadiation : public AtmosphereProcess { public: - using KT = ekat::KokkosTypes; - using real1dk = typename KT::template view_1d; - using real2dk = typename KT::template view_2d; - using real3dk = typename KT::template view_3d; - using creal1dk = typename KT::template view_1d; - using creal2dk = typename KT::template view_2d; - using creal3dk = typename KT::template view_3d; + using KT = ekat::KokkosTypes; +#ifdef RRTMGP_LAYOUT_LEFT + using layout_t = Kokkos::LayoutLeft; +#else + using layout_t = typename ekat::KokkosTypes::Layout; +#endif + using real1dk = Kokkos::View; + using real2dk = Kokkos::View; + using real3dk = Kokkos::View; + using creal1dk = Kokkos::View; + using creal2dk = Kokkos::View; + using creal3dk = Kokkos::View; using ureal1dk = Unmanaged; using ureal2dk = Unmanaged; using ureal3dk = Unmanaged; @@ -32,7 +43,8 @@ class RRTMGPRadiation : public AtmosphereProcess { using ci_string = ekat::CaseInsensitiveString; - using layout_t = typename ekat::KokkosTypes::Layout; + using lrreal2dk = typename KT::template view_2d; + using ulrreal2dk = Unmanaged; #ifdef RRTMGP_ENABLE_KOKKOS using interface_t = rrtmgp::rrtmgp_interface; @@ -97,7 +109,7 @@ class RRTMGPRadiation : public AtmosphereProcess { // These are the gases that we keep track of int m_ngas; std::vector m_gas_names; - real1dk m_gas_mol_weights; + real1dk m_gas_mol_weights; #ifdef RRTMGP_ENABLE_YAKL GasConcs m_gas_concs; #endif From a42b2385ce3accc95cc664a24e11f15139479031 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 29 Aug 2024 18:12:29 -0600 Subject: [PATCH 604/904] EAMxx: use portable format for unsigned std::int64_t in printf --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 37cb251d7796..fe613259b163 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %" PRIx64 " (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + fprintf(stderr, "bfbhash> %14d %" PRIx64 " (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } From 04872f3350c20c5e10faf14785c11d647f6bbe12 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 30 Aug 2024 03:00:57 +0000 Subject: [PATCH 605/904] Update Aurora machine config --- cime_config/allactive/config_pesall.xml | 8 ++-- .../cmake_macros/oneapi-ifxgpu_aurora.cmake | 1 - cime_config/machines/config_batch.xml | 2 +- cime_config/machines/config_machines.xml | 47 +++++++++---------- .../eamxx/cmake/machine-files/aurora.cmake | 14 ++++++ 5 files changed, 41 insertions(+), 31 deletions(-) create mode 100644 components/eamxx/cmake/machine-files/aurora.cmake diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 84c0e24d8c9b..3392ad71ca10 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -1472,9 +1472,9 @@ - + - sunspot: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 2 nodes pure-MPI + sunspot|aurora: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 2 nodes pure-MPI -2 -2 @@ -1759,9 +1759,9 @@ - + - allactive+sunspot: default, 96 mpi x 1 omp @ root 0 + allactive+sunspot|aurora: default, 96 mpi x 1 omp @ root 0 96 96 diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake index 16288ce4dee7..6835515164f0 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake @@ -3,6 +3,5 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 448eefece216..13cfdfb337b6 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -562,7 +562,7 @@ - /lus/gecko/projects/CSC249ADSE15_CNDA/tools/qsub/throttle + /lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle EarlyAppAccess workq-route diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index fed0110bc32b..f1bed254da48 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3474,14 +3474,14 @@ oneapi-ifx,oneapi-ifxgpu,gnu mpich CSC249ADSE15_CNDA - /lus/gecko/projects/CSC249ADSE15_CNDA/performance_archive + /lus/flare/projects/CSC249ADSE15_CNDA/performance_archive .* - /lus/gecko/projects/CSC249ADSE15_CNDA/$USER/scratch - /lus/gecko/projects/CSC249ADSE15_CNDA/inputdata - /lus/gecko/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 + /lus/flare/projects/CSC249ADSE15_CNDA/$USER/scratch + /lus/flare/projects/CSC249ADSE15_CNDA/inputdata + /lus/flare/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE - /lus/gecko/projects/CSC249ADSE15_CNDA/baselines/$COMPILER - /lus/gecko/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc + /lus/flare/projects/CSC249ADSE15_CNDA/baselines/$COMPILER + /lus/flare/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc 16 e3sm_developer 4 @@ -3490,7 +3490,7 @@ 208 104 104 - 12 + 48 FALSE mpiexec @@ -3498,46 +3498,40 @@ -np {{ total_tasks }} --label -ppn {{ tasks_per_node }} - --cpu-bind $ENV{RANKS_BIND} -envall + --cpu-bind depth -envall -d $ENV{OMP_NUM_THREADS} $ENV{GPU_TILE_COMPACT} - /lus/gecko/projects/CSC249ADSE15_CNDA/modules/lmod.sh + /lus/flare/projects/CSC249ADSE15_CNDA/modules/lmod.sh /soft/sunspot_migrate/soft/packaging/lmod/lmod/init/csh /soft/sunspot_migrate/soft/packaging/lmod/lmod/init/env_modules_python.py module module /soft/sunspot_migrate/soft/packaging/lmod/lmod/libexec/lmod python - - /soft/modulefiles - /soft/restricted/CNDA/updates/modulefiles - spack-pe-gcc cmake + cmake - oneapi/eng-compiler/2023.05.15.007 + oneapi/eng-compiler/2024.04.15.002 + + + kokkos/git.7ff87a5-omp-sycl spack-pe-gcc cmake gcc/10.3.0 - - cray-pals - libfabric/1.15.2.0 - cray-libpals/1.3.2 - $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007 - /lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007 - /lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007 - /lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007/lib:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007/lib:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007/lib:$ENV{LD_LIBRARY_PATH} - /lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007/bin:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007/bin:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007/bin:$ENV{PATH} - list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 + /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002 + /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002 + /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002 + /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/lib:$ENV{LD_LIBRARY_PATH} + /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/bin:$ENV{PATH} 1 @@ -3554,6 +3548,9 @@ /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh 131072 20 + $ENV{KOKKOS_ROOT} + 1 + 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4 0 diff --git a/components/eamxx/cmake/machine-files/aurora.cmake b/components/eamxx/cmake/machine-files/aurora.cmake new file mode 100644 index 000000000000..59157285bab7 --- /dev/null +++ b/components/eamxx/cmake/machine-files/aurora.cmake @@ -0,0 +1,14 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) + +set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE) +set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE) +set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "") +set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "") + +set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") +set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") +set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") From fd04726e9b49f12b1b9e0e533d1cffece64151ba Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 30 Aug 2024 08:09:18 -0500 Subject: [PATCH 606/904] Remove trailing whitespace --- .../src/framework/mpas_stream_manager.F | 302 +++++++++--------- 1 file changed, 151 insertions(+), 151 deletions(-) diff --git a/components/mpas-framework/src/framework/mpas_stream_manager.F b/components/mpas-framework/src/framework/mpas_stream_manager.F index 9f9b46adae54..e8705c51a6d6 100644 --- a/components/mpas-framework/src/framework/mpas_stream_manager.F +++ b/components/mpas-framework/src/framework/mpas_stream_manager.F @@ -146,7 +146,7 @@ end subroutine seed_random if (err_local /= MPAS_STREAM_LIST_NOERR) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR STREAM_ERROR_WRITE('Problems while creating stream list') - return + return end if ! @@ -156,7 +156,7 @@ end subroutine seed_random if (err_local /= MPAS_STREAM_LIST_NOERR) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR STREAM_ERROR_WRITE('Problems while creating input alarm list') - return + return end if ! @@ -166,7 +166,7 @@ end subroutine seed_random if (err_local /= MPAS_STREAM_LIST_NOERR) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR STREAM_ERROR_WRITE('Problems while creating output alarm list') - return + return end if ! @@ -204,7 +204,7 @@ subroutine MPAS_stream_mgr_finalize(manager, ierr)!{{{ threadNum = mpas_threading_get_thread_num() - STREAM_DEBUG_WRITE('-- Called MPAS_stream_mgr_finalize()') + STREAM_DEBUG_WRITE('-- Called MPAS_stream_mgr_finalize()') if (present(ierr)) ierr = MPAS_STREAM_MGR_NOERR @@ -264,16 +264,16 @@ end subroutine MPAS_stream_mgr_finalize!}}} !> \author Michael Duda, Doug Jacobsen !> \date 13 June 2014 !> \details - !> Creates a new stream within the stream manager. The "direction" - !> argument may be either MPAS_STREAM_INPUT, MPAS_STREAM_OUTPUT, - !> MPAS_STREAM_INPUT_OUTPUT, or MPAS_STREAM_NONE. The "filename" argument - !> is the template of the filenames that are associated with the stream. - !> Knowing the interval between files, and - !> the filename template, a "referenceTime" argument must be provided to - !> specify the first timestamp appearing in any of the files associated with - !> the stream, thereby determining where the "file breaks" will occur between - !> timestamps. If no "referenceTime" is specified, the start time of the - !> clock associated with the stream handler will be used as the reference + !> Creates a new stream within the stream manager. The "direction" + !> argument may be either MPAS_STREAM_INPUT, MPAS_STREAM_OUTPUT, + !> MPAS_STREAM_INPUT_OUTPUT, or MPAS_STREAM_NONE. The "filename" argument + !> is the template of the filenames that are associated with the stream. + !> Knowing the interval between files, and + !> the filename template, a "referenceTime" argument must be provided to + !> specify the first timestamp appearing in any of the files associated with + !> the stream, thereby determining where the "file breaks" will occur between + !> timestamps. If no "referenceTime" is specified, the start time of the + !> clock associated with the stream handler will be used as the reference !> time. Additionally, the interval between records in the file may be !> specified using the optional "recordInterval" argument; if this argument !> is not supplied, the stream manager will assume that this interval is @@ -284,8 +284,8 @@ end subroutine MPAS_stream_mgr_finalize!}}} !> MPAS_IO_DOUBLE_PRECISION, or MPAS_IO_NATIVE_PRECISION; if this argument is !> not supplied, native precision is assumed. !> The optional argument clobberMode determines how the stream manager will - !> deal with existing files; possible options include MPAS_STREAM_CLOBBER_NEVER, - !> MPAS_STREAM_CLOBBER_APPEND, MPAS_STREAM_CLOBBER_TRUNCATE, + !> deal with existing files; possible options include MPAS_STREAM_CLOBBER_NEVER, + !> MPAS_STREAM_CLOBBER_APPEND, MPAS_STREAM_CLOBBER_TRUNCATE, !> and MPAS_STREAM_CLOBBER_OVERWRITE. The default behavior is to never modify !> existing files (MPAS_STREAM_CLOBBER_NEVER). !> The optional argument ioType specifies the I/O type to use for the @@ -398,14 +398,14 @@ subroutine MPAS_stream_mgr_create_stream(manager, streamID, direction, filename, if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR STREAM_ERROR_WRITE('Problems while creating input alarm list') deallocate(new_stream) - return + return end if call MPAS_stream_list_create(new_stream % alarmList_out, ierr=err_local) if (err_local /= MPAS_STREAM_LIST_NOERR) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR STREAM_ERROR_WRITE('Problems while creating output alarm list') deallocate(new_stream) - return + return end if call mpas_pool_create_pool(new_stream % att_pool) call mpas_pool_clone_pool(manager % defaultAtts, new_stream % att_pool) @@ -424,7 +424,7 @@ subroutine MPAS_stream_mgr_create_stream(manager, streamID, direction, filename, STREAM_ERROR_WRITE('Problems while adding stream to list') return end if - + manager % numStreams = manager % numStreams + 1 end if @@ -635,7 +635,7 @@ subroutine MPAS_stream_mgr_destroy_stream(manager, streamID, ierr)!{{{ deallocate(stream % stream) end if deallocate(stream) - + manager % numStreams = manager % numStreams - 1 end if @@ -649,8 +649,8 @@ end subroutine MPAS_stream_mgr_destroy_stream!}}} !> \author Michael Duda !> \date 22 August 2014 !> \details - !> Returns a pointer to the clock associated with the stream manager, - !> in which any stream alarms should be defined before being added to + !> Returns a pointer to the clock associated with the stream manager, + !> in which any stream alarms should be defined before being added to !> the stream manager via the MPAS_stream_mgr_add_alarm() routine. ! !----------------------------------------------------------------------- @@ -1072,7 +1072,7 @@ end subroutine MPAS_stream_mgr_add_stream_fields!}}} ! !----------------------------------------------------------------------- subroutine MPAS_stream_mgr_remove_field(manager, streamID, fieldName, ierr)!{{{ - + implicit none character (len=*), parameter :: sub = 'MPAS_stream_mgr_remove_field' @@ -1232,7 +1232,7 @@ subroutine MPAS_stream_mgr_add_alarm(manager, streamID, alarmID, direction, ierr ! if ( threadNum == 0 ) then if (direction == MPAS_STREAM_INPUT .or. direction == MPAS_STREAM_INPUT_OUTPUT) then - + ! If alarm is not already defined, we need to create a new alarm node nullify(new_alarm) if (.not. MPAS_stream_list_query(manager % alarms_in, alarmID, new_alarm, ierr=err_local)) then @@ -1242,10 +1242,10 @@ subroutine MPAS_stream_mgr_add_alarm(manager, streamID, alarmID, direction, ierr if (err_local /= MPAS_STREAM_LIST_NOERR) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR STREAM_ERROR_WRITE('Problems while creating stream list for alarm') - return + return end if nullify(new_alarm % next) - + call MPAS_stream_list_insert(manager % alarms_in, new_alarm, ierr=err_local) if (err_local /= MPAS_STREAM_LIST_NOERR) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR @@ -1253,7 +1253,7 @@ subroutine MPAS_stream_mgr_add_alarm(manager, streamID, alarmID, direction, ierr return end if end if - + ! Add specified stream to alarm node stream list allocate(new_xref) new_xref % name = streamID @@ -1264,7 +1264,7 @@ subroutine MPAS_stream_mgr_add_alarm(manager, streamID, alarmID, direction, ierr STREAM_ERROR_WRITE('Problems while adding stream to alarm stream list') return end if - + ! Add alarm to stream alarm list allocate(new_xref) new_xref % name = alarmID @@ -1276,9 +1276,9 @@ subroutine MPAS_stream_mgr_add_alarm(manager, streamID, alarmID, direction, ierr return end if end if - + if (direction == MPAS_STREAM_OUTPUT .or. direction == MPAS_STREAM_INPUT_OUTPUT) then - + ! If alarm is not already defined, we need to create a new alarm node nullify(new_alarm) if (.not. MPAS_stream_list_query(manager % alarms_out, alarmID, new_alarm, ierr=err_local)) then @@ -1288,10 +1288,10 @@ subroutine MPAS_stream_mgr_add_alarm(manager, streamID, alarmID, direction, ierr if (err_local /= MPAS_STREAM_LIST_NOERR) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR STREAM_ERROR_WRITE('Problems while creating stream list for alarm') - return + return end if nullify(new_alarm % next) - + call MPAS_stream_list_insert(manager % alarms_out, new_alarm, ierr=err_local) if (err_local /= MPAS_STREAM_LIST_NOERR) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR @@ -1299,7 +1299,7 @@ subroutine MPAS_stream_mgr_add_alarm(manager, streamID, alarmID, direction, ierr return end if end if - + ! Add specified stream to alarm node stream list allocate(new_xref) new_xref % name = streamID @@ -1310,7 +1310,7 @@ subroutine MPAS_stream_mgr_add_alarm(manager, streamID, alarmID, direction, ierr STREAM_ERROR_WRITE('Problems while adding stream to alarm stream list') return end if - + ! Add alarm to stream alarm list allocate(new_xref) new_xref % name = alarmID @@ -1399,7 +1399,7 @@ subroutine MPAS_stream_mgr_remove_alarm(manager, streamID, alarmID, direction, i STREAM_ERROR_WRITE('Output alarm '//trim(alarmID)//' does not exist on stream '//trim(streamID)) end if return - end if + end if if (.not. associated(streamNode)) then if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR STREAM_ERROR_WRITE('Alarm '//trim(alarmID)//' does not have stream '//trim(streamID)//' on its stream list.') @@ -1433,7 +1433,7 @@ end subroutine MPAS_stream_mgr_remove_alarm!}}} !> Resets all alarms used by the stream manager. If the optional argument !> 'streamID' is provided, only alarms associated with streams that match !> the 'streamID' regular expression will be - !> reset. If the optional 'direction' argument is provided, only alarms + !> reset. If the optional 'direction' argument is provided, only alarms !> associated with that direction will be reset. ! !----------------------------------------------------------------------- @@ -1453,7 +1453,7 @@ subroutine MPAS_stream_mgr_reset_alarms(manager, streamID, direction, ierr)!{{{ logical :: resetAlarms threadNum = mpas_threading_get_thread_num() - + if (present(streamID)) then STREAM_DEBUG_WRITE('-- Called MPAS_stream_mgr_reset_alarms() for stream ' // trim(streamID)) else @@ -1486,7 +1486,7 @@ subroutine MPAS_stream_mgr_reset_alarms(manager, streamID, direction, ierr)!{{{ alarm_cursor => stream % alarmList_in % head do while (associated(alarm_cursor)) if (mpas_is_alarm_ringing(manager % streamClock, alarm_cursor % name, ierr=local_ierr)) then - call mpas_reset_clock_alarm(manager % streamClock, alarm_cursor % name, ierr=local_ierr) + call mpas_reset_clock_alarm(manager % streamClock, alarm_cursor % name, ierr=local_ierr) end if alarm_cursor => alarm_cursor % next end do @@ -1496,7 +1496,7 @@ subroutine MPAS_stream_mgr_reset_alarms(manager, streamID, direction, ierr)!{{{ alarm_cursor => stream % alarmList_out % head do while (associated(alarm_cursor)) if (mpas_is_alarm_ringing(manager % streamClock, alarm_cursor % name, ierr=local_ierr)) then - call mpas_reset_clock_alarm(manager % streamClock, alarm_cursor % name, ierr=local_ierr) + call mpas_reset_clock_alarm(manager % streamClock, alarm_cursor % name, ierr=local_ierr) end if alarm_cursor => alarm_cursor % next end do @@ -1514,7 +1514,7 @@ subroutine MPAS_stream_mgr_reset_alarms(manager, streamID, direction, ierr)!{{{ alarm_cursor => manager % alarms_in % head do while (associated(alarm_cursor)) if (mpas_is_alarm_ringing(manager % streamClock, alarm_cursor % name, ierr=local_ierr)) then - call mpas_reset_clock_alarm(manager % streamClock, alarm_cursor % name, ierr=local_ierr) + call mpas_reset_clock_alarm(manager % streamClock, alarm_cursor % name, ierr=local_ierr) end if alarm_cursor => alarm_cursor % next end do @@ -1525,7 +1525,7 @@ subroutine MPAS_stream_mgr_reset_alarms(manager, streamID, direction, ierr)!{{{ alarm_cursor => manager % alarms_out % head do while (associated(alarm_cursor)) if (mpas_is_alarm_ringing(manager % streamClock, alarm_cursor % name, ierr=local_ierr)) then - call mpas_reset_clock_alarm(manager % streamClock, alarm_cursor % name, ierr=local_ierr) + call mpas_reset_clock_alarm(manager % streamClock, alarm_cursor % name, ierr=local_ierr) end if alarm_cursor => alarm_cursor % next end do @@ -1543,10 +1543,10 @@ end subroutine MPAS_stream_mgr_reset_alarms!}}} !> \author Michael Duda !> \date 30 September 2014 !> \details - !> Tests whether any I/O alarms in a stream manager are ringing; if the optional - !> 'streamID' argument is given, only alarms for that stream are tested; if - !> the optional argument 'direction' is given, only alarms for the specified - !> direction are tested. If any of the tested alarms is ringing, the function + !> Tests whether any I/O alarms in a stream manager are ringing; if the optional + !> 'streamID' argument is given, only alarms for that stream are tested; if + !> the optional argument 'direction' is given, only alarms for the specified + !> direction are tested. If any of the tested alarms is ringing, the function !> returns .true.; otherwise, it returns .false.. !> Note: This function doesn't support streamID regular expressions ! @@ -1566,7 +1566,7 @@ logical function MPAS_stream_mgr_ringing_alarms(manager, streamID, direction, ie integer :: local_ierr, threadNum threadNum = mpas_threading_get_thread_num() - + STREAM_DEBUG_WRITE('-- Called MPAS_stream_mgr_ringing_alarms()') MPAS_stream_mgr_ringing_alarms = .false. @@ -1784,7 +1784,7 @@ subroutine MPAS_stream_mgr_set_property_int(manager, streamID, propertyName, pro if ( .not. setProperties ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_set_property().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if @@ -1848,7 +1848,7 @@ subroutine MPAS_stream_mgr_set_property_char(manager, streamID, propertyName, pr call mpas_set_time(stream_cursor % referenceTime, dateTimeString=propertyValue) case (MPAS_STREAM_PROPERTY_RECORD_INTV) - + ! The interval between records may not have been allocated if the optional recordInterval ! argument was not provided when the stream was created if (.not. associated(stream_cursor % recordInterval)) then @@ -1869,7 +1869,7 @@ subroutine MPAS_stream_mgr_set_property_char(manager, streamID, propertyName, pr if ( .not. setProperties ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_set_property().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if @@ -1945,7 +1945,7 @@ subroutine MPAS_stream_mgr_set_property_logical(manager, streamID, propertyName, if ( .not. setProperties ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_set_property().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if @@ -1990,7 +1990,7 @@ subroutine MPAS_stream_mgr_get_property_int(manager, streamID, propertyName, pro if (.not. MPAS_stream_list_query(manager % streams, streamID, stream_cursor, ierr=err_local)) then STREAM_ERROR_WRITE('Stream '//trim(streamID)//' does not exist in call to MPAS_stream_mgr_get_property().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if ! @@ -2057,7 +2057,7 @@ subroutine MPAS_stream_mgr_get_property_char(manager, streamID, propertyName, pr if (.not. MPAS_stream_list_query(manager % streams, streamID, stream_cursor, ierr=err_local)) then STREAM_ERROR_WRITE('Stream '//trim(streamID)//' does not exist in call to MPAS_stream_mgr_get_property().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if ! @@ -2081,9 +2081,9 @@ subroutine MPAS_stream_mgr_get_property_char(manager, streamID, propertyName, pr ! assume that the interval is the shortest interval between alarms on the stream; since ! recordInterval is only used for reading, use the input alarm list in this check. if (.not. associated(stream_cursor % recordInterval)) then - + ! - ! If no direction is specified, return the read interval, since this was the only historic + ! If no direction is specified, return the read interval, since this was the only historic ! use of the recordInterval for a stream. ! if (present(direction)) then @@ -2164,7 +2164,7 @@ subroutine MPAS_stream_mgr_get_property_logical(manager, streamID, propertyName, if (.not. MPAS_stream_list_query(manager % streams, streamID, stream_cursor, ierr=err_local)) then STREAM_ERROR_WRITE('Stream '//trim(streamID)//' does not exist in call to MPAS_stream_mgr_get_property().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if ! @@ -2245,14 +2245,14 @@ subroutine MPAS_stream_mgr_add_pkg(manager, streamID, packageName, ierr)!{{{ ! ! Add package to the packages pool for the stream ! - call mpas_pool_add_package(stream_cursor % pkg_pool, packageName, package) + call mpas_pool_add_package(stream_cursor % pkg_pool, packageName, package) end do if ( .not. addedPackages ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_add_pkg().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if @@ -2310,7 +2310,7 @@ subroutine MPAS_stream_mgr_remove_pkg(manager, streamID, packageName, ierr)!{{{ if ( .not. removedPackage ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_remove_pkg().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if @@ -2374,7 +2374,7 @@ subroutine MPAS_stream_mgr_add_att_int(manager, attName, attVal, streamID, ierr) if (mpas_pool_config_type(att_pool, attName) /= MPAS_POOL_FATAL) then STREAM_ERROR_WRITE('Attribute '//trim(attName)//' in stream '//trim(stream_cursor % name)//' is not of type integer.') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if call mpas_pool_add_config(att_pool, attName, attVal) else @@ -2385,7 +2385,7 @@ subroutine MPAS_stream_mgr_add_att_int(manager, attName, attVal, streamID, ierr) if ( .not. addedAttribute ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_add_att().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if else @@ -2403,7 +2403,7 @@ subroutine MPAS_stream_mgr_add_att_int(manager, attName, attVal, streamID, ierr) if (mpas_pool_config_type(att_pool, attName) /= MPAS_POOL_FATAL) then STREAM_ERROR_WRITE('Attribute '//trim(attName)//' in streamManager is not of type integer.') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if call mpas_pool_add_config(att_pool, attName, attVal) else @@ -2473,7 +2473,7 @@ subroutine MPAS_stream_mgr_add_att_real(manager, attName, attVal, streamID, ierr if (mpas_pool_config_type(att_pool, attName) /= MPAS_POOL_FATAL) then STREAM_ERROR_WRITE('Attribute '//trim(attName)//' in stream '//trim(stream_cursor % name)//' is not of type real.') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if call mpas_pool_add_config(att_pool, attName, attVal) else @@ -2484,7 +2484,7 @@ subroutine MPAS_stream_mgr_add_att_real(manager, attName, attVal, streamID, ierr if ( .not. addedAttribute ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_add_att().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if else @@ -2503,7 +2503,7 @@ subroutine MPAS_stream_mgr_add_att_real(manager, attName, attVal, streamID, ierr if (mpas_pool_config_type(att_pool, attName) /= MPAS_POOL_FATAL) then STREAM_ERROR_WRITE('Attribute '//trim(attName)//' in streamManager is not of type real.') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if call mpas_pool_add_config(att_pool, attName, attVal) else @@ -2573,7 +2573,7 @@ subroutine MPAS_stream_mgr_add_att_char(manager, attName, attVal, streamID, ierr if (mpas_pool_config_type(att_pool, attName) /= MPAS_POOL_FATAL) then STREAM_ERROR_WRITE('Attribute '//trim(attName)//' in stream '//trim(stream_cursor % name)//' is not of type character.') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if call mpas_pool_add_config(att_pool, attName, attVal) else @@ -2584,7 +2584,7 @@ subroutine MPAS_stream_mgr_add_att_char(manager, attName, attVal, streamID, ierr if ( .not. addedAttribute ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_add_att().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if else @@ -2603,7 +2603,7 @@ subroutine MPAS_stream_mgr_add_att_char(manager, attName, attVal, streamID, ierr if (mpas_pool_config_type(att_pool, attName) /= MPAS_POOL_FATAL) then STREAM_ERROR_WRITE('Attribute '//trim(attName)//' in streamManager is not of type character.') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if call mpas_pool_add_config(att_pool, attName, attVal) else @@ -2673,7 +2673,7 @@ subroutine MPAS_stream_mgr_add_att_logical(manager, attName, attVal, streamID, i if (mpas_pool_config_type(att_pool, attName) /= MPAS_POOL_FATAL) then STREAM_ERROR_WRITE('Attribute '//trim(attName)//' in stream '//trim(stream_cursor % name)//' is not of type logical.') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if call mpas_pool_add_config(att_pool, attName, attVal) else @@ -2684,7 +2684,7 @@ subroutine MPAS_stream_mgr_add_att_logical(manager, attName, attVal, streamID, i if ( .not. addedAttribute ) then STREAM_ERROR_WRITE('No stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_add_att().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if else @@ -2703,7 +2703,7 @@ subroutine MPAS_stream_mgr_add_att_logical(manager, attName, attVal, streamID, i if (mpas_pool_config_type(att_pool, attName) /= MPAS_POOL_FATAL) then STREAM_ERROR_WRITE('Attribute '//trim(attName)//' in streamManger is not of type logical.') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if call mpas_pool_add_config(att_pool, attName, attVal) else @@ -2727,11 +2727,11 @@ end subroutine MPAS_stream_mgr_add_att_logical!}}} !> stream is only written if any of its alarms are ringing. !> The "timeLevel" argument optionally specifies, for fields with multiple !> time levels, the time level from which fields should be written. - !> The "mgLevel" argument optionally specifies, for fields that exist for + !> The "mgLevel" argument optionally specifies, for fields that exist for !> multiple grid levels, the grid level from which fields should be written. - !> The "forceWriteNow" argument optionally specifies that all streams -- or - !> the stream specified by the "streamID" argument -- should be written by - !> the call regardless of whether any alarms associated with the stream(s) + !> The "forceWriteNow" argument optionally specifies that all streams -- or + !> the stream specified by the "streamID" argument -- should be written by + !> the call regardless of whether any alarms associated with the stream(s) !> are ringing. The "writeTime" argument optionally specifies a time stamp !> to be used for expanding a filename template, when it is not passed in, !> the current time of the streamManager's clock is used to expand filename @@ -2824,7 +2824,7 @@ subroutine MPAS_stream_mgr_write(manager, streamID, timeLevel, mgLevel, forceWri if ( .not. wroteStreams ) then STREAM_ERROR_WRITE('No output stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_write().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if else nullify(stream_cursor) @@ -2872,11 +2872,11 @@ end subroutine MPAS_stream_mgr_write !}}} !> stream is only written if any of its alarms are ringing. !> The "timeLevel" argument optionally specifies, for fields with multiple !> time levels, the time level from which fields should be written. - !> The "mgLevel" argument optionally specifies, for fields that exist for + !> The "mgLevel" argument optionally specifies, for fields that exist for !> multiple grid levels, the grid level from which fields should be written. - !> The "forceWriteNow" argument optionally specifies that all streams -- or - !> the stream specified by the "streamID" argument -- should be written by - !> the call regardless of whether any alarms associated with the stream(s) + !> The "forceWriteNow" argument optionally specifies that all streams -- or + !> the stream specified by the "streamID" argument -- should be written by + !> the call regardless of whether any alarms associated with the stream(s) !> are ringing. The "writeTime" argument optionally specifies a time stamp !> to be used for expanding a filename template, when it is not passed in, !> the current time of the streamManager's clock is used to expand filename @@ -3006,7 +3006,7 @@ subroutine MPAS_stream_mgr_block_write(manager, writeBlock, streamID, timeLevel, if ( .not. wroteStreams ) then STREAM_ERROR_WRITE('No output stream matching '//trim(streamID)//' exists in call to MPAS_stream_mgr_block_write().') if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if else nullify(stream_cursor) @@ -3048,7 +3048,7 @@ subroutine MPAS_stream_mgr_block_write(manager, writeBlock, streamID, timeLevel, call mpas_dmpar_finalize(debugContext % dminfo) deallocate(debugContext % dminfo) - deallocate(debugContext) + deallocate(debugContext) end if call mpas_threading_barrier() @@ -3180,7 +3180,7 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite ! Based on clobber_mode, determine if it matters if the file exists or not. if ( stream % clobber_mode == MPAS_STREAM_CLOBBER_OVERWRITE .or. stream % clobber_mode == MPAS_STREAM_CLOBBER_APPEND ) then STREAM_DEBUG_WRITE(' -- Cobber mode is overwrite or append...') - + ! Check if the file exists if (manager % ioContext % dminfo % my_proc_id == IO_NODE) then inquire(file=trim(stream % filename), exist=recordSeek) @@ -3217,12 +3217,12 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite if ( recordSeek ) then STREAM_DEBUG_WRITE(' -- File exists on disk: ' // trim(stream % filename)) call mpas_get_time(writeTime, dateTimeString=now_string) - + ! Look for exact record (in the case of overwriting) ! This also gets the number of records in the file. stream % nRecords = MPAS_seekStream(stream % stream, now_string, MPAS_STREAM_EXACT_TIME, actualWhen, maxRecords, local_ierr) STREAM_DEBUG_WRITE(' -- Seeked record is: $i with current records equal to $i and an error of $i' COMMA intArgs=(/stream % nRecords COMMA maxRecords COMMA local_ierr/)) - + if ( stream % nRecords == 0 ) then ! If we didn't find an exact time, set record to point to the end of the file. ! This might result in non-monotonic timestamps in the output file. @@ -3266,7 +3266,7 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite ! Based on clobber_mode, determine if it matters if the file exists or not. if ( stream % clobber_mode == MPAS_STREAM_CLOBBER_OVERWRITE .or. stream % clobber_mode == MPAS_STREAM_CLOBBER_APPEND ) then STREAM_DEBUG_WRITE(' -- Cobber mode is overwrite or append...') - + ! Check if the file exists if (manager % ioContext % dminfo % my_proc_id == IO_NODE) then inquire(file=trim(stream % filename), exist=recordSeek) @@ -3305,12 +3305,12 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite if ( recordSeek ) then STREAM_DEBUG_WRITE(' -- File exists on disk: ' // trim(stream % filename)) call mpas_get_time(writeTime, dateTimeString=now_string) - + ! Look for exact record (in the case of overwriting) ! This also gets the number of records in the file. stream % nRecords = MPAS_seekStream(stream % stream, now_string, MPAS_STREAM_EXACT_TIME, actualWhen, maxRecords, local_ierr) STREAM_DEBUG_WRITE(' -- Seeked record is: $i with current records equal to $i and an error of $i' COMMA intArgs=(/stream % nRecords COMMA maxRecords COMMA local_ierr/)) - + if ( stream % nRecords == 0 ) then ! If we didn't find an exact time, set record to point to the end of the file. ! This might result in non-monotonic timestamps in the output file. @@ -3330,12 +3330,12 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite stream % nRecords = stream % nRecords + 1 if ( stream % clobber_mode == MPAS_STREAM_CLOBBER_OVERWRITE .or. stream % clobber_mode == MPAS_STREAM_CLOBBER_APPEND ) then call mpas_get_time(writeTime, dateTimeString=now_string) - + ! Look for exact record (in the case of overwriting) ! This also gets the number of records in the file. tempRecord = MPAS_seekStream(stream % stream, now_string, MPAS_STREAM_EXACT_TIME, actualWhen, maxRecords, local_ierr) STREAM_DEBUG_WRITE(' -- Seeked record is: $i with current records equal to $i and an error of $i' COMMA intArgs=(/tempRecord COMMA maxRecords COMMA local_ierr/)) - + if ( tempRecord /= 0 .and. stream % nRecords < maxRecords ) then ! If we found an exact result ! This might result in non-monotonic timestamps in the output file. @@ -3375,9 +3375,9 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite call prewrite_reindex(manager % allFields, stream % field_pool) end if - ! + ! ! Write the stream - ! + ! STREAM_DEBUG_WRITE(' -- Writing stream ' // trim(stream % name)) call MPAS_writeStream(stream % stream, stream % nRecords, & useMissingValMask, ierr=local_ierr) @@ -3433,16 +3433,16 @@ end subroutine write_stream !}}} !> is only read if any of its alarms are ringing. !> The "timeLevel" argument optionally specifies, for fields with multiple !> time levels, the time level into which fields should be read. - !> The "mgLevel" argument optionally specifies, for fields that exist for + !> The "mgLevel" argument optionally specifies, for fields that exist for !> multiple grid levels, the grid level into which fields should be read. !> The "when" argument optionally specifies the timestamp from which fields !> are to be read. !> The "whence" argument optionally specifies the method for determining !> the timestamp to read from in case an exact match is not found for the !> read timestamp, which is the current time unless the optional "when" - !> argument is given; possible values are MPAS_STREAM_EXACT_TIME, - !> MPAS_STREAM_NEAREST, MPAS_STREAM_LATEST_BEFORE, - !> MPAS_STREAM_LATEST_STRICTLY_BEFORE, MPAS_STREAM_EARLIEST_AFTER, or + !> argument is given; possible values are MPAS_STREAM_EXACT_TIME, + !> MPAS_STREAM_NEAREST, MPAS_STREAM_LATEST_BEFORE, + !> MPAS_STREAM_LATEST_STRICTLY_BEFORE, MPAS_STREAM_EARLIEST_AFTER, or !> MPAS_STREAM_EARLIEST_STRICTLY_AFTER. !> The optional output argument "saveActualWhen" will save the actual time read !> from a stream in case an exact match for the "when" time is not found, @@ -3484,7 +3484,7 @@ subroutine MPAS_stream_mgr_read(manager, streamID, timeLevel, mgLevel, rightNow, integer :: local_whence integer :: local_ierr integer :: temp_ierr - type (MPAS_Time_type) :: now_time + type (MPAS_Time_type) :: now_time character (len=StrKIND) :: actualWhen_local logical :: local_saveActualWhen logical :: local_readInputOutputStreams @@ -3555,7 +3555,7 @@ subroutine MPAS_stream_mgr_read(manager, streamID, timeLevel, mgLevel, rightNow, nullify(stream_cursor) do while (MPAS_stream_list_query(manager % streams, streamID, stream_cursor, ierr=ierr)) STREAM_DEBUG_WRITE('-- Handling read of stream '//trim(stream_cursor % name)) - + ! Verify that the stream is an input stream if (local_readInputOutputStreams) then if (stream_cursor % direction == MPAS_STREAM_INPUT .or. stream_cursor % direction == MPAS_STREAM_INPUT_OUTPUT) then @@ -3590,7 +3590,7 @@ subroutine MPAS_stream_mgr_read(manager, streamID, timeLevel, mgLevel, rightNow, STREAM_ERROR_WRITE(' strictly INPUT direction were requested by this call.') endif if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - return + return end if else ! try to read any stream of input direction nullify(stream_cursor) @@ -3631,7 +3631,7 @@ subroutine MPAS_stream_mgr_read(manager, streamID, timeLevel, mgLevel, rightNow, local_ierr = MPAS_STREAM_MGR_ERROR end if end if - + stream_cursor => stream_cursor % next end do end if @@ -3747,7 +3747,7 @@ subroutine read_stream(manager, stream, timeLevel, mgLevel, forceReadNow, when, if ( stream % filename_interval /= 'none' ) then call mpas_set_time(now_time, dateTimeString=when, ierr=local_ierr) call mpas_set_timeInterval(filename_interval, timeString=stream % filename_interval) - + call mpas_build_stream_filename(stream % referenceTime, now_time, filename_interval, stream % filename_template, blockID_local, temp_filename, ierr=local_ierr) else call mpas_expand_string(when, blockID_local, stream % filename_template, temp_filename) @@ -3916,7 +3916,7 @@ subroutine read_stream(manager, stream, timeLevel, mgLevel, forceReadNow, when, call mpas_get_time(filenameTime, dateTimeString=test_when) call mpas_set_timeInterval(filename_interval, timeString=stream % filename_interval) - + call mpas_build_stream_filename(stream % referenceTime, filenameTime, filename_interval, stream % filename_template, blockID_local, test_filename, ierr=local_ierr) STREAM_DEBUG_WRITE(' --- Retesting filename is ' // trim(test_filename)) @@ -4031,9 +4031,9 @@ subroutine read_stream(manager, stream, timeLevel, mgLevel, forceReadNow, when, stream % timeLevel = timeLevel end if - ! + ! ! Read the stream - ! + ! call MPAS_readStream(stream % stream, stream % nRecords, ierr=local_ierr) if (local_ierr /= MPAS_STREAM_NOERR) then ierr = MPAS_STREAM_MGR_ERROR @@ -4068,13 +4068,13 @@ end subroutine read_stream !}}} !----------------------------------------------------------------------- ! routine MPAS_stream_mesg ! - !> \brief Write an error message (if the level requires it) to + !> \brief Write an error message (if the level requires it) to !> \author Michael Duda, Doug Jacobsen !> \date 07/16/2014 - !> \details Using the input error level, + !> \details Using the input error level, !----------------------------------------------------------------------- subroutine MPAS_stream_mesg(level, mesg)!{{{ - + use mpas_dmpar implicit none @@ -4171,7 +4171,7 @@ end subroutine mpas_get_stream_filename !}}} !> \brief Construct the filename that contains a specific time in a stream !> \author Michael Duda, Doug Jacobsen !> \date 21 August 2014 - !> \details + !> \details !> Given a filename template and the information necessary to determine the time !> in the stream that matches a time available in any of the files associated with !> the stream, returns a specific filename that should contain that time. @@ -4182,7 +4182,7 @@ end subroutine mpas_get_stream_filename !}}} !> !> This is a low level subroutine to complement the !> mpas_get_stream_Filename routine - !> + !> !> Return error codes: !> 0 no error !----------------------------------------------------------------------- @@ -4229,7 +4229,7 @@ subroutine mpas_build_stream_filename(ref_time, when, filename_interval, filenam call mpas_get_timeInterval(intv, timeString=temp_string) STREAM_DEBUG_WRITE(' ** intv is: ' // trim(temp_string)) - call mpas_interval_division(ref_time, intv, filename_interval, nrecs, rem) + call mpas_interval_division(ref_time, intv, filename_interval, nrecs, rem) ! STREAM_DEBUG_WRITE(' ** Divisions are: $i' COMMA intArgs=(/nrecs/)) @@ -4260,7 +4260,7 @@ end subroutine mpas_build_stream_filename !}}} !> \brief This is a utility routine to build a stream type from a pool representing a stream. !> \author Michael Duda, Doug Jacobsen !> \date 07/23/2014 - !> \details + !> \details !> This routine will take as input a pool representing a stream. !> It will then generate a stream type based on this pool, and return that. !----------------------------------------------------------------------- @@ -4465,8 +4465,8 @@ end subroutine build_stream !}}} !> \brief Updates the time level for fields in a stream !> \author Michael Duda, Doug Jacobsen !> \date 07/23/2014 - !> \details - !> For an existing stream, updates the time levels for all fields in + !> \details + !> For an existing stream, updates the time levels for all fields in !> the stream so that subsequent reads/writes of the stream will read !> from / write to the specified time level. !----------------------------------------------------------------------- @@ -4581,7 +4581,7 @@ end subroutine update_stream !}}} !> \brief Checks whether a stream has any active packages (or none at all) !> \author Michael Duda !> \date 23 September 2014 - !> \details + !> \details !> This function determines whether a stream has any active packages !> associated with it. If the stream has at least one active package, !> or no packages at all, the function returns true; else, if all packages @@ -4630,32 +4630,32 @@ end function stream_active_pkg_check !}}} !> \brief Parses a semi-colon-separated list of package names, indicating whether any are active !> \author Michael Duda !> \date 19 March 2015 - !> \details - !> This function determines whether any of the named strings in + !> \details + !> This function determines whether any of the named strings in !> the semi-colon-separated list provided in the 'packages' argument are - !> active. + !> active. !> If any of the packages does not exist in the package pool, the optional - !> argument ierr is set to a non-zero value; otherwise, if all packages exist, + !> argument ierr is set to a non-zero value; otherwise, if all packages exist, !> ierr will be set to zero upon return. ! !----------------------------------------------------------------------- logical function parse_package_list(package_pool, packages, ierr) result(active) - + implicit none - + type (mpas_pool_type), intent(in) :: package_pool character (len=*), intent(in) :: packages integer, intent(out), optional :: ierr - + integer :: i, j, slen integer :: err_level logical, pointer :: pkg_val - - + + if (present(ierr)) ierr = 0 - + slen = len_trim(packages) - + ! ! No packages @@ -4664,7 +4664,7 @@ logical function parse_package_list(package_pool, packages, ierr) result(active) active = .true. return end if - + active = .false. err_level = mpas_pool_get_error_level() @@ -4870,8 +4870,8 @@ end subroutine exch_all_halos !}}} !> \brief Determines whether a dimension represents a decomposed dimension or not !> \author Michael Duda !> \date 24 September 2014 - !> \details - !> This function determines whether the name of the input argument is + !> \details + !> This function determines whether the name of the input argument is !> a decompsed dimension or not. Currently in MPAS, the only decomposed !> dimensions are: !> nCells @@ -4906,8 +4906,8 @@ end function is_decomposed_dim !}}} !> \brief Reindex connectivity fields from local to global index space. !> \author Doug Jacobsen, Michael Duda !> \date 24 September 2014 - !> \details - !> For any connectivity fields contained in the stream to be written, + !> \details + !> For any connectivity fields contained in the stream to be written, !> whose fields include those in the streamFields pool, save the locally !> indexed fields in module variables *_save, and allocate new arrays for !> the fields, which are set to contain global indices. @@ -5203,12 +5203,12 @@ end subroutine prewrite_reindex !}}} !> \brief Reindex connectivity fields from global to local index space. !> \author Doug Jacobsen, Michael Duda !> \date 24 September 2014 - !> \details - !> For any connectivity fields contained in the stream to be written, + !> \details + !> For any connectivity fields contained in the stream to be written, !> whose fields include those in the streamFields pool, restore the locally !> indexed fields from module variables *_save. !> This routine should be called immediately after a write of a stream. - !> + !> !> NB: Even if the write of a stream fails, it is important to stil call !> this routine to reset the connectivity fields to contain local indices. ! @@ -5247,35 +5247,35 @@ subroutine postwrite_reindex(allFields, streamFields) !{{{ if (associated(cellsOnCell_save)) then cellsOnCell_ptr => cellsOnCell_save call mpas_pool_get_field(allFields, 'cellsOnCell', cellsOnCell) - end if + end if if (associated(edgesOnCell_save)) then edgesOnCell_ptr => edgesOnCell_save call mpas_pool_get_field(allFields, 'edgesOnCell', edgesOnCell) - end if + end if if (associated(verticesOnCell_save)) then verticesOnCell_ptr => verticesOnCell_save call mpas_pool_get_field(allFields, 'verticesOnCell', verticesOnCell) - end if + end if if (associated(cellsOnEdge_save)) then cellsOnEdge_ptr => cellsOnEdge_save call mpas_pool_get_field(allFields, 'cellsOnEdge', cellsOnEdge) - end if + end if if (associated(verticesOnEdge_save)) then verticesOnEdge_ptr => verticesOnEdge_save call mpas_pool_get_field(allFields, 'verticesOnEdge', verticesOnEdge) - end if + end if if (associated(edgesOnEdge_save)) then edgesOnEdge_ptr => edgesOnEdge_save call mpas_pool_get_field(allFields, 'edgesOnEdge', edgesOnEdge) - end if + end if if (associated(cellsOnVertex_save)) then cellsOnVertex_ptr => cellsOnVertex_save call mpas_pool_get_field(allFields, 'cellsOnVertex', cellsOnVertex) - end if + end if if (associated(edgesOnVertex_save)) then edgesOnVertex_ptr => edgesOnVertex_save call mpas_pool_get_field(allFields, 'edgesOnVertex', edgesOnVertex) - end if + end if ! ! Reset indices for connectivity arrays from global to local index space @@ -5378,8 +5378,8 @@ end subroutine postwrite_reindex !}}} !> \brief Reindex connectivity fields from global to local index space. !> \author Doug Jacobsen, Michael Duda !> \date 24 September 2014 - !> \details - !> For any connectivity fields contained in the stream that was read, + !> \details + !> For any connectivity fields contained in the stream that was read, !> whose fields include those in the streamFields pool, convert the !> globally indexed connectivity fields in the stream to local index space. !> This routine should be called immediately after a read of a stream. @@ -5606,13 +5606,13 @@ end subroutine postread_reindex !}}} !> \author Doug Jacobsen, Michael Duda !> \date 03/03/2015 !> \details - !> If the optional 'streamID' argument is provided, this routine resets - !> the iterator within a stream manager so that streams may subsequently - !> be iterated over using the MPAS_stream_mgr_get_next_stream function. + !> If the optional 'streamID' argument is provided, this routine resets + !> the iterator within a stream manager so that streams may subsequently + !> be iterated over using the MPAS_stream_mgr_get_next_stream function. !> !> If an optional stream name is provided via the 'streamID' argument, this - !> routine will reset the iterator for fields within the specified stream, - !> which may subsequently iterated over using the + !> routine will reset the iterator for fields within the specified stream, + !> which may subsequently iterated over using the !> MPAS_stream_mgr_get_next_field() routine. !> !> NOTE: This routine does not support regular expressions for StreamID @@ -5772,7 +5772,7 @@ end function MPAS_stream_mgr_get_next_stream !}}} !> the stream manager. !> !> This function returns .TRUE. if the stream contains another field, - !> whether active or not, in which case the output argument fieldName + !> whether active or not, in which case the output argument fieldName !> provides the name of this field, and .FALSE. otherwise. If a field name !> is returned, the optional logical argument isActive may be used to !> determine whether the field is currently active in the stream. @@ -5828,7 +5828,7 @@ logical function MPAS_stream_mgr_get_next_field(manager, streamID, fieldName, is else isActive = .true. end if - + call mpas_pool_set_error_level(err_level) end if @@ -5864,7 +5864,7 @@ logical function MPAS_stream_mgr_stream_exists(manager, streamID) result(validSt return end function MPAS_stream_mgr_stream_exists!}}} - + end module mpas_stream_manager @@ -5967,7 +5967,7 @@ subroutine stream_mgr_create_stream_c(manager_c, streamID_c, direction_c, filena end if STREAM_DEBUG_WRITE('Creating stream from c...') - + ! ! For immutable streams, the stream should have already been defined at this point, and ! all we need to do is update the stream's filename template; From 67556b4249d9a78299eb6f4c98870b352012f811 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 30 Aug 2024 10:25:08 -0600 Subject: [PATCH 607/904] Fix append feature --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index 3dfad8399b80..52787427846e 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 3dfad8399b803d724365c559c5b84e5546049e0c +Subproject commit 52787427846eec58038ab59664fd9c2706f23513 From 36bb7c01ad1577d84ebbe18808bed6495400913c Mon Sep 17 00:00:00 2001 From: Elizabeth Hunke Date: Fri, 30 Aug 2024 11:51:59 -0500 Subject: [PATCH 608/904] clean up debugging code and comments --- components/mpas-seaice/src/shared/mpas_seaice_icepack.F | 9 --------- 1 file changed, 9 deletions(-) diff --git a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F index c6386f2c1aa4..36899b4b10d7 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F @@ -10656,15 +10656,6 @@ subroutine init_icepack_package_configs(domain) call seaice_icepack_write_warnings(icepack_warnings_aborted()) -!note that icepack_recompute_constants, called from icepck_init_parameters above, recomputes Lfresh -! - make Lfresh optional for E3SM? - -! tcraig, this will write out icepack parameters to fort.101, need to uncomment 3 lines - if (mpas_log_info % taskID == 0) then - call icepack_write_parameters(101) - endif -! call seaice_icepack_write_warnings(icepack_warnings_aborted()) - call mpas_log_write(" ----- compare values after icepack init -----") ! Lfresh is derived in Icepack, not sent in from driver From 86eb592e18d8ebb59960500959e1862cc8c2b6a3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 30 Aug 2024 08:09:41 -0500 Subject: [PATCH 609/904] Warn on failure to clobber record in append mode We do not want this to be an error (even a non-fatal one). --- .../mpas-framework/src/framework/mpas_stream_manager.F | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/mpas-framework/src/framework/mpas_stream_manager.F b/components/mpas-framework/src/framework/mpas_stream_manager.F index e8705c51a6d6..fd393908a7de 100644 --- a/components/mpas-framework/src/framework/mpas_stream_manager.F +++ b/components/mpas-framework/src/framework/mpas_stream_manager.F @@ -3398,10 +3398,9 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite ! write(err_string,'(a,i4,a)') 'Writing to stream '''//trim(stream % name)//''' would overwrite record ', & stream % nRecords, ' in file '''//trim(stream % filename)//''',' - STREAM_ERROR_WRITE(trim(err_string)) + STREAM_WARNING_WRITE(trim(err_string)) write(err_string,'(a)') ' but clobber_mode is set to ''append''.' - STREAM_ERROR_WRITE(trim(err_string)) - ierr = MPAS_STREAM_MGR_ERR_CLOBBER_REC + STREAM_WARNING_WRITE(trim(err_string)) else ierr = MPAS_STREAM_MGR_ERROR end if From 6cbb5b0fc2f46b9d3c56e020ed80288a16ec4d23 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Fri, 30 Aug 2024 16:48:23 -0500 Subject: [PATCH 610/904] Homme: Fix planar-mode pgN get_latlon routines. Add is_planar block to gfr_f_get_(corner_)latlon. Add unit test for these routines. Clean up and add to some other unit tests. --- components/homme/src/share/gllfvremap_mod.F90 | 97 ++++++++++++++----- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e0e0fa6c4daa..e8013df217b1 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -2088,9 +2088,14 @@ subroutine gfr_f_get_latlon(ie, i, j, lat, lon) type (spherical_polar_t) :: p - p = change_coordinates(gfr%center_f(i,j,ie)) - lat = p%lat - lon = p%lon + if (gfr%is_planar) then + lon = gfr%center_f(i,j,ie)%x + lat = gfr%center_f(i,j,ie)%y + else + p = change_coordinates(gfr%center_f(i,j,ie)) + lat = p%lat + lon = p%lon + end if end subroutine gfr_f_get_latlon subroutine gfr_f_get_corner_latlon(ie, i, j, c, lat, lon) @@ -2103,9 +2108,14 @@ subroutine gfr_f_get_corner_latlon(ie, i, j, c, lat, lon) type (spherical_polar_t) :: p - p = change_coordinates(gfr%corners_f(c,i,j,ie)) - lat = p%lat - lon = p%lon + if (gfr%is_planar) then + lon = gfr%corners_f(c,i,j,ie)%x + lat = gfr%corners_f(c,i,j,ie)%y + else + p = change_coordinates(gfr%corners_f(c,i,j,ie)) + lat = p%lat + lon = p%lon + end if end subroutine gfr_f_get_corner_latlon subroutine gfr_f_get_cartesian3d(ie, i, j, p) @@ -2318,6 +2328,7 @@ subroutine set_ps_Q(elem, nets, nete, timeidx, qidx, nlev) ! Make up a test function for use in unit tests. use coordinate_systems_mod, only: cartesian3D_t, change_coordinates + use physical_constants, only: dd_pi, Lx, Ly type (element_t), intent(inout) :: elem(:) integer, intent(in) :: nets, nete, timeidx, qidx, nlev @@ -2329,7 +2340,13 @@ subroutine set_ps_Q(elem, nets, nete, timeidx, qidx, nlev) do ie = nets, nete do j = 1,np do i = 1,np - p = change_coordinates(elem(ie)%spherep(i,j)) + if (gfr%is_planar) then + p%x = elem(ie)%spherep(i,j)%lon*2*dd_pi/Lx + p%y = elem(ie)%spherep(i,j)%lat*2*dd_pi/Ly + p%z = 0 + else + p = change_coordinates(elem(ie)%spherep(i,j)) + end if elem(ie)%state%ps_v(i,j,timeidx) = & 1.0d3*(1 + 0.05*sin(2*p%x+0.5)*sin(p%y+1.5)*sin(3*p%z+2.5)) q = 0.5*(1 + sin(3*p%x)*sin(3*p%y)*sin(4*p%z)) @@ -2609,7 +2626,8 @@ function check(par, dom_mt, gfr, elem, verbose) result(nerr) logical, intent(in) :: verbose real(kind=real_kind) :: a, b, rd, x, y, f0(np*np), f1(np*np), g(np,np), & - wf(np*np), wg(np,np), qmin, qmax, qmin1, qmax1 + wf(np*np), wg(np,np), qmin, qmax, qmin1, qmax1, lat, lon, latext(2), & + lonext(2), tol integer :: nf, nf2, ie, i, j, k, iremap, info, ilimit, it real(kind=real_kind), allocatable :: Qdp_fv(:,:), ps_v_fv(:,:), & qmins(:,:,:), qmaxs(:,:,:) @@ -2669,6 +2687,31 @@ function check(par, dom_mt, gfr, elem, verbose) result(nerr) nerr = nerr + 1 write(iulog,*) 'gfr> Dinv', ie, rd end if + if (gfr%is_planar) then + lonext(1) = minval(elem(ie)%spherep(:,:)%lon) + lonext(2) = maxval(elem(ie)%spherep(:,:)%lon) + latext(1) = minval(elem(ie)%spherep(:,:)%lat) + latext(2) = maxval(elem(ie)%spherep(:,:)%lat) + tol = max(lonext(2) - lonext(1), latext(2) - latext(1))*1.e4*eps + do i = 1, nf + do j = 1, nf + call gfr_f_get_latlon(ie, i, j, lat, lon) + if ( lon < lonext(1) .or. lon > lonext(2) .or. & + lat < latext(1) .or. lat > latext(2)) then + write(iulog,*) 'gfr> planar latlon ctr', ie, latext, lonext, lat, lon + nerr = nerr + 1 + end if + do k = 1, 4 + call gfr_f_get_corner_latlon(ie, i, j, k, lat, lon) + if ( lon < lonext(1) - tol .or. lon > lonext(2) + tol .or. & + lat < latext(1) - tol .or. lat > latext(2) + tol) then + write(iulog,*) 'gfr> planar latlon crn', ie, latext, lonext, lat, lon + nerr = nerr + 1 + end if + end do + end do + end do + end if ! Check that FV -> GLL -> FV recovers the original FV values exactly ! (with no DSS and no limiter). @@ -2839,8 +2882,8 @@ function test_sphere2ref() result(nerr) use kinds, only: iulog type (cartesian3D_t) :: corners(4), sphere - real (real_kind) :: refin(2), refout(2), err - integer :: i, j, n, nerr + real (real_kind) :: refin(2), refout(2), err, tol + integer :: i, j, n, trial, nerr nerr = 0 @@ -2854,18 +2897,28 @@ function test_sphere2ref() result(nerr) do i = 1,n refin(1) = -1 + (1.0_real_kind/(n-1))*(i-1) do j = 1,n - refin(2) = -1 + (1.0_real_kind/(n-1))*(j-1) - call ref2sphere(corners, refin(1), refin(2), sphere) - call sphere2ref(corners, sphere, refout(1), refout(2)) - err = abs(refin(1) - refout(1)) + abs(refin(2) - refout(2)) - if (err > 15*eps .or. & - maxval(abs(refout)) > 1 + 5*eps .or. & - any(refout /= refout)) then - write(iulog,*) refin(1), refin(2) - write(iulog,*) refout(1), refout(2) - write(iulog,*) err - nerr = nerr + 1 - end if + do trial = 1, 2 + refin(2) = -1 + (1.0_real_kind/(n-1))*(j-1) + call ref2sphere(corners, refin(1), refin(2), sphere) + if (trial == 2) then + sphere%x = 0.9d0*sphere%x + sphere%y = 0.9d0*sphere%y + sphere%z = 0.9d0*sphere%z + end if + call sphere2ref(corners, sphere, refout(1), refout(2)) + err = abs(refin(1) - refout(1)) + abs(refin(2) - refout(2)) + tol = 15*eps + if (trial == 2) tol = 1e-6 + if (err > tol .or. & + maxval(abs(refout)) > 1 + 5*eps .or. & + any(refout /= refout)) then + write(iulog,*) 'gfr> test_sphere2ref trial', trial + write(iulog,*) refin(1), refin(2) + write(iulog,*) refout(1), refout(2) + write(iulog,*) err + nerr = nerr + 1 + end if + end do end do end do if (nerr /= 0) write(iulog,*) 'test_sphere2ref FAILED' From 4b5cf35b729a221770acd1134c720a7f8dff8410 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Sat, 31 Aug 2024 02:18:41 +0000 Subject: [PATCH 611/904] Remove "--cpu-bind depth" in Aurora mpiexec --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index f1bed254da48..e97c93957a8e 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3498,7 +3498,7 @@ -np {{ total_tasks }} --label -ppn {{ tasks_per_node }} - --cpu-bind depth -envall + -envall -d $ENV{OMP_NUM_THREADS} $ENV{GPU_TILE_COMPACT} From bdf22307e15e8cf3ca9f0cf58a9ceefae3d637d0 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 30 Aug 2024 22:28:18 -0500 Subject: [PATCH 612/904] change of parameters method create_moab_meshes is called from components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 in eamxx and from components/eam/src/dynamics/se/dyn_comp.F90 in eam in eamxx scenario, NPG (fv_nphys) is not visible in homme folder, so it has to be passed as a parameter to the routine This fix does not apply to eamxx, but when it will be merged back to master, it will need to be applied by eam, because homme implementation is changed --- components/eam/src/dynamics/se/dyn_comp.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/dynamics/se/dyn_comp.F90 b/components/eam/src/dynamics/se/dyn_comp.F90 index b420be4ee690..faff6ffd314b 100644 --- a/components/eam/src/dynamics/se/dyn_comp.F90 +++ b/components/eam/src/dynamics/se/dyn_comp.F90 @@ -269,7 +269,7 @@ subroutine dyn_init1(fh, NLFileName, dyn_in, dyn_out) end if #ifdef HAVE_MOAB - call create_moab_meshes(par, elem) + call create_moab_meshes(par, elem, fv_nphys) #endif ! Define the CAM grids (this has to be after dycore spinup). ! Physics-grid will be defined later by phys_grid_init From cf04c3890c3a99387401d4acdae3291514494a29 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 31 Aug 2024 09:49:28 -0500 Subject: [PATCH 613/904] add more comments for moab functions --- .../src/dynamics/homme/interface/phys_grid_mod.F90 | 5 ++++- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 index 1ece3bafa724..31629c7c26ba 100644 --- a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 @@ -652,7 +652,10 @@ subroutine phys_grid_init (pgN) write(iulog,*) "register MOAB app:", trim(appname), " MHPGID=", mhpgid write(iulog,*) " " endif - +! instance distributed moab meshes from elem structures +! 1 ) spectral coarse mesh +! 2 ) GLL fine quad mesh (used mostly for visualization) +! 3 ) pgN FV type mesh, (most of the time pg2 mesh), used for coupling with other components; call create_moab_meshes(par, elem, pgN) endif #endif diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 9f976d6f0c27..ca7877a7bc3a 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -235,6 +235,11 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call scream_set_cpl_indices (x2a, a2x) #ifdef HAVE_MOAB +! for MOAB driver, need to setup surface coupling for MOAB arrays that will hold +! fields data for import and export +! setup is similar to setup for the attribute vectors real data arrays (x2a%rAttr +! and a2x%rAttr) +! the major difference is that moab arrays are transposed compared to AVs arrays mblsize = lsize nsend = mct_avect_nRattr(a2x) totalmbls = mblsize * nsend ! size of the double array @@ -497,6 +502,11 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) end subroutine atm_domain_mct #ifdef HAVE_MOAB +! as part of initialization, moab version of the MCT type grid needs to be instanced too +! it is what we call "point cloud mesh", it contains just the vertices corresponding to the +! grid for surface coupling +! its tags are corresponding to the fields from a2x and x2a attribute vectors, and its +! GLOBAL_ID tag corresponds to the global degrees of freedom in global atmosphere subroutine moab_atm_phys_scream() use iMOAB, only : iMOAB_RegisterApplication, iMOAB_CreateVertices, iMOAB_WriteMesh, & From 215d225dcfc04c4e9ebffa9b63c05d77b993e03a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 31 Aug 2024 23:40:20 -0700 Subject: [PATCH 614/904] Fixes path for files and change c-style strings to std::strings --- ...eamxx_mam_constituent_fluxes_interface.cpp | 30 ++++++++++--------- .../CMakeLists.txt | 18 +++++------ .../input.yaml | 18 +++++------ .../mam/emissions/CMakeLists.txt | 18 +++++------ .../single-process/mam/emissions/input.yaml | 18 +++++------ 5 files changed, 52 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 3a1a1d39b63e..543a445da989 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -100,7 +100,7 @@ void MAMConstituentFluxes::set_grids( // number (n) mixing ratios for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = + const std::string int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); @@ -108,31 +108,31 @@ void MAMConstituentFluxes::set_grids( // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - const char *cld_nmr_field_name = + const std::string cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const std::string int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); - if(strlen(int_mmr_field_name) > 0) { + if(not int_mmr_field_name.empty()) { add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - const char *cld_mmr_field_name = + const std::string cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); - if(strlen(cld_mmr_field_name) > 0) { + if(not cld_mmr_field_name.empty()) { add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } // end for loop num species } // end for loop for num modes for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // end for loop num gases @@ -217,36 +217,38 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const std::string int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const std::string cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const std::string int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if(not int_mmr_field_name.empty()) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = + const std::string cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if(not cld_mmr_field_name.empty()) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); } } } for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); } diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt index 8d2f2653387e..886936ca0846 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt @@ -22,15 +22,15 @@ GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 24c1d9f886e2..55c62d69aa24 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -14,15 +14,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index 3d64bc5b2b27..c8e690b929fc 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -27,15 +27,15 @@ GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 06be9ec8a883..e7af45178aa5 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,15 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From b1fbda11b93ed28d292291b36db2f6059fa5bd1b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 31 Aug 2024 23:56:03 -0700 Subject: [PATCH 615/904] Replaces preprocess interface code --- ...eamxx_mam_constituent_fluxes_interface.cpp | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 543a445da989..3efd57864fa3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -281,8 +281,26 @@ void MAMConstituentFluxes::run_impl(const double dt) { // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ // etc. - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); + + auto lambda = + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int i = team.league_rank(); // column index + compute_dry_mixing_ratios(team, wet_atm_, // in + dry_atm_, // out + i); // in + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, // in + dry_atm_, // out + i); // in + compute_updraft_velocities(team, wet_atm_, // in + dry_atm_, // out + i); // in + }; + Kokkos::parallel_for("mam_cfi_compute_updraft", scan_policy, lambda); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, constituent_fluxes_, From c6d07118118c5241ba8a923d513412b449bc9fb0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 1 Sep 2024 00:15:47 -0700 Subject: [PATCH 616/904] Removes preprocess function from hpp and a calls from cpp interface files --- ...eamxx_mam_constituent_fluxes_functions.hpp | 10 ++--- ...eamxx_mam_constituent_fluxes_interface.cpp | 32 ++++++--------- ...eamxx_mam_constituent_fluxes_interface.hpp | 41 ------------------- 3 files changed, 16 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index 12c9c172f17d..602fc9ebf894 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -19,10 +19,7 @@ void update_gas_aerosols_using_constituents( static constexpr int pcnst = mam4::aero_model::pcnst; // Declare local variables - auto wet_aero_loc = wet_aero; - auto dry_atm_loc = dry_atm; - const int surface_lev = nlev - 1; - auto constituent_fluxes_loc = constituent_fluxes; + const int surface_lev = nlev - 1; // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); @@ -34,8 +31,7 @@ void update_gas_aerosols_using_constituents( // Create a policy to loop over columns annd number of constituents // to update // FIXME: TODO:We don't need a team for "nconstituents", so we can make the - // kookos_for - // simple by using just ncols + // kookos_for simple by using just ncols const auto policy = ekat::ExeSpaceUtils:: get_default_team_policy(ncol, nconstituents); @@ -58,7 +54,7 @@ void update_gas_aerosols_using_constituents( atmosphere_for_column(dry_atm, // output icol); // input - // Form state%q like array + // Form state%q like array at surface level Real state_q_at_surf_lev[pcnst] = {}; mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, // input diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 3efd57864fa3..c168c1ef0e39 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -253,53 +253,47 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(gas_mmr_field_name).get_view(); } - //----------------------------------------------------------------- - // Setup preprocessing and post processing - //----------------------------------------------------------------- - preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); - } // end initialize_impl() // ================================================================ // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in // E3SM. In EAMxx, all constituents are considered "wet" (or have wet // mixing ratios), we are *not* doing any wet to dry conversions in the - // "preprocess" for this process. We are simply updating the MAM4xx - // tracers using the "constituent fluxes". + // for this process. We are simply updating the MAM4xx tracers using the + // "constituent fluxes". // We are converting wet atm to dry atm. Since we do not use or update // any of the water constituents (qc, qv, qi etc.), we should be okay // to do this conversion. We need to do this conversion as our function // are built following HAERO data structures. // ------------------------------------------------------------------- - // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ - // etc. - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); - + // Compute vertical layer heights and updraft velocity. We need these to fully + // populate dry_atm_, so that we can form a HAERO atmosphere object. HAERO + // atmosphere object is used to for state%q like array. auto lambda = KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int i = team.league_rank(); // column index + const int icol = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_, // in dry_atm_, // out - i); // in + icol); // in team.team_barrier(); // vertical heights has to be computed after computing dry mixing ratios // for atmosphere compute_vertical_layer_heights(team, // in dry_atm_, // out - i); // in + icol); // in compute_updraft_velocities(team, wet_atm_, // in dry_atm_, // out - i); // in + icol); // in }; + // policy + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + Kokkos::parallel_for("mam_cfi_compute_updraft", scan_policy, lambda); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index c68065f21788..7d54761a2975 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -69,47 +69,6 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; - // Atmosphere processes often have a pre-processing step that constructs - // required variables from the set of fields stored in the field manager. - // This functor implements this step, which is called during run_impl. - struct Preprocess { - Preprocess() = default; - // on host: initializes preprocess functor with necessary state data - void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere &wet_atm, - const mam_coupling::DryAtmosphere &dry_atm) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; - dry_atm_pre_ = dry_atm; - } - - KOKKOS_INLINE_FUNCTION - void operator()( - const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index - - compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - team.team_barrier(); - // vertical heights has to be computed after computing dry mixing ratios - // for atmosphere - compute_vertical_layer_heights(team, dry_atm_pre_, i); - compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - } // operator() - - // local variables for preprocess struct - // number of horizontal columns and vertical levels - int ncol_pre_, nlev_pre_; - - // local atmospheric and aerosol state data - mam_coupling::WetAtmosphere wet_atm_pre_; - mam_coupling::DryAtmosphere dry_atm_pre_; - }; // Preprocess - - private: - // preprocessing scratch pads - Preprocess preprocess_; - }; // MAMConstituentFluxes } // namespace scream From 2d24c11c798a4a8844f38b4e4a0c4bf4fd34760b Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Sun, 1 Sep 2024 23:48:46 -0600 Subject: [PATCH 617/904] EAMxx: Fix PRIx64 usage to include width 16. --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index fe613259b163..533090e4e7cb 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %" PRIx64 " (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16" PRIx64 " (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %" PRIx64 " (%s)\n", + fprintf(stderr, "bfbhash> %14d %16" PRIx64 " (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } From 15d891a3b5ae59726c3f84b9306d508e31c204b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 10:23:29 +0000 Subject: [PATCH 618/904] Bump actions/setup-python from 5.1.1 to 5.2.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.1 to 5.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.1...v5.2.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 9ea25ae1864e..709fecc623ac 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index e6fb53ba29f5..7c9da3264d68 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index e0a8e19f9c01..fa84f9f96f01 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.11" - name: Run unit tests From dacbb43e3c985ea97dde3d0f5210e8fc60c50e84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:00:56 +0000 Subject: [PATCH 619/904] Bump actions/setup-python from 5.1.1 to 5.2.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.1 to 5.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.1...v5.2.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 9ea25ae1864e..709fecc623ac 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index e6fb53ba29f5..7c9da3264d68 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 7f8f2a177515..8decb315c9e2 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -22,7 +22,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.11" - name: Run unit tests From 76c1a9084fa954e3c360a3cbc8a2830ef1d115c1 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 2 Sep 2024 13:38:21 -0400 Subject: [PATCH 620/904] rename singularity2 to ghci-oci --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 16 ++++++++-------- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 16 ++++++++-------- cime_config/machines/config_machines.xml | 6 +++--- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index b90ed2f01198..adaadd68160f 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -27,14 +27,14 @@ jobs: fail-fast: false matrix: test: - - SMS_D_P4.ne4pg2_oQU480.F2010.singularity2_gnu - - SMS_P4.ne4pg2_oQU480.F2010.singularity2_gnu - - REP_P4.ne4pg2_oQU480.F2010.singularity2_gnu - - ERS_P4.ne4pg2_oQU480.F2010.singularity2_gnu - - ERS_P4.ne4pg2_oQU480.F2010.singularity2_gnu.eam-wcprod_F2010 - - ERP_P4.ne4pg2_oQU480.F2010.singularity2_gnu - - PET_P4.ne4pg2_oQU480.F2010.singularity2_gnu - - PEM_P4.ne4pg2_oQU480.F2010.singularity2_gnu + - SMS_D_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu + - SMS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu + - REP_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu + - ERS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu + - ERS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu.eam-wcprod_F2010 + - ERP_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu + - PET_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu + - PEM_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index c24604d4ad0e..a3ab08ddf696 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -19,14 +19,14 @@ jobs: fail-fast: false matrix: test: - - SMS_D_P8.ne4pg2_oQU480.WCYCL2010NS.singularity2_gnu - - SMS_P8.ne4pg2_oQU480.WCYCL2010NS.singularity2_gnu - - REP_P8.ne4pg2_oQU480.WCYCL2010NS.singularity2_gnu - - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.singularity2_gnu - - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.singularity2_gnu.allactive-wcprod_1850 - - ERP_P8.ne4pg2_oQU480.WCYCL2010NS.singularity2_gnu - - PET_P8.ne4pg2_oQU480.WCYCL2010NS.singularity2_gnu - - PEM_P8.ne4pg2_oQU480.WCYCL2010NS.singularity2_gnu + - SMS_D_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu + - SMS_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu + - REP_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu + - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu + - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 + - ERP_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu + - PET_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu + - PEM_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6cd9d585a25b..7f20b1b512b6 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1712,9 +1712,9 @@ - - Singularity container 2.0 - singularity2 + + OCI-based container 2.0 + ghci-oci LINUX gnu mpich From 0e3c779739a04d91c283a5730b9de10f5b56acc3 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 2 Sep 2024 13:40:48 -0400 Subject: [PATCH 621/904] reduce gh ci and ci-w burden --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 10 +++------- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index adaadd68160f..0af45e95d4ed 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -5,6 +5,7 @@ on: branches: [ master ] paths: # first, yes to these + - '.github/workflows/e3sm-gh-ci-cime-tests.yml' - 'cime_config/**' - 'components/eam/**' - 'components/elm/**' @@ -26,15 +27,10 @@ jobs: strategy: fail-fast: false matrix: - test: + test: + # format test like usual (TYPE_PX.GRID.COMPSET.MACHINE_COMPILER) - SMS_D_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - - SMS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - - REP_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - - ERS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - ERS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu.eam-wcprod_F2010 - - ERP_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - - PET_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - - PEM_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index a3ab08ddf696..b481119a85a5 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -18,15 +18,11 @@ jobs: strategy: fail-fast: false matrix: - test: + test: + # format test like usual (TYPE_PX.GRID.COMPSET.MACHINE_COMPILER) - SMS_D_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - - SMS_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - - REP_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 - - ERP_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - - PET_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - - PEM_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu + container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 From 22e94c1140a73b1f49bcaa0b98def0e6c890c56c Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 2 Sep 2024 13:42:53 -0400 Subject: [PATCH 622/904] add a screamv1 SMS test --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 1 + components/eamxx/cmake/machine-files/ghci-oci.cmake | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 components/eamxx/cmake/machine-files/ghci-oci.cmake diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 0af45e95d4ed..ed35f82a2a7f 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -31,6 +31,7 @@ jobs: # format test like usual (TYPE_PX.GRID.COMPSET.MACHINE_COMPILER) - SMS_D_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - ERS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu.eam-wcprod_F2010 + - SMS_P4.ne4pg2_oQU480.F2010-SCREAMv1.ghci-oci_gnu container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 diff --git a/components/eamxx/cmake/machine-files/ghci-oci.cmake b/components/eamxx/cmake/machine-files/ghci-oci.cmake new file mode 100644 index 000000000000..86a2fb1d5302 --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-oci.cmake @@ -0,0 +1,2 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() From 7d06bcfa09ecd6b0931034f6f8cd6fcb59eb796d Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 2 Sep 2024 18:01:40 -0500 Subject: [PATCH 623/904] add screamv1 prod testmod --- .../scream/v1prod/shell_commands | 97 +++++++++++++++++++ .../testmods_dirs/scream/v1prod/user_nl_cpl | 1 + .../testmods_dirs/scream/v1prod/user_nl_elm | 14 +++ ...cream_output.decadal.1dailyAVG_native.yaml | 16 +++ ...cream_output.decadal.1dailyMAX_native.yaml | 17 ++++ ...cream_output.decadal.1dailyMIN_native.yaml | 16 +++ ...scream_output.decadal.1hourlyINST_arm.yaml | 40 ++++++++ ...eam_output.decadal.1hourlyINST_native.yaml | 19 ++++ ...ream_output.decadal.3hourlyAVG_coarse.yaml | 69 +++++++++++++ ...eam_output.decadal.3hourlyINST_coarse.yaml | 68 +++++++++++++ ...ream_output.decadal.6hourlyAVG_coarse.yaml | 48 +++++++++ ...eam_output.decadal.6hourlyINST_coarse.yaml | 39 ++++++++ ...eam_output.decadal.6hourlyINST_native.yaml | 28 ++++++ ...scream_output.decadal.dailyAVG_coarse.yaml | 95 ++++++++++++++++++ 14 files changed, 567 insertions(+) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/shell_commands create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_cpl create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_elm create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/shell_commands new file mode 100644 index 000000000000..435556401a2b --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/shell_commands @@ -0,0 +1,97 @@ +#!/bin/bash + +cime_root=$(./xmlquery --value CIMEROOT) +input_data_dir=$(./xmlquery --value DIN_LOC_ROOT) +atmchange=$cime_root/../components/eamxx/scripts/atmchange + +# Change run length +./xmlchange RUN_STARTDATE="1994-10-01" + +# Turn on budget reporting +./xmlchange BUDGETS=TRUE + +# For big data +./xmlchange PIO_NETCDF_FORMAT="64bit_data" +./xmlchange PIO_TYPENAME=pnetcdf #adios #,PIO_TYPENAME_ATM=adios +./xmlchange PIO_REARRANGER=1 # use PIO_REARRANGER=3, for ADIOS; PIO_REARRANGER=1 for pnetcdf + +# Turn on cosp and set frequency +$atmchange -b physics::atm_procs_list="mac_aero_mic,rrtmgp,cosp" +$atmchange -b physics::cosp::cosp_frequency_units="hours" +$atmchange -b physics::cosp::cosp_frequency=1 + +# Need to explicitly turn on computing tendencies +$atmchange -b physics::mac_aero_mic::shoc::compute_tendencies=T_mid,qv +$atmchange -b physics::mac_aero_mic::p3::compute_tendencies=T_mid,qv +$atmchange -b physics::rrtmgp::compute_tendencies=T_mid +$atmchange -b homme::compute_tendencies=T_mid,qv + +# Set temperature cut off in dycore threshold to 180K +$atmchange -b vtheta_thresh=180 + +# Change lambda_high +$atmchange -b lambda_high=0.08 + +# use GHG levels more appropriate for sim +# Average from 19940101 - 20150101 +$atmchange -b co2vmr=377.2e-6 +$atmchange -b ch4vmr=1786.6e-9 +$atmchange -b n2ovmr=318.6e-9 +$atmchange -b orbital_year=-9999 +# use CO2 the same in land model +./xmlchange CCSM_CO2_PPMV=377.2 + +# determine grid and set remap files +atm_grid=$(./xmlquery --value ATM_GRID) +if [[ "${atm_grid}" = "ne30np4.pg2" ]]; then + hmapfile="${input_data_dir}/atm/scream/maps/map_ne30pg2_to_ne4pg2_20231201.nc" + armmapfile="${input_data_dir}/atm/scream/maps/map_ne30pg2_to_DecadalSites_c20240130.nc" + # Run with bugfixed SPA file + $atmchange -b spa_data_file="${input_data_dir}/atm/scream/init/spa_v3.LR.F2010.2011-2025.c_20240405.nc" +elif [[ "${atm_grid}" = "ne4np4.pg2" ]]; then + hmapfile="${input_data_dir}/atm/scream/maps/map_ne4pg2_to_ne2pg2_c20240902.nc" + echo "Note: arm remap only works for ne30pg2 atm grids for now" + armmapfile="not-supported-yet" + # Keep default SPA file + # ... (do nothing) +else + echo "Note: horiz/arm remaps only work for ne30pg2 and ne4pg2 atm grids for now" + hmapfile="not-supported-yet" + armmapfile="not-supported-yet" +fi + +# set the output yaml files +output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/ -maxdepth 1 -type f) +for file in ${output_yaml_files[@]}; do + # if the word "coarse" is in the file name, do nothing + if [[ "${file}" == *"_coarse.yaml" && "${hmapfile}" == "not-supported-yet" ]]; then + continue + elif [[ "${file}" == *"_arm.yaml" && "${armmapfile}" == "not-supported-yet" ]]; then + continue + else + # TODO: add remap file replacement for different grids + cp -v ${file} ./ + if [ "${file}" == "${output_yaml_files[0]}" ]; then + # First file, reset output list + $atmchange -b output_yaml_files="./$(basename ${file})" + else + # Append to output list + $atmchange -b output_yaml_files+="./$(basename ${file})" + fi + # Replace remap files + sed -i "s|horiz_remap_file:.*_to_ne30.*|horiz_remap_file: ${hmapfile}|" ./$(basename ${file}) + sed -i "s|horiz_remap_file:.*_to_DecadalSites.*|horiz_remap_file: ${armmapfile}|" ./$(basename ${file}) + fi +done + +# TODO: +# the only thing remaining is to set the SST data, but for some reason, this is proving difficult +# because it is not super important, leave it commented out for now, until a better solution can be found +# ... also, not sure if the below are specific to a given ocn/sea-ice grid? + +# # Point to new SST forcing +# ./xmlchange --file env_run.xml --id SSTICE_DATA_FILENAME --val "${input_data_dir}/atm/cam/sst/sst_ostia_3600x7200_19940930_20151231_c20240125.nc" +# ./xmlchange --file env_run.xml --id SSTICE_GRID_FILENAME --val "${input_data_dir}/ocn/docn7/domain.ocn.3600x7200.230522.nc" +# ./xmlchange --file env_run.xml --id SSTICE_YEAR_ALIGN --val 1994 +# ./xmlchange --file env_run.xml --id SSTICE_YEAR_START --val 1994 +# ./xmlchange --file env_run.xml --id SSTICE_YEAR_END --val 2015 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_cpl b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_cpl new file mode 100644 index 000000000000..3e3a05807216 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_cpl @@ -0,0 +1 @@ +ocn_surface_flux_scheme = 2 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_elm b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_elm new file mode 100644 index 000000000000..93b7ec9e277d --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_elm @@ -0,0 +1,14 @@ +! Override consistency checks since we know our surface data is inconsistent +check_finidat_fsurdat_consistency = .false. +check_finidat_year_consistency = .false. +check_finidat_pct_consistency = .false. +check_dynpft_consistency = .false. + +hist_dov2xy = .true.,.true. +hist_mfilt = 1,1 +hist_nhtfrq = -24,-24 +hist_avgflag_pertape = 'A','A' +hist_fincl1 = 'FIRE', 'FPSN', 'QDRAI', 'QRUNOFF', 'ZWT', 'FSAT', 'H2OSOI', 'EFLX_LH_TOT', + 'QVEGT', 'QVEGE', 'FSH', 'ALBD', 'ALBI', 'TBOT', 'QBOT', 'RAIN', 'SNOW', + 'FSDS', 'FSDSND', 'FSDSNI', 'FSDSVD', 'FSDSVI', 'FLDS' +hist_fincl2 = 'H2OSNO','SOILWATER_10CM','TG' diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml new file mode 100644 index 000000000000..4e1239b5c33c --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml @@ -0,0 +1,16 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.1dailyAVG_native.h +iotype: pnetcdf +Averaging Type: Average +Max Snapshots Per File: 1 +Fields: + Physics PG2: + Field Names: + - snow_depth_land +output_control: + Frequency: 1 + frequency_units: ndays + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml new file mode 100644 index 000000000000..a7d10efe0707 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml @@ -0,0 +1,17 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.1dailyMAX_native.h +iotype: pnetcdf +Averaging Type: Max +Max Snapshots Per File: 1 +Fields: + Physics PG2: + Field Names: + - T_2m + - precip_total_surf_mass_flux +output_control: + Frequency: 1 + frequency_units: ndays + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml new file mode 100644 index 000000000000..653e194d278b --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml @@ -0,0 +1,16 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.1dailyMIN_native.h +iotype: pnetcdf +Averaging Type: Min +Max Snapshots Per File: 1 +Fields: + Physics PG2: + Field Names: + - T_2m +output_control: + Frequency: 1 + frequency_units: ndays + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml new file mode 100644 index 000000000000..5bb07048aed6 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml @@ -0,0 +1,40 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.1hourlyINST_arm.h +iotype: pnetcdf +Averaging Type: Instant +Max Snapshots Per File: 24 # one file per day +horiz_remap_file: ${DIN_LOC_ROOT}/atm/scream/maps/map_ne1024pg2_to_DecadalSites_c20240130.nc +Fields: + Physics PG2: + Field Names: + # 3D + - T_mid + - RelativeHumidity + - qc + - qi + - qr + - qm + - ps + - cldfrac_tot_for_analysis + - cldfrac_liq + - cldfrac_ice_for_analysis + - omega + - tke + - eff_radius_qc + - eff_radius_qi + - nc + - ni + - nr + - bm + - p_mid + - z_mid + - U + - V + - landfrac +output_control: + Frequency: 1 + frequency_units: nhours + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml new file mode 100644 index 000000000000..7a221e89f1c3 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml @@ -0,0 +1,19 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.1hourlyINST_native.h +iotype: pnetcdf +Averaging Type: Instant +Max Snapshots Per File: 24 +Fields: + Physics PG2: + Field Names: + # For cloud morphology + - SW_flux_up_at_model_top + - LW_flux_up_at_model_top + - precip_total_surf_mass_flux +output_control: + Frequency: 1 + frequency_units: nhours + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml new file mode 100644 index 000000000000..665294c62273 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml @@ -0,0 +1,69 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.3hourlyAVG_coarse.h +iotype: pnetcdf +Averaging Type: Average +Max Snapshots Per File: 8 # one file per day +horiz_remap_file: ${DIN_LOC_ROOT}/atm/scream/maps/map_ne1024pg2_to_ne30pg2_mono.20230901.nc +Fields: + Physics PG2: + Field Names: + # 3D fields + - T_mid + - qv + - qc + - qr + - qi + - cldfrac_tot + - cldfrac_liq + - omega + - U + - V + - pseudo_density + - z_mid + # Surface + - surf_sens_flux + - surf_evap + - surface_upward_latent_heat_flux + - ps + - precip_liq_surf_mass_flux + - precip_ice_surf_mass_flux + - surf_mom_flux + - surf_radiative_T + - T_2m + - sfc_flux_dir_nir + - sfc_flux_dir_vis + - sfc_flux_dif_nir + - sfc_flux_dif_vis + - sfc_flux_sw_net + - sfc_flux_lw_dn + # Lowest Level + - U_at_model_bot + - V_at_model_bot + - T_mid_at_model_bot + - qv_at_model_bot + - qc_at_model_bot + - qi_at_model_bot + - qr_at_model_bot + - qm_at_model_bot + - bm_at_model_bot + - SW_flux_up_at_model_top + - SW_flux_dn_at_model_top + - LW_flux_up_at_model_top + - SW_clrsky_flux_up_at_model_top + - LW_clrsky_flux_up_at_model_top + - SW_flux_dn_at_model_bot + - SW_clrsky_flux_dn_at_model_bot + - SW_flux_up_at_model_bot + - SW_clrsky_flux_up_at_model_bot + - LW_flux_dn_at_model_bot + - LW_clrsky_flux_dn_at_model_bot + - LW_flux_up_at_model_bot + - LongwaveCloudForcing + - ShortwaveCloudForcing +output_control: + Frequency: 3 + frequency_units: nhours + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml new file mode 100644 index 000000000000..42c649545088 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml @@ -0,0 +1,68 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.3hourlyINST_coarse.h +iotype: pnetcdf +Averaging Type: Instant +Max Snapshots Per File: 8 # one file per day +horiz_remap_file: ${DIN_LOC_ROOT}/atm/scream/maps/map_ne1024pg2_to_ne30pg2_mono.20230901.nc +Fields: + Physics PG2: + Field Names: + # 2D vars for storm analysis + - SeaLevelPressure + - U_at_model_bot + - V_at_model_bot + - wind_speed_10m + - U_at_925hPa + - V_at_925hPa + - U_at_850hPa + - V_at_850hPa + - U_at_700hPa + - V_at_700hPa + - U_at_500hPa + - V_at_500hPa + - U_at_300hPa + - V_at_300hPa + - ZonalVapFlux + - MeridionalVapFlux + - T_2m + - qv_2m + - T_mid_at_925hPa + - T_mid_at_850hPa + - T_mid_at_700hPa + - T_mid_at_500hPa + - T_mid_at_300hPa + - RelativeHumidity_at_925hPa + - RelativeHumidity_at_850hPa + - RelativeHumidity_at_700hPa + - RelativeHumidity_at_500hPa + - RelativeHumidity_at_300hPa + - z_mid_at_925hPa + - z_mid_at_850hPa + - z_mid_at_700hPa + - z_mid_at_500hPa + - z_mid_at_300hPa + # 2D vars For ACI diagnostics + - SW_clrsky_flux_up_at_model_top + - SW_flux_up_at_model_top + - SW_flux_dn_at_model_top + - LiqWaterPath + - precip_liq_surf_mass_flux + - precip_ice_surf_mass_flux + - IceWaterPath + - cldfrac_tot_for_analysis + - cldfrac_liq + - omega_at_500hPa + - omega_at_700hPa + - omega_at_850hPa + - PotentialTemperature_at_700hPa + - PotentialTemperature_at_1000hPa + # 3D vars for nudging + - U + - V +output_control: + Frequency: 3 + frequency_units: nhours + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml new file mode 100644 index 000000000000..5e4aaed07384 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml @@ -0,0 +1,48 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.6hourlyAVG_coarse.h +iotype: pnetcdf +Averaging Type: Average +Max Snapshots Per File: 4 # one file per day +horiz_remap_file: ${DIN_LOC_ROOT}/atm/scream/maps/map_ne1024pg2_to_ne30pg2_mono.20230901.nc +Fields: + Physics PG2: + Field Names: + # Full 3D Profile + - p3_T_mid_tend + - shoc_T_mid_tend + - rrtmgp_T_mid_tend + - homme_T_mid_tend + - p3_qv_tend + - shoc_qv_tend + - homme_qv_tend + - SW_flux_dn + - SW_flux_up + - LW_flux_dn + - LW_flux_up + # Surface + - surf_sens_flux + - surf_evap + - surface_upward_latent_heat_flux + - ps + - precip_liq_surf_mass_flux + - precip_ice_surf_mass_flux + - surf_mom_flux + - surf_radiative_T + - T_2m + # Lowest Level + - U_at_model_bot + - V_at_model_bot + - SW_flux_dn_at_model_bot + - SW_flux_up_at_model_bot + - LW_flux_dn_at_model_bot + - LW_flux_up_at_model_bot + - SW_flux_up_at_model_top + - SW_flux_dn_at_model_top + - LW_flux_up_at_model_top +output_control: + Frequency: 6 + frequency_units: nhours + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml new file mode 100644 index 000000000000..e9e0f34d5e0f --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml @@ -0,0 +1,39 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.6hourlyINST_coarse.h +iotype: pnetcdf +Averaging Type: Instant +Max Snapshots Per File: 4 # one file per day +horiz_remap_file: ${DIN_LOC_ROOT}/atm/scream/maps/map_ne1024pg2_to_ne30pg2_mono.20230901.nc +Fields: + Physics PG2: + Field Names: + # 3D fields on model levels + - T_mid + - qv + - RelativeHumidity + - U + - V + - omega + - qc + - nc + - qr + - qi + - tke + - o3_volume_mix_ratio + # 2D fields + - VapWaterPath + - LiqWaterPath + - IceWaterPath + - surf_radiative_T + - ps + - qv_2m + - T_2m + - ocnfrac + - landfrac +output_control: + Frequency: 6 + frequency_units: nhours + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml new file mode 100644 index 000000000000..bb7fd275abf4 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml @@ -0,0 +1,28 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.6hourlyINST_native.h +iotype: pnetcdf +Averaging Type: Instant +Max Snapshots Per File: 4 # one file per day +Fields: + Physics PG2: + Field Names: + # For storm analysis + - SeaLevelPressure + - wind_speed_10m + - z_mid_at_500hPa + - z_mid_at_200hPa + # For wind turbine study + - wind_speed_at_100m_above_surface + # For atmospheric rivers + - ZonalVapFlux + - MeridionalVapFlux + # Others + - omega_at_500hPa + - omega_at_850hPa +output_control: + Frequency: 6 + frequency_units: nhours + MPI Ranks in Filename: false +Restart: + force_new_file: true diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml new file mode 100644 index 000000000000..7c1990a7b56e --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml @@ -0,0 +1,95 @@ +%YAML 1.1 +--- +filename_prefix: scream_output.decadal.dailyAVG_coarse.h +iotype: pnetcdf +Averaging Type: Average +Max Snapshots Per File: 1 +horiz_remap_file: ${DIN_LOC_ROOT}/atm/scream/maps/map_ne1024pg2_to_ne30pg2_mono.20230901.nc +Fields: + Physics PG2: + Field Names: + # 3D fields + - T_mid + - qv + - RelativeHumidity + - qc + - qi + - qr + - qm + - nc + - ni + - nr + - cldfrac_tot_for_analysis + - cldfrac_ice_for_analysis + - cldfrac_liq + - omega + - U + - V + - z_mid + - p_mid + - tke + # 2D fields + - SW_flux_up_at_model_top + - SW_flux_dn_at_model_top + - LW_flux_up_at_model_top + - SW_clrsky_flux_up_at_model_top + - SW_clrsky_flux_dn_at_model_top + - LW_clrsky_flux_up_at_model_top + - SW_flux_up_at_model_bot + - SW_flux_dn_at_model_bot + - LW_flux_up_at_model_bot + - LW_flux_dn_at_model_bot + - SW_clrsky_flux_up_at_model_bot + - SW_clrsky_flux_dn_at_model_bot + - LW_clrsky_flux_dn_at_model_bot + - ShortwaveCloudForcing + - LongwaveCloudForcing + - ps + - SeaLevelPressure + - T_2m + - qv_2m + - surf_radiative_T + - VapWaterPath + - IceWaterPath + - LiqWaterPath + - RainWaterPath + - ZonalVapFlux + - MeridionalVapFlux + - surf_evap + - surf_sens_flux + - surface_upward_latent_heat_flux + - precip_liq_surf_mass_flux + - precip_ice_surf_mass_flux + - landfrac + - ocnfrac + - PotentialTemperature_at_700hPa + - PotentialTemperature_at_850hPa + - PotentialTemperature_at_1000hPa + - PotentialTemperature_at_2m_above_surface + - omega_at_500hPa + - omega_at_700hPa + - omega_at_850hPa + - RelativeHumidity_at_700hPa + - RelativeHumidity_at_1000hPa + - RelativeHumidity_at_2m_above_surface + - wind_speed_10m + - z_mid_at_700hPa + - z_mid_at_1000hPa + - T_mid_at_850hPa + - T_mid_at_700hPa + # For SST advection + - U_at_10m_above_surface + - V_at_10m_above_surface + # COSP + - isccp_ctptau + - modis_ctptau + - misr_cthtau + - cosp_sunlit + - isccp_cldtot + +output_control: + Frequency: 1 + frequency_units: ndays + MPI Ranks in Filename: false +Restart: + force_new_file: true From 7b43a058dd90183f3bca87daa6fea67c358cedd4 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 2 Sep 2024 19:58:19 -0500 Subject: [PATCH 624/904] add scream-v1prod test to gh/ci --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 4 ++-- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index ed35f82a2a7f..62f6c1542e72 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -28,10 +28,10 @@ jobs: fail-fast: false matrix: test: - # format test like usual (TYPE_PX.GRID.COMPSET.MACHINE_COMPILER) - SMS_D_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - ERS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu.eam-wcprod_F2010 - - SMS_P4.ne4pg2_oQU480.F2010-SCREAMv1.ghci-oci_gnu + - SMS_D_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu + - ERS_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.scream-v1prod container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index b481119a85a5..178b9a6a4429 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -14,15 +14,14 @@ on: jobs: ci-w: + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: test: - # format test like usual (TYPE_PX.GRID.COMPSET.MACHINE_COMPILER) - SMS_D_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 - container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 From dd9b28d77a8a1b2c408662864185d7608b82b5fd Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 3 Sep 2024 08:36:55 -0600 Subject: [PATCH 625/904] Add white space at end of mkatmsrffile.py --- components/eam/tools/mkatmsrffile/mkatmsrffile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/tools/mkatmsrffile/mkatmsrffile.py b/components/eam/tools/mkatmsrffile/mkatmsrffile.py index 826544feb084..268bd4ddd86e 100644 --- a/components/eam/tools/mkatmsrffile/mkatmsrffile.py +++ b/components/eam/tools/mkatmsrffile/mkatmsrffile.py @@ -301,4 +301,4 @@ def apply_map_2D( data_out, ntime, n_s, wgt, row, col, data_in ): #--------------------------------------------------------------------------------------------------- if __name__ == '__main__': main() -#--------------------------------------------------------------------------------------------------- \ No newline at end of file +#--------------------------------------------------------------------------------------------------- From 47e474c6dc1284e9793c2d346cfcb2a5316a00a9 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Tue, 3 Sep 2024 10:47:43 -0500 Subject: [PATCH 626/904] Remove alternative aliases for v3 single-forcings compsets --- cime_config/allactive/config_compsets.xml | 40 ----------------------- 1 file changed, 40 deletions(-) diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index 28eefc45b765..ace33ae09172 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -93,81 +93,41 @@ 20TRSOI_EAM%CMIP6-GHG_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - hist-GHG - 20TRSOI_EAM%CMIP6-GHG_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - WCYCL20TR-aer 20TRSOI_EAM%CMIP6-AER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - hist-aer - 20TRSOI_EAM%CMIP6-AER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - WCYCL20TR-xGHG-xaer 20TRSOI_EAM%CMIP6-xGHG-xAER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - hist-all-xGHG-xaer - 20TRSOI_EAM%CMIP6-xGHG-xAER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - WCYCL20TR-nat 20TRSOI_EAM%CMIP6-NAT_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - hist-nat - 20TRSOI_EAM%CMIP6-NAT_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - WCYCL20TR-ozone 20TRSOI_EAM%CMIP6-OZONE_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - hist-ozone - 20TRSOI_EAM%CMIP6-OZONE_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - WCYCL20TR-lulc 20TRSOI_EAM%CMIP6-LULC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - hist-lulc - 20TRSOI_EAM%CMIP6-LULC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - WCYCL20TR-volc 20TRSOI_EAM%CMIP6-VOLC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - hist-volc - 20TRSOI_EAM%CMIP6-VOLC_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - WCYCL20TR-xaer 20TRSOI_EAM%CMIP6-xAER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - hist-all-xaer - 20TRSOI_EAM%CMIP6-xAER_ELM%CNPRDCTCBCTOP_MPASSI_MPASO_MOSART_SGLC_SWAV - - From ddc2f2347929d23704ec6582517689eabd120bea Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 3 Sep 2024 10:10:49 -0600 Subject: [PATCH 627/904] Delete components/eam/tools/mkatmsrffile/test.md --- components/eam/tools/mkatmsrffile/test.md | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 components/eam/tools/mkatmsrffile/test.md diff --git a/components/eam/tools/mkatmsrffile/test.md b/components/eam/tools/mkatmsrffile/test.md deleted file mode 100644 index bc0ef89536e0..000000000000 --- a/components/eam/tools/mkatmsrffile/test.md +++ /dev/null @@ -1,3 +0,0 @@ -# test - -Some text goes here. From 9034faa5cd2076c85651bf33b3fad4b2bce89f4f Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Mon, 17 Jun 2024 12:11:53 -0700 Subject: [PATCH 628/904] Adds LULC for NARRM A LULC file for 1850-2000 at r025 is added. [BFB] --- .../elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml index 4f48bf6326a8..92f186f1a38d 100644 --- a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml @@ -38,6 +38,7 @@ lnd/clm2/initdata_map/20180316.DECKv1b_A1.ne30_oEC.edison.clm2.r.1980-01-01-00000.8575c3f_c20190904.nc lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_hist_simyr1850-2015_c240308.nc +lnd/clm2/surfdata_map/landuse.timeseries_0.25x0.25_hist_simyr1850-2015_c240125.nc lnd/clm2/surfdata_map/landuse.timeseries_ne120np4_historical_simyr1850-2015_c190904.nc From 6ca2cfe9d1b7dd00cb2c63d77b9fb23a5e5b1812 Mon Sep 17 00:00:00 2001 From: noel Date: Tue, 3 Sep 2024 11:25:28 -0700 Subject: [PATCH 629/904] For pm-cpu, use 2 nodes for default with tests using l%360x720cru eg SMS.hcru_hcru.IELM --- components/elm/cime_config/config_pes.xml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/elm/cime_config/config_pes.xml b/components/elm/cime_config/config_pes.xml index 4af0da699656..9dbc776bcac4 100644 --- a/components/elm/cime_config/config_pes.xml +++ b/components/elm/cime_config/config_pes.xml @@ -239,14 +239,14 @@ elm: pm-cpu 2 nodes for grid l%360x720cru - 192 - 192 - 192 - 192 - 192 - 192 - 192 - 192 + -2 + -2 + -2 + -2 + -2 + -2 + -2 + -2 @@ -568,7 +568,7 @@ - + GIS 20km (low-res) testing config 128 @@ -606,7 +606,7 @@ - + GIS 1-to-10km (high-res) config 128 From fdb2218352fe267d10317a6598cb8dcdf0669e56 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 3 Sep 2024 14:21:37 -0500 Subject: [PATCH 630/904] Hommexx: Use portable PRIx64 instead of lx in printf. --- .../homme/src/share/cxx/utilities/InternalDiagnostics.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp b/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp index 3249966c0b5e..ffc9657f4108 100644 --- a/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp +++ b/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp @@ -13,6 +13,8 @@ #include "TimeLevel.hpp" #include "mpi/Comm.hpp" +#include + namespace Homme { namespace { @@ -50,10 +52,10 @@ void print_global_state_hash (const std::string& label) { all_reduce_HashType(comm.mpi_comm(), accum, gaccum, 5); if (comm.root()) { for (int i = 0; i < NUM_TIME_LEVELS; ++i) - fprintf(stderr, "hxxhash> %14d %1d %16lx (E %s)\n", + fprintf(stderr, "hxxhash> %14d %1d %16" PRIx64 " (E %s)\n", tl.nstep, i, gaccum[i], label.c_str()); for (int i = 0; i < Q_NUM_TIME_LEVELS; ++i) - fprintf(stderr, "hxxhash> %14d %1d %16lx (T %s)\n", + fprintf(stderr, "hxxhash> %14d %1d %16" PRIx64 " (T %s)\n", tl.nstep, i, gaccum[NUM_TIME_LEVELS+i], label.c_str()); } } From 73a96413bcf2c8d5195313ca2ece05ba47d2ee7f Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Tue, 3 Sep 2024 15:03:46 -0500 Subject: [PATCH 631/904] Add to list of the supported coupled compsets --- docs/user-guide/index.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index c2daf519a42d..01890a0d41cd 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -37,6 +37,16 @@ Below list the standard configuration compsets supported in the current version |`WCYCLSSP245` | Standard configuration with prescribed SSP-245 forcings | |`WCYCLSSP370` | Standard configuration with prescribed SSP-370 forcings | |`WCYCLSSP585` | Standard configuration with prescribed SSP-585 forcings | +| | +|Extra Compsets |For single-forcings simulations of the historical period (1850-2014) | +|`WCYCL20TR-GHG` | Greenhouse gases only configuration (`GHGs`)| +|`WCYCL20TR-aer` | Anthropogenic aerosols and precursors only configuration (`aer`)| +|`WCYCL20TR-xGHG-xaer` | All forcings except GHGs and aer (`solar irradiance, stratospheric ozone and volcanic emissions, land use land cover`) | +|`WCYCL20TR-xaer` | All forcings except aer (`GHGs, solar irradiance, stratospheric ozone and volcanic emissions, land use land cover`) | +|`WCYCL20TR-nat` | Natural-only configuration (`solar irradiance, stratospheric volcanic emissions, land use land cover`) | +|`WCYCL20TR-ozone` | Stratospheric ozone only configuration | +|`WCYCL20TR-lulc` | Land use land cover only configuration | +|`WCYCL20TR-volc` | Stratospheric volcanic emissions only configuration | The compsets for the other two science simulation campaigns are being finalized, with additional components and/or features. The compset naming follows the same convention, e.g., `CRYO1850` and `CRYO1850-4xCO2` are with prognostic ice-shelf melt fluxes for the `polar processes` simulation campaign. From 72f544fff76ae812174be4887568a704d14d7337 Mon Sep 17 00:00:00 2001 From: noel Date: Tue, 3 Sep 2024 14:51:44 -0700 Subject: [PATCH 632/904] Use 3 full nodes instead of 2 in last commit to avoid yet a different issue. --- components/elm/cime_config/config_pes.xml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/elm/cime_config/config_pes.xml b/components/elm/cime_config/config_pes.xml index 9dbc776bcac4..39e36221175b 100644 --- a/components/elm/cime_config/config_pes.xml +++ b/components/elm/cime_config/config_pes.xml @@ -237,16 +237,16 @@ - elm: pm-cpu 2 nodes for grid l%360x720cru - - -2 - -2 - -2 - -2 - -2 - -2 - -2 - -2 + elm: pm-cpu 3 nodes for grid l%360x720cru + + -3 + -3 + -3 + -3 + -3 + -3 + -3 + -3 From 6f601bfb6fe2948787de60d44de5a3fa2275bd28 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Tue, 3 Sep 2024 14:30:47 -0700 Subject: [PATCH 633/904] Adds notes about generating ELM land IC --- components/elm/docs/user-guide/index.md | 5 ++ components/elm/docs/user-guide/interpinic.md | 79 ++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 components/elm/docs/user-guide/interpinic.md diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index 7995e54acb36..d33278ce7617 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -76,3 +76,8 @@ Using the above-mentioned settings: [FATES](fates.md) can be run in various modes with ELM through the use of namelist settings. The [FATES User's Guide section on namelist options](https://fates-users-guide.readthedocs.io/en/latest/user/Namelist-Options-and-Run-Time-Modes.html) provides guidance on enabling these different FATES run modes. + +## Generate land initial condition + +Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). + diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md new file mode 100644 index 000000000000..63f8c30b03da --- /dev/null +++ b/components/elm/docs/user-guide/interpinic.md @@ -0,0 +1,79 @@ +# Creating an ELM initial condition file + +An ELM initial condition (IC) file can be created by remapping an existing IC file from +one resolution to another using the `interpinic`, located at +`components/elm/tools/interpinic`. An ELM IC file is in the same format as an ELM restart file. +The composet of the remapped IC file will be the same as that of the input IC file. +So, for a new ELM SP-mode IC file, use an ELM input file corresponding to the SP-mode. + +The steps involved in creating a new IC files are as follows: + +1. Identifying an input ELM IC or restart file that will be remapped. +2. Obtaining an ELM restart file at the new resolution. +3. Compiling `interpinic` on the machine of interest. +4. Running `interpinic` to perform the interpolation. + +The notes below provide an example of creating 1850 ELM IC file for the NARRM grid using E3SM v3 LR piControl from year = 0101. These notes are provided for Chrysalis. + +### 1. Identification of the input ELM IC file + +The identified input land condition file for this case is the following: + +``` +/lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc +``` + +## 2. Obtaining an ELM restart file + +Using an existing NARRM land IC and making a copy of it + +``` +cd components/elm/tools/interpinic + +cp /lcrc/group/e3sm/data/inputdata/lnd/clm2/initdata_map/elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1870-01-01-00000.c20240704.nc \ +elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c`date "+%Y%m%d"`.nc +``` + +## 3. Compiling `interpinic` + +``` +# Load relevant modules +cd +eval $(./cime/CIME/Tools/get_case_env/get_case_env) + +# change directory +cd components/elm/tools/interpinic/src + +export USER_LDFLAGS="-L$NETCDF_C_DIR/lib -lnetcdf -L$NETCDF_F_DIR/lib -lnetcdff -L$HDF5_DIR/lib -lhdf5" + +USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 + +``` + +## 4. Run `interpinic` + +The `interpinic ` can then be run via the following batch job (e.g., `remap.r025_RRSwISC6to18E3r4.1850.batch`) to generate the initial condition. + +``` +>cat remap.r025_RRSwISC6to18E3r4.1850.batch + +#!/bin/sh +#SBATCH --job-name=remap +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --time 24:00:00 +#SBATCH -p slurm +#SBATCH --account esmd + +# Load relevant modules. +cd +eval $(./cime/CIME/Tools/get_case_env/get_case_env) + +# Change dir to `interpinic` +cd components/elm/tools/interpinic/src + +srun -n 1 ./interpinic \ +-i /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc \ +-o elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c20240903.nc +``` + From 846531dc611d44d0d056dae0d0a9f68159c85a63 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 3 Sep 2024 18:19:51 -0500 Subject: [PATCH 634/904] fix minor bug and add mkatmsrffile workflow --- .../e3sm-gh-tools-mkatmsrffile-test.yml | 86 +++++++++++++++++++ .../eam/tools/mkatmsrffile/mkatmsrffile.py | 14 +-- 2 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml new file mode 100644 index 000000000000..13b73e4df3f4 --- /dev/null +++ b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml @@ -0,0 +1,86 @@ +name: mkatmsrffile + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + paths: + - 'components/eam/tools/mkatmsrffile/mkatmsrffile.py' + schedule: + - cron: '00 15 * * 2' + workflow_dispatch: + +jobs: + mkatmsrffile-test: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + outputs: + event_name: ${{ github.event_name }} + steps: + - + name: Repository checkout + uses: actions/checkout@v4 + with: + show-progress: false + submodules: false + - + name: Conda setup + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: "envmkatmsrffile" + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true + mamba-version: "*" + channel-priority: strict + auto-update-conda: true + python-version: 3.11 + - + name: Install dependencies + run: | + echo $CONDA_PREFIX + mamba install -y nco xarray numba numpy netcdf4 + - + name: Run tests + working-directory: components/eam/tools/mkatmsrffile + run: | + echo $CONDA_PREFIX + wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/clim_soilw.nc + wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/regrid_vegetation.nc + wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/map_1x1_to_ne30pg2_traave_c20240903.nc + python mkatmsrffile.py --map_file=map_1x1_to_ne30pg2_traave_c20240903.nc --vegetation_file=regrid_vegetation.nc --soil_water_file=clim_soilw.nc --dst_grid=ne30pg2 + + mkatmsrffile-notify: + needs: mkatmsrffile-test + if: ${{ failure() && needs.mkatmsrffile-test.outputs.event_name != 'pull_request' }} + runs-on: ubuntu-latest + steps: + - name: Create issue + run: | + previous_issue_number=$(gh issue list \ + --label "$LABELS" \ + --json number \ + --jq '.[0].number') + if [[ -n $previous_issue_number ]]; then + gh issue comment "$previous_issue_number" \ + --body "$BODY" + else + gh issue create \ + --title "$TITLE" \ + --assignee "$ASSIGNEES" \ + --label "$LABELS" \ + --body "$BODY" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TITLE: mkatmsrffile test failure + ASSIGNEES: whannah1 + LABELS: bug + BODY: | + Workflow failed! There's likely an issue in the mkatmsrffile tool! For more information, please see: + - Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (number ${{ github.run_number }}, attempt ${{ github.run_attempt }}) + - Workflow SHA: ${{ github.sha }} diff --git a/components/eam/tools/mkatmsrffile/mkatmsrffile.py b/components/eam/tools/mkatmsrffile/mkatmsrffile.py index 268bd4ddd86e..101671488f30 100644 --- a/components/eam/tools/mkatmsrffile/mkatmsrffile.py +++ b/components/eam/tools/mkatmsrffile/mkatmsrffile.py @@ -253,13 +253,13 @@ def main(): ds_out['lon'] = ds_map.xc_b.rename({'n_b':'ncol'}) ds_out['lat'] = ds_map.yc_b.rename({'n_b':'ncol'}) - ds.attrs['title'] = 'E3SM atmosphere surface data' - ds.attrs['source_code'] = source_code_meta - ds.attrs['hostname'] = str(host) - ds.attrs['history'] = f'created by {user}, '+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - ds.attrs['input_map_file'] = opts.map_file - ds.attrs['input_vegetation_file'] = opts.vegetation_file - ds.attrs['input_soil_water_file'] = opts.soil_water_file + ds_out.attrs['title'] = 'E3SM atmosphere surface data' + ds_out.attrs['source_code'] = source_code_meta + ds_out.attrs['hostname'] = str(host) + ds_out.attrs['history'] = f'created by {user}, '+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ds_out.attrs['input_map_file'] = opts.map_file + ds_out.attrs['input_vegetation_file'] = opts.vegetation_file + ds_out.attrs['input_soil_water_file'] = opts.soil_water_file ds_out.to_netcdf(path=output_file,mode='w') From 9c88f0db5b974e40a79e1ee1ca5a5b7c69b6cae8 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Tue, 3 Sep 2024 18:51:45 -0500 Subject: [PATCH 635/904] Update docs for the new compsets --- docs/user-guide/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 01890a0d41cd..d32d6c65b136 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -37,8 +37,7 @@ Below list the standard configuration compsets supported in the current version |`WCYCLSSP245` | Standard configuration with prescribed SSP-245 forcings | |`WCYCLSSP370` | Standard configuration with prescribed SSP-370 forcings | |`WCYCLSSP585` | Standard configuration with prescribed SSP-585 forcings | -| | -|Extra Compsets |For single-forcings simulations of the historical period (1850-2014) | +| |Compsets for single-forcings simulations of the historical period (1850-2014) | |`WCYCL20TR-GHG` | Greenhouse gases only configuration (`GHGs`)| |`WCYCL20TR-aer` | Anthropogenic aerosols and precursors only configuration (`aer`)| |`WCYCL20TR-xGHG-xaer` | All forcings except GHGs and aer (`solar irradiance, stratospheric ozone and volcanic emissions, land use land cover`) | From 5c8c7369208f42ef2df9719d5c449f8cb78519e1 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Tue, 3 Sep 2024 20:40:18 -0500 Subject: [PATCH 636/904] Fix markdown text in user's guide --- docs/user-guide/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index d32d6c65b136..8674926526e9 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -37,7 +37,7 @@ Below list the standard configuration compsets supported in the current version |`WCYCLSSP245` | Standard configuration with prescribed SSP-245 forcings | |`WCYCLSSP370` | Standard configuration with prescribed SSP-370 forcings | |`WCYCLSSP585` | Standard configuration with prescribed SSP-585 forcings | -| |Compsets for single-forcings simulations of the historical period (1850-2014) | +| | #### Compsets for single-forcings simulations of the historical period (1850-2014) | |`WCYCL20TR-GHG` | Greenhouse gases only configuration (`GHGs`)| |`WCYCL20TR-aer` | Anthropogenic aerosols and precursors only configuration (`aer`)| |`WCYCL20TR-xGHG-xaer` | All forcings except GHGs and aer (`solar irradiance, stratospheric ozone and volcanic emissions, land use land cover`) | From ac1f0eda2e37b29f090013cdef3bb66534362860 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Fri, 30 Aug 2024 16:48:23 -0500 Subject: [PATCH 637/904] Homme: Fix planar-mode pgN get_latlon routines. Add is_planar block to gfr_f_get_(corner_)latlon. Add unit test for these routines. Clean up and add to some other unit tests. --- components/homme/src/share/gllfvremap_mod.F90 | 97 ++++++++++++++----- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e0e0fa6c4daa..e8013df217b1 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -2088,9 +2088,14 @@ subroutine gfr_f_get_latlon(ie, i, j, lat, lon) type (spherical_polar_t) :: p - p = change_coordinates(gfr%center_f(i,j,ie)) - lat = p%lat - lon = p%lon + if (gfr%is_planar) then + lon = gfr%center_f(i,j,ie)%x + lat = gfr%center_f(i,j,ie)%y + else + p = change_coordinates(gfr%center_f(i,j,ie)) + lat = p%lat + lon = p%lon + end if end subroutine gfr_f_get_latlon subroutine gfr_f_get_corner_latlon(ie, i, j, c, lat, lon) @@ -2103,9 +2108,14 @@ subroutine gfr_f_get_corner_latlon(ie, i, j, c, lat, lon) type (spherical_polar_t) :: p - p = change_coordinates(gfr%corners_f(c,i,j,ie)) - lat = p%lat - lon = p%lon + if (gfr%is_planar) then + lon = gfr%corners_f(c,i,j,ie)%x + lat = gfr%corners_f(c,i,j,ie)%y + else + p = change_coordinates(gfr%corners_f(c,i,j,ie)) + lat = p%lat + lon = p%lon + end if end subroutine gfr_f_get_corner_latlon subroutine gfr_f_get_cartesian3d(ie, i, j, p) @@ -2318,6 +2328,7 @@ subroutine set_ps_Q(elem, nets, nete, timeidx, qidx, nlev) ! Make up a test function for use in unit tests. use coordinate_systems_mod, only: cartesian3D_t, change_coordinates + use physical_constants, only: dd_pi, Lx, Ly type (element_t), intent(inout) :: elem(:) integer, intent(in) :: nets, nete, timeidx, qidx, nlev @@ -2329,7 +2340,13 @@ subroutine set_ps_Q(elem, nets, nete, timeidx, qidx, nlev) do ie = nets, nete do j = 1,np do i = 1,np - p = change_coordinates(elem(ie)%spherep(i,j)) + if (gfr%is_planar) then + p%x = elem(ie)%spherep(i,j)%lon*2*dd_pi/Lx + p%y = elem(ie)%spherep(i,j)%lat*2*dd_pi/Ly + p%z = 0 + else + p = change_coordinates(elem(ie)%spherep(i,j)) + end if elem(ie)%state%ps_v(i,j,timeidx) = & 1.0d3*(1 + 0.05*sin(2*p%x+0.5)*sin(p%y+1.5)*sin(3*p%z+2.5)) q = 0.5*(1 + sin(3*p%x)*sin(3*p%y)*sin(4*p%z)) @@ -2609,7 +2626,8 @@ function check(par, dom_mt, gfr, elem, verbose) result(nerr) logical, intent(in) :: verbose real(kind=real_kind) :: a, b, rd, x, y, f0(np*np), f1(np*np), g(np,np), & - wf(np*np), wg(np,np), qmin, qmax, qmin1, qmax1 + wf(np*np), wg(np,np), qmin, qmax, qmin1, qmax1, lat, lon, latext(2), & + lonext(2), tol integer :: nf, nf2, ie, i, j, k, iremap, info, ilimit, it real(kind=real_kind), allocatable :: Qdp_fv(:,:), ps_v_fv(:,:), & qmins(:,:,:), qmaxs(:,:,:) @@ -2669,6 +2687,31 @@ function check(par, dom_mt, gfr, elem, verbose) result(nerr) nerr = nerr + 1 write(iulog,*) 'gfr> Dinv', ie, rd end if + if (gfr%is_planar) then + lonext(1) = minval(elem(ie)%spherep(:,:)%lon) + lonext(2) = maxval(elem(ie)%spherep(:,:)%lon) + latext(1) = minval(elem(ie)%spherep(:,:)%lat) + latext(2) = maxval(elem(ie)%spherep(:,:)%lat) + tol = max(lonext(2) - lonext(1), latext(2) - latext(1))*1.e4*eps + do i = 1, nf + do j = 1, nf + call gfr_f_get_latlon(ie, i, j, lat, lon) + if ( lon < lonext(1) .or. lon > lonext(2) .or. & + lat < latext(1) .or. lat > latext(2)) then + write(iulog,*) 'gfr> planar latlon ctr', ie, latext, lonext, lat, lon + nerr = nerr + 1 + end if + do k = 1, 4 + call gfr_f_get_corner_latlon(ie, i, j, k, lat, lon) + if ( lon < lonext(1) - tol .or. lon > lonext(2) + tol .or. & + lat < latext(1) - tol .or. lat > latext(2) + tol) then + write(iulog,*) 'gfr> planar latlon crn', ie, latext, lonext, lat, lon + nerr = nerr + 1 + end if + end do + end do + end do + end if ! Check that FV -> GLL -> FV recovers the original FV values exactly ! (with no DSS and no limiter). @@ -2839,8 +2882,8 @@ function test_sphere2ref() result(nerr) use kinds, only: iulog type (cartesian3D_t) :: corners(4), sphere - real (real_kind) :: refin(2), refout(2), err - integer :: i, j, n, nerr + real (real_kind) :: refin(2), refout(2), err, tol + integer :: i, j, n, trial, nerr nerr = 0 @@ -2854,18 +2897,28 @@ function test_sphere2ref() result(nerr) do i = 1,n refin(1) = -1 + (1.0_real_kind/(n-1))*(i-1) do j = 1,n - refin(2) = -1 + (1.0_real_kind/(n-1))*(j-1) - call ref2sphere(corners, refin(1), refin(2), sphere) - call sphere2ref(corners, sphere, refout(1), refout(2)) - err = abs(refin(1) - refout(1)) + abs(refin(2) - refout(2)) - if (err > 15*eps .or. & - maxval(abs(refout)) > 1 + 5*eps .or. & - any(refout /= refout)) then - write(iulog,*) refin(1), refin(2) - write(iulog,*) refout(1), refout(2) - write(iulog,*) err - nerr = nerr + 1 - end if + do trial = 1, 2 + refin(2) = -1 + (1.0_real_kind/(n-1))*(j-1) + call ref2sphere(corners, refin(1), refin(2), sphere) + if (trial == 2) then + sphere%x = 0.9d0*sphere%x + sphere%y = 0.9d0*sphere%y + sphere%z = 0.9d0*sphere%z + end if + call sphere2ref(corners, sphere, refout(1), refout(2)) + err = abs(refin(1) - refout(1)) + abs(refin(2) - refout(2)) + tol = 15*eps + if (trial == 2) tol = 1e-6 + if (err > tol .or. & + maxval(abs(refout)) > 1 + 5*eps .or. & + any(refout /= refout)) then + write(iulog,*) 'gfr> test_sphere2ref trial', trial + write(iulog,*) refin(1), refin(2) + write(iulog,*) refout(1), refout(2) + write(iulog,*) err + nerr = nerr + 1 + end if + end do end do end do if (nerr /= 0) write(iulog,*) 'test_sphere2ref FAILED' From e757ed8c6f8bfe1a1acc3de019c8193e1d996dfb Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 3 Sep 2024 17:43:07 -0500 Subject: [PATCH 638/904] rename v1prod to prod, add eamxx to F testing --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 5 ++++- cime_config/config_files.xml | 1 + .../{scream/v1prod => eamxx/prod}/shell_commands | 0 .../testmods_dirs/{scream/v1prod => eamxx/prod}/user_nl_cpl | 0 .../testmods_dirs/{scream/v1prod => eamxx/prod}/user_nl_elm | 0 .../yaml_outs/scream_output.decadal.1dailyAVG_native.yaml | 0 .../yaml_outs/scream_output.decadal.1dailyMAX_native.yaml | 0 .../yaml_outs/scream_output.decadal.1dailyMIN_native.yaml | 0 .../yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml | 0 .../yaml_outs/scream_output.decadal.1hourlyINST_native.yaml | 0 .../yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml | 0 .../yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml | 0 .../yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml | 0 .../yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml | 0 .../yaml_outs/scream_output.decadal.6hourlyINST_native.yaml | 0 .../yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml | 0 16 files changed, 5 insertions(+), 1 deletion(-) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/shell_commands (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/user_nl_cpl (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/user_nl_elm (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml (100%) rename components/eamxx/cime_config/testdefs/testmods_dirs/{scream/v1prod => eamxx/prod}/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml (100%) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 62f6c1542e72..f5c3ffcebf94 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -8,12 +8,15 @@ on: - '.github/workflows/e3sm-gh-ci-cime-tests.yml' - 'cime_config/**' - 'components/eam/**' + - 'components/eamxx/**' - 'components/elm/**' - 'driver-moab/**' - 'driver-mct/**' # second, no to these - '!components/eam/docs/**' - '!components/eam/mkdocs.yml' + - '!components/eamxx/docs/**' + - '!components/eamxx/mkdocs.yml' - '!components/elm/docs/**' - '!components/elm/mkdocs.yml' @@ -31,7 +34,7 @@ jobs: - SMS_D_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - ERS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu.eam-wcprod_F2010 - SMS_D_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu - - ERS_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.scream-v1prod + - ERS_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 diff --git a/cime_config/config_files.xml b/cime_config/config_files.xml index 281291798dbd..e88a9d68f62a 100644 --- a/cime_config/config_files.xml +++ b/cime_config/config_files.xml @@ -348,6 +348,7 @@ $COMP_ROOT_DIR_ICE/cime_config/testdefs/testmods_dirs $COMP_ROOT_DIR_ROF/cime_config/testdefs/testmods_dirs $COMP_ROOT_DIR_ATM/cime_config/testdefs/testmods_dirs + $COMP_ROOT_DIR_ATM/cime_config/testdefs/testmods_dirs $COMP_ROOT_DIR_OCN/cime_config/testdefs/testmods_dirs $COMP_ROOT_DIR_ICE/cime_config/testdefs/testmods_dirs $COMP_ROOT_DIR_WAV/cime_config/testdefs/testmods_dirs diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_cpl b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/user_nl_cpl similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_cpl rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/user_nl_cpl diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_elm b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/user_nl_elm similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/user_nl_elm rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/user_nl_elm diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml From 71326bde3bebf5569cc050adc65b7ce2cbd69b10 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 4 Sep 2024 10:14:27 -0600 Subject: [PATCH 639/904] Another fix for CIME mvk and pgn --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index 52787427846e..2a3142fc14b1 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 52787427846eec58038ab59664fd9c2706f23513 +Subproject commit 2a3142fc14b18fd58379f98d7d2a9a8b4b0d43a8 From 6a2bebe06909386fdce735bd0d94b2c6750e8219 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 8 Aug 2024 13:09:08 -0400 Subject: [PATCH 640/904] Delete crusher from config_machines. --- cime_config/machines/config_machines.xml | 295 ----------------------- 1 file changed, 295 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 889fc03834e1..84cbaacd5506 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1094,301 +1094,6 @@ - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu - mpich - cli115 - /lustre/orion/cli115/world-shared/crusher - .* - /lustre/orion/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli115/world-shared/e3sm/baselines/crusher/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - 8 - 8 - 8 - TRUE - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - -c $ENV{OMP_NUM_THREADS} - $ENV{NTASKS_PER_GPU} - $ENV{GPU_BIND_ARGS} - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/15.0.1 - - - craype craype/2.7.20 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.4.0 - - - craype-accel-amd-gfx90a - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - gcc gcc/11.2.0 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - cray-python/3.9.13.1 - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0.25 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} - --gpu-bind=closest - romio_cb_read=disable - 0 - - - 10 - 3 - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --threads-per-core=1 - -c $ENV{OMP_NUM_THREADS} - -m *:block - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/14.0.0 - - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.1.0 - - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - - - cray-mpich/8.1.12 - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-libsci/21.08.1.2 - cray-hdf5-parallel/1.12.1.1 - cray-netcdf-hdf5parallel/4.8.1.1 - cray-parallel-netcdf/1.12.1.7 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - - 0 - - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 64 - 8 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --gpus-per-node=8 --gpu-bind=closest - -c $ENV{OMP_NUM_THREADS} - - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray - - craype-accel-amd-gfx90a - rocm/5.1.0 - - cce/14.0.3 - - - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - 0 - - 1 - romio_cb_read=disable - - - - 128M - spread - threads - - - Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* From bcc5b62f37e14c69a62cd7f57c11a9ec5d4ba5d8 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 8 Aug 2024 14:09:47 -0400 Subject: [PATCH 641/904] Deleted crusher files from cime_config/machines. --- .../Depends.crusher-gpu.crayclang.cmake | 43 --------- .../cmake_macros/amdclang_crusher.cmake | 12 --- .../cmake_macros/amdclanggpu_crusher.cmake | 26 ----- .../crayclang-scream_crusher-scream-cpu.cmake | 12 --- .../crayclang-scream_crusher-scream-gpu.cmake | 28 ------ .../crayclang-scream_crusher-scream.cmake | 12 --- .../cmake_macros/crayclang_crusher.cmake | 10 -- .../cmake_macros/crayclanggpu_crusher.cmake | 63 ------------- .../machines/cmake_macros/gnu_crusher.cmake | 15 --- .../cmake_macros/gnugpu_crusher.cmake | 25 ----- cime_config/machines/syslog.crusher | 94 ------------------- 11 files changed, 340 deletions(-) delete mode 100644 cime_config/machines/Depends.crusher-gpu.crayclang.cmake delete mode 100644 cime_config/machines/cmake_macros/amdclang_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/gnu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/gnugpu_crusher.cmake delete mode 100755 cime_config/machines/syslog.crusher diff --git a/cime_config/machines/Depends.crusher-gpu.crayclang.cmake b/cime_config/machines/Depends.crusher-gpu.crayclang.cmake deleted file mode 100644 index e41d959b52b4..000000000000 --- a/cime_config/machines/Depends.crusher-gpu.crayclang.cmake +++ /dev/null @@ -1,43 +0,0 @@ -set(CICE_F90 - ice_FY.F90 - ice_aerosol.F90 - ice_age.F90 - ice_atmo.F90 - ice_blocks.F90 - ice_calendar.F90 - ice_diagnostics.F90 - ice_distribution.F90 - ice_domain.F90 - ice_domain_size.F90 - ice_dyn_evp.F90 - ice_fileunits.F90 - ice_flux.F90 - ice_forcing.F90 - ice_grid.F90 - ice_history.F90 - ice_history_fields.F90 - ice_init.F90 - ice_itd.F90 - ice_kinds_mod.F90 - ice_lvl.F90 - ice_mechred.F90 - ice_meltpond.F90 - ice_ocean.F90 - ice_orbital.F90 - ice_probability.F90 - ice_probability_tools.F90 - ice_read_write.F90 - ice_restoring.F90 - ice_shortwave.F90 - ice_spacecurve.F90 - ice_state.F90 - ice_step_mod.F90 - ice_therm_itd.F90 - ice_therm_vertical.F90 - ice_transport_driver.F90 - ice_transport_remap.F90 - ice_work.F90) - -foreach(ITEM IN LISTS CICE_F90) - e3sm_add_flags("cice/src/source/${ITEM}" "-O0") -endforeach() diff --git a/cime_config/machines/cmake_macros/amdclang_crusher.cmake b/cime_config/machines/cmake_macros/amdclang_crusher.cmake deleted file mode 100644 index 2df5074d11f3..000000000000 --- a/cime_config/machines/cmake_macros/amdclang_crusher.cmake +++ /dev/null @@ -1,12 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") diff --git a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake deleted file mode 100644 index 6f3c0074798e..000000000000 --- a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake +++ /dev/null @@ -1,26 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") - -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") - -set(MPICXX "hipcc") -set(SCXX "hipcc") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake deleted file mode 100644 index f8176ccb3d0b..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() - -set(PIO_FILESYSTEM_HINTS "gpfs") - -string(APPEND CPPDEFS " -DCPRCRAY") - - diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake deleted file mode 100644 index 18564d4d301f..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake +++ /dev/null @@ -1,28 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() - -set(PIO_FILESYSTEM_HINTS "gpfs") - -set(MPICXX "hipcc") -set(SCXX "hipcc") - -string(APPEND CMAKE_CXX_FLAGS " -I${MPICH_DIR}/include") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.16/gtl/lib -lmpi_gtl_hsa") - -# For YAKL's -lroctx64 -lrocfft; the rocm module doesn't set this. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib") - -# 'NOT DEBUG': this resolves a crash in mct in docn init -# 'DEBUG' casee, too: resolves a build error in elm/src/main/elm_varctl.F90 due to several OpenACC syntax errors -#if (NOT DEBUG) - string(APPEND CMAKE_C_FLAGS " -O2 -hnoacc -hfp0 -hipa0") - string(APPEND CMAKE_Fortran_FLAGS " -O2 -hnoacc -hfp0 -hipa0") -#endif() - -string(APPEND CPPDEFS " -DCPRCRAY") - -#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "See SCREAM issue #2080.") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake deleted file mode 100644 index 2a0bfd6217c8..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -if (COMP_NAME STREQUAL elm) - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") - -set(PIO_FILESYSTEM_HINTS "gpfs") diff --git a/cime_config/machines/cmake_macros/crayclang_crusher.cmake b/cime_config/machines/cmake_macros/crayclang_crusher.cmake deleted file mode 100644 index 7a5fb412cbb9..000000000000 --- a/cime_config/machines/cmake_macros/crayclang_crusher.cmake +++ /dev/null @@ -1,10 +0,0 @@ -if (COMP_NAME STREQUAL elm) - # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -# Disable ipa and zero initialization are for other NaN isues: -# https://github.com/E3SM-Project/E3SM/pull/5208 -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") -# -em -ef generates modulename.mod (lowercase files) to support -# Scorpio installs -string(APPEND CMAKE_Fortran_FLAGS " -em -ef") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake deleted file mode 100644 index 653938b81da9..000000000000 --- a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake +++ /dev/null @@ -1,63 +0,0 @@ -set(MPICC "cc") -set(MPICXX "hipcc") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "hipcc") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() - -if (compile_threaded) - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") -string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY") -string(APPEND CMAKE_Fortran_FLAGS " -f free -em") -if (NOT compile_threaded) - # -M1077 flag used to suppress message about OpenMP directives - # that are ignored for non-threaded builds. (-h omp inactive) - # Details: `explain ftn-1077` - string(APPEND CMAKE_Fortran_FLAGS " -M1077") -endif() -set(HAS_F2008_CONTIGUOUS "TRUE") - -# -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: -# /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' -# AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") - -# Switch to O3 for better performance -# Using O2 to ensure passing tests -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") - -if (COMP_NAME STREQUAL elm) - # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -# -em -ef generates modulename.mod (lowercase files) to support -# Scorpio installs -# Disable ipa and zero initialization are for other NaN isues: -# https://github.com/E3SM-Project/E3SM/pull/5208 -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") - -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") - -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/cmake_macros/gnu_crusher.cmake b/cime_config/machines/cmake_macros/gnu_crusher.cmake deleted file mode 100644 index 9b242243989b..000000000000 --- a/cime_config/machines/cmake_macros/gnu_crusher.cmake +++ /dev/null @@ -1,15 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") - -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") diff --git a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake b/cime_config/machines/cmake_macros/gnugpu_crusher.cmake deleted file mode 100644 index bb213014b2f5..000000000000 --- a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake +++ /dev/null @@ -1,25 +0,0 @@ -set(MPICC "cc") -set(MPICXX "hipcc") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "hipcc") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") - -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--copy-dt-needed-entries -L/opt/cray/pe/gcc-libs -lgfortran -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") - -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/syslog.crusher b/cime_config/machines/syslog.crusher deleted file mode 100755 index 25ef50e1e955..000000000000 --- a/cime_config/machines/syslog.crusher +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/csh -f -# pm-gpu syslog script: -# mach_syslog - -set sample_interval = $1 -set jid = $2 -set lid = $3 -set run = $4 -set timing = $5 -set dir = $6 - -# Wait until job task-to-node mapping information is output before saving output file. -# Target length was determined empirically (maximum number of lines before job mapping -# information starts + number of nodes), and it may need to be adjusted in the future. -# (Note that calling script 'touch'es the e3sm log file before spawning this script, so that 'wc' does not fail.) -set nnodes = `scontrol show jobid $jid | grep -F NumNodes | sed 's/ *NumNodes=\([0-9]*\) .*/\1/' ` -@ target_lines = 150 + $nnodes -sleep 10 -set outlth = `wc \-l $run/e3sm.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -while ($outlth < $target_lines) - sleep 60 - set outlth = `wc \-l $run/e3sm.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -end - -set TimeLimit = `scontrol show jobid $jid | grep -F TimeLimit | sed 's/^ *RunTime=.*TimeLimit=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` -set limit_hours = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` -set limit_mins = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` -set limit_secs = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` -if ("X$limit_hours" == "X") set limit_hours = 0 -if ("X$limit_mins" == "X") set limit_mins = 0 -if ("X$limit_secs" == "X") set limit_secs = 0 -@ limit = 3600 * $limit_hours + 60 * $limit_mins + $limit_secs - -set RunTime = `scontrol show jobid $jid | grep -F RunTime | sed 's/^ *RunTime=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` -set runt_hours = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` -set runt_mins = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` -set runt_secs = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` -if ("X$runt_hours" == "X") set runt_hours = 0 -if ("X$runt_mins" == "X") set runt_mins = 0 -if ("X$runt_secs" == "X") set runt_secs = 0 -@ runt = 3600 * $runt_hours + 60 * $runt_mins + $runt_secs - -@ remaining = $limit - $runt -cat > $run/Walltime.Remaining < $dir/squeuef.$lid.$remaining - squeue -s | grep -v -F extern > $dir/squeues.$lid.$remaining - # squeue -t R -o "%.10i %R" > $dir/squeueR.$lid.$remaining -endif - -while ($remaining > 0) - echo "Wallclock time remaining: $remaining" >> $dir/atm.log.$lid.step - grep -Fa -e "nstep" -e "model date" $run/*atm.log.$lid | tail -n 4 >> $dir/atm.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/lnd.log.$lid.step - grep -Fa -e "timestep" -e "model date" $run/*lnd.log.$lid | tail -n 4 >> $dir/lnd.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/ocn.log.$lid.step - grep -Fa -e "timestep" -e "Step number" -e "model date" $run/*ocn.log.$lid | tail -n 4 >> $dir/ocn.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/ice.log.$lid.step - grep -Fa -e "timestep" -e "istep" -e "model date" $run/*ice.log.$lid | tail -n 4 >> $dir/ice.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/rof.log.$lid.step - grep -Fa "model date" $run/*rof.log.$lid | tail -n 4 >> $dir/rof.log.$lid.step - grep -Fa "model date" $run/*cpl.log.$lid > $dir/cpl.log.$lid.step-all - echo "Wallclock time remaining: $remaining" >> $dir/cpl.log.$lid.step - tail -n 4 $dir/cpl.log.$lid.step-all >> $dir/cpl.log.$lid.step - /bin/cp --preserve=timestamps -u $timing/* $dir - # sqs -w -a | grep "^[0-9]* *R *"> $dir/sqswr.$lid.$remaining - squeue -t R -o "%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l" > $dir/squeuef.$lid.$remaining - squeue -s | grep -v -F extern > $dir/squeues.$lid.$remaining - # squeue -t R -o "%.10i %R" > $dir/squeueR.$lid.$remaining - chmod a+r $dir/* - # sleep $sample_interval - set sleep_remaining = $sample_interval - while ($sleep_remaining > 120) - sleep 120 - @ sleep_remaining = $sleep_remaining - 120 - end - sleep $sleep_remaining - set RunTime = `scontrol show jobid $jid | grep -F RunTime | sed 's/^ *RunTime=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` - set runt_hours = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` - set runt_mins = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` - set runt_secs = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` - if ("X$runt_hours" == "X") set runt_hours = 0 - if ("X$runt_mins" == "X") set runt_mins = 0 - if ("X$runt_secs" == "X") set runt_secs = 0 - @ runt = 3600 * $runt_hours + 60 * $runt_mins + $runt_secs - @ remaining = $limit - $runt - cat > $run/Walltime.Remaining << EOF2 -$remaining $sample_interval -EOF2 - -end From ea640364780427fe3f6eac0e5bf6a8c1f58a0459 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 22 Aug 2024 14:32:05 -0400 Subject: [PATCH 642/904] Successful build and run of ne30pg2_ne30pg2 with PrgEnv-gnu and rocm/6.2.0 on Frontier. --- .../machines/cmake_macros/craygnu-hipcc.cmake | 37 +++++++++++++++++++ cime_config/machines/config_machines.xml | 30 +++++++-------- components/eamxx/CMakeLists.txt | 4 +- .../machine-files/frontier-scream-gpu.cmake | 2 +- 4 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 cime_config/machines/cmake_macros/craygnu-hipcc.cmake diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake new file mode 100644 index 000000000000..18f5300fdedc --- /dev/null +++ b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake @@ -0,0 +1,37 @@ +set(MPICC "cc") +set(MPICXX "hipcc") # Needs MPICH_CXX to use hipcc +set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90 +set(SCC "cc") +set(SCXX "hipcc") +set(SFC "ftn") + +string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND CMAKE_Fortran_FLAGS " -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fallow-argument-mismatch") + +string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow") +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace") + +string(APPEND CMAKE_C_FLAGS_RELEASE " -g -O2") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -g -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -g -O2") + +if (COMP_NAME STREQUAL csm_share) + string(APPEND CMAKE_C_FLAGS " -std=c99") +endif() +string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") +string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") + +string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") + +set(E3SM_LINK_WITH_FORTRAN "TRUE") +string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") + +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") + +set(USE_HIP "TRUE") +string(APPEND CMAKE_HIP_FLAGS "$ENV{CXXFLAGS} --offload-arch=gfx90a -munsafe-fp-atomics") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 84cbaacd5506..88fad58bf3e3 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1098,7 +1098,7 @@ Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* CNL - crayclang-scream + craygnu-hipcc mpich cli115 /lustre/orion/proj-shared/cli115 @@ -1135,23 +1135,21 @@ module module /usr/share/lmod/lmod/libexec/lmod python - + - cpe/22.12 + + PrgEnv-gnu + cpe/24.07 craype-accel-amd-gfx90a - rocm/5.4.0 - libunwind/1.6.2 - cce/15.0.1 - libfabric/1.15.2.0 - craype/2.7.20 - cray-mpich/8.1.26 - cray-python/3.9.13.1 - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 + rocm/6.2.0 + libunwind + cray-python + subversion + git + cmake + cray-hdf5-parallel + cray-netcdf-hdf5parallel + cray-parallel-netcdf darshan-runtime diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 398efa3bc2df..efaf77514d91 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -550,9 +550,9 @@ DisableMpiCxxBindings() if (SCREAM_DOUBLE_PRECISION) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") set(SCREAM_Fortran_FLAGS -real-size 64) - elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") + #elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") #find what is crayclang id in cmake - set(SCREAM_Fortran_FLAGS -s default32 -eZ) + #set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() diff --git a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake index 14d1d501160d..8704d848334b 100644 --- a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake +++ b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake @@ -6,4 +6,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/hip.cmake) set(SCREAM_MPIRUN_EXE "srun" CACHE STRING "") set(SCREAM_MACHINE "frontier-scream-gpu" CACHE STRING "") -set(CMAKE_CXX_FLAGS "--amdgpu-target=gfx90a -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-Wno-mismatched-tags --offload-arch=gfx90a -munsafe-fp-atomics -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) From dba1054f184ea62fbfe5eb07889e3ffd38a17c82 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Mon, 26 Aug 2024 10:53:31 -0400 Subject: [PATCH 643/904] Changed Frontier build to use OpenMP and Adios2. --- cime_config/machines/cmake_macros/craygnu-hipcc.cmake | 9 +++++++-- cime_config/machines/config_machines.xml | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake index 18f5300fdedc..8322c5d3a945 100644 --- a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake +++ b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake @@ -25,12 +25,17 @@ endif() string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - set(E3SM_LINK_WITH_FORTRAN "TRUE") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +if (compile_threaded) + string(APPEND CMAKE_C_FLAGS " -fopenmp") + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp=libgomp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") set(USE_HIP "TRUE") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 88fad58bf3e3..2448e2f797fc 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1176,8 +1176,8 @@ threads - - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi} From cc81b1f6565ef058eed6340a365e9eadc4d2a759 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 29 Aug 2024 20:21:38 -0400 Subject: [PATCH 644/904] Add SCREAM_SYSTEM_WORKAROUND_P3_PART2. Return support for crayclang-scream. Rename craygnuamdgpu. --- ...raygnu-hipcc.cmake => craygnuamdgpu.cmake} | 2 +- cime_config/machines/config_machines.xml | 29 +++++++++++++++++-- components/eamxx/CMakeLists.txt | 5 ++-- .../p3/disp/p3_main_impl_part2_disp.cpp | 7 ++++- 4 files changed, 35 insertions(+), 8 deletions(-) rename cime_config/machines/cmake_macros/{craygnu-hipcc.cmake => craygnuamdgpu.cmake} (97%) diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnuamdgpu.cmake similarity index 97% rename from cime_config/machines/cmake_macros/craygnu-hipcc.cmake rename to cime_config/machines/cmake_macros/craygnuamdgpu.cmake index 8322c5d3a945..6cb79c0146b1 100644 --- a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake +++ b/cime_config/machines/cmake_macros/craygnuamdgpu.cmake @@ -5,7 +5,7 @@ set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") -string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU") +string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU -DSCREAM_SYSTEM_WORKAROUND_P3_PART2") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 2448e2f797fc..e2d4a50cab55 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1098,7 +1098,7 @@ Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* CNL - craygnu-hipcc + craygnuamdgpu,crayclang-scream mpich cli115 /lustre/orion/proj-shared/cli115 @@ -1135,7 +1135,7 @@ module module /usr/share/lmod/lmod/libexec/lmod python - + PrgEnv-gnu @@ -1152,6 +1152,25 @@ cray-parallel-netcdf darshan-runtime + + + cpe/22.12 + craype-accel-amd-gfx90a + rocm/5.4.0 + libunwind/1.6.2 + cce/15.0.1 + libfabric/1.15.2.0 + craype/2.7.20 + cray-mpich/8.1.26 + cray-python/3.9.13.1 + subversion/1.14.1 + git/2.36.1 + cmake/3.21.3 + cray-hdf5-parallel/1.12.2.1 + cray-netcdf-hdf5parallel/4.9.0.1 + cray-parallel-netcdf/1.12.3.1 + darshan-runtime + $CIME_OUTPUT_ROOT/$CASE/run @@ -1176,9 +1195,13 @@ threads - + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + + diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index efaf77514d91..4c80aa55b4fd 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -550,9 +550,8 @@ DisableMpiCxxBindings() if (SCREAM_DOUBLE_PRECISION) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") set(SCREAM_Fortran_FLAGS -real-size 64) - #elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") - #find what is crayclang id in cmake - #set(SCREAM_Fortran_FLAGS -s default32 -eZ) + elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Cray") + set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 2b619d54bf33..7dddaf343e80 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -10,6 +10,9 @@ namespace p3 { * this file, #include p3_functions.hpp instead. */ +#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2 +#pragma clang optimize off +#endif template <> void Functions ::p3_main_part2_disp( @@ -130,7 +133,9 @@ ::p3_main_part2_disp( if (!hydrometeorsPresent(i)) return; }); } - +#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2 +#pragma clang optimize on +#endif } // namespace p3 } // namespace scream From 72cebf6f1c6b70efdada9d7907f49c885d0a2f71 Mon Sep 17 00:00:00 2001 From: noel Date: Wed, 4 Sep 2024 10:26:32 -0700 Subject: [PATCH 645/904] fix white spaces only. see next commit for real change --- components/eam/src/dynamics/se/dyn_grid.F90 | 130 ++++++++++---------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/components/eam/src/dynamics/se/dyn_grid.F90 b/components/eam/src/dynamics/se/dyn_grid.F90 index c39b3aea7301..6294fafe20f1 100644 --- a/components/eam/src/dynamics/se/dyn_grid.F90 +++ b/components/eam/src/dynamics/se/dyn_grid.F90 @@ -1,28 +1,28 @@ module dyn_grid - !------------------------------------------------------------------------------------------------- - ! + !------------------------------------------------------------------------------------------------- + ! ! Purpose: Definition of dynamics computational grid. - ! - ! The grid used by the SE dynamics is called the GLL grid, which is - ! made up up of elements that correspond to "blocks". The columns + ! + ! The grid used by the SE dynamics is called the GLL grid, which is + ! made up up of elements that correspond to "blocks". The columns ! within each element are located at the Gauss-Lobatto-Legendre (GLL) ! quadrature points. The GLL nodes can be used as the physics columns ! as well, but this can lead to grid imprinting. The physics can also - ! be done on a quasi-equal area FV grid when fv_nphys>0, which will + ! be done on a quasi-equal area FV grid when fv_nphys>0, which will ! define fv_nphys^2 cells per element. - ! + ! ! Method: Variables are private; interface routines used to extract ! information for use in user code. Global column index range - ! defined using full (unreduced) grid. - ! + ! defined using full (unreduced) grid. + ! ! Entry points: - ! get_block_bounds_d get first and last indices in global + ! get_block_bounds_d get first and last indices in global ! block ordering ! get_block_gcol_d get column indices for given block ! get_block_gcol_cnt_d get number of columns in given block ! get_block_lvl_cnt_d get number of vertical levels in column ! get_block_levels_d get vertical levels in column - ! get_gcol_block_d get global block indices and local columns + ! get_gcol_block_d get global block indices and local columns ! index for given global column index ! get_gcol_block_cnt_d get number of blocks containing data ! from a given global column index @@ -31,14 +31,14 @@ module dyn_grid ! information ! get_horiz_grid_dim_d get horizontal dimensions of dynamics grid ! dyn_grid_get_pref get reference pressures for the dynamics grid - ! dyn_grid_get_elem_coords get coordinates of a specified block element + ! dyn_grid_get_elem_coords get coordinates of a specified block element ! of the dynamics grid ! dyn_grid_find_gcols finds nearest column for given lat/lon - ! dyn_grid_get_colndx get element block/column and MPI process indices + ! dyn_grid_get_colndx get element block/column and MPI process indices ! corresponding to a specified global column index ! ! Author: Jim Edwards and Patrick Worley - ! + ! !------------------------------------------------------------------------------------------------- use element_mod, only: element_t use cam_logfile, only: iulog @@ -60,7 +60,7 @@ module dyn_grid integer, parameter, public :: physgrid_d = 102 ! FV physics grid resolution (physics on GLL grid if NPG=0) - integer, parameter, public :: fv_nphys = NPG + integer, parameter, public :: fv_nphys = NPG integer :: ngcols_d = 0 ! number of dynamics columns logical :: gblocks_need_initialized = .true. @@ -113,9 +113,9 @@ end subroutine dyn_grid_init !================================================================================================= ! subroutine get_block_bounds_d(block_first,block_last) - !--------------------------------------------------------------------------- + !--------------------------------------------------------------------------- ! Purpose: Return first and last indices used in global block ordering - ! + ! ! Author: Jim Edwards !------------------------------Arguments------------------------------------ integer, intent(out) :: block_first ! first (global) index used for blocks @@ -129,9 +129,9 @@ end subroutine get_block_bounds_d !================================================================================================= ! subroutine get_block_gcol_d(blockid,size,cdex) - !--------------------------------------------------------------------------- + !--------------------------------------------------------------------------- ! Purpose: Return list of dynamics column indices in given block - ! + ! ! Author: Jim Edwards !--------------------------------------------------------------------------- implicit none @@ -160,9 +160,9 @@ end subroutine get_block_gcol_d !================================================================================================= ! integer function get_block_gcol_cnt_d(blockid) - !--------------------------------------------------------------------------- + !--------------------------------------------------------------------------- ! Purpose: Return number of dynamics columns in indicated block - ! + ! ! Author: Jim Edwards !------------------------------Arguments------------------------------------ integer, intent(in) :: blockid @@ -243,11 +243,11 @@ subroutine get_gcol_block_d(gcol, cnt, blockid, bcid, localblockid) use kinds, only: int_kind use cam_abortutils, only: endrun !--------------------------------------------------------------------------- - ! Purpose: Return global block index and local column index for + ! Purpose: Return global block index and local column index for ! global column index. Element array is naturally ordered. - ! + ! ! Author: Jim Edwards - ! Modified: replaced linear search with best guess and binary search + ! Modified: replaced linear search with best guess and binary search ! (Pat Worley, 7/2/09) !--------------------------------------------------------------------------- implicit none @@ -378,7 +378,7 @@ integer function get_gcol_block_cnt_d(gcol) !--------------------------------------------------------------------------- ! Purpose: Return number of blocks contain data for the vertical column ! with the given global column index - ! + ! ! Author: Patrick Worley !--------------------------------------------------------------------------- implicit none @@ -392,9 +392,9 @@ end function get_gcol_block_cnt_d !================================================================================================= ! integer function get_block_owner_d(blockid) - !--------------------------------------------------------------------------- + !--------------------------------------------------------------------------- ! Purpose: Return id of processor that "owns" the indicated block - ! + ! ! Author: Jim Edwards !--------------------------------------------------------------------------- implicit none @@ -402,7 +402,7 @@ integer function get_block_owner_d(blockid) integer, intent(in) :: blockid ! global block id !--------------------------------------------------------------------------- if (gblocks_need_initialized) call gblocks_init() - + if (gblocks(blockid)%Owner>-1) then get_block_owner_d=gblocks(blockid)%Owner else @@ -419,7 +419,7 @@ subroutine get_horiz_grid_dim_d(hdim1_d,hdim2_d) ! Purpose: Returns declared horizontal dimensions of computational grid. ! For non-lon/lat grids, declare grid to be one-dimensional, ! i.e., (ngcols_d x 1) - ! + ! ! Author: Patrick Worley !------------------------------Arguments------------------------------------ integer, intent(out) :: hdim1_d ! first horizontal dimension @@ -430,7 +430,7 @@ subroutine get_horiz_grid_dim_d(hdim1_d,hdim2_d) else hdim1_d = ngcols_d end if ! fv_nphys > 0 - + if (present(hdim2_d)) hdim2_d = 1 return @@ -441,7 +441,7 @@ end subroutine get_horiz_grid_dim_d subroutine set_horiz_grid_cnt_d(NumUniqueCols) !--------------------------------------------------------------------------- ! Purpose: Set number of columns in the dynamics computational grid - ! + ! ! Author: Jim Edwards !--------------------------------------------------------------------------- use dimensions_mod, only: nelem, nelemd, np, npsq @@ -454,7 +454,7 @@ subroutine set_horiz_grid_cnt_d(NumUniqueCols) integer :: ie, i, j, ii logical, save :: local_coords_initialized = .false. real(r8) :: areaw(np, np) - type(index_t), pointer :: idx + type(index_t), pointer :: idx !--------------------------------------------------------------------------- if (.not. local_coords_initialized) then ngcols_d = NumUniqueCols @@ -463,12 +463,12 @@ subroutine set_horiz_grid_cnt_d(NumUniqueCols) call endrun('set_horiz_grid_cnt_d: fdofp_local already defined') end if allocate(fdofp_local(npsq, nelemd)) - fdofp_local =0 + fdofp_local =0 do ie = 1, nelemd idx => elem(ie)%idxP do ii = 1, idx%NumUniquePts i = idx%ia(ii) - j = idx%ja(ii) + j = idx%ja(ii) fdofp_local((np*(j-1))+i,ie) = (idx%UniquePtoffset+ii-1) end do end do @@ -480,7 +480,7 @@ subroutine set_horiz_grid_cnt_d(NumUniqueCols) ! Now, fill in the appropriate values ii = 1 do ie = 1, nelemd - areaw = 1.0_r8 / elem(ie)%rspheremp(:,:) + areaw = 1.0_r8 / elem(ie)%rspheremp(:,:) pearea(ii:ii+npsq-1) = reshape(areaw, (/ np*np /)) pemap(ii:ii+npsq-1) = fdofp_local(:,ie) do j = 1, np @@ -503,14 +503,14 @@ end subroutine set_horiz_grid_cnt_d ! subroutine get_horiz_grid_d(nxy,clat_d_out,clon_d_out,area_d_out, & wght_d_out,lat_d_out,lon_d_out,cost_d_out) - !--------------------------------------------------------------------------- + !--------------------------------------------------------------------------- ! Purpose: Return latitude and longitude (in radians), column surface ! area (in radians squared) and surface integration weights ! for global column indices that will be passed to/from - ! physics. Optionally also return estimated physics - ! computational cost per global column for use in load + ! physics. Optionally also return estimated physics + ! computational cost per global column for use in load ! balancing. - ! + ! ! Author: Jim Edwards !------------------------------Arguments------------------------------------ integer, intent(in ) :: nxy ! array sizes @@ -572,7 +572,7 @@ subroutine get_horiz_grid_d(nxy,clat_d_out,clon_d_out,area_d_out, & end if if ( present(clat_d_out) .or. present(clon_d_out) ) then - + allocate( lat_d_rad_temp(nxy) ) allocate( lon_d_rad_temp(nxy) ) allocate( lat_d_deg_temp(nxy) ) @@ -628,7 +628,7 @@ subroutine define_cam_grids() !--------------------------------------------------------------------------- ! Create grid for data on HOMME GLL nodes !--------------------------------------------------------------------------- - ! When the FV physs grid is used the GLL grid output + ! When the FV physs grid is used the GLL grid output ! variables will use coordinates with the '_d' suffix. gridname = 'GLL' if (fv_nphys > 0) then @@ -670,7 +670,7 @@ subroutine define_cam_grids() call cam_grid_attribute_register(trim(gridname), trim(areaname), & 'gll grid areas', trim(ncolname), pearea, pemap) else - ! if single column model, then this attribute has to be handled + ! if single column model, then this attribute has to be handled ! by assigning just the SCM point. Else, the model will bomb out ! when writing the header information to history output area_scm(1) = 1.0_r8 / elem(1)%rspheremp(1,1) @@ -687,15 +687,15 @@ subroutine define_cam_grids() deallocate(pelon_deg) ! These can be nullified since the grid object has the reference - nullify(grid_map_d) + nullify(grid_map_d) nullify(pearea) nullify(pemap) !--------------------------------------------------------------------------- ! Create grid object for physics grid on the dynamics decomposition !--------------------------------------------------------------------------- - ! Following CAM6-SE the 'physgrid_d' grid is created and used to load the - ! PHIS field (i.e. surface topo) on the FV physics grid and then + ! Following CAM6-SE the 'physgrid_d' grid is created and used to load the + ! PHIS field (i.e. surface topo) on the FV physics grid and then ! interpolated to the GLL grid. This ensures consistent treatment of SGH. if (fv_nphys>0) then @@ -822,13 +822,13 @@ subroutine gblocks_init() end do end if - do ie = 1,nelemdmax + do ie = 1,nelemdmax if (ie<=nelemd) then rdispls(iam+1)=elem(ie)%idxP%UniquePtOffset-1 gid(iam+1)=elem(ie)%GlobalID lid(iam+1)=ie recvcounts(iam+1)=elem(ie)%idxP%NumUniquePts - else + else rdispls(iam+1) = 0 recvcounts(iam+1) = 0 gid(iam+1)=0 @@ -882,7 +882,7 @@ subroutine compute_global_area(area_d) end if if (fv_nphys > 0) then - + ncol_fv_gbl = fv_nphys*fv_nphys*nelem ncol_fv_lcl = fv_nphys*fv_nphys*nelemd allocate(rbuf(ncol_fv_gbl)) @@ -948,14 +948,14 @@ subroutine compute_global_area(area_d) deallocate(col_id_global) else ! physics is on GLL grid - + allocate(rbuf(ngcols_d)) - do ie = 1, nelemdmax + do ie = 1, nelemdmax if(ie <= nelemd) then displace(iam+1) = elem(ie)%idxp%UniquePtOffset-1 recvcnts(iam+1) = elem(ie)%idxP%NumUniquePts eb = displace(iam+1) + elem(ie)%idxp%NumUniquePts - areaw = 1.0_r8 / elem(ie)%rspheremp(:,:) + areaw = 1.0_r8 / elem(ie)%rspheremp(:,:) call UniquePoints(elem(ie)%idxP, areaw, area_d(displace(iam+1)+1:eb)) else displace(iam+1) = 0 @@ -1065,7 +1065,7 @@ subroutine compute_global_coords(clat, clon, lat_out, lon_out, corner_lat_out, c ! Gather global latitudes call mpi_allgatherv( lat_rad_local(:), recvcnts(iam+1), mpi_real8, rbuf, & recvcnts(:), displace(:), mpi_real8, mpicom, ierr) - + ! sort latitude according to global element ID do icol = 1,ncol_fv_gbl clat( col_id_global(icol) ) = rbuf(icol) @@ -1074,7 +1074,7 @@ subroutine compute_global_coords(clat, clon, lat_out, lon_out, corner_lat_out, c ! Gather global longitudes call mpi_allgatherv( lon_rad_local(:), recvcnts(iam+1), mpi_real8, rbuf, & recvcnts(:), displace(:), mpi_real8, mpicom, ierr) - + ! sort longitude according to global element ID do icol = 1,ncol_fv_gbl clon( col_id_global(icol) ) = rbuf(icol) @@ -1107,14 +1107,14 @@ subroutine compute_global_coords(clat, clon, lat_out, lon_out, corner_lat_out, c ! Gather latitude call mpi_allgatherv( corner_lat_rad_local(:,c), recvcnts(iam+1), mpi_real8, rbuf, & recvcnts(:), displace(:), mpi_real8, mpicom, ierr) - ! sort latitude + ! sort latitude do icol = 1,ncol_fv_gbl corner_lat_out( col_id_global(icol), c) = rbuf(icol) * rad2deg end do ! Gather longitude call mpi_allgatherv( corner_lon_rad_local(:,c), recvcnts(iam+1), mpi_real8, rbuf, & recvcnts(:), displace(:), mpi_real8, mpicom, ierr) - ! sort longitude + ! sort longitude do icol = 1,ncol_fv_gbl corner_lon_out( col_id_global(icol), c) = rbuf(icol) * rad2deg end do @@ -1179,7 +1179,7 @@ subroutine compute_global_coords(clat, clon, lat_out, lon_out, corner_lat_out, c call mpi_allgatherv(rbuf, recvcnts(iam+1), mpi_real8, clat, & recvcnts(:), displace(:), mpi_real8, mpicom, ierr) if (present(lat_out)) then - rbuf(1:recvcnts(iam+1)) = lat_out(sb:eb) + rbuf(1:recvcnts(iam+1)) = lat_out(sb:eb) call mpi_allgatherv(rbuf, recvcnts(iam+1), mpi_real8, lat_out, & recvcnts(:), displace(:), mpi_real8, mpicom, ierr) end if @@ -1188,12 +1188,12 @@ subroutine compute_global_coords(clat, clon, lat_out, lon_out, corner_lat_out, c call mpi_allgatherv(rbuf, recvcnts(iam+1), mpi_real8, clon, & recvcnts(:), displace(:), mpi_real8, mpicom, ierr) if (present(lon_out)) then - rbuf(1:recvcnts(iam+1)) = lon_out(sb:eb) + rbuf(1:recvcnts(iam+1)) = lon_out(sb:eb) call mpi_allgatherv(rbuf, recvcnts(iam+1), mpi_real8, lon_out, & recvcnts(:), displace(:), mpi_real8, mpicom, ierr) end if end do - + deallocate(rbuf) end if ! fv_nphys > 0 @@ -1275,7 +1275,7 @@ integer function get_dyn_grid_parm(name) result(ival) ival = fv_nphys*fv_nphys*nelem else ival = ngcols_d - end if + end if else if(name.eq.'plev') then ival = plev else if(name.eq.'plevp') then @@ -1318,10 +1318,10 @@ end subroutine dyn_grid_get_pref ! !================================================================================================= ! - subroutine dyn_grid_find_gcols( lat, lon, nclosest, owners, col, lbk, rlat, rlon, idyn_dists ) + subroutine dyn_grid_find_gcols( lat, lon, nclosest, owners, col, lbk, rlat, rlon, idyn_dists ) !--------------------------------------------------------------------------- - ! This returns the lat/lon information (and corresponding MPI task - ! numbers (owners)) of the global model grid columns nearest to the + ! This returns the lat/lon information (and corresponding MPI task + ! numbers (owners)) of the global model grid columns nearest to the ! input satellite coordinate (lat,lon) !--------------------------------------------------------------------------- use shr_const_mod, only: SHR_CONST_REARTH @@ -1415,7 +1415,7 @@ end subroutine dyn_grid_find_gcols ! !================================================================================================= ! - subroutine dyn_grid_get_colndx( igcol, nclosest, owners, col, lbk ) + subroutine dyn_grid_get_colndx( igcol, nclosest, owners, col, lbk ) !------------------------------Arguments------------------------------------ integer, intent(in) :: nclosest integer, intent(in) :: igcol(nclosest) @@ -1437,7 +1437,7 @@ subroutine dyn_grid_get_colndx( igcol, nclosest, owners, col, lbk ) call get_gcol_block_d( igcol(i), 1, blockid, bcid, lclblockid ) owners(i) = get_block_owner_d(blockid(1)) - + if (owners(i)==iam) then lbk(i) = lclblockid(1) ii = igcol(i)-elem(lbk(i))%idxp%UniquePtoffset+1 @@ -1448,7 +1448,7 @@ subroutine dyn_grid_get_colndx( igcol, nclosest, owners, col, lbk ) lbk(i) = -1 col(i) = -1 endif - + end do end subroutine dyn_grid_get_colndx From f0fa68c522af80e8390c69189fa5afa733179432 Mon Sep 17 00:00:00 2001 From: noel Date: Wed, 4 Sep 2024 10:30:29 -0700 Subject: [PATCH 646/904] create and use a module scope pointer to hold area for SCM that will survive until writing to netcdf file --- components/eam/src/dynamics/se/dyn_grid.F90 | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/components/eam/src/dynamics/se/dyn_grid.F90 b/components/eam/src/dynamics/se/dyn_grid.F90 index 6294fafe20f1..0faed9ea1d32 100644 --- a/components/eam/src/dynamics/se/dyn_grid.F90 +++ b/components/eam/src/dynamics/se/dyn_grid.F90 @@ -101,6 +101,7 @@ module dyn_grid real(r8), allocatable :: pelon_deg(:) ! pe-local longitudes (degrees) real(r8), pointer :: pearea(:) => null() ! pe-local areas integer(iMap), pointer :: pemap(:) => null() ! pe-local map for PIO decomp + real(r8), pointer :: pearea_scm(:) => null() ! special case: area for SCM (only 1 value) !=================================================================================================== contains @@ -618,7 +619,7 @@ subroutine define_cam_grids() integer(iMap), pointer :: grid_map_d(:,:) integer(iMap), pointer :: grid_map_p(:,:) integer :: ie, i, j, k, mapind ! Loop variables - real(r8) :: area_scm(1), lat, lon + real(r8) :: lat, lon integer :: ncols_p_lcl ! local column count integer :: ncols_p_gbl ! global column count integer(iMap), pointer :: physgrid_map(:) @@ -670,12 +671,13 @@ subroutine define_cam_grids() call cam_grid_attribute_register(trim(gridname), trim(areaname), & 'gll grid areas', trim(ncolname), pearea, pemap) else - ! if single column model, then this attribute has to be handled - ! by assigning just the SCM point. Else, the model will bomb out - ! when writing the header information to history output - area_scm(1) = 1.0_r8 / elem(1)%rspheremp(1,1) - call cam_grid_attribute_register(trim(gridname), trim(areaname), & - 'gll grid areas', trim(ncolname), area_scm) + ! If single column model, set pearea_scm(1) to be the area. + ! Then register attribute in same way as non-SCM to simplify. + allocate(pearea_scm(1)) + pearea_scm(1) = 1.0_r8 / elem(1)%rspheremp(1,1) + call cam_grid_attribute_register(trim(gridname), trim(areaname), & + 'gll grid areas', trim(ncolname), pearea_scm, pemap) + nullify(pearea_scm) end if ! .not. single_column call cam_grid_attribute_register(trim(gridname), 'np', '', np) From 04933000cb9e2390e5e2231ccba64fd49795e3f4 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Wed, 4 Sep 2024 14:52:58 -0500 Subject: [PATCH 647/904] Add single forcing table Add table explaining what is fixed and what is varying in the single forcing compsets. --- docs/user-guide/index.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 8674926526e9..25d2d1c74c07 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -24,7 +24,7 @@ is available. A *fully coupled* compset is one which has active components for at least the atmosphere, ocean, land surface, ocean and sea-ice all interacting. Each compset is associated with a specific forcing condition. -Coupled compsets in E3SM are developed for three science-driven simulation campaigns: `water cycle change and impacts`, `human-earth system feedbacks`, and `polar processes, sea-level rise and coastal impacts`. The standard coupled configurations -- which consist of prognostic atmosphere, land, river, ocean and sea-ice components -- form the base physical coupled system and are mainly designed for `water cycle change and impacts` simulation campaign. +Coupled compsets in E3SM are developed for three science-driven simulation campaigns: **water cycle change and impacts**, **human-earth system feedbacks**, and **polar processes, sea-level rise and coastal impacts**. The standard coupled configurations -- which consist of prognostic atmosphere, land, river, ocean and sea-ice components -- form the base physical coupled system and are mainly designed for `water cycle change and impacts` simulation campaign. Below list the standard configuration compsets supported in the current version of E3SM: |Compset alias | Description | @@ -37,7 +37,7 @@ Below list the standard configuration compsets supported in the current version |`WCYCLSSP245` | Standard configuration with prescribed SSP-245 forcings | |`WCYCLSSP370` | Standard configuration with prescribed SSP-370 forcings | |`WCYCLSSP585` | Standard configuration with prescribed SSP-585 forcings | -| | #### Compsets for single-forcings simulations of the historical period (1850-2014) | +| | **Compsets for single-forcings simulations of the historical period (1850-2014)** | |`WCYCL20TR-GHG` | Greenhouse gases only configuration (`GHGs`)| |`WCYCL20TR-aer` | Anthropogenic aerosols and precursors only configuration (`aer`)| |`WCYCL20TR-xGHG-xaer` | All forcings except GHGs and aer (`solar irradiance, stratospheric ozone and volcanic emissions, land use land cover`) | @@ -47,6 +47,25 @@ Below list the standard configuration compsets supported in the current version |`WCYCL20TR-lulc` | Land use land cover only configuration | |`WCYCL20TR-volc` | Stratospheric volcanic emissions only configuration | +The table below specifies which forcing category is fixed at 1850 conditions and which are allowed to vary over the historical period +for each of the historical ("20TR") compsets including the single-forcing compsets. + +|Compset alias | GHGs | Aerosol & precursors | Oxidants | Ozone (CI & Linoz) | Volcano | Solar | Land Use & ndep/popdensa | +|:------------ |:-----: | :---: | :---: | :---: | :---: | :---: | :---: | +|`WCYCL20TR` | varying | varying | varying | varying | varying | varying | varying | +|`WCYCL20TR-GHG`| varying | 1850 | 1850 | 1850 | 1850 | 1850 | 1850 | +|`WCYCL20TR-aer`| 1850 | varying | varying | 1850 | 1850 | 1850 | 1850 | +|`WCYCL20TR-xGHG-xaer`| 1850 | 1850 | 1850 | varying | varying | varying | varying | +|`WCYCL20TR-xaer`| varying | 1850 | 1850 | varying | varying | varying | varying | +|`WCYCL20TR-nat`| 1850 | 1850 | 1850 | 1850 | varying | varying | 1850 | +|`WCYCL20TR-ozone`| 1850 | 1850 | 1850 | varying | 1850 | 1850 | 1850 | +|`WCYCL20TR-lulc`| 1850 | 1850 | 1850 | 1850 | 1850 | 1850 | varying | +|`WCYCL20TR-volc`| 1850 | 1850 | 1850 | 1850 | varying | 1850 | 1850 | + + +- *Volcano* refers to stratospheric volcanic SO2 emissions; *1850* for *Volcano refers to background (average) stratospheric volcanic emissions used in pre-industrial control experiments +- *Oxidants* always follow *Aerosol & precursors* for fixed or varying. + The compsets for the other two science simulation campaigns are being finalized, with additional components and/or features. The compset naming follows the same convention, e.g., `CRYO1850` and `CRYO1850-4xCO2` are with prognostic ice-shelf melt fluxes for the `polar processes` simulation campaign. From 943ca3b61ec4f7298b0e90c5ecf30ab81ba80d32 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 30 Aug 2024 10:15:33 -0600 Subject: [PATCH 648/904] view alloc fixes for P3 small kernels --- .../src/physics/p3/disp/p3_main_impl_disp.cpp | 185 ++++++++++-------- .../physics/p3/eamxx_p3_process_interface.cpp | 124 +++++++++--- .../physics/p3/eamxx_p3_process_interface.hpp | 24 ++- .../eamxx/src/physics/p3/eamxx_p3_run.cpp | 6 +- .../src/physics/p3/impl/p3_main_impl.hpp | 30 +-- .../eamxx/src/physics/p3/p3_functions.hpp | 43 ++++ .../eamxx/src/physics/p3/p3_functions_f90.cpp | 37 +++- 7 files changed, 322 insertions(+), 127 deletions(-) diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index d20350b0788b..0f125b7ae960 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -32,16 +32,16 @@ ::p3_main_init_disp( const uview_2d& qv_supersat_i, const uview_2d& qtend_ignore, const uview_2d& ntend_ignore, const uview_2d& mu_c, const uview_2d& lamc, const uview_2d& rho_qi, const uview_2d& qv2qi_depos_tend, const uview_2d& precip_total_tend, const uview_2d& nevapr, const uview_2d& precip_liq_flux, const uview_2d& precip_ice_flux) -{ +{ using ExeSpace = typename KT::ExeSpace; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); Kokkos::parallel_for("p3_main_init", policy, KOKKOS_LAMBDA(const MemberType& team) { - - const Int i = team.league_rank(); - precip_liq_surf(i) = 0; - precip_ice_surf(i) = 0; + + const Int i = team.league_rank(); + precip_liq_surf(i) = 0; + precip_ice_surf(i) = 0; Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nk_pack), [&] (Int k) { @@ -107,6 +107,7 @@ ::p3_main_internal_disp( const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, + const P3Temporaries& temporaries, const WorkspaceManager& workspace_mgr, Int nj, Int nk, @@ -131,87 +132,99 @@ ::p3_main_internal_disp( view_1d nucleationPossible("nucleationPossible", nj); view_1d hydrometeorsPresent("hydrometeorsPresent", nj); - // - // Create temporary variables needed for p3 - // - view_2d - mu_r("mu_r", nj, nk_pack), // shape parameter of rain - T_atm("T_atm", nj, nk_pack), // temperature at the beginning of the microphysics step [K] - - // 2D size distribution and fallspeed parameters - lamr("lamr", nj, nk_pack), logn0r("logn0r", nj, nk_pack), nu("nu", nj, nk_pack), - cdist("cdist", nj, nk_pack), cdist1("cdist1", nj, nk_pack), cdistr("cdistr", nj, nk_pack), - - // Variables needed for in-cloud calculations - // Inverse cloud fractions (1/cld) - inv_cld_frac_i("inv_cld_frac_i", nj, nk_pack), inv_cld_frac_l("inv_cld_frac_l", nj, nk_pack), inv_cld_frac_r("inv_cld_frac_r", nj, nk_pack), - // In cloud mass-mixing ratios - qc_incld("qc_incld", nj, nk_pack), qr_incld("qr_incld", nj, nk_pack), qi_incld("qi_incld", nj, nk_pack), qm_incld("qm_incld", nj, nk_pack), - // In cloud number concentrations - nc_incld("nc_incld", nj, nk_pack), nr_incld("nr_incld", nj, nk_pack), ni_incld("ni_incld", nj, nk_pack), bm_incld("bm_incld", nj, nk_pack), - - // Other - inv_dz("inv_dz", nj, nk_pack), inv_rho("inv_rho", nj, nk_pack), ze_ice("ze_ice", nj, nk_pack), ze_rain("ze_rain", nj, nk_pack), - prec("prec", nj, nk_pack), rho("rho", nj, nk_pack), rhofacr("rhofacr", nj, nk_pack), rhofaci("rhofaci", nj, nk_pack), - acn("acn", nj, nk_pack), qv_sat_l("qv_sat", nj, nk_pack), qv_sat_i("qv_sat_i", nj, nk_pack), sup("sup", nj, nk_pack), - qv_supersat_i("qv_supersat", nj, nk_pack), tmparr2("tmparr2", nj, nk_pack), exner("exner", nj, nk_pack), - diag_equiv_reflectivity("diag_equiv_ref", nj, nk_pack), diag_vm_qi("diag_vm_qi", nj, nk_pack), diag_diam_qi("diag_diam_qi", nj, nk_pack), - pratot("pratot", nj, nk_pack), prctot("prctot", nj, nk_pack), - - // p3_tend_out, may not need these - qtend_ignore("qtend_ignore", nj, nk_pack), ntend_ignore("ntend_ignore", nj, nk_pack), - - // Variables still used in F90 but removed from C++ interface - mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), - qr_evap_tend("qr_evap_tend", nj, nk_pack), - - // cloud sedimentation - v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), - - // ice sedimentation - v_qit("v_qit", nj, nk_pack), v_nit("v_nit", nj, nk_pack), flux_nit("flux_nit", nj, nk_pack), flux_bir("flux_bir", nj, nk_pack), - flux_qir("flux_qir", nj, nk_pack), flux_qit("flux_qit", nj, nk_pack), - - // rain sedimentation - v_qr("v_qr", nj, nk_pack), v_nr("v_nr", nj, nk_pack); - // Get views of all inputs - auto pres = diagnostic_inputs.pres; - auto dz = diagnostic_inputs.dz; - auto nc_nuceat_tend = diagnostic_inputs.nc_nuceat_tend; - auto nccn_prescribed = diagnostic_inputs.nccn; - auto ni_activated = diagnostic_inputs.ni_activated; - auto inv_qc_relvar = diagnostic_inputs.inv_qc_relvar; - auto dpres = diagnostic_inputs.dpres; - auto inv_exner = diagnostic_inputs.inv_exner; - auto cld_frac_i = diagnostic_inputs.cld_frac_i; - auto cld_frac_l = diagnostic_inputs.cld_frac_l; - auto cld_frac_r = diagnostic_inputs.cld_frac_r; - auto col_location = infrastructure.col_location; - auto qc = prognostic_state.qc; - auto nc = prognostic_state.nc; - auto qr = prognostic_state.qr; - auto nr = prognostic_state.nr; - auto qi = prognostic_state.qi; - auto qm = prognostic_state.qm; - auto ni = prognostic_state.ni; - auto bm = prognostic_state.bm; - auto qv = prognostic_state.qv; - auto th = prognostic_state.th; - auto diag_eff_radius_qc = diagnostic_outputs.diag_eff_radius_qc; - auto diag_eff_radius_qi = diagnostic_outputs.diag_eff_radius_qi; - auto diag_eff_radius_qr = diagnostic_outputs.diag_eff_radius_qr; - auto qv2qi_depos_tend = diagnostic_outputs.qv2qi_depos_tend; - auto rho_qi = diagnostic_outputs.rho_qi; - auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; - auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; - auto precip_total_tend = diagnostic_outputs.precip_total_tend; - auto nevapr = diagnostic_outputs.nevapr; - auto qv_prev = diagnostic_inputs.qv_prev; - auto t_prev = diagnostic_inputs.t_prev; - auto liq_ice_exchange = history_only.liq_ice_exchange; - auto vap_liq_exchange = history_only.vap_liq_exchange; - auto vap_ice_exchange = history_only.vap_ice_exchange; + auto pres = diagnostic_inputs.pres; + auto dz = diagnostic_inputs.dz; + auto nc_nuceat_tend = diagnostic_inputs.nc_nuceat_tend; + auto nccn_prescribed = diagnostic_inputs.nccn; + auto ni_activated = diagnostic_inputs.ni_activated; + auto inv_qc_relvar = diagnostic_inputs.inv_qc_relvar; + auto dpres = diagnostic_inputs.dpres; + auto inv_exner = diagnostic_inputs.inv_exner; + auto cld_frac_i = diagnostic_inputs.cld_frac_i; + auto cld_frac_l = diagnostic_inputs.cld_frac_l; + auto cld_frac_r = diagnostic_inputs.cld_frac_r; + auto col_location = infrastructure.col_location; + auto qc = prognostic_state.qc; + auto nc = prognostic_state.nc; + auto qr = prognostic_state.qr; + auto nr = prognostic_state.nr; + auto qi = prognostic_state.qi; + auto qm = prognostic_state.qm; + auto ni = prognostic_state.ni; + auto bm = prognostic_state.bm; + auto qv = prognostic_state.qv; + auto th = prognostic_state.th; + auto diag_eff_radius_qc = diagnostic_outputs.diag_eff_radius_qc; + auto diag_eff_radius_qi = diagnostic_outputs.diag_eff_radius_qi; + auto diag_eff_radius_qr = diagnostic_outputs.diag_eff_radius_qr; + auto qv2qi_depos_tend = diagnostic_outputs.qv2qi_depos_tend; + auto rho_qi = diagnostic_outputs.rho_qi; + auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; + auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; + auto precip_total_tend = diagnostic_outputs.precip_total_tend; + auto nevapr = diagnostic_outputs.nevapr; + auto qv_prev = diagnostic_inputs.qv_prev; + auto t_prev = diagnostic_inputs.t_prev; + auto liq_ice_exchange = history_only.liq_ice_exchange; + auto vap_liq_exchange = history_only.vap_liq_exchange; + auto vap_ice_exchange = history_only.vap_ice_exchange; + auto mu_r = temporaries.mu_r; + auto T_atm = temporaries.T_atm; + auto lamr = temporaries.lamr; + auto logn0r = temporaries.logn0r; + auto nu = temporaries.nu; + auto cdist = temporaries.cdist; + auto cdist1 = temporaries.cdist1; + auto cdistr = temporaries.cdistr; + auto inv_cld_frac_i = temporaries.inv_cld_frac_i; + auto inv_cld_frac_l = temporaries.inv_cld_frac_l; + auto inv_cld_frac_r = temporaries.inv_cld_frac_r; + auto qc_incld = temporaries.qc_incld; + auto qr_incld = temporaries.qr_incld; + auto qi_incld = temporaries.qi_incld; + auto qm_incld = temporaries.qm_incld; + auto nc_incld = temporaries.nc_incld; + auto nr_incld = temporaries.nr_incld; + auto ni_incld = temporaries.ni_incld; + auto bm_incld = temporaries.bm_incld; + auto inv_dz = temporaries.inv_dz; + auto inv_rho = temporaries.inv_rho; + auto ze_ice = temporaries.ze_ice; + auto ze_rain = temporaries.ze_rain; + auto prec = temporaries.prec; + auto rho = temporaries.rho; + auto rhofacr = temporaries.rhofacr; + auto rhofaci = temporaries.rhofaci; + auto acn = temporaries.acn; + auto qv_sat_l = temporaries.qv_sat_l; + auto qv_sat_i = temporaries.qv_sat_i; + auto sup = temporaries.sup; + auto qv_supersat_i = temporaries.qv_supersat_i; + auto tmparr2 = temporaries.tmparr2; + auto exner = temporaries.exner; + auto diag_equiv_reflectivity = temporaries.diag_equiv_reflectivity; + auto diag_vm_qi = temporaries.diag_vm_qi; + auto diag_diam_qi = temporaries.diag_diam_qi; + auto pratot = temporaries.pratot; + auto prctot = temporaries.prctot; + auto qtend_ignore = temporaries.qtend_ignore; + auto ntend_ignore = temporaries.ntend_ignore; + auto mu_c = temporaries.mu_c; + auto lamc = temporaries.lamc; + auto qr_evap_tend = temporaries.qr_evap_tend; + auto v_qc = temporaries.v_qc; + auto v_nc = temporaries.v_nc; + auto flux_qx = temporaries.flux_qx; + auto flux_nx = temporaries.flux_nx; + auto v_qit = temporaries.v_qit; + auto v_nit = temporaries.v_nit; + auto flux_nit = temporaries.flux_nit; + auto flux_bir = temporaries.flux_bir; + auto flux_qir = temporaries.flux_qir; + auto flux_qit = temporaries.flux_qit; + auto v_qr = temporaries.v_qr; + auto v_nr = temporaries.v_nr; // we do not want to measure init stuff auto start = std::chrono::steady_clock::now(); @@ -242,7 +255,7 @@ ::p3_main_internal_disp( p3_main_part2_disp( nj, nk, runtime_options.max_total_ni, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, inv_dt, - lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, + lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, lookup_tables.revap_table_vals, pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index acaecd7eef70..c414d7093f62 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -152,10 +152,14 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) Real* mem = reinterpret_cast(buffer_manager.get_memory()); // 1d scalar views - m_buffer.precip_liq_surf_flux = decltype(m_buffer.precip_liq_surf_flux)(mem, m_num_cols); - mem += m_buffer.precip_liq_surf_flux.size(); - m_buffer.precip_ice_surf_flux = decltype(m_buffer.precip_ice_surf_flux)(mem, m_num_cols); - mem += m_buffer.precip_ice_surf_flux.size(); + using scalar_1d_view_t = decltype(m_buffer.precip_liq_surf_flux); + scalar_1d_view_t* _1d_scalar_view_ptrs[Buffer::num_1d_scalar] = { + &m_buffer.precip_liq_surf_flux, &m_buffer.precip_ice_surf_flux + }; + for (int i=0; isize(); + } // 2d scalar views m_buffer.col_location = decltype(m_buffer.col_location)(mem, m_num_cols, 3); @@ -167,26 +171,38 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) const Int nk_pack = ekat::npack(m_num_levs); const Int nk_pack_p1 = ekat::npack(m_num_levs+1); - m_buffer.inv_exner = decltype(m_buffer.inv_exner)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.inv_exner.size(); - m_buffer.th_atm = decltype(m_buffer.th_atm)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.th_atm.size(); - m_buffer.cld_frac_l = decltype(m_buffer.cld_frac_l)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.cld_frac_l.size(); - m_buffer.cld_frac_i = decltype(m_buffer.cld_frac_i)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.cld_frac_i.size(); - m_buffer.dz = decltype(m_buffer.dz)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.dz.size(); - m_buffer.qv2qi_depos_tend = decltype(m_buffer.qv2qi_depos_tend)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.qv2qi_depos_tend.size(); - m_buffer.rho_qi = decltype(m_buffer.rho_qi)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.rho_qi.size(); - m_buffer.precip_liq_flux = decltype(m_buffer.precip_liq_flux)(s_mem, m_num_cols, nk_pack_p1); - s_mem += m_buffer.precip_liq_flux.size(); - m_buffer.precip_ice_flux = decltype(m_buffer.precip_ice_flux)(s_mem, m_num_cols, nk_pack_p1); - s_mem += m_buffer.precip_ice_flux.size(); - m_buffer.unused = decltype(m_buffer.unused)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.unused.size(); + using spack_2d_view_t = decltype(m_buffer.inv_exner); + spack_2d_view_t* _2d_spack_mid_view_ptrs[Buffer::num_2d_vector] = { + &m_buffer.inv_exner, &m_buffer.th_atm, &m_buffer.cld_frac_l, &m_buffer.cld_frac_i, + &m_buffer.dz, &m_buffer.qv2qi_depos_tend, &m_buffer.rho_qi, &m_buffer.unused +#ifdef SCREAM_P3_SMALL_KERNELS + , &m_buffer.mu_r, &m_buffer.T_atm, &m_buffer.lamr, &m_buffer.logn0r, &m_buffer.nu, + &m_buffer.cdist, &m_buffer.cdist1, &m_buffer.cdistr, &m_buffer.inv_cld_frac_i, + &m_buffer.inv_cld_frac_l, &m_buffer.inv_cld_frac_r, &m_buffer.qc_incld, &m_buffer.qr_incld, + &m_buffer.qi_incld, &m_buffer.qm_incld, &m_buffer.nc_incld, &m_buffer.nr_incld, + &m_buffer.ni_incld, &m_buffer.bm_incld, &m_buffer.inv_dz, &m_buffer.inv_rho, &m_buffer.ze_ice, + &m_buffer.ze_rain, &m_buffer.prec, &m_buffer.rho, &m_buffer.rhofacr, &m_buffer.rhofaci, + &m_buffer.acn, &m_buffer.qv_sat_l, &m_buffer.qv_sat_i, &m_buffer.sup, &m_buffer.qv_supersat_i, + &m_buffer.tmparr2, &m_buffer.exner, &m_buffer.diag_equiv_reflectivity, &m_buffer.diag_vm_qi, + &m_buffer.diag_diam_qi, &m_buffer.pratot, &m_buffer.prctot, &m_buffer.qtend_ignore, + &m_buffer.ntend_ignore, &m_buffer.mu_c, &m_buffer.lamc, &m_buffer.qr_evap_tend, &m_buffer.v_qc, + &m_buffer.v_nc, &m_buffer.flux_qx, &m_buffer.flux_nx, &m_buffer.v_qit, &m_buffer.v_nit, + &m_buffer.flux_nit, &m_buffer.flux_bir, &m_buffer.flux_qir, &m_buffer.flux_qit, &m_buffer.v_qr, + &m_buffer.v_nr +#endif + }; + for (int i=0; isize(); + } + + spack_2d_view_t* _2d_spack_int_view_ptrs[Buffer::num_2dp1_vector] = { + &m_buffer.precip_liq_flux, &m_buffer.precip_ice_flux + }; + for (int i=0; isize(); + } // WSM data m_buffer.wsm_data = s_mem; @@ -324,6 +340,66 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) history_only.liq_ice_exchange = get_field_out("micro_liq_ice_exchange").get_view(); history_only.vap_liq_exchange = get_field_out("micro_vap_liq_exchange").get_view(); history_only.vap_ice_exchange = get_field_out("micro_vap_ice_exchange").get_view(); +#ifdef SCREAM_P3_SMALL_KERNELS + // Temporaries + temporaries.mu_r = m_buffer.mu_r; + temporaries.T_atm = m_buffer.T_atm; + temporaries.lamr = m_buffer.lamr; + temporaries.logn0r = m_buffer.logn0r; + temporaries.nu = m_buffer.nu; + temporaries.cdist = m_buffer.cdist; + temporaries.cdist1 = m_buffer.cdist1; + temporaries.cdistr = m_buffer.cdistr; + temporaries.inv_cld_frac_i = m_buffer.inv_cld_frac_i; + temporaries.inv_cld_frac_l = m_buffer.inv_cld_frac_l; + temporaries.inv_cld_frac_r = m_buffer.inv_cld_frac_r; + temporaries.qc_incld = m_buffer.qc_incld; + temporaries.qr_incld = m_buffer.qr_incld; + temporaries.qi_incld = m_buffer.qi_incld; + temporaries.qm_incld = m_buffer.qm_incld; + temporaries.nc_incld = m_buffer.nc_incld; + temporaries.nr_incld = m_buffer.nr_incld; + temporaries.ni_incld = m_buffer.ni_incld; + temporaries.bm_incld = m_buffer.bm_incld; + temporaries.inv_dz = m_buffer.inv_dz; + temporaries.inv_rho = m_buffer.inv_rho; + temporaries.ze_ice = m_buffer.ze_ice; + temporaries.ze_rain = m_buffer.ze_rain; + temporaries.prec = m_buffer.prec; + temporaries.rho = m_buffer.rho; + temporaries.rhofacr = m_buffer.rhofacr; + temporaries.rhofaci = m_buffer.rhofaci; + temporaries.acn = m_buffer.acn; + temporaries.qv_sat_l = m_buffer.qv_sat_l; + temporaries.qv_sat_i = m_buffer.qv_sat_i; + temporaries.sup = m_buffer.sup; + temporaries.qv_supersat_i = m_buffer.qv_supersat_i; + temporaries.tmparr2 = m_buffer.tmparr2; + temporaries.exner = m_buffer.exner; + temporaries.diag_equiv_reflectivity = m_buffer.diag_equiv_reflectivity; + temporaries.diag_vm_qi = m_buffer.diag_vm_qi; + temporaries.diag_diam_qi = m_buffer.diag_diam_qi; + temporaries.pratot = m_buffer.pratot; + temporaries.prctot = m_buffer.prctot; + temporaries.qtend_ignore = m_buffer.qtend_ignore; + temporaries.ntend_ignore = m_buffer.ntend_ignore; + temporaries.mu_c = m_buffer.mu_c; + temporaries.lamc = m_buffer.lamc; + temporaries.qr_evap_tend = m_buffer.qr_evap_tend; + temporaries.v_qc = m_buffer.v_qc; + temporaries.v_nc = m_buffer.v_nc; + temporaries.flux_qx = m_buffer.flux_qx; + temporaries.flux_nx = m_buffer.flux_nx; + temporaries.v_qit = m_buffer.v_qit; + temporaries.v_nit = m_buffer.v_nit; + temporaries.flux_nit = m_buffer.flux_nit; + temporaries.flux_bir = m_buffer.flux_bir; + temporaries.flux_qir = m_buffer.flux_qir; + temporaries.flux_qit = m_buffer.flux_qit; + temporaries.v_qr = m_buffer.v_qr; + temporaries.v_nr = m_buffer.v_nr; +#endif + // -- Set values for the post-amble structure p3_postproc.set_variables(m_num_cols,nk_pack, prog_state.th,pmid,pmid_dry,T_atm,t_prev, diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp index 47d543ded022..32e3b76d1a0d 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp @@ -158,7 +158,7 @@ class P3Microphysics : public AtmosphereProcess // Assigning local variables void set_variables(const int ncol, const int npack, const view_2d_const& pmid_, const view_2d_const& pmid_dry_, - const view_2d_const& pseudo_density_, + const view_2d_const& pseudo_density_, const view_2d_const& pseudo_density_dry_, const view_2d& T_atm_, const view_2d_const& cld_frac_t_, const view_2d& qv_, const view_2d& qc_, const view_2d& nc_, const view_2d& qr_, const view_2d& nr_, const view_2d& qi_, @@ -361,7 +361,11 @@ class P3Microphysics : public AtmosphereProcess // 1d view scalar, size (ncol) static constexpr int num_1d_scalar = 2; //no 2d vars now, but keeping 1d struct for future expansion // 2d view packed, size (ncol, nlev_packs) +#ifdef SCREAM_P3_SMALL_KERNELS + static constexpr int num_2d_vector = 64; +#else static constexpr int num_2d_vector = 8; +#endif static constexpr int num_2dp1_vector = 2; uview_1d precip_liq_surf_flux; @@ -377,6 +381,21 @@ class P3Microphysics : public AtmosphereProcess uview_2d precip_ice_flux; //nlev+1 uview_2d unused; +#ifdef SCREAM_P3_SMALL_KERNELS + uview_2d + mu_r, T_atm, lamr, logn0r, nu, cdist, cdist1, cdistr, + inv_cld_frac_i, inv_cld_frac_l, inv_cld_frac_r, + qc_incld, qr_incld, qi_incld, qm_incld, + nc_incld, nr_incld, ni_incld, bm_incld, + inv_dz, inv_rho, ze_ice, ze_rain, prec, rho, rhofacr, + rhofaci, acn, qv_sat_l, qv_sat_i, sup, qv_supersat_i, + tmparr2, exner, diag_equiv_reflectivity, diag_vm_qi, + diag_diam_qi, pratot, prctot, qtend_ignore, ntend_ignore, + mu_c, lamc, qr_evap_tend, v_qc, v_nc, flux_qx, flux_nx, + v_qit, v_nit, flux_nit, flux_bir, flux_qir, flux_qit, + v_qr, v_nr; +#endif + suview_2d col_location; Spack* wsm_data; @@ -410,6 +429,9 @@ class P3Microphysics : public AtmosphereProcess P3F::P3DiagnosticOutputs diag_outputs; P3F::P3HistoryOnly history_only; P3F::P3LookupTables lookup_tables; +#ifdef SCREAM_P3_SMALL_KERNELS + P3F::P3Temporaries temporaries; +#endif P3F::P3Infrastructure infrastructure; P3F::P3Runtime runtime_options; p3_preamble p3_preproc; diff --git a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp index 9ccfdaf20220..cdb156d71f62 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp @@ -29,7 +29,11 @@ void P3Microphysics::run_impl (const double dt) get_field_out("micro_vap_ice_exchange").deep_copy(0.0); P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, m_num_cols, m_num_levs, m_p3constants); + history_only, lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + temporaries, +#endif + workspace_mgr, m_num_cols, m_num_levs, m_p3constants); // Conduct the post-processing of the p3_main output. Kokkos::parallel_for( diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index c98a37d0557a..270042c99986 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -162,10 +162,10 @@ ::p3_main_internal( &inv_dz, &inv_rho, &ze_ice, &ze_rain, &prec, &rho, &rhofacr, &rhofaci, &acn, &qv_sat_l, &qv_sat_i, &sup, &qv_supersat_i, &tmparr1, &exner, &diag_equiv_reflectivity, &diag_vm_qi, &diag_diam_qi, - &pratot, &prctot, &qtend_ignore, &ntend_ignore, + &pratot, &prctot, &qtend_ignore, &ntend_ignore, &mu_c, &lamc, &qr_evap_tend }); - + // Get single-column subviews of all inputs, shouldn't need any i-indexing // after this. const auto opres = ekat::subview(diagnostic_inputs.pres, i); @@ -344,22 +344,15 @@ ::p3_main( const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + const P3Temporaries& temporaries, +#endif const WorkspaceManager& workspace_mgr, Int nj, Int nk, const physics::P3_Constants & p3constants) { -#ifndef SCREAM_P3_SMALL_KERNELS - return p3_main_internal(runtime_options, - prognostic_state, - diagnostic_inputs, - diagnostic_outputs, - infrastructure, - history_only, - lookup_tables, - workspace_mgr, - nj, nk, p3constants); -#else +#ifdef SCREAM_P3_SMALL_KERNELS return p3_main_internal_disp(runtime_options, prognostic_state, diagnostic_inputs, @@ -367,8 +360,19 @@ ::p3_main( infrastructure, history_only, lookup_tables, + temporaries, workspace_mgr, nj, nk, p3constants); +#else + return p3_main_internal(runtime_options, + prognostic_state, + diagnostic_inputs, + diagnostic_outputs, + infrastructure, + history_only, + lookup_tables, + workspace_mgr, + nj, nk, p3constants); #endif } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 3a5218f5c001..791538d67dcb 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -246,6 +246,45 @@ struct Functions view_dnu_table dnu_table_vals; }; +#ifdef SCREAM_P3_SMALL_KERNELS + struct P3Temporaries { + P3Temporaries() = default; + // shape parameter of rain + view_2d mu_r; + // temperature at the beginning of the microphysics step [K] + view_2d T_atm; + // 2D size distribution and fallspeed parameters + view_2d lamr, logn0r, nu; + view_2d cdist, cdist1, cdistr; + // Variables needed for in-cloud calculations + // Inverse cloud fractions (1/cld) + view_2d inv_cld_frac_i, inv_cld_frac_l, inv_cld_frac_r; + // In cloud mass-mixing ratios + view_2d qc_incld, qr_incld, qi_incld, qm_incld; + // In cloud number concentrations + view_2d nc_incld, nr_incld, ni_incld, bm_incld; + // Other + view_2d inv_dz, inv_rho, ze_ice, ze_rain; + view_2d prec, rho, rhofacr, rhofaci; + view_2d acn, qv_sat_l, qv_sat_i, sup; + view_2d qv_supersat_i, tmparr2, exner; + view_2d diag_equiv_reflectivity, diag_vm_qi, diag_diam_qi; + view_2d pratot, prctot; + // p3_tend_out, may not need these + view_2d qtend_ignore, ntend_ignore; + // Variables still used in F90 but removed from C++ interface + view_2d mu_c, lamc; + view_2d qr_evap_tend; + // cloud sedimentation + view_2d v_qc, v_nc, flux_qx, flux_nx; + // ice sedimentation + view_2d v_qit, v_nit, flux_nit, flux_bir; + view_2d flux_qir, flux_qit; + // rain sedimentation + view_2d v_qr, v_nr; + }; +#endif + // -- Table3 -- struct Table3 { @@ -1301,6 +1340,9 @@ struct Functions const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + const P3Temporaries& temporaries, +#endif const WorkspaceManager& workspace_mgr, Int nj, // number of columns Int nk, // number of vertical cells per column @@ -1328,6 +1370,7 @@ struct Functions const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, + const P3Temporaries& temporaries, const WorkspaceManager& workspace_mgr, Int nj, // number of columns Int nk, // number of vertical cells per column diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index eff3102f0e44..b8f6e826fc14 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -2054,6 +2054,36 @@ Int p3_main_f( P3F::P3HistoryOnly history_only{liq_ice_exchange_d, vap_liq_exchange_d, vap_ice_exchange_d}; + const Int nk_pack = ekat::npack(nk); +#ifdef SCREAM_P3_SMALL_KERNELS + view_2d + mu_r("mu_r", nj, nk_pack), T_atm("T_atm", nj, nk_pack), lamr("lamr", nj, nk_pack), logn0r("logn0r", nj, nk_pack), nu("nu", nj, nk_pack), + cdist("cdist", nj, nk_pack), cdist1("cdist1", nj, nk_pack), cdistr("cdistr", nj, nk_pack), inv_cld_frac_i("inv_cld_frac_i", nj, nk_pack), + inv_cld_frac_l("inv_cld_frac_l", nj, nk_pack), inv_cld_frac_r("inv_cld_frac_r", nj, nk_pack), qc_incld("qc_incld", nj, nk_pack), + qr_incld("qr_incld", nj, nk_pack), qi_incld("qi_incld", nj, nk_pack), qm_incld("qm_incld", nj, nk_pack), nc_incld("nc_incld", nj, nk_pack), + nr_incld("nr_incld", nj, nk_pack), ni_incld("ni_incld", nj, nk_pack), bm_incld("bm_incld", nj, nk_pack), inv_dz("inv_dz", nj, nk_pack), + inv_rho("inv_rho", nj, nk_pack), ze_ice("ze_ice", nj, nk_pack), ze_rain("ze_rain", nj, nk_pack), prec("prec", nj, nk_pack), + rho("rho", nj, nk_pack), rhofacr("rhofacr", nj, nk_pack), rhofaci("rhofaci", nj, nk_pack), acn("acn", nj, nk_pack), qv_sat_l("qv_sat", nj, nk_pack), + qv_sat_i("qv_sat_i", nj, nk_pack), sup("sup", nj, nk_pack), qv_supersat_i("qv_supersat", nj, nk_pack), tmparr2("tmparr2", nj, nk_pack), + exner("exner", nj, nk_pack), diag_equiv_reflectivity("diag_equiv_ref", nj, nk_pack), diag_vm_qi("diag_vm_qi", nj, nk_pack), + diag_diam_qi("diag_diam_qi", nj, nk_pack), pratot("pratot", nj, nk_pack), prctot("prctot", nj, nk_pack), qtend_ignore("qtend_ignore", nj, nk_pack), + ntend_ignore("ntend_ignore", nj, nk_pack), mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), qr_evap_tend("qr_evap_tend", nj, nk_pack), + v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), v_qit("v_qit", nj, nk_pack), + v_nit("v_nit", nj, nk_pack), flux_nit("flux_nit", nj, nk_pack), flux_bir("flux_bir", nj, nk_pack), flux_qir("flux_qir", nj, nk_pack), + flux_qit("flux_qit", nj, nk_pack), v_qr("v_qr", nj, nk_pack), v_nr("v_nr", nj, nk_pack); + + P3F::P3Temporaries temporaries{ + mu_r, T_atm, lamr, logn0r, nu, cdist, cdist1, cdistr, inv_cld_frac_i, + inv_cld_frac_l, inv_cld_frac_r, qc_incld, qr_incld, qi_incld, qm_incld, + nc_incld, nr_incld, ni_incld, bm_incld, inv_dz, inv_rho, ze_ice, ze_rain, + prec, rho, rhofacr, rhofaci, acn, qv_sat_l, qv_sat_i, sup, qv_supersat_i, + tmparr2, exner, diag_equiv_reflectivity, diag_vm_qi, diag_diam_qi, + pratot, prctot, qtend_ignore, ntend_ignore, mu_c, lamc, qr_evap_tend, + v_qc, v_nc, flux_qx, flux_nx, v_qit, v_nit, flux_nit, flux_bir, flux_qir, + flux_qit, v_qr, v_nr + }; +#endif + // load tables view_1d_table mu_r_table_vals; view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; @@ -2068,12 +2098,15 @@ Int p3_main_f( P3F::P3Runtime runtime_options{740.0e3}; // Create local workspace - const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); ekat::WorkspaceManager workspace_mgr(nk_pack, 52, policy); auto elapsed_microsec = P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, nj, nk, physics::P3_Constants()); + history_only, lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + temporaries, +#endif + workspace_mgr, nj, nk, physics::P3_Constants()); Kokkos::parallel_for(nj, KOKKOS_LAMBDA(const Int& i) { precip_liq_surf_temp_d(0, i / Spack::n)[i % Spack::n] = precip_liq_surf_d(i); From 9438a980a0caebe562568beccb9b1c4b614a390f Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 3 Sep 2024 10:54:25 -0600 Subject: [PATCH 649/904] Remove latent_heat_* in/out variables Replace with constant scalars --- .../eamxx/src/physics/p3/CMakeLists.txt | 13 +- .../physics/p3/disp/p3_ice_sed_impl_disp.cpp | 7 +- .../src/physics/p3/disp/p3_main_impl_disp.cpp | 13 +- .../p3/disp/p3_main_impl_part1_disp.cpp | 11 +- .../p3/disp/p3_main_impl_part2_disp.cpp | 25 +- .../p3/disp/p3_main_impl_part3_disp.cpp | 10 +- .../src/physics/p3/eti/p3_get_latent_heat.cpp | 14 - .../p3/impl/p3_evaporate_rain_impl.hpp | 8 +- .../p3/impl/p3_get_latent_heat_impl.hpp | 24 -- .../p3_get_time_space_phys_variables_impl.hpp | 12 +- .../p3/impl/p3_ice_cldliq_wet_growth_impl.hpp | 8 +- .../physics/p3/impl/p3_ice_melting_impl.hpp | 8 +- .../src/physics/p3/impl/p3_ice_sed_impl.hpp | 6 +- .../p3_ice_supersat_conservation_impl.hpp | 7 +- .../src/physics/p3/impl/p3_main_impl.hpp | 17 +- .../physics/p3/impl/p3_main_impl_part1.hpp | 13 +- .../physics/p3/impl/p3_main_impl_part2.hpp | 35 +- .../physics/p3/impl/p3_main_impl_part3.hpp | 10 +- .../p3_prevent_liq_supersaturation_impl.hpp | 20 +- .../p3/impl/p3_update_prognostics_impl.hpp | 15 +- .../eamxx/src/physics/p3/p3_functions.hpp | 39 +-- .../eamxx/src/physics/p3/p3_functions_f90.cpp | 309 ++++++++---------- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 10 +- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 1 - .../p3/tests/p3_evaporate_rain_unit_tests.cpp | 52 +-- .../tests/p3_get_latent_heat_unit_tests.cpp | 17 +- .../p3_ice_cldliq_wet_growth_unit_tests.cpp | 47 +-- .../p3/tests/p3_ice_melting_unit_tests.cpp | 46 +-- .../p3/tests/p3_ice_sed_unit_tests.cpp | 10 +- .../p3_ice_supersat_conservation_tests.cpp | 12 +- .../physics/p3/tests/p3_main_unit_tests.cpp | 48 ++- .../p3_prevent_liq_supersaturation_tests.cpp | 29 +- .../src/physics/p3/tests/p3_unit_tests.cpp | 125 ++++--- 33 files changed, 477 insertions(+), 544 deletions(-) delete mode 100644 components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp delete mode 100644 components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index fc0293cb35d0..a1dfc946267d 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -45,7 +45,6 @@ if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos eti/p3_ice_relaxation_timescale.cpp eti/p3_ice_nucleation.cpp eti/p3_ice_cldliq_wet_growth.cpp - eti/p3_get_latent_heat.cpp eti/p3_check_values.cpp eti/p3_incloud_mixingratios.cpp eti/p3_subgrid_variance_scaling.cpp @@ -63,13 +62,13 @@ endif() # List of dispatch source files if monolithic kernels are off set(P3_SK_SRCS - disp/p3_check_values_impl_disp.cpp - disp/p3_ice_sed_impl_disp.cpp - disp/p3_main_impl_part1_disp.cpp + disp/p3_check_values_impl_disp.cpp + disp/p3_ice_sed_impl_disp.cpp + disp/p3_main_impl_part1_disp.cpp disp/p3_main_impl_part3_disp.cpp - disp/p3_cloud_sed_impl_disp.cpp - disp/p3_main_impl_disp.cpp - disp/p3_main_impl_part2_disp.cpp + disp/p3_cloud_sed_impl_disp.cpp + disp/p3_main_impl_disp.cpp + disp/p3_main_impl_part2_disp.cpp disp/p3_rain_sed_impl_disp.cpp ) diff --git a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp index 0d2aad5247dd..f1e84750c250 100644 --- a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp @@ -46,8 +46,8 @@ ::ice_sedimentation_disp( // Ice sedimentation: (adaptive substepping) ice_sedimentation( - ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(cld_frac_i, i), - ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, + ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(cld_frac_i, i), + ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, ekat::subview(qi, i), ekat::subview(qi_incld, i), ekat::subview(ni, i), ekat::subview(ni_incld, i), ekat::subview(qm, i), ekat::subview(qm_incld, i), ekat::subview(bm, i), ekat::subview(bm_incld, i), ekat::subview(qi_tend, i), ekat::subview(ni_tend, i), ice_table_vals, precip_ice_surf(i), p3constants); @@ -60,7 +60,6 @@ void Functions ::homogeneous_freezing_disp( const uview_2d& T_atm, const uview_2d& inv_exner, - const uview_2d& latent_heat_fusion, const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_2d& qc, const uview_2d& nc, @@ -89,7 +88,7 @@ ::homogeneous_freezing_disp( // homogeneous freezing of cloud and rain homogeneous_freezing( - ekat::subview(T_atm, i), ekat::subview(inv_exner, i), ekat::subview(latent_heat_fusion, i), team, nk, ktop, kbot, kdir, + ekat::subview(T_atm, i), ekat::subview(inv_exner, i), team, nk, ktop, kbot, kdir, ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(th_atm, i)); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index 0f125b7ae960..b79b4f6d072c 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -115,10 +115,6 @@ ::p3_main_internal_disp( { using ExeSpace = typename KT::ExeSpace; - view_2d latent_heat_sublim("latent_heat_sublim", nj, nk), latent_heat_vapor("latent_heat_vapor", nj, nk), latent_heat_fusion("latent_heat_fusion", nj, nk); - - get_latent_heat(nj, nk, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion); - const Int nk_pack = ekat::npack(nk); // load constants into local vars @@ -244,7 +240,7 @@ ::p3_main_internal_disp( p3_main_part1_disp( nj, nk, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, pres, dpres, dz, nc_nuceat_tend, nccn_prescribed, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, - inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, + inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, @@ -259,8 +255,7 @@ ::p3_main_internal_disp( lookup_tables.revap_table_vals, pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - qv, th, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, - latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, + qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, @@ -301,7 +296,7 @@ ::p3_main_internal_disp( // homogeneous freezing f cloud and rain homogeneous_freezing_disp( - T_atm, inv_exner, latent_heat_fusion, nj, nk, ktop, kbot, kdir, qc, nc, qr, nr, qi, + T_atm, inv_exner, nj, nk, ktop, kbot, kdir, qc, nc, qr, nr, qi, ni, qm, bm, th, nucleationPossible, hydrometeorsPresent); // @@ -311,7 +306,7 @@ ::p3_main_internal_disp( p3_main_part3_disp( nj, nk_pack, runtime_options.max_total_ni, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, rho, inv_rho, rhofaci, qv, th, qc, nc, qr, nr, qi, ni, - qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, + qm, bm, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent, p3constants); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp index 3dae805255b0..28db7a6348dc 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp @@ -31,9 +31,6 @@ ::p3_main_part1_disp( const uview_2d& inv_cld_frac_l, const uview_2d& inv_cld_frac_i, const uview_2d& inv_cld_frac_r, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& T_atm, const uview_2d& rho, const uview_2d& inv_rho, @@ -73,17 +70,17 @@ ::p3_main_part1_disp( policy, KOKKOS_LAMBDA(const MemberType& team) { const Int i = team.league_rank(); - + p3_main_part1( team, nk, predictNc, prescribedCCN, dt, ekat::subview(pres, i), ekat::subview(dpres, i), ekat::subview(dz, i), ekat::subview(nc_nuceat_tend, i), ekat::subview(nccn_prescribed, i), ekat::subview(inv_exner, i), ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), - ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), - ekat::subview(latent_heat_fusion, i), ekat::subview(T_atm, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), + ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), + ekat::subview(T_atm, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(qc_incld, i), ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), - ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), + ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), nucleationPossible(i), hydrometeorsPresent(i), p3constants); }); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 2b619d54bf33..5aa32bf817f5 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -59,9 +59,6 @@ ::p3_main_part2_disp( const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& qc_incld, const uview_2d& qr_incld, const uview_2d& qi_incld, @@ -104,25 +101,25 @@ ::p3_main_part2_disp( const Int i = team.league_rank(); if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { - return; + return; } // ------------------------------------------------------------------------------------------ // main k-loop (for processes): p3_main_part2( team, nk_pack, max_total_ni, predictNc, do_prescribed_CCN, dt, inv_dt, - dnu_table_vals, ice_table_vals, collect_table_vals, revap_table_vals, + dnu_table_vals, ice_table_vals, collect_table_vals, revap_table_vals, ekat::subview(pres, i), ekat::subview(dpres, i), ekat::subview(dz, i), ekat::subview(nc_nuceat_tend, i), ekat::subview(inv_exner, i), - ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), - ekat::subview(ni_activated, i), ekat::subview(inv_qc_relvar, i), ekat::subview(cld_frac_i, i), ekat::subview(cld_frac_l, i), - ekat::subview(cld_frac_r, i), ekat::subview(qv_prev, i), ekat::subview(t_prev, i), ekat::subview(T_atm, i), ekat::subview(rho, i), - ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), - ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), - ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), - ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), ekat::subview(latent_heat_fusion, i), ekat::subview(qc_incld, i), - ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), + ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), + ekat::subview(ni_activated, i), ekat::subview(inv_qc_relvar, i), ekat::subview(cld_frac_i, i), ekat::subview(cld_frac_l, i), + ekat::subview(cld_frac_r, i), ekat::subview(qv_prev, i), ekat::subview(t_prev, i), ekat::subview(T_atm, i), ekat::subview(rho, i), + ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), + ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), + ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), + ekat::subview(qc_incld, i), + ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(cdist, i), - ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), + ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), ekat::subview(qv2qi_depos_tend, i), ekat::subview(precip_total_tend, i), ekat::subview(nevapr, i), ekat::subview(qr_evap_tend, i), ekat::subview(vap_liq_exchange, i), ekat::subview(vap_ice_exchange, i), ekat::subview(liq_ice_exchange, i), ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk, p3constants); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp index 6d591a7e9328..fd41fb5f0fe1 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp @@ -38,8 +38,6 @@ ::p3_main_part3_disp( const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, const uview_2d& mu_c, const uview_2d& nu, const uview_2d& lamc, @@ -77,11 +75,11 @@ ::p3_main_part3_disp( // p3_main_part3( team, nk_pack, max_total_ni, dnu_table_vals, ice_table_vals, ekat::subview(inv_exner, i), ekat::subview(cld_frac_l, i), ekat::subview(cld_frac_r, i), - ekat::subview(cld_frac_i, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(qv, i), - ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), - ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), + ekat::subview(cld_frac_i, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(qv, i), + ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), + ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), - ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), + ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), ekat::subview(diag_diam_qi, i), ekat::subview(rho_qi, i), ekat::subview(diag_equiv_reflectivity, i), ekat::subview(diag_eff_radius_qc, i), ekat::subview(diag_eff_radius_qr, i), p3constants); diff --git a/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp b/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp deleted file mode 100644 index 87e41c934587..000000000000 --- a/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "p3_get_latent_heat_impl.hpp" - -namespace scream { -namespace p3 { - -/* - * Explicit instantiation for doing conservation functions on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace p3 -} // namespace scream diff --git a/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp index 5455bfaa4c14..5560fcfe9123 100644 --- a/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp @@ -69,7 +69,7 @@ ::evaporate_rain( const Spack& cld_frac_l, const Spack& cld_frac_r, const Spack& qv, const Spack& qv_prev, const Spack& qv_sat_l, const Spack& qv_sat_i, const Spack& ab, const Spack& abi, const Spack& epsr, const Spack& epsi_tot, const Spack& t_atm, const Spack& t_atm_prev, - const Spack& latent_heat_sublim, const Spack& dqsdt, const Scalar& dt, + const Spack& dqsdt, const Scalar& dt, Spack& qr2qv_evap_tend, Spack& nr_evap_tend, const Smask& context) { @@ -92,6 +92,8 @@ ::evaporate_rain( constexpr Scalar QSMALL = C::QSMALL; constexpr Scalar Tmelt = C::Tmelt; constexpr Scalar inv_cp = 1/C::Cpair; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //Compute absolute supersaturation. //Ignore the difference between clear-sky and cell-ave qv and T @@ -133,9 +135,9 @@ ::evaporate_rain( const Smask not_freezing = !is_freezing && context; Spack eps_eff, A_c; if (is_freezing.any()){ - eps_eff.set(is_freezing,epsr + epsi_tot*(1 + latent_heat_sublim*inv_cp*dqsdt)/abi); + eps_eff.set(is_freezing,epsr + epsi_tot*(1 + (latvap+latice)*inv_cp*dqsdt)/abi); A_c.set(is_freezing,(qv - qv_prev)*inv_dt - dqsdt*(t_atm-t_atm_prev)*inv_dt - - (qv_sat_l - qv_sat_i)*(1 + latent_heat_sublim*inv_cp*dqsdt)/abi*epsi_tot ); + - (qv_sat_l - qv_sat_i)*(1 + (latvap+latice)*inv_cp*dqsdt)/abi*epsi_tot ); } if (not_freezing.any()){ eps_eff.set(not_freezing,epsr); diff --git a/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp deleted file mode 100644 index 439a1e2c84d6..000000000000 --- a/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef P3_GET_LATENT_HEAT_IMPL_HPP -#define P3_GET_LATENT_HEAT_IMPL_HPP - -#include "p3_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace p3 { - -template -void Functions -::get_latent_heat(const Int& nj, const Int& nk, view_2d& v, view_2d& s, view_2d& f) -{ - constexpr Scalar latvap = C::LatVap; - constexpr Scalar latice = C::LatIce; - - Kokkos::deep_copy(v, latvap); - Kokkos::deep_copy(s, latvap + latice); - Kokkos::deep_copy(f, latice); -} - -} // namespace p3 -} // namespace scream - -#endif // P3_GET_LATENT_HEAT_IMPL_HPP diff --git a/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp index f46c4aaebb5f..ea42d7995ba1 100644 --- a/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp @@ -11,7 +11,7 @@ template KOKKOS_FUNCTION void Functions ::get_time_space_phys_variables( - const Spack& T_atm, const Spack& pres, const Spack& rho, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, + const Spack& T_atm, const Spack& pres, const Spack& rho, const Spack& qv_sat_l, const Spack& qv_sat_i, Spack& mu, Spack& dv, Spack& sc, Spack& dqsdt, Spack& dqsidt, Spack& ab, Spack& abi, Spack& kap, Spack& eii, const Smask& context) @@ -23,15 +23,17 @@ ::get_time_space_phys_variables( constexpr Scalar RV = C::RV; constexpr Scalar INV_CP = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; constexpr Scalar tval1 = 253.15; constexpr Scalar tval2 = 273.15; constexpr Scalar dtval = 20; //this is tval2-tval1, but specifying here as int to be BFB with F90. const auto dum = 1/(RV*square(T_atm)); - dqsdt.set(context, latent_heat_vapor*qv_sat_l*dum); - dqsidt.set(context, latent_heat_sublim*qv_sat_i*dum); - ab.set(context, 1+dqsdt*latent_heat_vapor*INV_CP); - abi.set(context, 1+dqsidt*latent_heat_sublim*INV_CP); + dqsdt.set(context, latvap*qv_sat_l*dum); + dqsidt.set(context, (latvap+latice)*qv_sat_i*dum); + ab.set(context, 1+dqsdt*latvap*INV_CP); + abi.set(context, 1+dqsidt*(latvap+latice)*INV_CP); kap.set(context, sp(1.414e+3)*mu); //very simple temperature dependent aggregation efficiency diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp index cd8d9ce7ef0e..319d10f64ef7 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp @@ -13,7 +13,7 @@ KOKKOS_FUNCTION void Functions ::ice_cldliq_wet_growth( const Spack& rho, const Spack& temp, const Spack& pres, const Spack& rhofaci, const Spack& table_val_qi2qr_melting, - const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, const Spack& dv, + const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& kap, const Spack& mu, const Spack& sc, const Spack& qv, const Spack& qc_incld, const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Smask& log_wetgrowth, Spack& qr2qi_collect_tend, Spack& qc2qi_collect_tend, Spack& qc_growth_rate, Spack& nr_ice_shed_tend, Spack& qc2qr_ice_shed_tend, const Smask& context) @@ -26,6 +26,8 @@ ::ice_cldliq_wet_growth( constexpr Scalar zero = C::ZERO; constexpr Scalar one = C::ONE; constexpr Scalar cpw = C::CpLiq; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; const auto t_is_negative = temp < tmelt; const auto qi_incld_ge_small = qi_incld >= qsmall; @@ -46,8 +48,8 @@ ::ice_cldliq_wet_growth( qc_growth_rate.set(any_if, ((table_val_qi2qr_melting+table_val_qi2qr_vent_melt*cbrt(sc)*sqrt(rhofaci*rho/mu))* - twopi*(rho*latent_heat_vapor*dv*(qsat0-qv)-(temp-tmelt)*kap)/ - (latent_heat_fusion+cpw*(temp-tmelt)))*ni_incld); + twopi*(rho*latvap*dv*(qsat0-qv)-(temp-tmelt)*kap)/ + (latice+cpw*(temp-tmelt)))*ni_incld); qc_growth_rate.set(any_if, max(qc_growth_rate, zero)); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp index dc6c8d16256f..a4b86806ad0c 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp @@ -13,7 +13,7 @@ KOKKOS_FUNCTION void Functions ::ice_melting( const Spack& rho, const Spack& T_atm, const Spack& pres, const Spack& rhofaci, - const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, + const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& sc, const Spack& mu, const Spack& kap, const Spack& qv, const Spack& qi_incld, const Spack& ni_incld, Spack& qi2qr_melt_tend, Spack& ni2nr_melt_tend, const Smask& context) @@ -29,6 +29,8 @@ ::ice_melting( const auto Pi = C::Pi; const auto QSMALL = C::QSMALL; const auto Tmelt = C::Tmelt; + const auto latvap = C::LatVap; + const auto latice = C::LatIce; //Find cells above freezing AND which have ice const auto has_melt_qi = (qi_incld >= QSMALL ) && (T_atm > Tmelt) && context; @@ -38,8 +40,8 @@ ::ice_melting( const auto qsat0 = physics::qv_sat_dry(Spack(Tmelt), pres, false, context, physics::MurphyKoop, "p3::ice_melting"); //"false" here means NOT saturation w/ respect to ice. qi2qr_melt_tend.set(has_melt_qi, ( (table_val_qi2qr_melting+table_val_qi2qr_vent_melt*cbrt(sc)*sqrt(rhofaci*rho/mu)) - *((T_atm-Tmelt)*kap-rho*latent_heat_vapor*dv*(qsat0-qv)) - * 2 * Pi /latent_heat_fusion)*ni_incld ); + *((T_atm-Tmelt)*kap-rho*latvap*dv*(qsat0-qv)) + * 2 * Pi /latice)*ni_incld ); //make sure qi2qr_melt_tend is always negative qi2qr_melt_tend = max(qi2qr_melt_tend, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp index 29232c69e68f..1273d7bbc206 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp @@ -210,7 +210,6 @@ void Functions ::homogeneous_freezing( const uview_1d& T_atm, const uview_1d& inv_exner, - const uview_1d& latent_heat_fusion, const MemberType& team, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_1d& qc, @@ -228,6 +227,7 @@ ::homogeneous_freezing( constexpr Scalar T_homogfrz = C::T_homogfrz; constexpr Scalar inv_rho_rimeMax = C::INV_RHO_RIMEMAX; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latice = C::LatIce; const Int kmin_scalar = ( kdir == 1 ? kbot : ktop); const Int kmax_scalar = ( kdir == 1 ? ktop : kbot); @@ -254,13 +254,13 @@ ::homogeneous_freezing( qi(pk).set(qc_ge_small, qi(pk) + Qc_nuc); bm(pk).set(qc_ge_small, bm(pk) + Qc_nuc*inv_rho_rimeMax); ni(pk).set(qc_ge_small, ni(pk) + Nc_nuc); - th_atm(pk).set (qc_ge_small, th_atm(pk) + inv_exner(pk)*Qc_nuc*latent_heat_fusion(pk)*inv_cp); + th_atm(pk).set (qc_ge_small, th_atm(pk) + inv_exner(pk)*Qc_nuc*latice*inv_cp); qm(pk).set(qr_ge_small, qm(pk) + Qr_nuc); qi(pk).set(qr_ge_small, qi(pk) + Qr_nuc); bm(pk).set(qr_ge_small, bm(pk) + Qr_nuc*inv_rho_rimeMax); ni(pk).set(qr_ge_small, ni(pk) + Nr_nuc); - th_atm(pk).set (qr_ge_small, th_atm(pk) + inv_exner(pk)*Qr_nuc*latent_heat_fusion(pk)*inv_cp); + th_atm(pk).set (qr_ge_small, th_atm(pk) + inv_exner(pk)*Qr_nuc*latice*inv_cp); qc(pk).set(qc_ge_small, 0); nc(pk).set(qc_ge_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp index 1bed57bad630..091aa20f58d4 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp @@ -13,11 +13,14 @@ namespace p3 { template KOKKOS_FUNCTION -void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& qv2qi_nucleat_tend, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& latent_heat_sublim, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context) +void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& qv2qi_nucleat_tend, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar cp = C::CP; constexpr Scalar rv = C::RH2O; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + constexpr Scalar latsublim2 = (latvap+latice)*(latvap+latice); const auto qv_sink = qv2qi_vapdep_tend + qv2qi_nucleat_tend; // in [kg/kg] cell-avg values @@ -25,7 +28,7 @@ void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& if (mask.any()) { // --- Available water vapor for deposition/nucleation auto qv_avail = (qv + (qi2qv_sublim_tend+qr2qv_evap_tend)*dt - qv_sat_i) / - (1 + square(latent_heat_sublim)*qv_sat_i / (cp*rv*square(t_atm)) ) / dt; + (1 + latsublim2*qv_sat_i / (cp*rv*square(t_atm)) ) / dt; // --- Only excess water vapor can be limited qv_avail = max(qv_avail, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index 270042c99986..a36a79364044 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -88,10 +88,6 @@ ::p3_main_internal( { using ExeSpace = typename KT::ExeSpace; - view_2d latent_heat_sublim("latent_heat_sublim", nj, nk), latent_heat_vapor("latent_heat_vapor", nj, nk), latent_heat_fusion("latent_heat_fusion", nj, nk); - - get_latent_heat(nj, nk, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion); - const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); @@ -202,9 +198,6 @@ ::p3_main_internal( const auto oliq_ice_exchange = ekat::subview(history_only.liq_ice_exchange, i); const auto ovap_liq_exchange = ekat::subview(history_only.vap_liq_exchange, i); const auto ovap_ice_exchange = ekat::subview(history_only.vap_ice_exchange, i); - const auto olatent_heat_vapor = ekat::subview(latent_heat_vapor, i); - const auto olatent_heat_sublim = ekat::subview(latent_heat_sublim, i); - const auto olatent_heat_fusion = ekat::subview(latent_heat_fusion, i); const auto oqv_prev = ekat::subview(diagnostic_inputs.qv_prev, i); const auto ot_prev = ekat::subview(diagnostic_inputs.t_prev, i); @@ -232,7 +225,7 @@ ::p3_main_internal( p3_main_part1( team, nk, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, opres, odpres, odz, onc_nuceat_tend, onccn_prescribed, oinv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, - inv_cld_frac_r, olatent_heat_vapor, olatent_heat_sublim, olatent_heat_fusion, + inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, @@ -251,8 +244,8 @@ ::p3_main_internal( lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, lookup_tables.revap_table_vals, opres, odpres, odz, onc_nuceat_tend, oinv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, oni_activated, oinv_qc_relvar, ocld_frac_i, ocld_frac_l, ocld_frac_r, oqv_prev, ot_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, olatent_heat_vapor, - olatent_heat_sublim, olatent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, + oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, + qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, oqv2qi_depos_tend, oprecip_total_tend, onevapr, qr_evap_tend, ovap_liq_exchange, ovap_ice_exchange, oliq_ice_exchange, @@ -295,7 +288,7 @@ ::p3_main_internal( // homogeneous freezing of cloud and rain homogeneous_freezing( - T_atm, oinv_exner, olatent_heat_fusion, team, nk, ktop, kbot, kdir, oqc, onc, oqr, onr, oqi, + T_atm, oinv_exner, team, nk, ktop, kbot, kdir, oqc, onc, oqr, onr, oqi, oni, oqm, obm, oth); // @@ -305,7 +298,7 @@ ::p3_main_internal( p3_main_part3( team, nk_pack, runtime_options.max_total_ni, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, oinv_exner, ocld_frac_l, ocld_frac_r, ocld_frac_i, rho, inv_rho, rhofaci, oqv, oth, oqc, onc, oqr, onr, oqi, oni, - oqm, obm, olatent_heat_vapor, olatent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, + oqm, obm, mu_c, nu, lamc, mu_r, lamr, ovap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, odiag_eff_radius_qi, diag_diam_qi, orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr, p3constants); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp index 439b0544adf9..6771a48d4c68 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp @@ -34,9 +34,6 @@ ::p3_main_part1( const uview_1d& inv_cld_frac_l, const uview_1d& inv_cld_frac_i, const uview_1d& inv_cld_frac_r, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& T_atm, const uview_1d& rho, const uview_1d& inv_rho, @@ -80,6 +77,8 @@ ::p3_main_part1( constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; const Scalar p3_spa_to_nc = p3constants.p3_spa_to_nc; @@ -124,7 +123,7 @@ ::p3_main_part1( auto drymass = qc(k) < qsmall; auto not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qc(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qc(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qc(k) * latvap * inv_cp); qc(k).set(drymass, 0); nc(k).set(drymass, 0); if ( not_drymass.any() ) { @@ -147,7 +146,7 @@ ::p3_main_part1( drymass = qr(k) < qsmall; not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qr(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qr(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qr(k) * latvap * inv_cp); qr(k).set(drymass, 0); nr(k).set(drymass, 0); if ( not_drymass.any() ) { @@ -157,7 +156,7 @@ ::p3_main_part1( drymass = (qi(k) < qsmall || (qi(k) < 1.e-8 && qv_supersat_i(k) < -0.1)); not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qi(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_sublim(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * (latvap+latice) * inv_cp); qi(k).set(drymass, 0); ni(k).set(drymass, 0); qm(k).set(drymass, 0); @@ -168,7 +167,7 @@ ::p3_main_part1( drymass = (qi(k) >= qsmall && qi(k) < 1.e-8 && T_atm(k) >= T_zerodegc); qr(k).set(drymass, qr(k) + qi(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_fusion(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latice * inv_cp); qi(k).set(drymass, 0); ni(k).set(drymass, 0); qm(k).set(drymass, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp index c32fb7202ebb..d19dc579b7e9 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp @@ -65,9 +65,6 @@ ::p3_main_part2( const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& qc_incld, const uview_1d& qr_incld, const uview_1d& qi_incld, @@ -104,6 +101,8 @@ ::p3_main_part2( constexpr Scalar f2r = C::f2r; constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; team.team_barrier(); hydrometeorsPresent = false; @@ -115,12 +114,12 @@ ::p3_main_part2( //compute mask to identify padded values in packs, which shouldn't be used in calculations const auto range_pack = ekat::range(k*Spack::n); const auto range_mask = range_pack < nk; - + // if relatively dry and no hydrometeors at this level, skip to end of k-loop (i.e. skip this level) const auto skip_all = ( !range_mask || (qc(k)= qsmall && not_skip_all; qv(k).set(qc_small, qv(k) + qc(k)); - th_atm(k).set(qc_small, th_atm(k) - inv_exner(k) * qc(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(qc_small, th_atm(k) - inv_exner(k) * qc(k) * latvap * inv_cp); qc(k).set(qc_small, 0); nc(k).set(qc_small, 0); @@ -464,7 +463,7 @@ ::p3_main_part2( } qv(k).set(qr_small, qv(k) + qr(k)); - th_atm(k).set(qr_small, th_atm(k) - inv_exner(k) * qr(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(qr_small, th_atm(k) - inv_exner(k) * qr(k) * latvap * inv_cp); qr(k).set(qr_small, 0); nr(k).set(qr_small, 0); @@ -473,7 +472,7 @@ ::p3_main_part2( } qv(k).set(qi_small, qv(k) + qi(k)); - th_atm(k).set(qi_small, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_sublim(k) * inv_cp); + th_atm(k).set(qi_small, th_atm(k) - inv_exner(k) * qi(k) * (latvap+latice) * inv_cp); qi(k).set(qi_small, 0); ni(k).set(qi_small, 0); qm(k).set(qi_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp index 441bcb9feb9b..febb00ef4d31 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp @@ -41,8 +41,6 @@ ::p3_main_part3( const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, const uview_1d& mu_c, const uview_1d& nu, const uview_1d& lamc, @@ -63,6 +61,8 @@ ::p3_main_part3( constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; constexpr Scalar nsmall = C::NSMALL; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nk_pack), [&] (Int k) { @@ -93,7 +93,7 @@ ::p3_main_part3( } if (qc_small.any()) { qv(k) .set(qc_small, qv(k)+qc(k)); - th_atm(k) .set(qc_small, th_atm(k)-inv_exner(k)*qc(k)*latent_heat_vapor(k)*inv_cp); + th_atm(k) .set(qc_small, th_atm(k)-inv_exner(k)*qc(k)*latvap*inv_cp); vap_liq_exchange(k) .set(qc_small, vap_liq_exchange(k) - qc(k)); qc(k) .set(qc_small, 0); nc(k) .set(qc_small, 0); @@ -123,7 +123,7 @@ ::p3_main_part3( if (qr_small.any()) { qv(k) .set(qr_small, qv(k) + qr(k)); - th_atm(k) .set(qr_small, th_atm(k) - inv_exner(k)*qr(k)*latent_heat_vapor(k)*inv_cp); + th_atm(k) .set(qr_small, th_atm(k) - inv_exner(k)*qr(k)*latvap*inv_cp); vap_liq_exchange(k).set(qr_small, vap_liq_exchange(k) - qr(k)); qr(k) .set(qr_small, 0); nr(k) .set(qr_small, 0); @@ -185,7 +185,7 @@ ::p3_main_part3( ze_ice(k).set(qi_gt_small, ze_ice(k)*cld_frac_i(k)); qv(k).set(qi_small, qv(k) + qi(k)); - th_atm(k).set(qi_small, th_atm(k) - inv_exner(k)*qi(k)*latent_heat_sublim(k)*inv_cp); + th_atm(k).set(qi_small, th_atm(k) - inv_exner(k)*qi(k)*(latvap+latice)*inv_cp); qi(k).set(qi_small, 0); ni(k).set(qi_small, 0); qm(k).set(qi_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp index a9f265d8dcdd..10f728f1a079 100644 --- a/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp @@ -14,7 +14,7 @@ namespace p3 { template KOKKOS_FUNCTION -void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Scalar& dt, const Spack& qv2qi_vapdep_tend, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context) +void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Scalar& dt, const Spack& qv2qi_vapdep_tend, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context) // Note: context masks cells which are just padding for packs or which don't have any condensate worth // performing calculations on. { @@ -23,6 +23,8 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& constexpr Scalar inv_cp = C::INV_CP; constexpr Scalar rv = C::RV; constexpr Scalar qsmall = C::QSMALL; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; Spack qv_sinks, qv_sources, qv_endstep, T_endstep, A, frac; @@ -37,8 +39,8 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& //Actual qv and T after microphys step qv_endstep.set(has_sources,qv - qv_sinks*dt + qv_sources*dt); - T_endstep.set(has_sources,t_atm + ( (qv_sinks-qi2qv_sublim_tend)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend*latent_heat_vapor*inv_cp )*dt); + T_endstep.set(has_sources,t_atm + ( (qv_sinks-qi2qv_sublim_tend)*(latvap+latice)*inv_cp + - qr2qv_evap_tend*latvap*inv_cp )*dt); //qv we would have at end of step if we were saturated with respect to liquid const auto qsl = physics::qv_sat_dry(T_endstep,pres,false,has_sources,physics::MurphyKoop,"p3::prevent_liq_supersaturation"); //"false" means NOT sat w/ respect to ice @@ -46,18 +48,18 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& //The balance we seek is: // qv-qv_sinks*dt+qv_sources*frac*dt=qsl+dqsl_dT*(T correction due to conservation) // where the T correction for conservation is: - // dt*[latent_heat_sublim/cp*(qi2qv_sublim_tend-frac*qi2qv_sublim_tend) - // +latent_heat_vapor/cp*(qr2qv_evap_tend -frac*qr2qv_evap_tend)] - // =(1-frac)*dt/cp*(latent_heat_sublim*qi2qv_sublim_tend + latent_heat_vap*qr2qv_evap_tend). + // dt*[(latvap+latice)/cp*(qi2qv_sublim_tend-frac*qi2qv_sublim_tend) + // +latvap/cp*(qr2qv_evap_tend -frac*qr2qv_evap_tend)] + // =(1-frac)*dt/cp*((latvap+latice)*qi2qv_sublim_tend + latvap*qr2qv_evap_tend). // Note T correction is positive because frac *reduces* evaporative cooling. Note as well that // dqsl_dt comes from linearization of qsl around the end-of-step T computed before temperature // correction. dqsl_dt should be computed with respect to *liquid* even though frac also adjusts // sublimation because we want to be saturated with respect to liquid at the end of the step. - // dqsl_dt=Latent_heat_vapor*qsl/rv*T^2 following Clausius Clapeyron. Combining and solving for + // dqsl_dt=latvap*qsl/rv*T^2 following Clausius Clapeyron. Combining and solving for // frac yields: - A.set(has_sources,latent_heat_vapor*qsl*dt*inv_cp/(rv*T_endstep*T_endstep) - * (latent_heat_sublim*qi2qv_sublim_tend + latent_heat_vapor*qr2qv_evap_tend) ); + A.set(has_sources,latvap*qsl*dt*inv_cp/(rv*T_endstep*T_endstep) + * ((latvap+latice)*qi2qv_sublim_tend + latvap*qr2qv_evap_tend) ); frac.set(has_sources, (qsl-qv+qv_sinks*dt + A)/(qv_sources*dt + A) ); diff --git a/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp index 3f8bcab2cd51..86feaa04f499 100644 --- a/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp @@ -15,7 +15,7 @@ ::update_prognostic_ice( const Spack& qr2qi_immers_freeze_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nr_ice_shed_tend, const Spack& qi2qr_melt_tend, const Spack& ni2nr_melt_tend, const Spack& qi2qv_sublim_tend, const Spack& qv2qi_vapdep_tend, const Spack& qv2qi_nucleat_tend, const Spack& ni_nucleat_tend, const Spack& ni_selfcollect_tend, const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, - const Spack& inv_exner, const Spack& latent_heat_sublim, const Spack& latent_heat_fusion, const bool do_predict_nc, + const Spack& inv_exner, const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, const Scalar& nmltratio, const Spack& rho_qm_cloud, Spack& th_atm, Spack& qv, Spack& qi, Spack& ni, Spack& qm, Spack& bm, Spack& qc, Spack& nc, Spack& qr, Spack& nr, @@ -23,6 +23,8 @@ ::update_prognostic_ice( { constexpr Scalar QSMALL = C::QSMALL; constexpr Scalar INV_RHO_RIMEMAX = C::INV_RHO_RIMEMAX; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; qc.set(context, qc + (-qc2qi_hetero_freeze_tend-qc2qi_collect_tend-qc2qr_ice_shed_tend-qc2qi_berg_tend)*dt); if ( do_predict_nc ){ @@ -76,9 +78,9 @@ ::update_prognostic_ice( qv.set(context, qv + (-qv2qi_vapdep_tend+qi2qv_sublim_tend-qv2qi_nucleat_tend)*dt); constexpr Scalar INV_CP = C::INV_CP; - th_atm.set(context, th_atm + inv_exner * ((qv2qi_vapdep_tend - qi2qv_sublim_tend + qv2qi_nucleat_tend) * latent_heat_sublim * INV_CP + - (qr2qi_collect_tend + qc2qi_collect_tend + qc2qi_hetero_freeze_tend + qr2qi_immers_freeze_tend - - qi2qr_melt_tend + qc2qi_berg_tend) * latent_heat_fusion * INV_CP) * dt); + th_atm.set(context, th_atm + inv_exner * ((qv2qi_vapdep_tend - qi2qv_sublim_tend + qv2qi_nucleat_tend) * (latvap+latice) * INV_CP + + (qr2qi_collect_tend + qc2qi_collect_tend + qc2qi_hetero_freeze_tend + qr2qi_immers_freeze_tend - + qi2qr_melt_tend + qc2qi_berg_tend) * latice * INV_CP) * dt); } template @@ -88,13 +90,14 @@ ::update_prognostic_liquid( const Spack& qc2qr_accret_tend, const Spack& nc_accret_tend, const Spack& qc2qr_autoconv_tend,const Spack& nc2nr_autoconv_tend, const Spack& ncautr, const Spack& nc_selfcollect_tend, const Spack& qr2qv_evap_tend, const Spack& nr_evap_tend, const Spack& nr_selfcollect_tend, - const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Spack& latent_heat_vapor, + const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Scalar dt, Spack& th_atm, Spack& qv, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context) { constexpr Scalar NCCNST = C::NCCNST; constexpr int IPARAM = C::IPARAM; constexpr Scalar INV_CP = C::INV_CP; + constexpr Scalar latvap = C::LatVap; qc.set(context, qc + (-qc2qr_accret_tend-qc2qr_autoconv_tend)*dt); qr.set(context, qr + (qc2qr_accret_tend+qc2qr_autoconv_tend-qr2qv_evap_tend)*dt); @@ -115,7 +118,7 @@ ::update_prognostic_liquid( qv.set(context, qv + qr2qv_evap_tend *dt); - th_atm.set(context, th_atm + inv_exner*(-qr2qv_evap_tend * latent_heat_vapor * INV_CP) * dt); + th_atm.set(context, th_atm + inv_exner*(-qr2qv_evap_tend * latvap * INV_CP) * dt); } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 791538d67dcb..81a492688007 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -588,7 +588,6 @@ struct Functions static void homogeneous_freezing( const uview_1d& T_atm, const uview_1d& inv_exner, - const uview_1d& latent_heat_fusion, const MemberType& team, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_1d& qc, @@ -605,7 +604,6 @@ struct Functions static void homogeneous_freezing_disp( const uview_2d& T_atm, const uview_2d& inv_exner, - const uview_2d& latent_heat_fusion, const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_2d& qc, const uview_2d& nc, @@ -767,8 +765,8 @@ struct Functions const Spack& qr2qi_collect_tend, const Spack& nr_collect_tend, const Spack& qr2qi_immers_freeze_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nr_ice_shed_tend, const Spack& qi2qr_melt_tend, const Spack& ni2nr_melt_tend, const Spack& qi2qv_sublim_tend, const Spack& qv2qi_vapdep_tend, const Spack& qv2qi_nucleat_tend, const Spack& ni_nucleat_tend, const Spack& ni_selfcollect_tend, - const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, const Spack& inv_exner, const Spack& latent_heat_sublim, - const Spack& latent_heat_fusion, const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, + const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, const Spack& inv_exner, + const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, const Scalar& nmltratio, const Spack& rho_qm_cloud, Spack& th_atm, Spack& qv, Spack& qi, Spack& ni, Spack& qm, Spack& bm, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context = Smask(true)); @@ -776,7 +774,6 @@ struct Functions // TODO (comments) KOKKOS_FUNCTION static void get_time_space_phys_variables(const Spack& T_atm, const Spack& pres, const Spack& rho, - const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Spack& qv_sat_l, const Spack& qv_sat_i, Spack& mu, Spack& dv, Spack& sc, Spack& dqsdt, Spack& dqsidt, Spack& ab, Spack& abi, Spack& kap, Spack& eii, @@ -835,14 +832,14 @@ struct Functions const Spack& cld_frac_l, const Spack& cld_frac_r, const Spack& qv, const Spack& qv_prev, const Spack& qv_sat_l, const Spack& qv_sat_i, const Spack& ab, const Spack& abi, const Spack& epsr, const Spack & epsi_tot, const Spack& t, const Spack& t_prev, - const Spack& latent_heat_sublim, const Spack& dqsdt, const Scalar& dt, + const Spack& dqsdt, const Scalar& dt, Spack& qr2qv_evap_tend, Spack& nr_evap_tend, const Smask& context = Smask(true)); //get number and mass tendencies due to melting ice KOKKOS_FUNCTION static void ice_melting(const Spack& rho, const Spack& T_atm, const Spack& pres, const Spack& rhofaci, - const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, + const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& sc, const Spack& mu, const Spack& kap, const Spack& qv, const Spack& qi_incld, const Spack& ni_incld, Spack& qi2qr_melt_tend, Spack& ni2nr_melt_tend, const Smask& context = Smask(true)); @@ -852,7 +849,7 @@ struct Functions static void update_prognostic_liquid(const Spack& qc2qr_accret_tend, const Spack& nc_accret_tend, const Spack& qc2qr_autoconv_tend,const Spack& nc2nr_autoconv_tend, const Spack& ncautr, const Spack& nc_selfcollect_tend, const Spack& qr2qv_evap_tend, const Spack& nr_evap_tend, const Spack& nr_selfcollect_tend, - const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Spack& latent_heat_vapor, + const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Scalar dt, Spack& th_atm, Spack& qv, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context = Smask(true)); @@ -893,15 +890,12 @@ struct Functions KOKKOS_FUNCTION static void ice_cldliq_wet_growth(const Spack& rho, const Spack& temp, const Spack& pres, const Spack& rhofaci, const Spack& table_val_qi2qr_melting, - const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, const Spack& dv, + const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& kap, const Spack& mu, const Spack& sc, const Spack& qv, const Spack& qc_incld, const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Smask& log_wetgrowth, Spack& qr2qi_collect_tend, Spack& qc2qi_collect_tend, Spack& qc_growth_rate, Spack& nr_ice_shed_tend, Spack& qc2qr_ice_shed_tend, const Smask& context = Smask(true)); - // Note: not a kernel function - static void get_latent_heat(const Int& nj, const Int& nk, view_2d& v, view_2d& s, view_2d& f); - KOKKOS_FUNCTION static void check_values(const uview_1d& qv, const uview_1d& temp, const Int& ktop, const Int& kbot, const Int& timestepcount, const bool& force_abort, const Int& source_ind, const MemberType& team, @@ -993,9 +987,6 @@ struct Functions const uview_1d& inv_cld_frac_l, const uview_1d& inv_cld_frac_i, const uview_1d& inv_cld_frac_r, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& T_atm, const uview_1d& rho, const uview_1d& inv_rho, @@ -1044,9 +1035,6 @@ struct Functions const uview_2d& inv_cld_frac_l, const uview_2d& inv_cld_frac_i, const uview_2d& inv_cld_frac_r, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& T_atm, const uview_2d& rho, const uview_2d& inv_rho, @@ -1127,9 +1115,6 @@ struct Functions const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& qc_incld, const uview_1d& qr_incld, const uview_1d& qi_incld, @@ -1208,9 +1193,6 @@ struct Functions const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& qc_incld, const uview_2d& qr_incld, const uview_2d& qi_incld, @@ -1266,8 +1248,6 @@ struct Functions const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, const uview_1d& mu_c, const uview_1d& nu, const uview_1d& lamc, @@ -1309,8 +1289,6 @@ struct Functions const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, const uview_2d& mu_c, const uview_2d& nu, const uview_2d& lamc, @@ -1378,7 +1356,7 @@ struct Functions #endif KOKKOS_FUNCTION - static void ice_supersat_conservation(Spack& qidep, Spack& qinuc, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& latent_heat_sublim, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context = Smask(true)); + static void ice_supersat_conservation(Spack& qidep, Spack& qinuc, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context = Smask(true)); KOKKOS_FUNCTION static void nc_conservation(const Spack& nc, const Spack& nc_selfcollect_tend, const Real& dt, Spack& nc_collect_tend, Spack& nc2ni_immers_freeze_tend, Spack& nc_accret_tend, Spack& nc2nr_autoconv_tend, const Smask& context = Smask(true)); @@ -1390,7 +1368,7 @@ struct Functions static void ni_conservation(const Spack& ni, const Spack& ni_nucleat_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nc2ni_immers_freeze_tend, const Real& dt, Spack& ni2nr_melt_tend, Spack& ni_sublim_tend, Spack& ni_selfcollect_tend, const Smask& context = Smask(true)); KOKKOS_FUNCTION - static void prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Scalar& dt, const Spack& qidep, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context = Smask(true) ); + static void prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Scalar& dt, const Spack& qidep, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context = Smask(true) ); }; // struct Functions template @@ -1438,7 +1416,6 @@ void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, # include "p3_ice_melting_impl.hpp" # include "p3_calc_liq_relaxation_timescale_impl.hpp" # include "p3_ice_cldliq_wet_growth_impl.hpp" -# include "p3_get_latent_heat_impl.hpp" # include "p3_check_values_impl.hpp" # include "p3_incloud_mixingratios_impl.hpp" # include "p3_subgrid_variance_scaling_impl.hpp" diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index b8f6e826fc14..40d82d4f6529 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -1417,7 +1417,7 @@ void rain_sedimentation_f( void homogeneous_freezing_f( Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, + Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm) { using P3F = Functions; @@ -1440,31 +1440,31 @@ void homogeneous_freezing_f( const Int nk_pack = ekat::npack(nk); // Set up views - std::vector temp_d(HomogeneousFreezingData::NUM_ARRAYS); + std::vector temp_d(HomogeneousFreezingData::NUM_ARRAYS-1); - ekat::host_to_device({T_atm, inv_exner, latent_heat_fusion, qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, + ekat::host_to_device({T_atm, inv_exner, qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, temp_d); + int current_index = 0; view_1d - t_d (temp_d[0]), - inv_exner_d (temp_d[1]), - latent_heat_fusion_d (temp_d[2]), - qc_d (temp_d[3]), - nc_d (temp_d[4]), - qr_d (temp_d[5]), - nr_d (temp_d[6]), - qi_d (temp_d[7]), - ni_d (temp_d[8]), - qm_d (temp_d[9]), - bm_d (temp_d[10]), - th_atm_d (temp_d[11]); + t_d (temp_d[current_index++]), + inv_exner_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]); // Call core function from kernel auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { P3F::homogeneous_freezing( - t_d, inv_exner_d, latent_heat_fusion_d, + t_d, inv_exner_d, team, nk, ktop, kbot, kdir, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, th_atm_d); @@ -1476,37 +1476,6 @@ void homogeneous_freezing_f( ekat::device_to_host({qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, inout_views); } -void get_latent_heat_f(Int its, Int ite, Int kts, Int kte, Real* v, Real* s, Real* f) -{ - using P3F = Functions; - using Spack = typename P3F::Spack; - using view_2d = typename P3F::view_2d; - - EKAT_REQUIRE_MSG(kte >= kts, - "kte must be >= kts, kts=" << kts << " kte=" << kte); - - EKAT_REQUIRE_MSG(ite >= its, - "ite must be >= its, its=" << its << " ite=" << ite); - - kts -= 1; - kte -= 1; - its -= 1; - ite -= 1; - - Int nk = (kte - kts) + 1; - Int nj = (ite - its) + 1; - - // Set up views - view_2d v_d("v_d", nj, nk), - s_d("s_d", nj, nk), - f_d("f_d", nj, nk); - - P3F::get_latent_heat(nj, nk, v_d, s_d, f_d); - - std::vector out_views = {v_d, s_d, f_d}; - ekat::device_to_host({v, s, f}, nj, nk, out_views, true); -} - void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc) { @@ -1545,7 +1514,7 @@ void p3_main_part1_f( bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, + Real* inv_cld_frac_r, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, @@ -1576,7 +1545,7 @@ void p3_main_part1_f( ekat::host_to_device({pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, - acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, + acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, nccn_prescribed}, nk, temp_d); @@ -1609,18 +1578,15 @@ void p3_main_part1_f( ni_d (temp_d[25]), qm_d (temp_d[26]), bm_d (temp_d[27]), - latent_heat_vapor_d (temp_d[28]), - latent_heat_sublim_d (temp_d[29]), - latent_heat_fusion_d (temp_d[30]), - qc_incld_d (temp_d[31]), - qr_incld_d (temp_d[32]), - qi_incld_d (temp_d[33]), - qm_incld_d (temp_d[34]), - nc_incld_d (temp_d[35]), - nr_incld_d (temp_d[36]), - ni_incld_d (temp_d[37]), - bm_incld_d (temp_d[38]), - nccn_prescribed_d (temp_d[39]); + qc_incld_d (temp_d[28]), + qr_incld_d (temp_d[29]), + qi_incld_d (temp_d[30]), + qm_incld_d (temp_d[31]), + nc_incld_d (temp_d[32]), + nr_incld_d (temp_d[33]), + ni_incld_d (temp_d[34]), + bm_incld_d (temp_d[35]), + nccn_prescribed_d (temp_d[36]); // Call core function from kernel bview_1d bools_d("bools", 2); @@ -1630,7 +1596,7 @@ void p3_main_part1_f( P3F::p3_main_part1( team, nk, do_predict_nc, do_prescribed_CCN, dt, pres_d, dpres_d, dz_d, nc_nuceat_tend_d, nccn_prescribed_d, inv_exner_d, exner_d, inv_cld_frac_l_d, inv_cld_frac_i_d, - inv_cld_frac_r_d, latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, + inv_cld_frac_r_d, t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, @@ -1661,7 +1627,7 @@ void p3_main_part2_f( Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, + Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present) @@ -1688,82 +1654,80 @@ void p3_main_part2_f( const Real max_total_ni = 740.0e3; // Hard-code this value for F90 comparison // Set up views - std::vector temp_d(P3MainPart2Data::NUM_ARRAYS); + std::vector temp_d(P3MainPart2Data::NUM_ARRAYS-3); ekat::host_to_device({pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, + qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, prctot, qv_prev, t_prev }, nk, temp_d); + int current_index = 0; view_1d - pres_d (temp_d[0]), - dpres_d (temp_d[1]), - dz_d (temp_d[2]), - nc_nuceat_tend_d (temp_d[3]), - inv_exner_d (temp_d[4]), - exner_d (temp_d[5]), - inv_cld_frac_l_d (temp_d[6]), - inv_cld_frac_i_d (temp_d[7]), - inv_cld_frac_r_d (temp_d[8]), - ni_activated_d (temp_d[9]), - inv_qc_relvar_d (temp_d[10]), - cld_frac_i_d (temp_d[11]), - cld_frac_l_d (temp_d[12]), - cld_frac_r_d (temp_d[13]), - t_d (temp_d[14]), - rho_d (temp_d[15]), - inv_rho_d (temp_d[16]), - qv_sat_l_d (temp_d[17]), - qv_sat_i_d (temp_d[18]), - qv_supersat_i_d (temp_d[19]), - rhofacr_d (temp_d[20]), - rhofaci_d (temp_d[21]), - acn_d (temp_d[22]), - qv_d (temp_d[23]), - th_atm_d (temp_d[24]), - qc_d (temp_d[25]), - nc_d (temp_d[26]), - qr_d (temp_d[27]), - nr_d (temp_d[28]), - qi_d (temp_d[29]), - ni_d (temp_d[30]), - qm_d (temp_d[31]), - bm_d (temp_d[32]), - latent_heat_vapor_d (temp_d[33]), - latent_heat_sublim_d(temp_d[34]), - latent_heat_fusion_d(temp_d[35]), - qc_incld_d (temp_d[36]), - qr_incld_d (temp_d[37]), - qi_incld_d (temp_d[38]), - qm_incld_d (temp_d[39]), - nc_incld_d (temp_d[40]), - nr_incld_d (temp_d[41]), - ni_incld_d (temp_d[42]), - bm_incld_d (temp_d[43]), - mu_c_d (temp_d[44]), - nu_d (temp_d[45]), - lamc_d (temp_d[46]), - cdist_d (temp_d[47]), - cdist1_d (temp_d[48]), - cdistr_d (temp_d[49]), - mu_r_d (temp_d[50]), - lamr_d (temp_d[51]), - logn0r_d (temp_d[52]), - qv2qi_depos_tend_d (temp_d[53]), - precip_total_tend_d (temp_d[54]), - nevapr_d (temp_d[55]), - qr_evap_tend_d (temp_d[56]), - vap_liq_exchange_d (temp_d[57]), - vap_ice_exchange_d (temp_d[58]), - liq_ice_exchange_d (temp_d[59]), - pratot_d (temp_d[60]), - prctot_d (temp_d[61]), - qv_prev_d (temp_d[62]), - t_prev_d (temp_d[63]); + pres_d (temp_d[current_index++]), + dpres_d (temp_d[current_index++]), + dz_d (temp_d[current_index++]), + nc_nuceat_tend_d (temp_d[current_index++]), + inv_exner_d (temp_d[current_index++]), + exner_d (temp_d[current_index++]), + inv_cld_frac_l_d (temp_d[current_index++]), + inv_cld_frac_i_d (temp_d[current_index++]), + inv_cld_frac_r_d (temp_d[current_index++]), + ni_activated_d (temp_d[current_index++]), + inv_qc_relvar_d (temp_d[current_index++]), + cld_frac_i_d (temp_d[current_index++]), + cld_frac_l_d (temp_d[current_index++]), + cld_frac_r_d (temp_d[current_index++]), + t_d (temp_d[current_index++]), + rho_d (temp_d[current_index++]), + inv_rho_d (temp_d[current_index++]), + qv_sat_l_d (temp_d[current_index++]), + qv_sat_i_d (temp_d[current_index++]), + qv_supersat_i_d (temp_d[current_index++]), + rhofacr_d (temp_d[current_index++]), + rhofaci_d (temp_d[current_index++]), + acn_d (temp_d[current_index++]), + qv_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + qc_incld_d (temp_d[current_index++]), + qr_incld_d (temp_d[current_index++]), + qi_incld_d (temp_d[current_index++]), + qm_incld_d (temp_d[current_index++]), + nc_incld_d (temp_d[current_index++]), + nr_incld_d (temp_d[current_index++]), + ni_incld_d (temp_d[current_index++]), + bm_incld_d (temp_d[current_index++]), + mu_c_d (temp_d[current_index++]), + nu_d (temp_d[current_index++]), + lamc_d (temp_d[current_index++]), + cdist_d (temp_d[current_index++]), + cdist1_d (temp_d[current_index++]), + cdistr_d (temp_d[current_index++]), + mu_r_d (temp_d[current_index++]), + lamr_d (temp_d[current_index++]), + logn0r_d (temp_d[current_index++]), + qv2qi_depos_tend_d (temp_d[current_index++]), + precip_total_tend_d (temp_d[current_index++]), + nevapr_d (temp_d[current_index++]), + qr_evap_tend_d (temp_d[current_index++]), + vap_liq_exchange_d (temp_d[current_index++]), + vap_ice_exchange_d (temp_d[current_index++]), + liq_ice_exchange_d (temp_d[current_index++]), + pratot_d (temp_d[current_index++]), + prctot_d (temp_d[current_index++]), + qv_prev_d (temp_d[current_index++]), + t_prev_d (temp_d[current_index++]); // Call core function from kernel const auto dnu = P3GlobalForFortran::dnu(); @@ -1780,7 +1744,7 @@ void p3_main_part2_f( inv_cld_frac_i_d, inv_cld_frac_r_d, ni_activated_d, inv_qc_relvar_d, cld_frac_i_d, cld_frac_l_d, cld_frac_r_d, qv_prev_d, t_prev_d, t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, - latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, qc_incld_d, qr_incld_d, qi_incld_d, + qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, @@ -1791,7 +1755,7 @@ void p3_main_part2_f( std::vector inout_views = { t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, - latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, + qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, vap_ice_exchange_d, @@ -1800,7 +1764,7 @@ void p3_main_part2_f( ekat::device_to_host({ T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, - qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, + qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, @@ -1818,7 +1782,7 @@ void p3_main_part3_f( Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, - Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* mu_c, Real* nu, Real* lamc, + Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr) @@ -1844,50 +1808,49 @@ void p3_main_part3_f( const Real max_total_ni = 740.0e3; // Hard-code this value for F90 comparison // Set up views - std::vector temp_d(P3MainPart3Data::NUM_ARRAYS); + std::vector temp_d(P3MainPart3Data::NUM_ARRAYS-2); ekat::host_to_device({ inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, rho, inv_rho, rhofaci, qv, th_atm, qc, - nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, + nc, qr, nr, qi, ni, qm, bm, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr}, nk, temp_d); + int current_index = 0; view_1d - inv_exner_d (temp_d[0]), - cld_frac_l_d (temp_d[1]), - cld_frac_r_d (temp_d[2]), - cld_frac_i_d (temp_d[3]), - rho_d (temp_d[4]), - inv_rho_d (temp_d[5]), - rhofaci_d (temp_d[6]), - qv_d (temp_d[7]), - th_atm_d (temp_d[8]), - qc_d (temp_d[9]), - nc_d (temp_d[10]), - qr_d (temp_d[11]), - nr_d (temp_d[12]), - qi_d (temp_d[13]), - ni_d (temp_d[14]), - qm_d (temp_d[15]), - bm_d (temp_d[16]), - latent_heat_vapor_d (temp_d[17]), - latent_heat_sublim_d (temp_d[18]), - mu_c_d (temp_d[19]), - nu_d (temp_d[20]), - lamc_d (temp_d[21]), - mu_r_d (temp_d[22]), - lamr_d (temp_d[23]), - vap_liq_exchange_d (temp_d[24]), - ze_rain_d (temp_d[25]), - ze_ice_d (temp_d[26]), - diag_vm_qi_d (temp_d[27]), - diag_eff_radius_qi_d (temp_d[28]), - diag_diam_qi_d (temp_d[29]), - rho_qi_d (temp_d[30]), - diag_equiv_reflectivity_d (temp_d[31]), - diag_eff_radius_qc_d (temp_d[32]), - diag_eff_radius_qr_d (temp_d[33]); + inv_exner_d (temp_d[current_index++]), + cld_frac_l_d (temp_d[current_index++]), + cld_frac_r_d (temp_d[current_index++]), + cld_frac_i_d (temp_d[current_index++]), + rho_d (temp_d[current_index++]), + inv_rho_d (temp_d[current_index++]), + rhofaci_d (temp_d[current_index++]), + qv_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + mu_c_d (temp_d[current_index++]), + nu_d (temp_d[current_index++]), + lamc_d (temp_d[current_index++]), + mu_r_d (temp_d[current_index++]), + lamr_d (temp_d[current_index++]), + vap_liq_exchange_d (temp_d[current_index++]), + ze_rain_d (temp_d[current_index++]), + ze_ice_d (temp_d[current_index++]), + diag_vm_qi_d (temp_d[current_index++]), + diag_eff_radius_qi_d (temp_d[current_index++]), + diag_diam_qi_d (temp_d[current_index++]), + rho_qi_d (temp_d[current_index++]), + diag_equiv_reflectivity_d (temp_d[current_index++]), + diag_eff_radius_qc_d (temp_d[current_index++]), + diag_eff_radius_qr_d (temp_d[current_index++]); // Call core function from kernel const auto dnu = P3GlobalForFortran::dnu(); @@ -1898,8 +1861,8 @@ void p3_main_part3_f( P3F::p3_main_part3(team, nk_pack, max_total_ni, dnu, ice_table_vals, inv_exner_d, cld_frac_l_d, cld_frac_r_d, cld_frac_i_d, rho_d, inv_rho_d, rhofaci_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, - qi_d, ni_d, qm_d, bm_d, latent_heat_vapor_d, - latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, + qi_d, ni_d, qm_d, bm_d, + mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d, physics::P3_Constants()); @@ -1908,14 +1871,14 @@ void p3_main_part3_f( // Sync back to host std::vector inout_views = { rho_d, inv_rho_d, rhofaci_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, - ni_d, qm_d, bm_d, latent_heat_vapor_d, latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, + ni_d, qm_d, bm_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d }; ekat::device_to_host({ rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, - latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, + mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr }, diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 926c9ab47017..d1c29d3de0de 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -908,11 +908,9 @@ void rain_sedimentation_f( void homogeneous_freezing_f( Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, + Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); -void get_latent_heat_f(Int its, Int ite, Int kts, Int kte, Real* v, Real* s, Real* f); - void check_values_f(Real* Qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); @@ -921,7 +919,7 @@ void p3_main_part1_f( bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, + Real* inv_cld_frac_r, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, @@ -931,7 +929,7 @@ void p3_main_part2_f( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, + Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present); @@ -939,7 +937,7 @@ void p3_main_part2_f( void p3_main_part3_f( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, - Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, + Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 912c7cedac66..217c2945e489 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -26,7 +26,6 @@ set(P3_TESTS_SRCS p3_ice_melting_unit_tests.cpp p3_evaporate_rain_unit_tests.cpp p3_ice_cldliq_wet_growth_unit_tests.cpp - p3_get_latent_heat_unit_tests.cpp p3_subgrid_variance_scaling_unit_tests.cpp p3_check_values_unit_tests.cpp p3_incloud_mixingratios_unit_tests.cpp diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index 24cc3a848188..f8398135a19e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -82,7 +82,6 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Spack epsi_tot(1./60.); Spack t(287); Spack t_prev(285); - Spack latent_heat_sublim(3.34e5); Spack dqsdt(1e-3); Scalar dt=60; Spack qrtend; @@ -92,7 +91,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip constexpr Scalar QSMALL = C::QSMALL; Functions::evaporate_rain(Spack(QSMALL/2),qc_incld,nr_incld,qi_incld, //qr_incld->QSMALL/2 cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -101,7 +100,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Spack qr_tiny=Spack(5e-13); Functions::evaporate_rain(qr_tiny,qc_incld,nr_incld,qi_incld, //qr_incld->_tiny cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0] - qr_tiny[0]/dt *(cld_frac_r[0]-cld_frac_l[0])/cld_frac_r[0])<1e-8 ); @@ -111,7 +110,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip //if no rainy areas outside cloud, don't evap Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_r,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, //cld_frac_l->_r - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -120,7 +119,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, //set qv->qv_sat_l*2 in next line to ensure supersaturated. cld_frac_l,cld_frac_r,qv_sat_l*2,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -129,7 +128,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, //qv -> qv*0.1 to encourage lots of rain evap cld_frac_l,cld_frac_r,qv*0.1,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( qrtend[0] <= qr_incld[0]/dt); REQUIRE( nrtend[0] <= nr_incld[0]/dt); //keep end-of-step nr positive. Should always be true. @@ -137,6 +136,8 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip }; //end run_property static void run_bfb(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following //This subroutine has 20 args, only 18 are supplied here for invoking it as last 2 are intent-outs @@ -152,23 +153,23 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip //rows 9-16: random junk but ensured cld_frac_r>cld_frac_l and subsaturated. EvapRainData espd[max_pack_size] = { //qr_incld, qc_incld, nr_incld, qi_incld, cld_frac_l, cld_frac_r, qv, qv_prev, qv_sat_l, qv_sat_i, ab, abi, epsr, epsi_tot, t, t_prev, lat_ht_sublim, dqsdt, dt - {4.634940e-03,1.215335e-03,6.073270e+07,3.594486e-04,6.134229e-01,9.134229e-01,2.747871e-03,1.911238e-03,5.913313e-03,1.057645e-03,1.782748e+00,1.571392e+00,3.868229e+02,2.248689e+02,3.101180e+02,1.395063e+02,3.335413e+05,5.494606e-03,6.000000e+02}, - {6.175320e-13,4.432407e-03,8.029967e+07,1.905151e-03,2.190099e-01,7.031070e-01,4.172977e-05,7.315360e-03,7.280063e-03,1.378543e-03,1.461443e+00,1.507382e+00,8.452377e+02,1.971876e+02,2.389249e+02,1.497752e+02,3.578466e+05,5.107905e-03,6.000000e+02}, - {4.519798e-03,7.348916e-03,7.420725e+07,2.220971e-03,1.882608e-01,2.934182e-01,4.957590e-03,2.550256e-03,3.136926e-03,4.498115e-03,1.433526e+00,1.207516e+00,9.716844e+02,5.602546e+01,1.389465e+02,1.075863e+02,3.570404e+05,6.771428e-03,6.000000e+02}, - {7.169182e-03,6.657331e-03,9.807967e+07,7.981196e-03,2.914473e-01,6.375719e-01,2.420032e-03,1.223012e-03,7.685516e-03,5.207024e-03,1.644865e+00,1.433872e+00,3.825069e+02,6.550300e+02,1.833466e+02,1.741918e+02,3.295800e+05,3.792982e-03,6.000000e+02}, - {1.103118e-03,9.158125e-03,3.136196e+07,4.286154e-03,2.699078e-01,4.668103e-01,9.645460e-03,6.379119e-03,8.283285e-03,3.342400e-03,1.546698e+00,1.417916e+00,9.289270e+02,9.844129e+02,2.543202e+02,1.932996e+02,3.327786e+05,2.693119e-03,6.000000e+02}, - {4.308000e-03,8.168535e-03,7.439969e+07,5.131497e-03,6.851225e-01,3.298025e-01,4.331812e-03,2.814373e-03,3.592807e-03,1.527499e-03,1.856943e+00,1.003269e+00,9.165690e+02,9.379921e+02,2.163204e+02,3.165814e+02,3.874371e+05,6.801393e-03,6.000000e+02}, - {0.000000e-00,3.318968e-03,4.664041e+07,8.737282e-03,2.585907e-01,6.297295e-02,8.747418e-03,2.710437e-03,2.164895e-03,9.455725e-03,1.241506e+00,1.561393e+00,2.492674e+02,6.546182e+02,2.228772e+02,2.147968e+02,3.590381e+05,5.903261e-03,6.000000e+02}, - {7.677170e-03,6.069057e-05,6.404241e+07,3.094233e-03,3.755403e-01,5.026876e-01,4.723817e-03,1.204228e-03,6.156526e-03,8.194797e-03,1.361509e+00,1.772751e+00,6.420537e+01,4.043364e+02,2.833110e+02,3.314521e+02,3.427004e+05,2.996696e-03,6.000000e+02}, - - {9.999294e-03,3.138400e-03,2.355097e+07,9.897893e-03,7.667177e-01,9.739270e-01,4.221430e-03,3.570130e-03,8.370033e-03,9.527208e-03,1.597218e+00,1.111438e+00,7.832357e+02,8.364566e+02,2.854867e+02,2.340771e+02,3.198709e+05,7.235757e-03,6.000000e+02}, - {8.841793e-03,3.530456e-03,9.618284e+07,9.311658e-03,3.458590e-01,6.978258e-01,1.279864e-03,4.652008e-03,1.869728e-03,8.931663e-03,1.712564e+00,1.223882e+00,9.692403e+02,2.358558e+02,3.204043e+02,1.827677e+02,3.220502e+05,7.646405e-03,6.000000e+02}, - {1.425612e-03,6.653411e-04,2.843806e+07,1.922560e-03,9.100262e-01,0.996264e-01,8.973183e-04,9.857420e-03,6.221419e-03,8.133433e-03,1.815337e+00,1.885506e+00,5.508742e+02,1.612139e+02,2.798523e+02,2.631136e+02,3.045141e+05,4.148666e-03,6.000000e+02}, - {4.125177e-04,4.056163e-03,2.716439e+07,6.484214e-03,1.658752e-01,2.859102e-01,5.724081e-03,6.282997e-03,7.313187e-03,6.049825e-03,1.140910e+00,1.145941e+00,7.490652e+02,5.011633e+02,1.986541e+02,2.745566e+02,3.371001e+05,6.784784e-03,6.000000e+02}, - {5.010628e-03,2.863789e-04,8.953841e+07,3.953058e-03,1.135952e-01,9.718675e-01,1.846157e-03,5.743094e-03,2.842649e-03,8.155366e-03,1.227867e+00,1.894249e+00,1.161776e+02,3.578576e+02,1.240083e+02,1.639791e+02,3.167181e+05,4.497257e-03,6.000000e+02}, - {9.487866e-03,6.584660e-03,6.149682e+06,9.413342e-03,4.757261e-01,6.503885e-01,1.078922e-03,3.489665e-03,3.059596e-03,9.285703e-03,1.192620e+00,1.967205e+00,5.085628e+02,3.741816e+01,1.196252e+02,2.904002e+02,3.637035e+05,2.566077e-03,6.000000e+02}, - {3.241928e-03,7.024929e-03,2.212493e+07,8.600485e-03,3.963690e-01,4.834201e-01,3.736511e-03,5.724475e-03,4.790239e-03,2.766218e-03,1.151150e+00,1.150516e+00,2.089426e+02,8.666450e+02,1.898220e+02,2.862496e+02,3.056143e+05,7.039800e-03,6.000000e+02}, - {4.617594e-03,3.157739e-03,5.569465e+07,8.221076e-03,7.918279e-01,9.995014e-01,1.338309e-04,1.319707e-03,2.896082e-03,4.359171e-03,1.007827e+00,1.812954e+00,5.332209e+02,2.973599e+02,3.271466e+02,2.622351e+02,3.821569e+05,1.407429e-03,6.000000e+02} + {4.634940e-03,1.215335e-03,6.073270e+07,3.594486e-04,6.134229e-01,9.134229e-01,2.747871e-03,1.911238e-03,5.913313e-03,1.057645e-03,1.782748e+00,1.571392e+00,3.868229e+02,2.248689e+02,3.101180e+02,1.395063e+02,latvap+latice,5.494606e-03,6.000000e+02}, + {6.175320e-13,4.432407e-03,8.029967e+07,1.905151e-03,2.190099e-01,7.031070e-01,4.172977e-05,7.315360e-03,7.280063e-03,1.378543e-03,1.461443e+00,1.507382e+00,8.452377e+02,1.971876e+02,2.389249e+02,1.497752e+02,latvap+latice,5.107905e-03,6.000000e+02}, + {4.519798e-03,7.348916e-03,7.420725e+07,2.220971e-03,1.882608e-01,2.934182e-01,4.957590e-03,2.550256e-03,3.136926e-03,4.498115e-03,1.433526e+00,1.207516e+00,9.716844e+02,5.602546e+01,1.389465e+02,1.075863e+02,latvap+latice,6.771428e-03,6.000000e+02}, + {7.169182e-03,6.657331e-03,9.807967e+07,7.981196e-03,2.914473e-01,6.375719e-01,2.420032e-03,1.223012e-03,7.685516e-03,5.207024e-03,1.644865e+00,1.433872e+00,3.825069e+02,6.550300e+02,1.833466e+02,1.741918e+02,latvap+latice,3.792982e-03,6.000000e+02}, + {1.103118e-03,9.158125e-03,3.136196e+07,4.286154e-03,2.699078e-01,4.668103e-01,9.645460e-03,6.379119e-03,8.283285e-03,3.342400e-03,1.546698e+00,1.417916e+00,9.289270e+02,9.844129e+02,2.543202e+02,1.932996e+02,latvap+latice,2.693119e-03,6.000000e+02}, + {4.308000e-03,8.168535e-03,7.439969e+07,5.131497e-03,6.851225e-01,3.298025e-01,4.331812e-03,2.814373e-03,3.592807e-03,1.527499e-03,1.856943e+00,1.003269e+00,9.165690e+02,9.379921e+02,2.163204e+02,3.165814e+02,latvap+latice,6.801393e-03,6.000000e+02}, + {0.000000e-00,3.318968e-03,4.664041e+07,8.737282e-03,2.585907e-01,6.297295e-02,8.747418e-03,2.710437e-03,2.164895e-03,9.455725e-03,1.241506e+00,1.561393e+00,2.492674e+02,6.546182e+02,2.228772e+02,2.147968e+02,latvap+latice,5.903261e-03,6.000000e+02}, + {7.677170e-03,6.069057e-05,6.404241e+07,3.094233e-03,3.755403e-01,5.026876e-01,4.723817e-03,1.204228e-03,6.156526e-03,8.194797e-03,1.361509e+00,1.772751e+00,6.420537e+01,4.043364e+02,2.833110e+02,3.314521e+02,latvap+latice,2.996696e-03,6.000000e+02}, + + {9.999294e-03,3.138400e-03,2.355097e+07,9.897893e-03,7.667177e-01,9.739270e-01,4.221430e-03,3.570130e-03,8.370033e-03,9.527208e-03,1.597218e+00,1.111438e+00,7.832357e+02,8.364566e+02,2.854867e+02,2.340771e+02,latvap+latice,7.235757e-03,6.000000e+02}, + {8.841793e-03,3.530456e-03,9.618284e+07,9.311658e-03,3.458590e-01,6.978258e-01,1.279864e-03,4.652008e-03,1.869728e-03,8.931663e-03,1.712564e+00,1.223882e+00,9.692403e+02,2.358558e+02,3.204043e+02,1.827677e+02,latvap+latice,7.646405e-03,6.000000e+02}, + {1.425612e-03,6.653411e-04,2.843806e+07,1.922560e-03,9.100262e-01,0.996264e-01,8.973183e-04,9.857420e-03,6.221419e-03,8.133433e-03,1.815337e+00,1.885506e+00,5.508742e+02,1.612139e+02,2.798523e+02,2.631136e+02,latvap+latice,4.148666e-03,6.000000e+02}, + {4.125177e-04,4.056163e-03,2.716439e+07,6.484214e-03,1.658752e-01,2.859102e-01,5.724081e-03,6.282997e-03,7.313187e-03,6.049825e-03,1.140910e+00,1.145941e+00,7.490652e+02,5.011633e+02,1.986541e+02,2.745566e+02,latvap+latice,6.784784e-03,6.000000e+02}, + {5.010628e-03,2.863789e-04,8.953841e+07,3.953058e-03,1.135952e-01,9.718675e-01,1.846157e-03,5.743094e-03,2.842649e-03,8.155366e-03,1.227867e+00,1.894249e+00,1.161776e+02,3.578576e+02,1.240083e+02,1.639791e+02,latvap+latice,4.497257e-03,6.000000e+02}, + {9.487866e-03,6.584660e-03,6.149682e+06,9.413342e-03,4.757261e-01,6.503885e-01,1.078922e-03,3.489665e-03,3.059596e-03,9.285703e-03,1.192620e+00,1.967205e+00,5.085628e+02,3.741816e+01,1.196252e+02,2.904002e+02,latvap+latice,2.566077e-03,6.000000e+02}, + {3.241928e-03,7.024929e-03,2.212493e+07,8.600485e-03,3.963690e-01,4.834201e-01,3.736511e-03,5.724475e-03,4.790239e-03,2.766218e-03,1.151150e+00,1.150516e+00,2.089426e+02,8.666450e+02,1.898220e+02,2.862496e+02,latvap+latice,7.039800e-03,6.000000e+02}, + {4.617594e-03,3.157739e-03,5.569465e+07,8.221076e-03,7.918279e-01,9.995014e-01,1.338309e-04,1.319707e-03,2.896082e-03,4.359171e-03,1.007827e+00,1.812954e+00,5.332209e+02,2.973599e+02,3.271466e+02,2.622351e+02,latvap+latice,1.407429e-03,6.000000e+02} }; // Sync to device @@ -191,7 +192,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip // Init pack inputs Spack qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt; + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt; Scalar dt; @@ -215,7 +216,6 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip epsi_tot[s] = espd_device(vs).epsi_tot; t[s] = espd_device(vs).t; t_prev[s] = espd_device(vs).t_prev; - latent_heat_sublim[s]=espd_device(vs).latent_heat_sublim; dqsdt[s]=espd_device(vs).dqsdt; dt=espd_device(vs).dt; //qr2qv_evap_tend[s] = espd_device(vs).qr2qv_evap_tend; //PMC shouldn't have to init output vars. @@ -224,7 +224,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qr2qv_evap_tend,nr_evap_tend); // Copy results back into views diff --git a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp index 3ce02ced4597..5d7b4de4ea7e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp @@ -24,12 +24,12 @@ struct UnitWrap::UnitTest::TestLatentHeat { static void run_latent_heat_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + LatentHeatData latent_fortran[] = { // its, ite, kts, kte LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), }; static constexpr Int num_runs = sizeof(latent_fortran) / sizeof(LatentHeatData); @@ -45,16 +45,11 @@ struct UnitWrap::UnitTest::TestLatentHeat { LatentHeatData& h = latent_fortran[i]; get_latent_heat(h); - LatentHeatData& d = latent_cxx[i]; - get_latent_heat_f(d.its, d.ite, d.kts, d.kte, d.v, d.s, d.f); - if (SCREAM_BFB_TESTING) { - REQUIRE(h.total(h.v) == d.total(d.v)); - for (Int j = 0; j < h.total(h.v); ++j) { - REQUIRE(d.v[j] == h.v[j]); - REQUIRE(d.s[j] == h.s[j]); - REQUIRE(d.f[j] == h.f[j]); + REQUIRE(h.v[j] == latvap); + REQUIRE(h.s[j] == (latvap+latice)); + REQUIRE(h.f[j] == latice); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index 78f7ab986c66..caf6638d5cd5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -25,27 +25,30 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { { using KTH = KokkosTypes; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + IceWetGrowthData self[max_pack_size] = { // rho,temp,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,kap,mu,sc,qv,qc_incld,qi_incld,ni_incld,qr_incld,log_wetgrowth,qr2qi_collect_tend,qc2qi_collect_tend,qc_growth_rate,nr_ice_shed_tend,qc2qr_ice_shed_tend - {4.056000E-03, 1.023000E+02, 1.201000E+02, 9.002000E-04, 8.215000E-04, 8.852000E-01, 0.174000E+00, 1.221000E-14, 5.100000E-03, 9.558000E-04, 1.213000E-03, 9.653000E-04, 1.023000E-01, 4.098000E-02, 2.098000E-02, 9.952000E+03, 1.023000E-05, false, 1.241456E-04, 9.021345E-02, 1.043000E-01, 1.921000E-02, 0.242000E-02}, - {6.852000E-02, 1.120000E+02, 2.450000E+02, 9.321000E-04, 9.124000E-04, 8.852000E-01, 0.374000E+00, 1.221000E-13, 4.100000E-03, 9.558000E-04, 2.560000E-03, 1.764000E-03, 2.346000E-01, 5.632000E-02, 3.024000E-02, 9.952000E+03, 2.093000E-05, false, 2.341678E-04, 1.092432E-02, 2.903000E-01, 2.125000E-02, 0.342000E-02}, - {8.852000E-02, 1.210000E+02, 3.420000E+02, 9.623000E-04, 9.432000E-04, 8.900000E-01, 0.123000E+00, 1.221000E-12, 3.100000E-03, 9.558000E-04, 3.211000E-03, 3.421000E-03, 3.421000E-01, 6.542000E-02, 4.567000E-02, 9.952000E+03, 3.091000E-05, false, 3.215234E-04, 2.098987E-02, 3.450000E-01, 3.490000E-02, 0.932000E-02}, - {1.902000E-01, 1.326000E+02, 4.321000E+02, 9.982000E-04, 9.623000E-04, 9.900000E-01, 0.123000E+00, 1.221000E-11, 2.100000E-03, 9.558000E-04, 4.121000E-03, 4.569000E-03, 4.673000E-01, 7.902000E-02, 5.321000E-02, 9.952000E+03, 4.521000E-05, false, 4.675567E-04, 3.214982E-02, 4.290000E-01, 4.590000E-02, 1.025000E-01}, - - {2.201000E-01, 1.456000E+02, 5.670000E+02, 1.234000E-03, 9.723000E-04, 0.100000E+01, 0.174000E+00, 1.221000E-10, 1.100000E-03, 2.550008E-05, 4.980000E-03, 5.621000E-03, 5.420000E-01, 8.021000E-02, 6.902000E-02, 9.952000E+04, 5.678000E-05, false, 5.389236E-04, 4.125969E-02, 5.098000E-01, 5.921000E-02, 2.031000E-01}, - {3.502000E-01, 1.780009E+02, 6.832000E+02, 1.562000E-03, 1.024000E-03, 0.100000E+01, 0.374000E+00, 1.221000E-09, 8.100000E-04, 2.558000E-05, 5.643000E-03, 7.367000E-03, 6.782000E-01, 9.253000E-02, 8.045000E-02, 9.952000E+04, 6.902000E-05, false, 6.432654E-04, 5.389457E-02, 6.723000E-01, 6.093000E-02, 4.098000E-01}, - {4.852000E-01, 2.100009E+02, 7.090000E+02, 2.101000E-03, 1.235000E-03, 0.100000E+01, 0.123000E+00, 1.221000E-08, 4.100000E-04, 2.558000E-05, 7.892000E-03, 9.087000E-03, 8.213000E-01, 1.256000E-01, 9.134000E-02, 9.952000E+04, 8.367000E-05, false, 7.210983E-04, 6.476985E-02, 8.902000E-01, 8.345000E-02, 8.023000E-01}, - {5.852000E-01, 2.310000E+02, 9.215000E+02, 2.312000E-03, 1.456000E-03, 0.100000E+01, 0.123000E+00, 1.221000E-07, 2.100000E-04, 2.558000E-05, 9.321000E-03, 1.245000E-02, 1.067000E-00, 2.347000E-01, 1.092000E-01, 9.952000E+04, 9.098000E-05, false, 8.543367E-04, 8.213186E-02, 9.021000E-01, 9.321000E-02, 9.098000E-01}, - - {6.852000E-01, 2.563000E+02, 1.089000E+03, 3.601000E-03, 1.864000E-03, 0.950000E+00, 0.150000E+00, 1.221000E-06, 9.952000E-05, 4.596000E-05, 1.453000E-02, 2.543000E-02, 2.345000E-00, 3.578000E-01, 2.873000E-01, 1.734000E+04, 1.023000E-04, false, 9.021215E-04, 9.023367E-02, 1.023000E-00, 1.056000E-01, 1.256000E-00}, - {7.852000E-01, 2.789000E+02, 3.754000E+03, 3.891000E-03, 2.093000E-03, 0.950000E+00, 0.374000E+00, 1.221000E-05, 4.952000E-05, 4.596000E-05, 2.789000E-02, 4.367000E-02, 3.890000E-00, 4.980000E-01, 3.468000E-01, 1.734000E+04, 2.146000E-04, false, 1.043468E-05, 1.094854E-02, 2.012000E-00, 2.893000E-01, 2.903000E-00}, - {8.852000E-01, 3.123000E+02, 8.902000E+03, 4.872000E-03, 2.345000E-03, 0.950000E+00, 0.123000E+00, 1.221000E-04, 1.952000E-05, 4.596000E-05, 4.256000E-02, 6.324000E-02, 4.120000E-00, 6.321000E-01, 4.890000E-01, 1.734000E+04, 4.321000E-04, false, 2.341763E-05, 2.126247E-03, 3.120000E-00, 3.456000E-01, 3.912000E-00}, - {9.852000E-01, 4.981000E+02, 1.092000E+04, 5.210000E-03, 3.210000E-03, 0.950000E+00, 0.123000E+00, 1.221000E-03, 9.952000E-06, 4.596000E-05, 6.821000E-02, 8.789000E-02, 5.320000E-00, 7.982000E-01, 6.921000E-01, 1.734000E+04, 5.821000E-04, false, 3.901479E-05, 3.874763E-03, 5.902000E-00, 5.092000E-01, 4.821000E-00}, - - {1.002000E+01, 1.234000E+03, 2.125000E+04, 6.012000E-03, 5.902000E-03, 1.069000E+00, 0.174000E+00, 1.221000E-02, 6.952000E-06, 6.596000E-05, 8.472000E-02, 1.543000E-01, 6.012000E-00, 8.902000E-01, 9.210000E-01, 1.734000E+04, 6.921000E-04, false, 4.521923E-05, 4.592698E-03, 6.091000E-00, 6.743000E-01, 5.602000E-00}, - {1.152000E+01, 2.120000E+03, 4.568000E+04, 6.342000E-03, 9.210000E-03, 1.069000E+00, 0.374000E+00, 1.221000E-02, 3.952000E-06, 6.596000E-05, 1.098000E-01, 3.456000E-01, 7.241000E-00, 9.102000E-01, 1.002000E-00, 1.734000E+04, 7.901000E-04, false, 5.236542E-05, 5.678873E-03, 7.231000E-00, 8.321000E-01, 6.092000E-00}, - {1.252000E+01, 3.145000E+03, 8.213000E+04, 9.290000E-03, 1.034000E-02, 1.069000E+00, 0.123000E+00, 1.221000E-02, 1.952000E-06, 6.596000E-05, 2.340006E-01, 5.632000E-01, 8.452000E-00, 1.003000E-01, 2.145000E-00, 1.734000E+04, 9.212000E-04, false, 6.732276E-05, 7.321873E-03, 8.234000E-00, 9.023000E-01, 7.201000E-00}, - {1.352000E+01, 4.742000E+03, 1.014000E+05, 1.234000E-02, 1.456000E-02, 1.069000E+00, 0.123000E+00, 1.221000E-02, 9.952000E-07, 6.596000E-05, 4.123000E-01, 6.128000E-01, 9.076000E-00, 2.831000E-01, 3.902000E-00, 1.734000E+04, 1.023000E-03, false, 7.902887E-05, 9.032908E-03, 9.021000E-00, 1.092000E-01, 8.096000E-00} + {4.056000E-03, 1.023000E+02, 1.201000E+02, 9.002000E-04, 8.215000E-04, 8.852000E-01, latvap, latice, 5.100000E-03, 9.558000E-04, 1.213000E-03, 9.653000E-04, 1.023000E-01, 4.098000E-02, 2.098000E-02, 9.952000E+03, 1.023000E-05, false, 1.241456E-04, 9.021345E-02, 1.043000E-01, 1.921000E-02, 0.242000E-02}, + {6.852000E-02, 1.120000E+02, 2.450000E+02, 9.321000E-04, 9.124000E-04, 8.852000E-01, latvap, latice, 4.100000E-03, 9.558000E-04, 2.560000E-03, 1.764000E-03, 2.346000E-01, 5.632000E-02, 3.024000E-02, 9.952000E+03, 2.093000E-05, false, 2.341678E-04, 1.092432E-02, 2.903000E-01, 2.125000E-02, 0.342000E-02}, + {8.852000E-02, 1.210000E+02, 3.420000E+02, 9.623000E-04, 9.432000E-04, 8.900000E-01, latvap, latice, 3.100000E-03, 9.558000E-04, 3.211000E-03, 3.421000E-03, 3.421000E-01, 6.542000E-02, 4.567000E-02, 9.952000E+03, 3.091000E-05, false, 3.215234E-04, 2.098987E-02, 3.450000E-01, 3.490000E-02, 0.932000E-02}, + {1.902000E-01, 1.326000E+02, 4.321000E+02, 9.982000E-04, 9.623000E-04, 9.900000E-01, latvap, latice, 2.100000E-03, 9.558000E-04, 4.121000E-03, 4.569000E-03, 4.673000E-01, 7.902000E-02, 5.321000E-02, 9.952000E+03, 4.521000E-05, false, 4.675567E-04, 3.214982E-02, 4.290000E-01, 4.590000E-02, 1.025000E-01}, + + {2.201000E-01, 1.456000E+02, 5.670000E+02, 1.234000E-03, 9.723000E-04, 0.100000E+01, latvap, latice, 1.100000E-03, 2.550008E-05, 4.980000E-03, 5.621000E-03, 5.420000E-01, 8.021000E-02, 6.902000E-02, 9.952000E+04, 5.678000E-05, false, 5.389236E-04, 4.125969E-02, 5.098000E-01, 5.921000E-02, 2.031000E-01}, + {3.502000E-01, 1.780009E+02, 6.832000E+02, 1.562000E-03, 1.024000E-03, 0.100000E+01, latvap, latice, 8.100000E-04, 2.558000E-05, 5.643000E-03, 7.367000E-03, 6.782000E-01, 9.253000E-02, 8.045000E-02, 9.952000E+04, 6.902000E-05, false, 6.432654E-04, 5.389457E-02, 6.723000E-01, 6.093000E-02, 4.098000E-01}, + {4.852000E-01, 2.100009E+02, 7.090000E+02, 2.101000E-03, 1.235000E-03, 0.100000E+01, latvap, latice, 4.100000E-04, 2.558000E-05, 7.892000E-03, 9.087000E-03, 8.213000E-01, 1.256000E-01, 9.134000E-02, 9.952000E+04, 8.367000E-05, false, 7.210983E-04, 6.476985E-02, 8.902000E-01, 8.345000E-02, 8.023000E-01}, + {5.852000E-01, 2.310000E+02, 9.215000E+02, 2.312000E-03, 1.456000E-03, 0.100000E+01, latvap, latice, 2.100000E-04, 2.558000E-05, 9.321000E-03, 1.245000E-02, 1.067000E-00, 2.347000E-01, 1.092000E-01, 9.952000E+04, 9.098000E-05, false, 8.543367E-04, 8.213186E-02, 9.021000E-01, 9.321000E-02, 9.098000E-01}, + + {6.852000E-01, 2.563000E+02, 1.089000E+03, 3.601000E-03, 1.864000E-03, 0.950000E+00, latvap, latice, 9.952000E-05, 4.596000E-05, 1.453000E-02, 2.543000E-02, 2.345000E-00, 3.578000E-01, 2.873000E-01, 1.734000E+04, 1.023000E-04, false, 9.021215E-04, 9.023367E-02, 1.023000E-00, 1.056000E-01, 1.256000E-00}, + {7.852000E-01, 2.789000E+02, 3.754000E+03, 3.891000E-03, 2.093000E-03, 0.950000E+00, latvap, latice, 4.952000E-05, 4.596000E-05, 2.789000E-02, 4.367000E-02, 3.890000E-00, 4.980000E-01, 3.468000E-01, 1.734000E+04, 2.146000E-04, false, 1.043468E-05, 1.094854E-02, 2.012000E-00, 2.893000E-01, 2.903000E-00}, + {8.852000E-01, 3.123000E+02, 8.902000E+03, 4.872000E-03, 2.345000E-03, 0.950000E+00, latvap, latice, 1.952000E-05, 4.596000E-05, 4.256000E-02, 6.324000E-02, 4.120000E-00, 6.321000E-01, 4.890000E-01, 1.734000E+04, 4.321000E-04, false, 2.341763E-05, 2.126247E-03, 3.120000E-00, 3.456000E-01, 3.912000E-00}, + {9.852000E-01, 4.981000E+02, 1.092000E+04, 5.210000E-03, 3.210000E-03, 0.950000E+00, latvap, latice, 9.952000E-06, 4.596000E-05, 6.821000E-02, 8.789000E-02, 5.320000E-00, 7.982000E-01, 6.921000E-01, 1.734000E+04, 5.821000E-04, false, 3.901479E-05, 3.874763E-03, 5.902000E-00, 5.092000E-01, 4.821000E-00}, + + {1.002000E+01, 1.234000E+03, 2.125000E+04, 6.012000E-03, 5.902000E-03, 1.069000E+00, latvap, latice, 6.952000E-06, 6.596000E-05, 8.472000E-02, 1.543000E-01, 6.012000E-00, 8.902000E-01, 9.210000E-01, 1.734000E+04, 6.921000E-04, false, 4.521923E-05, 4.592698E-03, 6.091000E-00, 6.743000E-01, 5.602000E-00}, + {1.152000E+01, 2.120000E+03, 4.568000E+04, 6.342000E-03, 9.210000E-03, 1.069000E+00, latvap, latice, 3.952000E-06, 6.596000E-05, 1.098000E-01, 3.456000E-01, 7.241000E-00, 9.102000E-01, 1.002000E-00, 1.734000E+04, 7.901000E-04, false, 5.236542E-05, 5.678873E-03, 7.231000E-00, 8.321000E-01, 6.092000E-00}, + {1.252000E+01, 3.145000E+03, 8.213000E+04, 9.290000E-03, 1.034000E-02, 1.069000E+00, latvap, latice, 1.952000E-06, 6.596000E-05, 2.340006E-01, 5.632000E-01, 8.452000E-00, 1.003000E-01, 2.145000E-00, 1.734000E+04, 9.212000E-04, false, 6.732276E-05, 7.321873E-03, 8.234000E-00, 9.023000E-01, 7.201000E-00}, + {1.352000E+01, 4.742000E+03, 1.014000E+05, 1.234000E-02, 1.456000E-02, 1.069000E+00, latvap, latice, 9.952000E-07, 6.596000E-05, 4.123000E-01, 6.128000E-01, 9.076000E-00, 2.831000E-01, 3.902000E-00, 1.734000E+04, 1.023000E-03, false, 7.902887E-05, 9.032908E-03, 9.021000E-00, 1.092000E-01, 8.096000E-00} }; // Sync to device @@ -64,7 +67,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { const Int offset = i * Spack::n; // Init pack inputs - Spack rho,temp, pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,kap,mu,sc, + Spack rho,temp, pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,kap,mu,sc, qv,qc_incld,qi_incld,ni_incld,qr_incld; Smask log_wetgrowth; @@ -78,8 +81,6 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { rhofaci[s] = self_device(vs).rhofaci; table_val_qi2qr_melting[s] = self_device(vs).table_val_qi2qr_melting; table_val_qi2qr_vent_melt[s] = self_device(vs).table_val_qi2qr_vent_melt; - latent_heat_vapor[s] = self_device(vs).latent_heat_vapor; - latent_heat_fusion[s] = self_device(vs).latent_heat_fusion; dv[s] = self_device(vs).dv; kap[s] = self_device(vs).kap; mu[s] = self_device(vs).mu; @@ -97,7 +98,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { log_wetgrowth.set(s, self_device(vs).log_wetgrowth); } - Functions::ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, + Functions::ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, dv, kap, mu, sc, qv, qc_incld, qi_incld, ni_incld, qr_incld, log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend); diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 3e340f68710a..28c6582f8a14 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -23,29 +23,31 @@ struct UnitWrap::UnitTest::TestP3IceMelting { static void ice_melting_bfb(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; // make array of input data (why not pass actual variables?). Copied 1st 4 rows 4x to fill pack size. IceMeltingData IceMelt[max_pack_size] = { //rho, T_atm, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, sc, mu, kap, qv, qi_incld,ni_incld - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05} + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05} }; // Sync to device @@ -65,7 +67,7 @@ static void ice_melting_bfb(){ const Int offset = i * Spack::n; // Init pack inputs - Spack rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend; + Spack rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { rho[s] = IceMelt_device(vs).rho; T_atm[s] = IceMelt_device(vs).T_atm; @@ -73,8 +75,6 @@ static void ice_melting_bfb(){ rhofaci[s] = IceMelt_device(vs).rhofaci; table_val_qi2qr_melting[s] = IceMelt_device(vs).table_val_qi2qr_melting; table_val_qi2qr_vent_melt[s] = IceMelt_device(vs).table_val_qi2qr_vent_melt; - latent_heat_vapor[s] = IceMelt_device(vs).latent_heat_vapor; - latent_heat_fusion[s] = IceMelt_device(vs).latent_heat_fusion; dv[s] = IceMelt_device(vs).dv; sc[s] = IceMelt_device(vs).sc; mu[s] = IceMelt_device(vs).mu; @@ -86,7 +86,7 @@ static void ice_melting_bfb(){ ni2nr_melt_tend[s] = IceMelt_device(vs).ni2nr_melt_tend; } - Functions::ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend); + Functions::ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { IceMelt_device(vs).qi2qr_melt_tend = qi2qr_melt_tend[s]; diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 02167261bf86..93c06a9054d7 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -187,6 +187,8 @@ static void run_bfb_ice_sed() static void run_bfb_homogeneous_freezing() { + constexpr Scalar latice = C::LatIce; + auto engine = setup_random_test(); HomogeneousFreezingData hfds_fortran[] = { @@ -203,6 +205,12 @@ static void run_bfb_homogeneous_freezing() for (auto& d : hfds_fortran) { const auto qsmall_r = std::make_pair(C::QSMALL/2, C::QSMALL*2); d.randomize(engine, { {d.T_atm, {C::T_homogfrz - 10, C::T_homogfrz + 10}}, {d.qc, qsmall_r}, {d.qr, qsmall_r} }); + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + for (int k=0; k::TestIceSupersatConservation { static void run_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + auto engine = setup_random_test(); IceSupersatConservationData f90_data[max_pack_size]; @@ -27,6 +30,10 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { for (auto& d : f90_data) { d.randomize(engine); d.dt = f90_data[0].dt; // hold this fixed, it is not packed data + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + d.latent_heat_sublim = latvap+latice; } // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that @@ -46,10 +53,9 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { const Int offset = i * Spack::n; // Init pack inputs - Spack cld_frac_i, latent_heat_sublim, qidep, qinuc, qv, qv_sat_i, t_atm, qi2qv_sublim_tend, qr2qv_evap_tend; + Spack cld_frac_i, qidep, qinuc, qv, qv_sat_i, t_atm, qi2qv_sublim_tend, qr2qv_evap_tend; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { cld_frac_i[s] = cxx_device(vs).cld_frac_i; - latent_heat_sublim[s] = cxx_device(vs).latent_heat_sublim; qidep[s] = cxx_device(vs).qidep; qinuc[s] = cxx_device(vs).qinuc; qv[s] = cxx_device(vs).qv; @@ -59,7 +65,7 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { qr2qv_evap_tend[s] = cxx_device(vs).qr2qv_evap_tend; } - Functions::ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, cxx_device(offset).dt, qi2qv_sublim_tend, qr2qv_evap_tend); + Functions::ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, t_atm, cxx_device(offset).dt, qi2qv_sublim_tend, qr2qv_evap_tend); // Copy spacks back into cxx_device view for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index bad0ca7014da..a804bd8756d5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -57,6 +57,8 @@ static void run_bfb_p3_main_part1() constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar sup_upper = -0.05; constexpr Scalar sup_lower = -0.1; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; P3MainPart1Data isds_fortran[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt @@ -74,6 +76,14 @@ static void run_bfb_p3_main_part1() {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, {d.qv_supersat_i, {sup_lower -.05, sup_upper + .05}}, {d.qc, qsmall_r}, {d.qr, qsmall_r}, {d.qi, qsmall_r} }); + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + for (int k=0; k(); } diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index 55501a30aa33..a1b270562819 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -25,14 +25,14 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { using physics = scream::physics::Functions; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //Start with reasonable values //============================ Spack pres(100000); Spack t_atm(270); Spack qv(1.7e-3); - Spack latent_heat_vapor(2.5e6); - Spack latent_heat_sublim(2.838e6); Scalar dt=60; Spack qidep(0); Spack qinuc(0); //dep and nuc only used together, so only fiddle with one @@ -50,8 +50,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Spack qv_sinks=qidep + qinuc; Spack qv_sources=qi2qv_sublim_tend_tmp + qr2qv_evap_tend_tmp; Spack qv_endstep=qv - qv_sinks*dt + qv_sources*dt; - Spack T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend_tmp*latent_heat_vapor*inv_cp )*dt; + Spack T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*(latvap+latice)*inv_cp + - qr2qv_evap_tend_tmp*latvap*inv_cp )*dt; Spack qsl = physics::qv_sat_dry(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice //just require index 0 since all entries are identical @@ -59,7 +59,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { REQUIRE(qv_endstep[0]>qsl[0]); // inputs to this test are testing what we want. //Now update sublim and evap tends using prevent_liq_supersaturation - Functions::prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, + Functions::prevent_liq_supersaturation(pres, t_atm, qv, dt, qidep, qinuc, qi2qv_sublim_tend_tmp, qr2qv_evap_tend_tmp); //Finally, recompute liquid saturation @@ -67,8 +67,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { qv_sinks=qidep + qinuc; qv_sources=qi2qv_sublim_tend_tmp + qr2qv_evap_tend_tmp; qv_endstep=qv - qv_sinks*dt + qv_sources*dt; - T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend_tmp*latent_heat_vapor*inv_cp )*dt; + T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*(latvap+latice)*inv_cp + - qr2qv_evap_tend_tmp*latvap*inv_cp )*dt; qsl = physics::qv_sat_dry(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice //just require index 0 since all entries are identical @@ -80,7 +80,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Spack qi2qv_sublim_tend_tmp2(1e-4); Spack qr2qv_evap_tend_tmp2(1e-4); - Functions::prevent_liq_supersaturation(pres, t_atm, qv_tmp, latent_heat_vapor, latent_heat_sublim, + Functions::prevent_liq_supersaturation(pres, t_atm, qv_tmp, dt, qidep, qinuc, qi2qv_sublim_tend_tmp2, qr2qv_evap_tend_tmp2); //just require index 0 since all entries are identical. REQUIRE( qi2qv_sublim_tend_tmp2[0] ==0 ); @@ -90,6 +90,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { static void run_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; auto engine = setup_random_test(); @@ -100,6 +102,11 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { for (auto& d : f90_data) { d.randomize(engine); d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + d.latent_heat_vapor = latvap; + d.latent_heat_sublim = latvap+latice; } // Create copies of data for use by cxx and sync it to device. Needs to happen before @@ -127,11 +134,9 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { // Init pack inputs Scalar dt; - Spack latent_heat_sublim, latent_heat_vapor, pres, qi2qv_sublim_tend, qidep, qinuc, qr2qv_evap_tend, qv, t_atm; + Spack pres, qi2qv_sublim_tend, qidep, qinuc, qr2qv_evap_tend, qv, t_atm; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { dt = cxx_device(vs).dt; //dt is scalar but PreventLiqSupersaturationData has diff val for each row. - latent_heat_sublim[s] = cxx_device(vs).latent_heat_sublim; - latent_heat_vapor[s] = cxx_device(vs).latent_heat_vapor; pres[s] = cxx_device(vs).pres; qi2qv_sublim_tend[s] = cxx_device(vs).qi2qv_sublim_tend; qidep[s] = cxx_device(vs).qidep; @@ -141,7 +146,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { t_atm[s] = cxx_device(vs).t_atm; } - Functions::prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend); + Functions::prevent_liq_supersaturation(pres, t_atm, qv, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend); // Copy spacks back into cxx_device view for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index df5b80cdc19e..3b994cfdad5d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -480,103 +480,105 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar dt = 1.8000E+03; constexpr bool do_predict_nc = true; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following P3UpdatePrognosticIceData pupidc[max_pack_size] = { {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, }; @@ -599,9 +601,9 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce const Int offset = i * Spack::n; // Init pack inputs - Spack qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, - ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, latent_heat_fusion, latent_heat_sublim, + Spack qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, + qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, + ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, rho_qm_cloud, th_atm, qv, qc, nc, qr, nr, qi, ni, qm, bm; Scalar dt; bool do_predict_nc; @@ -633,8 +635,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce ni_sublim_tend[s] = pupidc_device(vs).ni_sublim_tend; qc2qi_berg_tend[s] = pupidc_device(vs).qc2qi_berg_tend; inv_exner[s] = pupidc_device(vs).inv_exner; - latent_heat_fusion[s] = pupidc_device(vs).latent_heat_fusion; - latent_heat_sublim[s] = pupidc_device(vs).latent_heat_sublim; rho_qm_cloud[s] = pupidc_device(vs).rho_qm_cloud; th_atm[s] = pupidc_device(vs).th_atm; @@ -654,7 +654,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce Functions::update_prognostic_ice(qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend,ncshdc, qr2qi_collect_tend, nr_collect_tend, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, ni_nucleat_tend, - ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, latent_heat_sublim, latent_heat_fusion, + ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, do_predict_nc, log_wetgrowth, dt, pupidc_device(0).nmltratio, rho_qm_cloud, th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr); @@ -705,26 +705,28 @@ template struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables { static void get_time_space_phys_variables_unit_bfb_tests(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following GetTimeSpacePhysVarsData gtspvd[max_pack_size] = { // T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i - {2.9792E+02, 9.8711E+04, 1.1532E+00, 2.5010E+06, 2.8347E+06, 2.0321E-02, 2.0321E-02}, - {2.9792E+02, 9.8711E+04, 1.1532E+00, 2.5010E+06, 2.8347E+06, 2.0321E-02, 2.0321E-02}, - {2.9583E+02, 9.7322E+04, 1.1449E+00, 2.5010E+06, 2.8347E+06, 1.8120E-02, 1.8120E-02}, - {2.9375E+02, 9.5933E+04, 1.1366E+00, 2.5010E+06, 2.8347E+06, 1.6134E-02, 1.6134E-02}, - {2.8959E+02, 9.3156E+04, 1.1196E+00, 2.5010E+06, 2.8347E+06, 1.2729E-02, 1.2729E-02}, - {2.8750E+02, 9.1767E+04, 1.1109E+00, 2.5010E+06, 2.8347E+06, 1.1279E-02, 1.1279E-02}, - {2.8542E+02, 9.0378E+04, 1.1020E+00, 2.5010E+06, 2.8347E+06, 9.9759E-03, 9.9759E-03}, - {2.8334E+02, 8.8989E+04, 1.0931E+00, 2.5010E+06, 2.8347E+06, 8.8076E-03, 8.8076E-03}, - {2.8125E+02, 8.7600E+04, 1.0840E+00, 2.5010E+06, 2.8347E+06, 7.7615E-03, 7.7615E-03}, - {2.7917E+02, 8.6211E+04, 1.0748E+00, 2.5010E+06, 2.8347E+06, 6.8265E-03, 6.8265E-03}, - {2.7709E+02, 8.4822E+04, 1.0654E+00, 2.5010E+06, 2.8347E+06, 5.9921E-03, 5.9921E-03}, - {2.7501E+02, 8.3433E+04, 1.0559E+00, 2.5010E+06, 2.8347E+06, 5.2488E-03, 5.2488E-03}, - {2.7292E+02, 8.2044E+04, 1.0463E+00, 2.5010E+06, 2.8347E+06, 4.5879E-03, 4.5766E-03}, - {2.7084E+02, 8.0656E+04, 1.0365E+00, 2.5010E+06, 2.8347E+06, 4.0015E-03, 3.9112E-03}, - {2.6876E+02, 7.9267E+04, 1.0265E+00, 2.5010E+06, 2.8347E+06, 3.4821E-03, 3.3349E-03}, - {2.6667E+02, 7.7878E+04, 1.0164E+00, 2.5010E+06, 2.8347E+06, 3.0231E-03, 2.8368E-03}, + {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, + {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, + {2.9583E+02, 9.7322E+04, 1.1449E+00, latvap, latvap+latice, 1.8120E-02, 1.8120E-02}, + {2.9375E+02, 9.5933E+04, 1.1366E+00, latvap, latvap+latice, 1.6134E-02, 1.6134E-02}, + {2.8959E+02, 9.3156E+04, 1.1196E+00, latvap, latvap+latice, 1.2729E-02, 1.2729E-02}, + {2.8750E+02, 9.1767E+04, 1.1109E+00, latvap, latvap+latice, 1.1279E-02, 1.1279E-02}, + {2.8542E+02, 9.0378E+04, 1.1020E+00, latvap, latvap+latice, 9.9759E-03, 9.9759E-03}, + {2.8334E+02, 8.8989E+04, 1.0931E+00, latvap, latvap+latice, 8.8076E-03, 8.8076E-03}, + {2.8125E+02, 8.7600E+04, 1.0840E+00, latvap, latvap+latice, 7.7615E-03, 7.7615E-03}, + {2.7917E+02, 8.6211E+04, 1.0748E+00, latvap, latvap+latice, 6.8265E-03, 6.8265E-03}, + {2.7709E+02, 8.4822E+04, 1.0654E+00, latvap, latvap+latice, 5.9921E-03, 5.9921E-03}, + {2.7501E+02, 8.3433E+04, 1.0559E+00, latvap, latvap+latice, 5.2488E-03, 5.2488E-03}, + {2.7292E+02, 8.2044E+04, 1.0463E+00, latvap, latvap+latice, 4.5879E-03, 4.5766E-03}, + {2.7084E+02, 8.0656E+04, 1.0365E+00, latvap, latvap+latice, 4.0015E-03, 3.9112E-03}, + {2.6876E+02, 7.9267E+04, 1.0265E+00, latvap, latvap+latice, 3.4821E-03, 3.3349E-03}, + {2.6667E+02, 7.7878E+04, 1.0164E+00, latvap, latvap+latice, 3.0231E-03, 2.8368E-03}, }; // Sync to device @@ -745,14 +747,12 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables const Int offset = i * Spack::n; // Init pack inputs - Spack T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; + Spack T_atm, pres, rho, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { T_atm[s] = gtspvd_device(vs).T_atm; pres[s] = gtspvd_device(vs).pres; rho[s] = gtspvd_device(vs).rho; - latent_heat_vapor[s] = gtspvd_device(vs).latent_heat_vapor; - latent_heat_sublim[s] = gtspvd_device(vs).latent_heat_sublim; qv_sat_l[s] = gtspvd_device(vs).qv_sat_l; qv_sat_i[s] = gtspvd_device(vs).qv_sat_i; @@ -767,7 +767,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables eii[s] = gtspvd_device(vs).eii; } - Functions::get_time_space_phys_variables(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, + Functions::get_time_space_phys_variables(T_atm, pres, rho, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii); // Copy results back into views @@ -775,8 +775,6 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables gtspvd_device(vs).T_atm = T_atm[s]; gtspvd_device(vs).pres = pres[s]; gtspvd_device(vs).rho = rho[s]; - gtspvd_device(vs).latent_heat_vapor = latent_heat_vapor[s]; - gtspvd_device(vs).latent_heat_sublim = latent_heat_sublim[s]; gtspvd_device(vs).qv_sat_l = qv_sat_l[s]; gtspvd_device(vs).qv_sat_i = qv_sat_i[s]; @@ -820,72 +818,73 @@ template struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq { static void update_prognostic_liquid_unit_bfb_tests(){ + constexpr Scalar latvap = C::LatVap; //fortran generated data is input to the following P3UpdatePrognosticLiqData pupldc[max_pack_size] = { {1.0631E-12, 1.0631E+00, 1.5833E-12, 1.5833E+00, 2.4190E-02, 0.0000E+00, 0.0000E+00, 0.0000E+00, 4.2517E+00, - true , true , 8.6718E-01, 1.0037E+00, 2.5010E+06, 1.8000E+03, 2.9902E+02, 5.0000E-02, 1.0000E-06, 1.0000E+06, 1.0010E-06, + true , true , 8.6718E-01, 1.0037E+00, latvap, 1.8000E+03, 2.9902E+02, 5.0000E-02, 1.0000E-06, 1.0000E+06, 1.0010E-06, 6.3726E+05}, {3.2784E-08, 1.8780E+07, 2.1753E-11, 1.2461E+04, 7.8657E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8748E+04, - true , true , 9.8387E-01, 1.0741E+00, 2.5010E+06, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, + true , true , 9.8387E-01, 1.0741E+00, latvap, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, -8.6159E+07}, {3.2796E-09, 1.8778E+07, 1.8830E-12, 1.0782E+04, 6.8061E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.3698E+04, - true , true , 9.0740E-01, 1.0293E+00, 2.5010E+06, 1.8000E+03, 2.9376E+02, 5.0000E-03, 5.9067E-06,-6.9543E+09, 1.0439E-04, + true , true , 9.0740E-01, 1.0293E+00, latvap, 1.8000E+03, 2.9376E+02, 5.0000E-03, 5.9067E-06,-6.9543E+09, 1.0439E-04, -1.6967E+07}, {6.5634E-09, 1.8778E+07, 3.8238E-12, 1.0940E+04, 6.9061E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.3181E+04, - true , true , 9.1484E-01, 1.0339E+00, 2.5010E+06, 1.8000E+03, 2.9291E+02, 5.0000E-03, 1.1821E-05,-6.9282E+09, 1.0615E-04, + true , true , 9.1484E-01, 1.0339E+00, latvap, 1.8000E+03, 2.9291E+02, 5.0000E-03, 1.1821E-05,-6.9282E+09, 1.0615E-04, -2.8223E+07}, {9.8516E-09, 1.8779E+07, 5.8258E-12, 1.1105E+04, 7.0101E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.2655E+04, - true , true , 9.2251E-01, 1.0386E+00, 2.5010E+06, 1.8000E+03, 2.9206E+02, 5.0000E-03, 1.7743E-05,-6.9009E+09, 1.0790E-04, + true , true , 9.2251E-01, 1.0386E+00, latvap, 1.8000E+03, 2.9206E+02, 5.0000E-03, 1.7743E-05,-6.9009E+09, 1.0790E-04, -3.9628E+07}, {1.3145E-08, 1.8779E+07, 7.8929E-12, 1.1276E+04, 7.1180E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.2122E+04, - true , true , 9.3043E-01, 1.0433E+00, 2.5010E+06, 1.8000E+03, 2.9123E+02, 5.0000E-03, 2.3674E-05,-6.8725E+09, 1.0963E-04, + true , true , 9.3043E-01, 1.0433E+00, latvap, 1.8000E+03, 2.9123E+02, 5.0000E-03, 2.3674E-05,-6.8725E+09, 1.0963E-04, -5.1189E+07}, {1.6443E-08, 1.8779E+07, 1.0029E-11, 1.1454E+04, 7.2303E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.1581E+04, - true , true , 9.3860E-01, 1.0482E+00, 2.5010E+06, 1.8000E+03, 2.9040E+02, 5.0000E-03, 2.9615E-05,-6.8428E+09, 1.1136E-04, + true , true , 9.3860E-01, 1.0482E+00, latvap, 1.8000E+03, 2.9040E+02, 5.0000E-03, 2.9615E-05,-6.8428E+09, 1.1136E-04, -6.2915E+07}, {1.9746E-08, 1.8779E+07, 1.2238E-11, 1.1639E+04, 7.3471E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.1031E+04, - true , true , 9.4705E-01, 1.0531E+00, 2.5010E+06, 1.8000E+03, 2.8958E+02, 5.0000E-03, 3.5565E-05,-6.8117E+09, 1.1308E-04, + true , true , 9.4705E-01, 1.0531E+00, latvap, 1.8000E+03, 2.8958E+02, 5.0000E-03, 3.5565E-05,-6.8117E+09, 1.1308E-04, -7.4813E+07}, {2.3047E-08, 1.8779E+07, 1.4521E-11, 1.1832E+04, 7.4688E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.0474E+04, - true , true , 9.5579E-01, 1.0582E+00, 2.5010E+06, 1.8000E+03, 2.8941E+02, 4.7949E-03, 4.1510E-05,-6.7792E+09, 1.4787E-05, + true , true , 9.5579E-01, 1.0582E+00, latvap, 1.8000E+03, 2.8941E+02, 4.7949E-03, 4.1510E-05,-6.7792E+09, 1.4787E-05, -8.2885E+07}, {2.6289E-08, 1.8779E+07, 1.6845E-11, 1.2033E+04, 7.5955E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.9907E+04, - true , true , 9.6483E-01, 1.0634E+00, 2.5010E+06, 1.8000E+03, 2.8972E+02, 4.4341E-03, 4.7350E-05,-6.7452E+09,-4.7350E-05, + true , true , 9.6483E-01, 1.0634E+00, latvap, 1.8000E+03, 2.8972E+02, 4.4341E-03, 4.7350E-05,-6.7452E+09,-4.7350E-05, -8.3634E+07}, {2.9533E-08, 1.8779E+07, 1.9253E-11, 1.2242E+04, 7.7277E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.9332E+04, - true , false , 9.7418E-01, 1.0686E+00, 2.5010E+06, 1.8000E+03, 2.9002E+02, 4.0751E-03, 5.3194E-05,-6.7096E+09,-5.3194E-05, + true , false , 9.7418E-01, 1.0686E+00, latvap, 1.8000E+03, 2.9002E+02, 4.0751E-03, 5.3194E-05,-6.7096E+09,-5.3194E-05, -8.4862E+07}, {3.2784E-08, 1.8780E+07, 2.1753E-11, 1.2461E+04, 7.8657E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8748E+04, - true , false , 9.8387E-01, 1.0741E+00, 2.5010E+06, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, + true , false , 9.8387E-01, 1.0741E+00, latvap, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, -8.6159E+07}, {3.6045E-08, 1.8780E+07, 2.4356E-11, 1.2689E+04, 8.0098E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8154E+04, - true , false , 9.9391E-01, 1.0796E+00, 2.5010E+06, 1.8000E+03, 2.9063E+02, 3.3756E-03, 6.4925E-05,-6.6333E+09,-6.4925E-05, + true , false , 9.9391E-01, 1.0796E+00, latvap, 1.8000E+03, 2.9063E+02, 3.3756E-03, 6.4925E-05,-6.6333E+09,-6.4925E-05, -8.7530E+07}, {3.9321E-08, 1.8780E+07, 2.7069E-11, 1.2928E+04, 8.1605E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.7552E+04, - true , false , 1.0043E+00, 1.0853E+00, 2.5010E+06, 1.8000E+03, 2.9092E+02, 3.0417E-03, 7.0827E-05,-6.5924E+09,-7.0827E-05, + true , false , 1.0043E+00, 1.0853E+00, latvap, 1.8000E+03, 2.9092E+02, 3.0417E-03, 7.0827E-05,-6.5924E+09,-7.0827E-05, -8.8982E+07}, {4.2614E-08, 1.8780E+07, 2.9903E-11, 1.3178E+04, 8.3182E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.6939E+04, - true , false , 1.0151E+00, 1.0911E+00, 2.5010E+06, 1.8000E+03, 2.9119E+02, 2.7224E-03, 7.6760E-05,-6.5494E+09,-7.6760E-05, + true , false , 1.0151E+00, 1.0911E+00, latvap, 1.8000E+03, 2.9119E+02, 2.7224E-03, 7.6760E-05,-6.5494E+09,-7.6760E-05, -9.0523E+07}, {4.5927E-08, 1.8780E+07, 3.2867E-11, 1.3440E+04, 8.4833E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.6317E+04, - true , false , 1.0263E+00, 1.0970E+00, 2.5010E+06, 1.8000E+03, 2.9143E+02, 2.4202E-03, 8.2728E-05,-6.5044E+09,-8.2728E-05, + true , false , 1.0263E+00, 1.0970E+00, latvap, 1.8000E+03, 2.9143E+02, 2.4202E-03, 8.2728E-05,-6.5044E+09,-8.2728E-05, -9.0778E+07}, }; @@ -908,7 +907,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq // Init pack inputs Spack qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, inv_rho, - inv_exner, latent_heat_vapor, th_atm, qv, qc, nc, qr, nr; + inv_exner, th_atm, qv, qc, nc, qr, nr; bool do_predict_nc, do_prescribed_CCN; Scalar dt; @@ -929,7 +928,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq nr_selfcollect_tend[s] = pupldc_device(vs).nr_selfcollect_tend; inv_rho[s] = pupldc_device(vs).inv_rho; inv_exner[s] = pupldc_device(vs).inv_exner; - latent_heat_vapor[s] = pupldc_device(vs).latent_heat_vapor; th_atm[s] = pupldc_device(vs).th_atm; qv[s] = pupldc_device(vs).qv; @@ -941,7 +939,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq Functions::update_prognostic_liquid(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, - latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr); + dt, th_atm, qv, qc, nc, qr, nr); // Copy results back into views pupldc_device(0).dt = dt; @@ -959,7 +957,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq pupldc_device(vs).nr_selfcollect_tend = nr_selfcollect_tend[s]; pupldc_device(vs).inv_rho = inv_rho[s]; pupldc_device(vs).inv_exner = inv_exner[s]; - pupldc_device(vs).latent_heat_vapor = latent_heat_vapor[s]; pupldc_device(vs).th_atm = th_atm[s]; pupldc_device(vs).qv = qv[s]; From 3a85062a5d11f0353705be47ed8902388dd5d390 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 3 Sep 2024 13:14:25 -0600 Subject: [PATCH 650/904] Use kokkos scratch pad for local per-team bools --- .../eamxx/src/physics/p3/impl/p3_main_impl.hpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index a36a79364044..964ec21be700 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -87,9 +87,11 @@ ::p3_main_internal( const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; + using ScratchViewType = Kokkos::View; const Int nk_pack = ekat::npack(nk); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + const auto scratch_size = ScratchViewType::shmem_size(2); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack).set_scratch_size(0, Kokkos::PerTeam(scratch_size)); // load constants into local vars const Scalar inv_dt = 1 / infrastructure.dt; @@ -98,9 +100,6 @@ ::p3_main_internal( const Int kbot = kdir == -1 ? nk-1 : 0; constexpr bool debug_ABORT = false; - // per-column bools - view_2d bools("bools", nj, 2); - // we do not want to measure init stuff auto start = std::chrono::steady_clock::now(); @@ -201,9 +200,11 @@ ::p3_main_internal( const auto oqv_prev = ekat::subview(diagnostic_inputs.qv_prev, i); const auto ot_prev = ekat::subview(diagnostic_inputs.t_prev, i); - // Need to watch out for race conditions with these shared variables - bool &nucleationPossible = bools(i, 0); - bool &hydrometeorsPresent = bools(i, 1); + // Use Kokkos' scratch pad for allocating 2 bools + // per team to determine early exits + ScratchViewType bools(team.team_scratch(0), 2); + bool &nucleationPossible = bools(0); + bool &hydrometeorsPresent = bools(1); view_1d_ptr_array zero_init = { &mu_r, &lamr, &logn0r, &nu, &cdist, &cdist1, &cdistr, From 344bacac97aa21aa158bce89d1680241be3079df Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Wed, 4 Sep 2024 15:16:50 -0500 Subject: [PATCH 651/904] Fix linting of markdown Fix linting of markdown --- docs/user-guide/index.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 25d2d1c74c07..53fa28d64e25 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -62,8 +62,9 @@ for each of the historical ("20TR") compsets including the single-forcing compse |`WCYCL20TR-lulc`| 1850 | 1850 | 1850 | 1850 | 1850 | 1850 | varying | |`WCYCL20TR-volc`| 1850 | 1850 | 1850 | 1850 | varying | 1850 | 1850 | - -- *Volcano* refers to stratospheric volcanic SO2 emissions; *1850* for *Volcano refers to background (average) stratospheric volcanic emissions used in pre-industrial control experiments + +- *Volcano* refers to stratospheric volcanic SO2 emissions; *1850* for *Volcano* refers to background (average) stratospheric volcanic emissions used in pre-industrial control experiments + - *Oxidants* always follow *Aerosol & precursors* for fixed or varying. The compsets for the other two science simulation campaigns are being finalized, with additional components and/or features. From c23345d82da5667a9ebaacf05a7f1d87a7789fbb Mon Sep 17 00:00:00 2001 From: noel Date: Wed, 4 Sep 2024 13:20:22 -0700 Subject: [PATCH 652/904] use different routine signature -- tested correct what looks like an incorrect if conditional for scm and convert it to positive. only nullify is scm --- components/eam/src/dynamics/se/dyn_grid.F90 | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/components/eam/src/dynamics/se/dyn_grid.F90 b/components/eam/src/dynamics/se/dyn_grid.F90 index 0faed9ea1d32..125cfbca7a60 100644 --- a/components/eam/src/dynamics/se/dyn_grid.F90 +++ b/components/eam/src/dynamics/se/dyn_grid.F90 @@ -667,17 +667,16 @@ subroutine define_cam_grids() ! The native HOMME GLL grid call cam_grid_register(trim(gridname), dyn_decomp, lat_coord, lon_coord, & grid_map_d,block_indexed=.false., unstruct=.true.) - if (.not.single_column .or. scm_multcols) then - call cam_grid_attribute_register(trim(gridname), trim(areaname), & - 'gll grid areas', trim(ncolname), pearea, pemap) - else - ! If single column model, set pearea_scm(1) to be the area. - ! Then register attribute in same way as non-SCM to simplify. + if (single_column .or. scm_multcols) then + ! If single column model, set pearea_scm(1) to be the area as 1 value to simplify allocate(pearea_scm(1)) pearea_scm(1) = 1.0_r8 / elem(1)%rspheremp(1,1) call cam_grid_attribute_register(trim(gridname), trim(areaname), & - 'gll grid areas', trim(ncolname), pearea_scm, pemap) - nullify(pearea_scm) + 'gll grid areas', trim(ncolname), pearea_scm) + else + call cam_grid_attribute_register(trim(gridname), trim(areaname), & + 'gll grid areas', trim(ncolname), pearea, pemap) + end if ! .not. single_column call cam_grid_attribute_register(trim(gridname), 'np', '', np) @@ -692,6 +691,9 @@ subroutine define_cam_grids() nullify(grid_map_d) nullify(pearea) nullify(pemap) + if (single_column .or. scm_multcols) then + nullify(pearea_scm) + endif !--------------------------------------------------------------------------- ! Create grid object for physics grid on the dynamics decomposition From cbf149b928e4e804cd478d5df4a7386362da3a45 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 3 Sep 2024 14:21:37 -0500 Subject: [PATCH 653/904] Hommexx: Use portable PRIx64 instead of lx in printf. --- .../homme/src/share/cxx/utilities/InternalDiagnostics.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp b/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp index 3249966c0b5e..ffc9657f4108 100644 --- a/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp +++ b/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp @@ -13,6 +13,8 @@ #include "TimeLevel.hpp" #include "mpi/Comm.hpp" +#include + namespace Homme { namespace { @@ -50,10 +52,10 @@ void print_global_state_hash (const std::string& label) { all_reduce_HashType(comm.mpi_comm(), accum, gaccum, 5); if (comm.root()) { for (int i = 0; i < NUM_TIME_LEVELS; ++i) - fprintf(stderr, "hxxhash> %14d %1d %16lx (E %s)\n", + fprintf(stderr, "hxxhash> %14d %1d %16" PRIx64 " (E %s)\n", tl.nstep, i, gaccum[i], label.c_str()); for (int i = 0; i < Q_NUM_TIME_LEVELS; ++i) - fprintf(stderr, "hxxhash> %14d %1d %16lx (T %s)\n", + fprintf(stderr, "hxxhash> %14d %1d %16" PRIx64 " (T %s)\n", tl.nstep, i, gaccum[NUM_TIME_LEVELS+i], label.c_str()); } } From 25fe7324339bc2aef8a4b9f37b84d72ca5a21350 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 7 Aug 2024 13:30:47 -0400 Subject: [PATCH 654/904] Update Frontier machine/compilers following system update - Retain current software modules instead of updating to the latest versions. - Add linker options to use GCC 12.2, addressing linker errors. - Utilize Fortran linker to resolve additional linker errors. - Replace hipcc with mpicxx for MPICXX macro in the GPU compiler definitions. - Adjust compiler priority to prioritize reliability over performance. - Temporarily comment out ADIOS2 configurations --- .../cmake_macros/amdclang_frontier.cmake | 5 ++++- .../cmake_macros/amdclanggpu_frontier.cmake | 13 ++++++------- .../cmake_macros/crayclang_frontier.cmake | 2 ++ .../cmake_macros/crayclanggpu_frontier.cmake | 11 +++++++---- .../machines/cmake_macros/gnu_frontier.cmake | 5 +++++ .../machines/cmake_macros/gnugpu_frontier.cmake | 16 +++++++++++----- cime_config/machines/config_machines.xml | 14 ++++++++------ 7 files changed, 43 insertions(+), 23 deletions(-) diff --git a/cime_config/machines/cmake_macros/amdclang_frontier.cmake b/cime_config/machines/cmake_macros/amdclang_frontier.cmake index 2df5074d11f3..3727164933dc 100644 --- a/cime_config/machines/cmake_macros/amdclang_frontier.cmake +++ b/cime_config/machines/cmake_macros/amdclang_frontier.cmake @@ -9,4 +9,7 @@ string(APPEND CPPDEFS " -DLINUX") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") + +# to support Fortran specific compiler intrinsic functions +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake index 6f3c0074798e..4412ea0de7b3 100644 --- a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake @@ -1,8 +1,8 @@ set(MPICC "cc") -set(MPICXX "CC") +set(MPICXX "mpicxx") set(MPIFC "ftn") set(SCC "cc") -set(SCXX "CC") +set(SCXX "hipcc") set(SFC "ftn") string(APPEND CPPDEFS " -DLINUX") @@ -14,13 +14,12 @@ string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") -set(MPICXX "hipcc") -set(SCXX "hipcc") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") +string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") set(USE_HIP "TRUE") string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/crayclang_frontier.cmake b/cime_config/machines/cmake_macros/crayclang_frontier.cmake index 6c8d4164cb28..76ec41bdbe72 100644 --- a/cime_config/machines/cmake_macros/crayclang_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclang_frontier.cmake @@ -16,5 +16,7 @@ string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") # Scorpio installs string(APPEND CMAKE_Fortran_FLAGS " -em -ef") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") + # to support Fortran specific compiler intrinsic functions set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake index 32f125947b5c..a37ccde439e5 100644 --- a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake @@ -1,5 +1,6 @@ set(MPICC "cc") -set(MPICXX "hipcc") +set(MPICXX "mpicxx") +#set(MPICXX "CC") set(MPIFC "ftn") set(SCC "cc") set(SCXX "hipcc") @@ -33,7 +34,7 @@ set(HAS_F2008_CONTIGUOUS "TRUE") # -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: # /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' # AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") +#string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") # Switching to O3 for performance benchmarking # Will revisit any failing tests @@ -53,11 +54,13 @@ string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") +string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") set(USE_HIP "TRUE") string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/gnu_frontier.cmake b/cime_config/machines/cmake_macros/gnu_frontier.cmake index 9b242243989b..ee44a2ca08f5 100644 --- a/cime_config/machines/cmake_macros/gnu_frontier.cmake +++ b/cime_config/machines/cmake_macros/gnu_frontier.cmake @@ -13,3 +13,8 @@ string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") + +# to support Fortran specific compiler intrinsic functions +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake index 174f8207d0d0..6ca4b83d9c2f 100644 --- a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake @@ -1,6 +1,6 @@ set(MPICC "cc") -set(MPICXX "hipcc") # Needs MPICH_CXX to use hipcc -set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90 +set(MPICXX "mpicxx") +set(MPIFC "ftn") set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") @@ -16,11 +16,17 @@ string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -set(E3SM_LINK_WITH_FORTRAN "TRUE") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/11.2.0/snos/lib64/ -lgfortran -L/opt/rocm-5.4.0/lib -lhsa-runtime64 -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") +string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") +# +#string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/rocm-5.4.0/lib -lhsa-runtime64 -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") set(USE_HIP "TRUE") string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") + + +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6cd9d585a25b..2841df7200c7 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -980,7 +980,7 @@ Frontier exascale supercomputer at ORNL. 9408 nodes, Node: 4 AMD MI250X GPUs (2 GCDs) ~ 8 GPUs, 512 GB HDB2E, AMD EPYC 64 cores, 512GB DDR4 .*frontier.* Linux - crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu + gnu,amdclang,crayclang,gnugpu,amdclanggpu,crayclanggpu mpich cli115 /lustre/orion/cli115/world-shared/frontier @@ -1042,8 +1042,7 @@ PrgEnv-cray PrgEnv-gnu/8.3.3 - - gcc gcc/11.2.0 + gcc gcc/12.2.0 craype-accel-amd-gfx90a @@ -1051,6 +1050,7 @@ cray-python/3.9.13.1 + cray-libsci subversion/1.14.1 git/2.36.1 cmake/3.21.3 @@ -1066,14 +1066,12 @@ 0.25 0 + $ENV{CRAY_LIBSCI_PREFIX_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH} $ENV{NETCDF_DIR} $ENV{PNETCDF_DIR} - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} --gpu-bind=closest @@ -1090,6 +1088,9 @@ spread threads + From 8e6f5ecd8c5870d271790ddaedaa849b60fe7d89 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 5 Sep 2024 13:23:32 -0500 Subject: [PATCH 655/904] remove CMIP6 from MMF compsets --- cime_config/allactive/config_compsets.xml | 6 +++--- components/eam/cime_config/config_compsets.xml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index ace33ae09172..e66f1e473d85 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -133,18 +133,18 @@ WCYCL1950-MMF1 - 1950SOI_EAM%CMIP6-MMF1_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV + 1950SOI_EAM%MMF1_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV WCYCL20TR-MMF1 - 20TRSOI_EAM%CMIP6-MMF1_ELM%SP_MPASSI_MPASO_MOSART_SGLC_SWAV + 20TRSOI_EAM%MMF1_ELM%SP_MPASSI_MPASO_MOSART_SGLC_SWAV WCYCL20TRNS-MMF1 - 20TR_EAM%CMIP6-MMF1_ELM%SP_MPASSI_MPASO_MOSART_SGLC_SWAV + 20TR_EAM%MMF1_ELM%SP_MPASSI_MPASO_MOSART_SGLC_SWAV diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 39971fbb0945..7050bd0c8f90 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -272,12 +272,12 @@ F20TR-MMF1 - 20TR_EAM%CMIP6-MMF1_ELM%SP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV + 20TR_EAM%MMF1_ELM%SP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV F20TR-MMF1 - 20TR_EAM%CMIP6-MMF2_ELM%SP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV + 20TR_EAM%MMF2_ELM%SP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV From 26a70a5b4db09539acc25bbd873862d7189a7228 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 5 Sep 2024 16:19:58 -0400 Subject: [PATCH 656/904] Fix frontier builds One function that was being called from kernels was missing KOKKOS_INLINE_FUNCTION. --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b8ac7de768db..64ae65714c59 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -90,6 +90,7 @@ srfEmissFunctions::create_srfEmiss_data_reader( template template +KOKKOS_INLINE_FUNCTION ScalarX srfEmissFunctions::linear_interp(const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { From e950299cd9e91622308d3a3318b33ee01025b700 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Thu, 5 Sep 2024 17:00:26 -0500 Subject: [PATCH 657/904] Add eddy product AM to RRSwISC6to18 --- .../mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index c7441c6b2214..64211f42c39f 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -1111,6 +1111,7 @@ .false. +.true. '0000-00-00_01:00:00' 'eddyProductVariablesOutput' .true. From f88f96472d48567aafab8a125048bc1e2d14996b Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 5 Sep 2024 13:35:14 -0400 Subject: [PATCH 658/904] update all actions to use major pins and minor other fixes --- .github/workflows/e3sm-gh-pages.yml | 6 +++--- .github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 4 +++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 709fecc623ac..ebd2ac9c1e97 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -30,10 +30,10 @@ jobs: submodules: true - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - - name: Set up Python 3.10 - uses: actions/setup-python@v5.2.0 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.11" - name: Install python deps run: python3 -m pip install mkdocs-material pymdown-extensions mkdocs-monorepo-plugin mdutils mkdocs-bibtex # build every time (PR or push to master) diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml index 13b73e4df3f4..8fe212886d9f 100644 --- a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml +++ b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml @@ -79,7 +79,7 @@ jobs: GH_REPO: ${{ github.repository }} TITLE: mkatmsrffile test failure ASSIGNEES: whannah1 - LABELS: bug + LABELS: bug,notify-mkatmsrffile-gh-action BODY: | Workflow failed! There's likely an issue in the mkatmsrffile tool! For more information, please see: - Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (number ${{ github.run_number }}, attempt ${{ github.run_attempt }}) diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index 7c9da3264d68..2ce70d97843c 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index fa84f9f96f01..950f335700ee 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -5,6 +5,8 @@ on: branches: [ master ] pull_request: branches: [ master ] + paths: + - 'components/eamxx/cime_config/namelist_defaults_scream.xml' schedule: - cron: '00 00 * * *' workflow_dispatch: @@ -21,7 +23,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5 with: python-version: "3.11" - name: Run unit tests From 74a090ce88ae02645ef2b1a9c912dccc4372b590 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 17:07:50 -0600 Subject: [PATCH 659/904] EAMxx: call MPI_Abort in CIME runs when an exception is thrown inside EAMxx This ensures that the whole job is terminated. The MPI standard does not guarantee that an uncaught exception will cause the whole MPI job to terminate. --- .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index e9129e6fcea3..006bea1bef6a 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -57,10 +57,15 @@ void fpe_guard_wrapper (const Lambda& f) { try { f(); } catch (std::exception &e) { + // Print exception msg, then call MPI_Abort fprintf(stderr, "%s\n", e.what()); + + // Get raw comm before cleaning up singleton auto& c = ScreamContext::singleton(); + auto raw_comm = c.get().mpi_comm(); c.clean_up(); - throw; + + MPI_Abort (raw_comm,1); } // Restore the FPE flag as it was when control was handed to us. From 70222af46f2d41d247ce0cda4c327c439b7e6e04 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 17:30:34 -0600 Subject: [PATCH 660/904] EAMxx: in the fpe_guard_wrapper for CIME runs, capture all exceptions --- .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 006bea1bef6a..3e1514b38651 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -65,6 +65,14 @@ void fpe_guard_wrapper (const Lambda& f) { auto raw_comm = c.get().mpi_comm(); c.clean_up(); + MPI_Abort (raw_comm,1); + } catch (...) { + + // Get raw comm before cleaning up singleton + auto& c = ScreamContext::singleton(); + auto raw_comm = c.get().mpi_comm(); + c.clean_up(); + MPI_Abort (raw_comm,1); } From 2a4d9610f84e603a61eb0d1f81dfd04f5e9147e7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 17:59:10 -0600 Subject: [PATCH 661/904] Temporarily point to a COSP submodule branch with a bug fix We will revert once it gets properly integrated in COSP repo, and the COSP submodule is updated in E3SM. --- .gitmodules | 4 ++-- components/eam/src/physics/cosp2/external | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 69d56e140361..b36d90a00a42 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,8 +22,8 @@ branch = scorpio_classic [submodule "cosp2"] path = components/eam/src/physics/cosp2/external - url = git@github.com:CFMIP/COSPv2.0.git - branch = CESM_v2.1.4 + url = git@github.com:bartgol/COSPv2.0.git + branch = bartgol/fix-cosp_optical_inputs [submodule "cime"] path = cime url = git@github.com:ESMCI/cime.git diff --git a/components/eam/src/physics/cosp2/external b/components/eam/src/physics/cosp2/external index 9d910acba3e3..2deb41975faa 160000 --- a/components/eam/src/physics/cosp2/external +++ b/components/eam/src/physics/cosp2/external @@ -1 +1 @@ -Subproject commit 9d910acba3e3a3151de231184d4b109f65e28aee +Subproject commit 2deb41975faa4f5eacfc8b8f12b85acf6583d407 From 835ca84d8a97613f9553b2203957b052caea7057 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 6 Sep 2024 09:39:52 -0500 Subject: [PATCH 662/904] simplify CMIP6 physics default config for EAM --- components/eam/cime_config/config_component.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 655d7be5503c..d20a8c400cc1 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -51,10 +51,7 @@ -mach $MACH -phys default - &eamv3_phys_defaults; &eamv3_chem_defaults; - &eamv3_phys_defaults; &eamv3_chem_defaults; - &eamv3_phys_defaults; &eamv3_chem_defaults; - &eamv3_phys_defaults; &eamv3_chem_defaults; + &eamv3_phys_defaults; &eamv3_chem_defaults; -bc_dep_to_snow_updates -co2_cycle From 8208d4bd376f0cb8b51658f077b3941121ec1d8d Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 6 Sep 2024 09:40:29 -0500 Subject: [PATCH 663/904] specify ELM use case for MMF transient compset --- components/elm/cime_config/config_component.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/elm/cime_config/config_component.xml b/components/elm/cime_config/config_component.xml index 00518b5dc780..4e637ef55802 100755 --- a/components/elm/cime_config/config_component.xml +++ b/components/elm/cime_config/config_component.xml @@ -81,6 +81,7 @@ 20thC_CMIP6_transient 20thC_CMIP6_transient 20thC_CMIP6_transient + 20thC_CMIP6_transient 20thC_CMIP6bgc_transient 20thC_CMIP6bgc_transient 20thC_bgc_transient From c346b79fe9162b73d897e9ace9e3e6d1a8d05935 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 6 Sep 2024 14:22:08 -0600 Subject: [PATCH 664/904] Test fix --- .../single-process/rrtmgp/rrtmgp_standalone_unit.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 75feeb1b8dd4..dc1ece59b287 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -422,9 +422,14 @@ TEST_CASE("rrtmgp_scream_standalone", "") { #endif #ifdef RRTMGP_ENABLE_KOKKOS TEST_CASE("rrtmgp_scream_standalone_k", "") { - using interface_t = scream::rrtmgp::rrtmgp_interface<>; +#ifdef RRTMGP_LAYOUT_LEFT + using layout_t = Kokkos::LayoutLeft; +#else + using layout_t = typename ekat::KokkosTypes::Layout; +#endif + using interface_t = scream::rrtmgp::rrtmgp_interface; using MDRP = interface_t::MDRP; - using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using utils_t = rrtmgpTest::rrtmgp_test_utils; using real1dk = interface_t::view_t; using real2dk = interface_t::view_t; using real3dk = interface_t::view_t; From c99179079aee1d1b793ecf07530252046e2e40d6 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:19:19 -0600 Subject: [PATCH 665/904] HOMME: add aurora, polaris, spot machine files --- .../homme/cmake/machineFiles/aurora-aot.cmake | 64 ++++++++++++++++ .../homme/cmake/machineFiles/aurora-jit.cmake | 58 +++++++++++++++ .../cmake/machineFiles/chrysalis-bfb.cmake | 2 + .../homme/cmake/machineFiles/chrysalis.cmake | 2 + .../homme/cmake/machineFiles/polaris-a100.sh | 74 +++++++++++++++++++ .../cmake/machineFiles/spot-aot-AB2.cmake | 63 ++++++++++++++++ 6 files changed, 263 insertions(+) create mode 100644 components/homme/cmake/machineFiles/aurora-aot.cmake create mode 100644 components/homme/cmake/machineFiles/aurora-jit.cmake create mode 100644 components/homme/cmake/machineFiles/polaris-a100.sh create mode 100644 components/homme/cmake/machineFiles/spot-aot-AB2.cmake diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake new file mode 100644 index 000000000000..b6fe34a78d72 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -0,0 +1,64 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + + +SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/mar05-aot/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +# -fsycl-link-huge-device-code for theta to get build +#JIT flags +#SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +#SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") + +#AOT flags +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-link-huge-device-code -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + diff --git a/components/homme/cmake/machineFiles/aurora-jit.cmake b/components/homme/cmake/machineFiles/aurora-jit.cmake new file mode 100644 index 000000000000..1941fa9eb3f3 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-jit.cmake @@ -0,0 +1,58 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + + + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/jan03-2024/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +# -fsycl-link-huge-device-code for theta to get build +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") + +SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index b9f0d41a0606..fa1f1ac545c5 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -17,6 +17,8 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") +SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") + # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index 68ff76ec8082..97bc682c9546 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -17,6 +17,8 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") +SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") + # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/polaris-a100.sh b/components/homme/cmake/machineFiles/polaris-a100.sh new file mode 100644 index 000000000000..2b63c61a55e7 --- /dev/null +++ b/components/homme/cmake/machineFiles/polaris-a100.sh @@ -0,0 +1,74 @@ +#Currently Loaded Modules: +# 1) craype-x86-rome 6) craype/2.7.15 11) cray-libpals/1.1.7 16) nvhpc-mixed/21.9 +# 2) libfabric/1.11.0.4.125 7) cray-dsmml/0.2.2 12) PrgEnv-gnu/8.3.3 17) cudatoolkit-standalone/11.6.2 +# 3) craype-network-ofi 8) cray-pmi/6.1.2 13) gnu-parallel/2021-09-22 18) cmake/3.23.2 +# 4) perftools-base/22.05.0 9) cray-pmi-lib/6.0.17 14) gcc/11.2.0 +# 5) craype-accel-nvidia80 10) cray-pals/1.1.7 15) cray-mpich/8.1.16 + + + +#SET(HOMMEXX_EXEC_SPACE CUDA CACHE STRING "") +#SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") +#SET(HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING "") + +# cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 +#SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET(PNETCDF_DIR $ENV{CRAY_PARALLEL_NETCDF_DIR} CACHE FILEPATH "") +#SET(HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "") + +#for scorpio +#SET (NetCDF_C_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET (NetCDF_Fortran_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK FALSE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +SET(CUDA_BUILD TRUE CACHE BOOL "") + +#SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "") +SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") +#SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on +#SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") +SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") +SET(CMAKE_C_COMPILER "cc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/kokkos/bin/nvcc_wrapper" CACHE STRING "") + +# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER + +SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") + +SET(ENABLE_OPENMP OFF CACHE BOOL "") +SET(ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +SET(ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +SET(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "") + +#SET(HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +SET(USE_NUM_PROCS 4 CACHE STRING "") + +SET(USE_MPIEXEC "srun" CACHE STRING "") +#SET(CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/spot-aot-AB2.cmake b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake new file mode 100644 index 000000000000..23fad2361ccf --- /dev/null +++ b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake @@ -0,0 +1,63 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + +SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") + +SET (HOMMEXX_MPI_ON_DEVICE TRUE CACHE BOOL "") + +#SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/june22-2024-aot/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET (NetCDF_Fortran_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +SET (NetCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +#SET(CMAKE_CXX_STANDARD 17) +SET(CMAKE_CXX_STANDARD 17 CACHE STRING "CXX Standard") + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +#-fpscomp does not actually solve the issue with bools in here,another suggestion was -fp-model=precise, not working either +SET(ADD_Fortran_FLAGS " -fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS " -std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + From 1379db52a29ff31764b6268eae1f88fbedbae252 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:29:27 -0600 Subject: [PATCH 666/904] HOMME: CMake and CPP mods to support SYCL backend --- components/homme/CMakeLists.txt | 8 +++++--- components/homme/cmake/HommeMacros.cmake | 8 +++++++- components/homme/src/share/cxx/Config.hpp | 2 +- components/homme/src/share/cxx/ExecSpaceDefs.hpp | 4 ++++ .../homme/test_execs/share_kokkos_ut/CMakeLists.txt | 4 ++-- .../homme/test_execs/thetal_kokkos_ut/CMakeLists.txt | 3 +++ 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 80a89a296910..6fe81180ab54 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -206,7 +206,9 @@ IF (HOMME_USE_KOKKOS) STRING (TOUPPER ${HOMMEXX_EXEC_SPACE} HOMMEXX_EXEC_SPACE_UPPER) - IF (HOMMEXX_EXEC_SPACE_UPPER STREQUAL "HIP") + IF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "SYCL") + SET (HOMMEXX_SYCL_SPACE ON) + ELSEIF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "HIP") SET (HOMMEXX_HIP_SPACE ON) ELSEIF (HOMMEXX_EXEC_SPACE_UPPER STREQUAL "CUDA") SET (HOMMEXX_CUDA_SPACE ON) @@ -303,7 +305,7 @@ SET (HOMMEXX_ENABLE_GPU_F90 FALSE) IF (HOMME_USE_KOKKOS) - IF (CUDA_BUILD OR HIP_BUILD) + IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET (DEFAULT_VECTOR_SIZE 1) SET (HOMMEXX_ENABLE_GPU TRUE) SET (HOMMEXX_ENABLE_GPU_F90 TRUE) @@ -312,7 +314,7 @@ IF (HOMME_USE_KOKKOS) ENDIF() SET (HOMMEXX_VECTOR_SIZE ${DEFAULT_VECTOR_SIZE} CACHE STRING - "If AVX or Cuda or HIP don't take priority, use this software vector size.") + "If AVX or Cuda or HIP or SYCL don't take priority, use this software vector size.") IF (CMAKE_BUILD_TYPE_UPPER MATCHES "DEBUG" OR CMAKE_BUILD_TYPE_UPPER MATCHES "RELWITHDEBINFO") SET (HOMMEXX_DEBUG ON) diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 6d073dbbe83b..5610947cb299 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,7 +112,13 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + + if(SUNSPOT_MACHINE) + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) + else() + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + endif() + IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() diff --git a/components/homme/src/share/cxx/Config.hpp b/components/homme/src/share/cxx/Config.hpp index 684f9143beaf..b204b1dbd047 100644 --- a/components/homme/src/share/cxx/Config.hpp +++ b/components/homme/src/share/cxx/Config.hpp @@ -21,7 +21,7 @@ # endif #endif -#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE +#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE && ! defined HOMMEXX_SYCL_SPACE # define HOMMEXX_DEFAULT_SPACE #endif diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.hpp b/components/homme/src/share/cxx/ExecSpaceDefs.hpp index cd6649c7ab2d..82f5e803801c 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.hpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.hpp @@ -34,6 +34,10 @@ using HommexxGPU = Kokkos::Cuda; using HommexxGPU = Kokkos::Experimental::HIP; #endif +#ifdef KOKKOS_ENABLE_SYCL +using HommexxGPU = Kokkos::Experimental::SYCL; +#endif + #else using HommexxGPU = void; #endif diff --git a/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt index 3fbeff9f6f2c..bc788462ce6e 100644 --- a/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt @@ -10,7 +10,7 @@ SET(UTILS_TIMING_DIRS ${UTILS_TIMING_SRC_DIR} ${UTILS_TIMING_BIN_DIR}) # Note: need CUDA_BUILD and HOMMEXX_BFB_TESTING here, since the share # unit tests do not include a config.h file SET (COMMON_DEFINITIONS NP=4 NC=4) -IF (CUDA_BUILD OR HIP_BUILD) +IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET(COMMON_DEFINITIONS ${COMMON_DEFINITIONS} HOMMEXX_ENABLE_GPU_F90) ENDIF() IF (HOMMEXX_BFB_TESTING) @@ -158,7 +158,7 @@ ELSE() SET (NUM_CPUS 1) ENDIF() cxx_unit_test (sphere_op_ut "${SPHERE_OP_UT_F90_SRCS}" "${SPHERE_OP_UT_CXX_SRCS}" "${SPHERE_OP_UT_INCLUDE_DIRS}" "${CONFIG_DEFINES}" ${NUM_CPUS}) -endif () +endif () #BFB ### Limiters unit test ### diff --git a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt index 205635e918cc..e8bf5e20bd03 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt @@ -11,6 +11,8 @@ SET(UTILS_TIMING_BIN_DIR ${HOMME_BINARY_DIR}/utils/cime/CIME/non_py/src/timing) THETAL_KOKKOS_SETUP() # This is needed to compile the lib and test executables with the correct options +#these vars shared between all targets, so changing one var +#for one test only won't work, config is built once and for the last test SET(THIS_CONFIG_IN ${HOMME_SOURCE_DIR}/src/theta-l_kokkos/config.h.cmake.in) SET(THIS_CONFIG_HC ${CMAKE_CURRENT_BINARY_DIR}/config.h.c) SET(THIS_CONFIG_H ${CMAKE_CURRENT_BINARY_DIR}/config.h) @@ -18,6 +20,7 @@ SET (NUM_POINTS 4) SET (NUM_PLEV 12) SET (QSIZE_D 4) SET (PIO_INTERP TRUE) + HommeConfigFile (${THIS_CONFIG_IN} ${THIS_CONFIG_HC} ${THIS_CONFIG_H} ) ADD_LIBRARY(thetal_kokkos_ut_lib From c7320a6c5e31c161cf8c48ecb72d0ad792506ddf Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:30:22 -0600 Subject: [PATCH 667/904] HOMME: replace some printf with Kokkos::printf --- components/homme/src/share/cxx/utilities/BfbUtils.hpp | 2 +- .../homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp | 4 ++-- .../homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/BfbUtils.hpp b/components/homme/src/share/cxx/utilities/BfbUtils.hpp index e3570874e266..7fb4d042f7f2 100644 --- a/components/homme/src/share/cxx/utilities/BfbUtils.hpp +++ b/components/homme/src/share/cxx/utilities/BfbUtils.hpp @@ -64,7 +64,7 @@ KOKKOS_INLINE_FUNCTION ScalarType int_pow (ScalarType val, int k) { constexpr int max_shift = 30; if (k<0) { - printf ("k = %d\n",k); + Kokkos::printf ("k = %d\n",k); Kokkos::abort("int_pow implemented only for k>=0.\n"); } diff --git a/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp index ace1ba920141..d16769079729 100644 --- a/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp @@ -382,8 +382,8 @@ struct DirkFunctorImpl { kv.team_barrier(); if (it >= maxiter) { - printf("[DIRK] WARNING! Newton reached max iteration count," - " with deltaerr = %3.17f\n", deltaerr); + Kokkos::printf("[DIRK] WARNING! Newton reached max iteration count," + " with deltaerr = %3.17f\n", deltaerr); nerr = 1; } diff --git a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp index cd3bf7c32526..7914c0a60e3a 100644 --- a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp @@ -141,8 +141,8 @@ struct LimiterFunctor { [&](const int k,Real& result) { #ifndef HOMMEXX_BFB_TESTING if(diff_as_real(k) < 0){ - printf("WARNING:CAAR: dp3d too small. k=%d, dp3d(k)=%f, dp0=%f \n", - k+1,dp_as_real(k),dp0_as_real(k)); + Kokkos::printf("WARNING:CAAR: dp3d too small. k=%d, dp3d(k)=%f, dp0=%f \n", + k+1,dp_as_real(k),dp0_as_real(k)); } #endif result = result<=diff_as_real(k) ? result : diff_as_real(k); @@ -202,8 +202,8 @@ struct LimiterFunctor { for (int ivec=0; ivec Date: Thu, 5 Sep 2024 16:33:09 -0600 Subject: [PATCH 668/904] HOMME: prefer to use int to bool in a few places --- components/homme/src/share/control_mod.F90 | 1 + components/homme/src/share/cxx/GllFvRemap.cpp | 4 +- components/homme/src/share/cxx/GllFvRemap.hpp | 4 +- .../homme/src/share/cxx/GllFvRemapImpl.cpp | 8 ++-- .../homme/src/share/cxx/GllFvRemapImpl.hpp | 5 ++- .../homme/src/share/cxx/SimulationParams.hpp | 4 +- components/homme/src/share/gllfvremap_mod.F90 | 14 +++---- components/homme/src/share/namelist_mod.F90 | 6 +++ .../src/theta-l_kokkos/cxx/CamForcing.cpp | 2 +- .../theta-l_kokkos/cxx/EquationOfState.hpp | 4 +- .../src/theta-l_kokkos/cxx/ForcingFunctor.hpp | 4 +- .../cxx/HyperviscosityFunctorImpl.cpp | 8 +++- .../cxx/HyperviscosityFunctorImpl.hpp | 2 +- .../cxx/cxx_f90_interface_theta.cpp | 19 +++++----- .../src/theta-l_kokkos/prim_driver_mod.F90 | 38 ++++++++++++------- .../src/theta-l_kokkos/theta_f2c_mod.F90 | 12 +++--- 16 files changed, 79 insertions(+), 56 deletions(-) diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 0e9494f5a6cd..9c3c599b2324 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -43,6 +43,7 @@ module control_mod ! flag used by preqx, theta-l and theta-c models ! should be renamed to "hydrostatic_mode" logical, public :: theta_hydrostatic_mode + integer, public :: theta_hydrostatic_mode_integer integer, public :: tstep_type= 5 ! preqx timestepping options diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index e36dbc14d74f..a8f564958d46 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -16,7 +16,7 @@ namespace Homme { void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, - bool theta_hydrostatic_mode, + int theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f) { auto& c = Context::singleton(); @@ -52,7 +52,7 @@ void GllFvRemap::init_boundary_exchanges () { } void GllFvRemap -::init_data (const int nf, const int nf_max, bool theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f) { m_impl->init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index 07e4bf58a903..2adff0aeaa96 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -40,7 +40,7 @@ class GllFvRemap { typedef Phys2T::const_type CPhys2T; typedef Phys3T::const_type CPhys3T; - void init_data(const int nf, const int nf_max, bool theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f); @@ -81,7 +81,7 @@ class GllFvRemap { extern "C" void init_gllfvremap_c(int nelemd, int np, int nf, int nf_max, - const bool theta_hydrostatic_mode, + const int theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index 6148f69cfa9c..d4ab5c89f510 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -131,7 +131,7 @@ void GllFvRemapImpl::init_boundary_exchanges () { template using FV = Kokkos::View; void GllFvRemapImpl -::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r) { using Kokkos::create_mirror_view; @@ -142,7 +142,7 @@ ::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, " nf must be > 1.", Errors::err_not_implemented); auto& sp = Context::singleton().get(); - m_data.use_moisture = sp.moisture == MoistDry::MOIST; + m_data.use_moisture = sp.use_moisture; // Only in the unit test gllfvremap_ut does theta_hydrostatic_mode not already // == sp.theta_hydrostatic_mode. m_data.theta_hydrostatic_mode = sp.theta_hydrostatic_mode = theta_hydrostatic_mode; @@ -395,7 +395,7 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c const auto hvcoord = m_hvcoord; const bool use_moisture = m_data.use_moisture; - const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; const bool want_dp_fv_out = dp_fv_out_ptr != nullptr; VPhys2T dp_fv_out; @@ -605,7 +605,7 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, const auto fT = m_forcing.m_ft; const auto hvcoord = m_hvcoord; const auto dp3d = m_state.m_dp3d; - const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; EquationOfState eos; eos.init(theta_hydrostatic_mode, hvcoord); ElementOps ops; ops.init(hvcoord); const auto tu_ne = m_tu_ne; diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 11738b2bf455..7388fddb1231 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -60,7 +60,8 @@ struct GllFvRemapImpl { struct Data { int nelemd, qsize, nf2, n_dss_fld; - bool use_moisture, theta_hydrostatic_mode; + bool use_moisture; + int theta_hydrostatic_mode; static constexpr int nbuf1 = 2, nbuf2 = 1; Buf1 buf1[nbuf1]; @@ -107,7 +108,7 @@ struct GllFvRemapImpl { void init_buffers(const FunctorsBuffersManager& fbm); void init_boundary_exchanges(); - void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r); diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index b435911da2e6..4f36962b16c3 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -23,7 +23,7 @@ struct SimulationParams void print(std::ostream& out = std::cout); TimeStepType time_step_type; - MoistDry moisture; + bool use_moisture; RemapAlg remap_alg; TestCase test_case; ForcingAlg ftype = ForcingAlg::FORCING_OFF; @@ -77,7 +77,7 @@ inline void SimulationParams::print (std::ostream& out) { out << "\n************** CXX SimulationParams **********************\n\n"; out << " time_step_type: " << etoi(time_step_type) << "\n"; - out << " moisture: " << (moisture==MoistDry::DRY ? "dry" : "moist") << "\n"; + out << " use_moisture: " << (use_moisture ? "moist" : "dry") << "\n"; out << " remap_alg: " << etoi(remap_alg) << "\n"; out << " test case: " << etoi(test_case) << "\n"; out << " ftype: " << etoi(ftype) << "\n"; diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e0e0fa6c4daa..e927f04aba06 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -265,22 +265,22 @@ end subroutine gfr_init subroutine gfr_init_hxx() bind(c) #if KOKKOS_TARGET - use control_mod, only: theta_hydrostatic_mode - use iso_c_binding, only: c_bool + use control_mod, only: theta_hydrostatic_mode_integer + use iso_c_binding, only: c_int interface - subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode, & + subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_integer, & fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f) bind(c) - use iso_c_binding, only: c_bool, c_int, c_double + use iso_c_binding, only: c_int, c_double integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max - logical (c_bool), value, intent(in) :: theta_hydrostatic_mode + integer (c_int), value, intent(in) :: theta_hydrostatic_mode_integer real (c_double), dimension(nf*nf,nelemd), intent(in) :: fv_metdet real (c_double), dimension(np,np,nf_max*nf_max), intent(in) :: g2f_remapd real (c_double), dimension(nf_max*nf_max,np,np), intent(in) :: f2g_remapd real (c_double), dimension(nf*nf,2,2,nelemd), intent(in) :: D_f, Dinv_f end subroutine init_gllfvremap_c end interface - logical (c_bool) :: thm - thm = theta_hydrostatic_mode + integer (c_int) :: thm + thm = theta_hydrostatic_mode_integer call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, thm, & gfr%fv_metdet, gfr%g2f_remapd, gfr%f2g_remapd, gfr%D_f, gfr%Dinv_f) #endif diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 1d47090182ba..a3edaa07e235 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -41,6 +41,7 @@ module namelist_mod runtype, & integration, & ! integration method theta_hydrostatic_mode, & + theta_hydrostatic_mode_integer, & transport_alg , & ! SE Eulerian, classical SL, cell-integrated SL semi_lagrange_cdr_alg, & ! see control_mod for semi_lagrange_* descriptions semi_lagrange_cdr_check, & @@ -452,8 +453,10 @@ subroutine readnl(par) planar_slice = .false. theta_hydrostatic_mode = .true. ! for preqx, this must be .true. + theta_hydrostatic_mode_integer = 1 ! for preqx, this must be .true. #if ( defined MODEL_THETA_C || defined MODEL_THETA_L ) theta_hydrostatic_mode = .false. ! default NH + theta_hydrostatic_mode_integer = 0 ! default NH #endif @@ -850,7 +853,10 @@ subroutine readnl(par) call MPI_bcast(case_planar_bubble,1,MPIlogical_t,par%root,par%comm,ierr) #endif +if(theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 1 +if(.not. theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 0 call MPI_bcast(theta_hydrostatic_mode ,1,MPIlogical_t,par%root,par%comm,ierr) + call MPI_bcast(theta_hydrostatic_mode_integer ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(transport_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_check ,1,MPIlogical_t,par%root,par%comm,ierr) diff --git a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp index 02b999db16e9..bd7cee3e7c0a 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp @@ -33,7 +33,7 @@ static void apply_cam_forcing_tracers(const Real dt, ForcingFunctor& ff, if ( p.ftype == ForcingAlg::FORCING_2) adjustment = true; #endif - ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.moisture); + ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.use_moisture); GPTLstop("ApplyCAMForcing_tracers"); } diff --git a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp index dd97720f1be2..a50a28d58f55 100644 --- a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp @@ -23,7 +23,7 @@ class EquationOfState { EquationOfState () = default; - void init (const bool theta_hydrostatic_mode, + void init (const int theta_hydrostatic_mode, const HybridVCoord& hvcoord) { m_theta_hydrostatic_mode = theta_hydrostatic_mode; m_hvcoord = hvcoord; @@ -250,7 +250,7 @@ class EquationOfState { public: - bool m_theta_hydrostatic_mode; + int m_theta_hydrostatic_mode; HybridVCoord m_hvcoord; }; diff --git a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp index 28a702c1d273..00fa1deef667 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp @@ -236,7 +236,7 @@ class ForcingFunctor }); } - void tracers_forcing (const Real dt, const int np1, const int np1_qdp, const bool adjustment, const MoistDry moisture) { + void tracers_forcing (const Real dt, const int np1, const int np1_qdp, const bool adjustment, const bool use_moisture) { // The Functor needs to be fully setup to use this function assert (is_setup); @@ -245,7 +245,7 @@ class ForcingFunctor m_np1_qdp = np1_qdp; m_adjustment = adjustment; - m_moist = (moisture==MoistDry::MOIST); + m_moist = use_moisture; Kokkos::parallel_for("temperature, NH perturb press, FQps",m_policy_tracers_pre,*this); Kokkos::fence(); diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 046e6f9956d4..55792051d33b 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -118,9 +118,13 @@ void HyperviscosityFunctorImpl::init_params(const SimulationParams& params) m_eos.init(params.theta_hydrostatic_mode,m_hvcoord); #ifdef HOMMEXX_BFB_TESTING - m_process_nh_vars = true; + m_process_nh_vars = 1; #else - m_process_nh_vars = !params.theta_hydrostatic_mode; + if (params.theta_hydrostatic_mode){ + m_process_nh_vars = 0; + }else{ + m_process_nh_vars = 1; + } #endif } diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp index a55ecbb365f9..993d525422f5 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp @@ -397,7 +397,7 @@ class HyperviscosityFunctorImpl Buffers m_buffers; HybridVCoord m_hvcoord; - bool m_process_nh_vars; + int m_process_nh_vars; // Policies Kokkos::TeamPolicy m_policy_update_states; diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index ec4e2cbe6328..40c4ae64dc98 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -43,12 +43,13 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const Real& nu, const Real& nu_p, const Real& nu_q, const Real& nu_s, const Real& nu_div, const Real& nu_top, const int& hypervis_order, const int& hypervis_subcycle, const int& hypervis_subcycle_tom, const double& hypervis_scaling, const double& dcmip16_mu, - const int& ftype, const int& theta_adv_form, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, - const bool& use_cpstar, const int& transport_alg, const bool& theta_hydrostatic_mode, const char** test_case, + const int& ftype, const int& theta_adv_form, const int& prescribed_wind, const int& use_moisture, const int& disable_diagnostics, + const int& use_cpstar, const int& transport_alg, const int& theta_hydrostatic_mode, const char** test_case, const int& dt_remap_factor, const int& dt_tracer_factor, - const double& scale_factor, const double& laplacian_rigid_factor, const int& nsplit, const bool& pgrad_correction, + const double& scale_factor, const double& laplacian_rigid_factor, const int& nsplit, const int& pgrad_correction, const double& dp3d_thresh, const double& vtheta_thresh, const int& internal_diagnostics_level) { + // Check that the simulation options are supported. This helps us in the future, since we // are currently 'assuming' some option have/not have certain values. As we support for more // options in the C++ build, we will remove some checks @@ -111,16 +112,16 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_subcycle = hypervis_subcycle; params.hypervis_subcycle_tom = hypervis_subcycle_tom; params.hypervis_scaling = hypervis_scaling; - params.disable_diagnostics = disable_diagnostics; - params.moisture = (moisture ? MoistDry::MOIST : MoistDry::DRY); - params.use_cpstar = use_cpstar; + params.disable_diagnostics = (bool)disable_diagnostics; + params.use_moisture = (bool)use_moisture; + params.use_cpstar = (bool)use_cpstar; params.transport_alg = transport_alg; - params.theta_hydrostatic_mode = theta_hydrostatic_mode; + params.theta_hydrostatic_mode = (bool)theta_hydrostatic_mode; params.dcmip16_mu = dcmip16_mu; params.nsplit = nsplit; params.scale_factor = scale_factor; params.laplacian_rigid_factor = laplacian_rigid_factor; - params.pgrad_correction = pgrad_correction; + params.pgrad_correction = (bool)pgrad_correction; params.dp3d_thresh = dp3d_thresh; params.vtheta_thresh = vtheta_thresh; params.internal_diagnostics_level = internal_diagnostics_level; @@ -304,7 +305,7 @@ void init_elements_c (const int& num_elems) c.create_ref(e.m_forcing); } -void init_functors_c (const bool& allocate_buffer) +void init_functors_c (const int& allocate_buffer) { auto& c = Context::singleton(); diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 96b42314453f..262ba19f4b7a 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -64,7 +64,7 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) end subroutine prim_init2 subroutine prim_create_c_data_structures (tl, hvcoord, mp) - use iso_c_binding, only : c_loc, c_ptr, c_bool, C_NULL_CHAR + use iso_c_binding, only : c_loc, c_ptr, C_NULL_CHAR use theta_f2c_mod, only : init_reference_element_c, init_simulation_params_c, & init_time_level_c, init_hvcoord_c, init_elements_c use time_mod, only : TimeLevel_t, nsplit @@ -73,7 +73,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) nu, nu_p, nu_q, nu_s, nu_div, nu_top, vert_remap_q_alg, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom,& hypervis_scaling, & - ftype, prescribed_wind, moisture, disable_diagnostics, & + ftype, prescribed_wind, use_moisture, disable_diagnostics, & use_cpstar, transport_alg, theta_hydrostatic_mode, & dcmip16_mu, theta_advect_form, test_case, & MAX_STRING_LEN, dt_remap_factor, dt_tracer_factor, & @@ -93,6 +93,8 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) type (c_ptr) :: hybrid_am_ptr, hybrid_ai_ptr, hybrid_bm_ptr, hybrid_bi_ptr character(len=MAX_STRING_LEN), target :: test_name + integer :: disable_diagnostics_int, theta_hydrostatic_mode_int, use_moisture_int + ! Initialize the C++ reference element structure (i.e., pseudo-spectral deriv matrix and ref element mass matrix) dvv = deriv1%dvv elem_mp = mp @@ -100,22 +102,30 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) ! Fill the simulation params structures in C++ test_name = TRIM(test_case) // C_NULL_CHAR + + if (disable_diagnostics) disable_diagnostics_int=1 + if (.not.disable_diagnostics) disable_diagnostics_int=0 + if (use_moisture) use_moisture_int=1 + if (.not.use_moisture) use_moisture_int=0 + if(theta_hydrostatic_mode) theta_hydrostatic_mode_int=1 + if(.not.theta_hydrostatic_mode) theta_hydrostatic_mode_int=0 + call init_simulation_params_c (vert_remap_q_alg, limiter_option, rsplit, qsplit, tstep_type, & qsize, statefreq, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & hypervis_scaling, & dcmip16_mu, ftype, theta_advect_form, & - LOGICAL(prescribed_wind==1,c_bool), & - LOGICAL(moisture/="dry",c_bool), & - LOGICAL(disable_diagnostics,c_bool), & - LOGICAL(use_cpstar==1,c_bool), & + prescribed_wind, & + use_moisture_int, & + disable_diagnostics_int, & + use_cpstar, & transport_alg, & - LOGICAL(theta_hydrostatic_mode,c_bool), & + theta_hydrostatic_mode_int, & c_loc(test_name), & dt_remap_factor, dt_tracer_factor, & scale_factor, laplacian_rigid_factor, & nsplit, & - LOGICAL(pgrad_correction==1,c_bool), & + pgrad_correction, & dp3d_thresh, vtheta_thresh, internal_diagnostics_level) ! Initialize time level structure in C++ @@ -343,21 +353,21 @@ subroutine prim_init_elements_views (elem) end subroutine prim_init_elements_views subroutine prim_init_kokkos_functors (allocate_buffer) - use iso_c_binding, only : c_bool + use iso_c_binding, only : c_int use theta_f2c_mod, only : init_functors_c, init_boundary_exchanges_c - ! ! Optional Input ! - logical(kind=c_bool), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally - + integer, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + integer(kind=c_int) :: dummy ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, ! let Homme internally allocate buffers if (present(allocate_buffer)) then - call init_functors_c (logical(allocate_buffer,c_bool)) + call init_functors_c (allocate_buffer) else - call init_functors_c (logical(.true.,c_bool)) + dummy=1; + call init_functors_c (dummy) endif ! Initialize boundary exchange structure in C++ diff --git a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 index 7a4c0424807b..ba39bb03c22b 100644 --- a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 +++ b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 @@ -11,14 +11,14 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, qsize, state_frequency, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & hypervis_scaling, & - dcmip16_mu, ftype, theta_adv_form, prescribed_wind, moisture, & + dcmip16_mu, ftype, theta_adv_form, prescribed_wind, use_moisture, & disable_diagnostics, use_cpstar, transport_alg, & theta_hydrostatic_mode, test_case_name, dt_remap_factor, & dt_tracer_factor, scale_factor, laplacian_rigid_factor, & nsplit, pgrad_correction, dp3d_thresh, vtheta_thresh, & internal_diagnostics_level) bind(c) - use iso_c_binding, only: c_int, c_bool, c_double, c_ptr + use iso_c_binding, only: c_int, c_double, c_ptr ! ! Inputs ! @@ -29,8 +29,8 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, scale_factor, laplacian_rigid_factor, dp3d_thresh, vtheta_thresh integer(kind=c_int), intent(in) :: hypervis_order, hypervis_subcycle, hypervis_subcycle_tom integer(kind=c_int), intent(in) :: ftype, theta_adv_form - logical(kind=c_bool), intent(in) :: prescribed_wind, moisture, disable_diagnostics, use_cpstar - logical(kind=c_bool), intent(in) :: theta_hydrostatic_mode, pgrad_correction + integer(kind=c_int), intent(in) :: prescribed_wind, use_moisture, disable_diagnostics, use_cpstar + integer(kind=c_int), intent(in) :: theta_hydrostatic_mode, pgrad_correction type(c_ptr), intent(in) :: test_case_name end subroutine init_simulation_params_c @@ -138,11 +138,11 @@ end subroutine init_reference_element_c ! Create C++ functors subroutine init_functors_c (allocate_buffer) bind(c) - use iso_c_binding, only: c_bool + use iso_c_binding, only: c_int ! ! Inputs ! - logical(kind=c_bool), intent(in) :: allocate_buffer + integer(kind=c_int), intent(in) :: allocate_buffer end subroutine init_functors_c ! Initialize C++ boundary exchange structures From 0200065081910233ce6c861471b062d3e0608a2a Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:33:51 -0600 Subject: [PATCH 669/904] HOMME: some SYCL related mods in kokkos initialization and team policy defaults --- .../homme/src/share/cxx/ExecSpaceDefs.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 8d496bff5d16..c9ca8a0ecd93 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -21,6 +21,10 @@ #include #endif +#ifdef KOKKOS_ENABLE_SYCL +#include +#endif + namespace Homme { // Since we're initializing from inside a Fortran code and don't have access to @@ -52,7 +56,16 @@ void initialize_kokkos () { // It isn't a big deal if we can't get the device count. nd = 1; } +#elif defined(KOKKOS_ENABLE_SYCL) + +//https://developer.codeplay.com/products/computecpp/ce/2.11.0/guides/sycl-for-cuda-developers/migrating-from-cuda-to-sycl + +//to make it build + int nd = 1; + #endif + + #ifdef HOMMEXX_ENABLE_GPU std::stringstream ss; ss << "--kokkos-num-devices=" << nd; @@ -117,6 +130,7 @@ team_num_threads_vectors_for_gpu ( assert(num_warps_total >= max_num_warps); assert(tp.max_threads_usable >= 1 && tp.max_vectors_usable >= 1); +#ifndef KOKKOS_ENABLE_SYCL int num_warps; if (tp.prefer_larger_team) { const int num_warps_usable = @@ -161,6 +175,9 @@ team_num_threads_vectors_for_gpu ( return std::make_pair( num_device_threads / num_vectors, num_vectors ); } +#else + return std::make_pair(4,16); +#endif } } // namespace Parallel From 5a43946dcf66085a54e64af76c8c5fade9a0e575 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:34:39 -0600 Subject: [PATCH 670/904] HOMME: do not add dependency on cprnc if building without PIO --- components/homme/test_execs/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/homme/test_execs/CMakeLists.txt b/components/homme/test_execs/CMakeLists.txt index a3113921b029..a007a5532b61 100644 --- a/components/homme/test_execs/CMakeLists.txt +++ b/components/homme/test_execs/CMakeLists.txt @@ -142,8 +142,11 @@ ADD_CUSTOM_TARGET(test-execs) ADD_CUSTOM_TARGET(check COMMAND ${CMAKE_CTEST_COMMAND} "--output-on-failure") +if(NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) # Force cprnc to be built when make check is run ADD_DEPENDENCIES(check cprnc) +endif() + # Create a target for making the reference data ADD_CUSTOM_TARGET(baseline From 132b3a506b8a90274bcb9cc5f7c67cf2eafdfb20 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:30:50 -0600 Subject: [PATCH 671/904] HOMME: minor GPTL timing related mods * Disable timing in prim_main for the first two teps * Add some timers in EulerStepFunctor --- components/homme/src/prim_main.F90 | 7 ++++++- components/homme/src/share/cxx/EulerStepFunctorImpl.hpp | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/components/homme/src/prim_main.F90 b/components/homme/src/prim_main.F90 index bfbe57e8b317..d6901151d365 100644 --- a/components/homme/src/prim_main.F90 +++ b/components/homme/src/prim_main.F90 @@ -20,7 +20,7 @@ program prim_main use element_mod, only: element_t use common_io_mod, only: output_dir, infilenames use common_movie_mod, only: nextoutputstep - use perf_mod, only: t_initf, t_prf, t_finalizef, t_startf, t_stopf ! _EXTERNAL + use perf_mod, only: t_initf, t_prf, t_finalizef, t_startf, t_stopf, t_disablef, t_enablef ! _EXTERNAL use restart_io_mod , only: restartheader_t, writerestart use hybrid_mod, only: hybrid_create #if (defined MODEL_THETA_L && defined ARKODE) @@ -240,6 +240,11 @@ end subroutine finalize_kokkos_f90 nstep = nextoutputstep(tl) do while(tl%nstep= 2) call t_enablef() call t_startf('prim_run') call prim_run_subcycle(elem, hybrid,nets,nete, tstep, .false., tl, hvcoord,1) call t_stopf('prim_run') diff --git a/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp b/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp index f3029764dac3..f87bb108bebf 100644 --- a/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp +++ b/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp @@ -652,7 +652,10 @@ class EulerStepFunctorImpl { minmax_and_biharmonic(); } } + + GPTLstart("tl-at adv-n-limit"); advect_and_limit(); + GPTLstop("tl-at adv-n-limit"); exchange_qdp_dss_var(); } @@ -667,6 +670,7 @@ class EulerStepFunctorImpl { void run_tracer_phase (const KernelVariables& kv) const { compute_qtens(kv); kv.team_barrier(); + if (m_data.limiter_option == 8) { limiter_optim_iter_full(kv); kv.team_barrier(); @@ -674,6 +678,7 @@ class EulerStepFunctorImpl { limiter_clip_and_sum(kv); kv.team_barrier(); } + apply_spheremp(kv); } From 8f670095dcbf1d7fe22470031fdbdf8f2e0a8eff Mon Sep 17 00:00:00 2001 From: noel Date: Sat, 7 Sep 2024 11:36:45 -0700 Subject: [PATCH 672/904] correct my logic mistake in previous commit --- components/eam/src/dynamics/se/dyn_grid.F90 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/eam/src/dynamics/se/dyn_grid.F90 b/components/eam/src/dynamics/se/dyn_grid.F90 index 125cfbca7a60..a4a8424f887b 100644 --- a/components/eam/src/dynamics/se/dyn_grid.F90 +++ b/components/eam/src/dynamics/se/dyn_grid.F90 @@ -667,16 +667,15 @@ subroutine define_cam_grids() ! The native HOMME GLL grid call cam_grid_register(trim(gridname), dyn_decomp, lat_coord, lon_coord, & grid_map_d,block_indexed=.false., unstruct=.true.) - if (single_column .or. scm_multcols) then + if (.not.single_column .or. scm_multcols) then + call cam_grid_attribute_register(trim(gridname), trim(areaname), & + 'gll grid areas', trim(ncolname), pearea, pemap) + else ! If single column model, set pearea_scm(1) to be the area as 1 value to simplify allocate(pearea_scm(1)) pearea_scm(1) = 1.0_r8 / elem(1)%rspheremp(1,1) call cam_grid_attribute_register(trim(gridname), trim(areaname), & 'gll grid areas', trim(ncolname), pearea_scm) - else - call cam_grid_attribute_register(trim(gridname), trim(areaname), & - 'gll grid areas', trim(ncolname), pearea, pemap) - end if ! .not. single_column call cam_grid_attribute_register(trim(gridname), 'np', '', np) @@ -691,7 +690,8 @@ subroutine define_cam_grids() nullify(grid_map_d) nullify(pearea) nullify(pemap) - if (single_column .or. scm_multcols) then + if (.not.single_column .or. scm_multcols) then + else nullify(pearea_scm) endif From 2e56ac054e317d047ccee5e37c40dbff2e9c7cdc Mon Sep 17 00:00:00 2001 From: noel Date: Sat, 7 Sep 2024 11:45:46 -0700 Subject: [PATCH 673/904] upgrade to intel/2023.2.0 on pm-cpu --- cime_config/machines/config_machines.xml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6cd9d585a25b..5b693e455268 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -223,8 +223,8 @@ - PrgEnv-intel/8.3.3 - intel/2023.1.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -239,13 +239,25 @@ cray-libsci/23.02.1.1 - + + craype-accel-host + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + + + craype-accel-host craype/2.7.20 cray-mpich/8.1.25 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + + + cmake/3.24.3 evp-patch From 1ead26f943a8d38c510eebb27872f7ebb238f092 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Sun, 8 Sep 2024 15:05:47 -0500 Subject: [PATCH 674/904] Update ndep and popdens files for all configs including default --- components/elm/bld/namelist_files/namelist_defaults.xml | 6 +++--- .../namelist_files/use_cases/2015-2100_SSP245_transient.xml | 6 ++++-- .../namelist_files/use_cases/2015-2100_SSP370_transient.xml | 6 ++++-- .../namelist_files/use_cases/2015-2100_SSP585_transient.xml | 6 ++++-- .../bld/namelist_files/use_cases/20thC_CMIP6_transient.xml | 6 +++--- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index e76a5647b4e7..e66ebfa637ba 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -666,7 +666,7 @@ this mask will have smb calculated over the entire global land surface 2000 2100 -lnd/clm2/ndepdata/fndep_elm_cbgc_exp_simyr1849-2101_1.9x2.5_c190103.nc +lnd/clm2/ndepdata/fndep_elm_cbgc_exp_simyr1849-2101_1.9x2.5_ssp245_c240903.nc lnd/clm2/ndepdata/fndep_clm_rcp8.5_simyr1849-2106_1.9x2.5_c100428.nc lnd/clm2/ndepdata/fndep_clm_rcp6.0_simyr1849-2106_1.9x2.5_c100810.nc lnd/clm2/ndepdata/fndep_clm_rcp4.5_simyr1849-2106_1.9x2.5_c100428.nc @@ -899,8 +899,8 @@ this mask will have smb calculated over the entire global land surface 1850 2010 -lnd/clm2/firedata/elmforc.ssp5_hdm_0.5x0.5_simyr1850-2100_c190109.nc -lnd/clm2/firedata/elmforc.ssp5_hdm_0.5x0.5_simyr1850-2100_c190109.nc +lnd/clm2/firedata/elmforc.Li_20181205_mod_hist_SSP2_CMIP6_hdm_0.5x0.5_AVHRR_simyr1850-2100_c240906.nc +lnd/clm2/firedata/elmforc.Li_20181205_mod_hist_SSP2_CMIP6_hdm_0.5x0.5_AVHRR_simyr1850-2100_c240906.nc bilinear diff --git a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP245_transient.xml b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP245_transient.xml index e313dab6b16a..6e664162609f 100755 --- a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP245_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP245_transient.xml @@ -17,8 +17,10 @@ + + 1850 -2005 +2101 1850 2000 @@ -26,7 +28,7 @@ 2000 1850 -2010 +2100 1850 diff --git a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP370_transient.xml b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP370_transient.xml index 0e8415d3c1ab..0c54a8ce6039 100755 --- a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP370_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP370_transient.xml @@ -18,16 +18,18 @@ 1850 -2005 +2101 1850 +lnd/clm2/ndepdata/fndep_elm_cbgc_exp_simyr1849-2101_1.9x2.5_ssp370_c220614.nc 2000 2000 2000 1850 -2010 +2100 1850 +lnd/clm2/firedata/elmforc.Li_20181205_mod_hist_SSP3_CMIP6_hdm_0.5x0.5_AVHRR_simyr1850-2100_c240906.nc diff --git a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP585_transient.xml b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP585_transient.xml index 2db3549ce5f0..c297dffe1138 100644 --- a/components/elm/bld/namelist_files/use_cases/2015-2100_SSP585_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/2015-2100_SSP585_transient.xml @@ -18,16 +18,18 @@ 1850 -2005 +2101 1850 +lnd/clm2/ndepdata/fndep_elm_cbgc_exp_simyr1849-2101_1.9x2.5_ssp585_c190103.nc 2000 2000 2000 1850 -2010 +2100 1850 +lnd/clm2/firedata/elmforc.Li_20181205_mod_hist_SSP5_CMIP6_hdm_0.5x0.5_AVHRR_simyr1850-2100_c240906.nc diff --git a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml index 92f186f1a38d..a919a4847910 100644 --- a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml @@ -9,7 +9,7 @@ 1850 -1850-2000 +1850-2015 arb_ic @@ -18,7 +18,7 @@ 1850 -2005 +2101 1850 @@ -27,7 +27,7 @@ 2000 1850 -2010 +2100 1850 From 84de5e161bc93c8980d20eafaa5ae9deea2edda5 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Sun, 8 Sep 2024 15:06:31 -0500 Subject: [PATCH 675/904] Reset default/placeholder co2vmr value for ssp245 --- driver-mct/cime_config/config_component_e3sm.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index 08758f935313..8647bdda189f 100755 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -753,8 +753,7 @@ 312.821 388.717 388.717 - 0.000001 - 0.000001 + 0.000001 284.317 284.317 284.317 From a0e3ffd9bab97e983078c5ee321370ac3356c451 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Sun, 8 Sep 2024 21:30:03 -0700 Subject: [PATCH 676/904] Explicitly set dust_emis_scheme=2 for MMF --- driver-mct/cime_config/config_component_e3sm.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index acdca60ae2a6..feaa8f270a1b 100755 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -863,6 +863,7 @@ 1 2 + 2 shr_dust_nl env_run.xml From e7eb37a35330c10a76d5b0a0ef808b6f6ef2de82 Mon Sep 17 00:00:00 2001 From: Jinyun Tang Date: Mon, 9 Sep 2024 11:40:32 -0500 Subject: [PATCH 677/904] bug fix for N balance error due to spval value for land use simulation When the land use data is on, at the begining of every year, the model may crash with N balance error due to including spval value in N leaching flux update. This update fixes this error by keeping those spval values away. The fix should be BFB for simulations do not involve land use data. --- .../elm/src/data_types/ColumnDataType.F90 | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/components/elm/src/data_types/ColumnDataType.F90 b/components/elm/src/data_types/ColumnDataType.F90 index 724cefb5cfdf..d0a4a10cd3d6 100644 --- a/components/elm/src/data_types/ColumnDataType.F90 +++ b/components/elm/src/data_types/ColumnDataType.F90 @@ -7538,14 +7538,16 @@ subroutine col_cf_summary(this, bounds, num_soilc, filter_soilc, isotope) c = filter_soilc(fc) this%decomp_cpools_leached(c,l) = 0._r8 end do - do j = 1, nlev - do fc = 1,num_soilc + if(l /= i_cwd)then + do j = 1, nlev + do fc = 1,num_soilc c = filter_soilc(fc) this%decomp_cpools_leached(c,l) = & this%decomp_cpools_leached(c,l) + & this%decomp_cpools_transport_tendency(c,j,l) * dzsoi_decomp(j) - end do - end do + end do + end do + endif do fc = 1,num_soilc c = filter_soilc(fc) this%som_c_leached(c) = & @@ -9784,8 +9786,9 @@ subroutine col_nf_summary(this, bounds, num_soilc, filter_soilc) c = filter_soilc(fc) this%decomp_npools_leached(c,l) = 0._r8 end do - do j = 1, nlev - do fc = 1,num_soilc + if(l /= i_cwd)then + do j = 1, nlev + do fc = 1,num_soilc c = filter_soilc(fc) this%decomp_npools_leached(c,l) = & this%decomp_npools_leached(c,l) + & @@ -9793,8 +9796,9 @@ subroutine col_nf_summary(this, bounds, num_soilc, filter_soilc) this%bgc_npool_inputs(c,l) = this%bgc_npool_inputs(c,l) + & (this%bgc_npool_ext_inputs_vr(c,j,l)-this%bgc_npool_ext_loss_vr(c,j,l))*dzsoi_decomp(j) - end do - end do + end do + end do + endif do fc = 1,num_soilc c = filter_soilc(fc) this%som_n_leached(c) = & @@ -11314,16 +11318,16 @@ subroutine col_pf_summary(this, bounds, num_soilc, filter_soilc) c = filter_soilc(fc) this%decomp_ppools_leached(c,l) = 0._r8 end do - - do j = 1, nlevdecomp - do fc = 1,num_soilc + if(l /= i_cwd)then + do j = 1, nlevdecomp + do fc = 1,num_soilc c = filter_soilc(fc) this%decomp_ppools_leached(c,l) = & this%decomp_ppools_leached(c,l) + & this%decomp_ppools_transport_tendency(c,j,l) * dzsoi_decomp(j) - end do - end do - + end do + end do + endif do fc = 1,num_soilc c = filter_soilc(fc) this%som_p_leached(c) = & From 67cda37cb5c617f760793d04b2ee4a8e7731de14 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 9 Sep 2024 13:19:32 -0600 Subject: [PATCH 678/904] Update CIME submodule with a new fix commit That fixes BFAILs not being considered DIFFs [BFB] --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index 2a3142fc14b1..0cdd4b1c5c5e 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 2a3142fc14b18fd58379f98d7d2a9a8b4b0d43a8 +Subproject commit 0cdd4b1c5c5eb2e29c6ec64667724af434847bcf From 30b9f60a441723c13a6e111767b234f8eb482f73 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 9 Sep 2024 14:14:42 -0500 Subject: [PATCH 679/904] Hommexx: Isolate int-bool workaround to just the C++-F90 interface code. Also fix preqx's use of use_moisture. Remove MOIST-DRY enum to avoid confusion, since it's no longer used. --- .../homme/src/preqx_kokkos/cxx/CamForcing.cpp | 6 ++--- .../cxx/cxx_f90_interface_preqx.cpp | 4 +-- .../src/preqx_kokkos/cxx/prim_advance_exp.cpp | 2 +- components/homme/src/share/control_mod.F90 | 1 - components/homme/src/share/cxx/GllFvRemap.cpp | 6 ++--- components/homme/src/share/cxx/GllFvRemap.hpp | 2 +- .../homme/src/share/cxx/GllFvRemapImpl.cpp | 6 ++--- .../homme/src/share/cxx/GllFvRemapImpl.hpp | 5 ++-- .../homme/src/share/cxx/HommexxEnums.hpp | 5 ---- components/homme/src/share/gllfvremap_mod.F90 | 9 ++++--- components/homme/src/share/namelist_mod.F90 | 6 ----- .../theta-l_kokkos/cxx/EquationOfState.hpp | 4 +-- .../cxx/HyperviscosityFunctorImpl.cpp | 6 +---- .../cxx/HyperviscosityFunctorImpl.hpp | 2 +- .../src/theta-l_kokkos/prim_driver_mod.F90 | 25 +++++++++---------- .../thetal_kokkos_ut/forcing_ut.cpp | 8 +++--- .../thetal_kokkos_ut/gllfvremap_ut.cpp | 4 +-- 17 files changed, 42 insertions(+), 59 deletions(-) diff --git a/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp b/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp index 2b1e6514389e..36ca5f4a95f4 100644 --- a/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp @@ -51,7 +51,7 @@ void state_forcing( void tracer_forcing( const ExecViewUnmanaged &f_q, const HybridVCoord &hvcoord, const TimeLevel &tl, const int &num_q, - const MoistDry &moisture, const double &dt, + const bool &use_moisture, const double &dt, const ExecViewManaged &ps_v, const ExecViewManaged< Scalar * [Q_NUM_TIME_LEVELS][QSIZE_D][NP][NP][NUM_LEV]> &qdp, @@ -61,7 +61,7 @@ void tracer_forcing( const int np1 = tl.n0; const int np1_qdp = tl.n0_qdp; - if (moisture == MoistDry::MOIST) { + if (use_moisture) { // Remove the m_fq_ps_v buffer since it's not actually needed. // Instead apply the forcing to m_ps_v directly // Bonus - one less parallel reduce in dry cases! @@ -161,7 +161,7 @@ void apply_cam_forcing(const Real &dt) { tracers.fq = decltype(tracers.fq)("fq", elems.num_elems(),tracers.num_tracers()); } tracer_forcing(tracers.fq, hvcoord, tl, tracers.num_tracers(), - sim_params.moisture, dt, elems.m_state.m_ps_v, tracers.qdp, tracers.Q); + sim_params.use_moisture, dt, elems.m_state.m_ps_v, tracers.qdp, tracers.Q); GPTLstop("ApplyCAMForcing"); } diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index c75143a9836a..b433a48c2abc 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -37,7 +37,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const int& time_step_type, const int& qsize, const int& state_frequency, const Real& nu, const Real& nu_p, const Real& nu_q, const Real& nu_s, const Real& nu_div, const Real& nu_top, const int& hypervis_order, const int& hypervis_subcycle, const double& hypervis_scaling, - const int& ftype, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, + const int& ftype, const bool& prescribed_wind, const bool& use_moisture, const bool& disable_diagnostics, const bool& use_cpstar, const int& transport_alg, const int& dt_remap_factor, const int& dt_tracer_factor, const double& scale_factor, const double& laplacian_rigid_factor) @@ -90,7 +90,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_subcycle = hypervis_subcycle; params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = disable_diagnostics; - params.moisture = (moisture ? MoistDry::MOIST : MoistDry::DRY); + params.use_moisture = use_moisture; params.use_cpstar = use_cpstar; params.transport_alg = transport_alg; // SphereOperators parameters; preqx supports only the sphere. diff --git a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp index f7c7600aab8d..58e58f0160bf 100644 --- a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp @@ -34,7 +34,7 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost // Determine the tracers time level tl.n0_qdp= -1; - if (params.moisture == MoistDry::MOIST) { + if (params.use_moisture) { tl.update_tracers_levels(params.qsplit); } diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 9c3c599b2324..0e9494f5a6cd 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -43,7 +43,6 @@ module control_mod ! flag used by preqx, theta-l and theta-c models ! should be renamed to "hydrostatic_mode" logical, public :: theta_hydrostatic_mode - integer, public :: theta_hydrostatic_mode_integer integer, public :: tstep_type= 5 ! preqx timestepping options diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index a8f564958d46..7b0400427f38 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -21,8 +21,8 @@ void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f) { auto& c = Context::singleton(); auto& g = c.get(); - g.init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, g2f_remapd, - f2g_remapd, D_f, Dinv_f); + const bool thm = static_cast(theta_hydrostatic_mode); + g.init_data(nf, nf_max, thm, fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f); } GllFvRemap::GllFvRemap () { @@ -52,7 +52,7 @@ void GllFvRemap::init_boundary_exchanges () { } void GllFvRemap -::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f) { m_impl->init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index 2adff0aeaa96..7ebf5a82b71a 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -40,7 +40,7 @@ class GllFvRemap { typedef Phys2T::const_type CPhys2T; typedef Phys3T::const_type CPhys3T; - void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index d4ab5c89f510..ea1a52f5efdf 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -131,7 +131,7 @@ void GllFvRemapImpl::init_boundary_exchanges () { template using FV = Kokkos::View; void GllFvRemapImpl -::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r) { using Kokkos::create_mirror_view; @@ -395,7 +395,7 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c const auto hvcoord = m_hvcoord; const bool use_moisture = m_data.use_moisture; - const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; const bool want_dp_fv_out = dp_fv_out_ptr != nullptr; VPhys2T dp_fv_out; @@ -605,7 +605,7 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, const auto fT = m_forcing.m_ft; const auto hvcoord = m_hvcoord; const auto dp3d = m_state.m_dp3d; - const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; EquationOfState eos; eos.init(theta_hydrostatic_mode, hvcoord); ElementOps ops; ops.init(hvcoord); const auto tu_ne = m_tu_ne; diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 7388fddb1231..11738b2bf455 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -60,8 +60,7 @@ struct GllFvRemapImpl { struct Data { int nelemd, qsize, nf2, n_dss_fld; - bool use_moisture; - int theta_hydrostatic_mode; + bool use_moisture, theta_hydrostatic_mode; static constexpr int nbuf1 = 2, nbuf2 = 1; Buf1 buf1[nbuf1]; @@ -108,7 +107,7 @@ struct GllFvRemapImpl { void init_buffers(const FunctorsBuffersManager& fbm); void init_boundary_exchanges(); - void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r); diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 59c8f3c9652c..06abbf35adbc 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -47,11 +47,6 @@ enum class ForcingAlg : int { FORCING_2 = 2, // TODO: Rename FORCING_1 and FORCING_2 to something more descriptive }; -enum class MoistDry { - MOIST, - DRY -}; - enum class AdvectionForm { Conservative, NonConservative diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e927f04aba06..a5f9b3033c96 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -265,14 +265,14 @@ end subroutine gfr_init subroutine gfr_init_hxx() bind(c) #if KOKKOS_TARGET - use control_mod, only: theta_hydrostatic_mode_integer + use control_mod, only: theta_hydrostatic_mode use iso_c_binding, only: c_int interface - subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_integer, & + subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode, & fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f) bind(c) use iso_c_binding, only: c_int, c_double integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max - integer (c_int), value, intent(in) :: theta_hydrostatic_mode_integer + integer (c_int), value, intent(in) :: theta_hydrostatic_mode real (c_double), dimension(nf*nf,nelemd), intent(in) :: fv_metdet real (c_double), dimension(np,np,nf_max*nf_max), intent(in) :: g2f_remapd real (c_double), dimension(nf_max*nf_max,np,np), intent(in) :: f2g_remapd @@ -280,7 +280,8 @@ subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_inte end subroutine init_gllfvremap_c end interface integer (c_int) :: thm - thm = theta_hydrostatic_mode_integer + thm = 0 + if (theta_hydrostatic_mode) thm = 1 call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, thm, & gfr%fv_metdet, gfr%g2f_remapd, gfr%f2g_remapd, gfr%D_f, gfr%Dinv_f) #endif diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index a3edaa07e235..1d47090182ba 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -41,7 +41,6 @@ module namelist_mod runtype, & integration, & ! integration method theta_hydrostatic_mode, & - theta_hydrostatic_mode_integer, & transport_alg , & ! SE Eulerian, classical SL, cell-integrated SL semi_lagrange_cdr_alg, & ! see control_mod for semi_lagrange_* descriptions semi_lagrange_cdr_check, & @@ -453,10 +452,8 @@ subroutine readnl(par) planar_slice = .false. theta_hydrostatic_mode = .true. ! for preqx, this must be .true. - theta_hydrostatic_mode_integer = 1 ! for preqx, this must be .true. #if ( defined MODEL_THETA_C || defined MODEL_THETA_L ) theta_hydrostatic_mode = .false. ! default NH - theta_hydrostatic_mode_integer = 0 ! default NH #endif @@ -853,10 +850,7 @@ subroutine readnl(par) call MPI_bcast(case_planar_bubble,1,MPIlogical_t,par%root,par%comm,ierr) #endif -if(theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 1 -if(.not. theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 0 call MPI_bcast(theta_hydrostatic_mode ,1,MPIlogical_t,par%root,par%comm,ierr) - call MPI_bcast(theta_hydrostatic_mode_integer ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(transport_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_check ,1,MPIlogical_t,par%root,par%comm,ierr) diff --git a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp index a50a28d58f55..dd97720f1be2 100644 --- a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp @@ -23,7 +23,7 @@ class EquationOfState { EquationOfState () = default; - void init (const int theta_hydrostatic_mode, + void init (const bool theta_hydrostatic_mode, const HybridVCoord& hvcoord) { m_theta_hydrostatic_mode = theta_hydrostatic_mode; m_hvcoord = hvcoord; @@ -250,7 +250,7 @@ class EquationOfState { public: - int m_theta_hydrostatic_mode; + bool m_theta_hydrostatic_mode; HybridVCoord m_hvcoord; }; diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 55792051d33b..d160e114475b 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -120,11 +120,7 @@ void HyperviscosityFunctorImpl::init_params(const SimulationParams& params) #ifdef HOMMEXX_BFB_TESTING m_process_nh_vars = 1; #else - if (params.theta_hydrostatic_mode){ - m_process_nh_vars = 0; - }else{ - m_process_nh_vars = 1; - } + m_process_nh_vars = not params.theta_hydrostatic_mode; #endif } diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp index 993d525422f5..a55ecbb365f9 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp @@ -397,7 +397,7 @@ class HyperviscosityFunctorImpl Buffers m_buffers; HybridVCoord m_hvcoord; - int m_process_nh_vars; + bool m_process_nh_vars; // Policies Kokkos::TeamPolicy m_policy_update_states; diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 262ba19f4b7a..eae8544ca865 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -103,12 +103,12 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) ! Fill the simulation params structures in C++ test_name = TRIM(test_case) // C_NULL_CHAR - if (disable_diagnostics) disable_diagnostics_int=1 - if (.not.disable_diagnostics) disable_diagnostics_int=0 - if (use_moisture) use_moisture_int=1 - if (.not.use_moisture) use_moisture_int=0 - if(theta_hydrostatic_mode) theta_hydrostatic_mode_int=1 - if(.not.theta_hydrostatic_mode) theta_hydrostatic_mode_int=0 + disable_diagnostics_int = 0 + if (disable_diagnostics) disable_diagnostics_int = 1 + use_moisture_int = 0 + if (use_moisture) use_moisture_int = 1 + theta_hydrostatic_mode_int = 0 + if (theta_hydrostatic_mode) theta_hydrostatic_mode_int = 1 call init_simulation_params_c (vert_remap_q_alg, limiter_option, rsplit, qsplit, tstep_type, & qsize, statefreq, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & @@ -358,17 +358,16 @@ subroutine prim_init_kokkos_functors (allocate_buffer) ! ! Optional Input ! - integer, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally - integer(kind=c_int) :: dummy + logical, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + integer(kind=c_int) :: ab ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, ! let Homme internally allocate buffers + ab = 1 if (present(allocate_buffer)) then - call init_functors_c (allocate_buffer) - else - dummy=1; - call init_functors_c (dummy) - endif + if (.not. allocate_buffer) ab = 0 + end if + call init_functors_c (ab) ! Initialize boundary exchange structure in C++ call init_boundary_exchanges_c () diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index 5e4c51c7ca11..fb301166f429 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -160,8 +160,8 @@ TEST_CASE("forcing", "forcing") { std::cout << "Testing tracers forcing.\n"; for (const bool hydrostatic : {true,false}) { std::cout << " -> hydrostatic mode: " << (hydrostatic ? "true" : "false") << "\n"; - for (const MoistDry moisture : {MoistDry::DRY,MoistDry::MOIST}) { - std::cout << " -> moisture: " << (moisture==MoistDry::MOIST ? "moist" : "dry") << "\n"; + for (const bool use_moisture: {false,true}) { + std::cout << " -> moisture: " << (use_moisture ? "moist" : "dry") << "\n"; for (const bool adjustment : {true,false}) { std::cout << " -> adjustment: " << (adjustment ? "true" : "false") << "\n"; @@ -200,8 +200,8 @@ TEST_CASE("forcing", "forcing") { ff.init_buffers(fbm); // Run tracers forcing (cxx and f90) - ff.tracers_forcing(dt,np1,np1_qdp,adjustment,moisture); - tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,moisture==MoistDry::MOIST,adjustment); + ff.tracers_forcing(dt,np1,np1_qdp,adjustment,use_moisture); + tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,use_moisture,adjustment); // Compare answers Kokkos::deep_copy(h_dp,state.m_dp3d); diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp index 0f14b0c3e55a..cf9db941ea1c 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp @@ -183,7 +183,7 @@ struct Session { p.qsize = qsize; p.hypervis_scaling = 0; p.transport_alg = 0; - p.moisture = MoistDry::MOIST; + p.use_moisture = true; p.theta_hydrostatic_mode = false; p.scale_factor = is_sphere ? PhysicalConstants::rearth0 : 1; p.laplacian_rigid_factor = is_sphere ? 1/p.scale_factor : 0; @@ -725,7 +725,7 @@ static void test_get_temperature (Session& s) { const auto& sp = c.get(); EquationOfState eos; eos.init(theta_hydrostatic_mode, s.h); ElementOps ops; ops.init(s.h); - const bool use_moisture = sp.moisture == MoistDry::MOIST; + const bool use_moisture = sp.use_moisture; const auto state = c.get(); const auto tracers = c.get(); const auto dp3d = state.m_dp3d; From d1c60b4ff8e12cd3e88c31890cdc22816ac593dd Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Mon, 9 Sep 2024 15:49:31 -0500 Subject: [PATCH 680/904] move ww3 build to live in the run directory --- components/ww3/cime_config/buildlib_cmake | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/components/ww3/cime_config/buildlib_cmake b/components/ww3/cime_config/buildlib_cmake index 4f15ea2d58e3..9df52bc35c67 100755 --- a/components/ww3/cime_config/buildlib_cmake +++ b/components/ww3/cime_config/buildlib_cmake @@ -43,14 +43,23 @@ def buildlib(bldroot, installpath, case): # Define WW3 repository directories repodir = "{}/components/ww3/src".format(srcroot) modeldir = "{}/WW3/model".format(repodir) + builddir = "{}/wav".format(exeroot) # TODO: these work dirs will have to be changed to live in the binary/build area. # Achieving this will probably require a significant refactor of the ww3 infrastructure. # Doing this stuff in-source not only clutters the repo, but also introduces potential race # conditions if we were to try to build multiple ww3 cases simultaneously. - bindir = "{}/bin".format(modeldir) - exedir = "{}/exe".format(modeldir) - tmpdir = "{}/tmp".format(modeldir) + bindir1 = "{}/bin".format(modeldir) + bindir = "{}/wav/bin".format(exeroot) + shutil.copytree(bindir1, bindir) + auxdir1 = "{}/aux".format(modeldir) + auxdir = "{}/wav/aux".format(exeroot) + shutil.copytree(auxdir1, auxdir) + ftndir1 = "{}/ftn".format(modeldir) + ftndir = "{}/wav/ftn".format(exeroot) + shutil.copytree(ftndir1, ftndir) + + tmpdir = "{}/wav/tmp".format(exeroot) # Run w3_setup to create wwatch3.env file env_file = os.path.join(bindir, "wwatch3.env") @@ -77,7 +86,7 @@ def buildlib(bldroot, installpath, case): y """.format(sf90, scc, tmpdir)) - run_bld_cmd_ensure_logging("./w3_setup {} -s E3SM < w3_setup.inp".format(modeldir), logger, from_dir=bindir) + run_bld_cmd_ensure_logging("./w3_setup {} -s E3SM < w3_setup.inp".format(builddir), logger, from_dir=bindir) os.remove(inp_file) # Generate pre-processed WW3 source code @@ -85,7 +94,7 @@ y for exe in ww3_exe: run_bld_cmd_ensure_logging("./w3_source {}".format(exe), logger, from_dir=bindir) for exe in ww3_exe: - tarfile = "{}/work/{}.tar.gz".format(modeldir,exe) + tarfile = "{}/work/{}.tar.gz".format(builddir,exe) shutil.move(tarfile, tmpdir) run_bld_cmd_ensure_logging("tar -xzvf {}.tar.gz".format(exe), logger, from_dir=tmpdir) run_bld_cmd_ensure_logging("rm ww3_shel.F90", logger, from_dir=tmpdir) From a8019dc9b3b2d6b18ab1871e991c373638dfc9f2 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 9 Sep 2024 17:59:35 -0500 Subject: [PATCH 681/904] build error when built without MOABCOMP --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index ca7877a7bc3a..5e1b762270b2 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -308,10 +308,13 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) #ifdef MOABCOMP use mct_mod use seq_comm_mct, only : num_moab_exports +#endif + integer :: ent_type +#ifdef MOABCOMP real(r8) :: difference type(mct_list) :: temp_list - integer :: size_list, index_list, ent_type + integer :: size_list, index_list type(mct_string) :: mctOStr ! character(CXX) :: mct_field, modelStr #endif From 7bca46932236ffaa4af1287358d7e03613996501 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 9 Sep 2024 21:39:42 -0500 Subject: [PATCH 682/904] DPxx: Remove unused use_moisture line. --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 74579df552bd..cf70dd49f564 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -268,8 +268,6 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); - // Load data from IOP files, if necessary m_iop->read_iop_file_data(timestamp()); From 674d6adaa9062df8c549d598218008b98b7c5996 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Tue, 10 Sep 2024 11:21:19 -0500 Subject: [PATCH 683/904] Add grid support for ERA5r025 configurations --- cime_config/config_grids.xml | 67 +++++++++++++++++++ .../elm/cime_config/config_compsets.xml | 5 ++ 2 files changed, 72 insertions(+) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 0f1c3512ec8a..3e47fd4d754d 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -1979,6 +1979,26 @@ IcoswISC30E3r5 + + ERA5r025 + r05 + IcoswISC30E3r5 + r05 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + + + ERA5r025 + r025 + IcoswISC30E3r5 + r025 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + ne120np4.pg2 r0125 @@ -2801,6 +2821,14 @@ TL319 is JRA lat/lon grid: + + 1440 + 721 + $DIN_LOC_ROOT/share/domains/domain.lnd.ERA5r025_IcoswISC30E3r5.240903.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ERA5r025_IcoswISC30E3r5.240903.nc + ERA5r025 is the lat/lon cap grid used by ERA5 data: + + @@ -4732,6 +4760,38 @@ cpl/gridmaps/TL319/map_r05_to_TL319_traave.20240212.nc + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5-nomask_trbilin.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5_esmfpatch.20240903.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ERA5r025_traave.20240903.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ERA5r025_traave.20240903.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_traave.20240905.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_esmfbilin.20240907.nc + cpl/gridmaps/r05/map_r05_to_ERA5r025_traave.20240905.nc + cpl/gridmaps/r05/map_r05_to_ERA5r025_traave.20240905.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_traave.20240905.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_esmfbilin.20240907.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_trbilin.20240903.nc + cpl/gridmaps/r025/map_r025_to_ERA5r025_traave.20240903.nc + cpl/gridmaps/r025/map_r025_to_ERA5r025_traave.20240903.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_trbilin.20240903.nc + + cpl/cpl6/map_T31_to_gx3v7_aave_da_090903.nc cpl/cpl6/map_T31_to_gx3v7_patch_090903.nc @@ -5635,6 +5695,13 @@ cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r05_traave.20240403.nc + + cpl/gridmaps/r025/map_r025_to_gis1to10kmR2_traave.20240903.nc + cpl/gridmaps/r025/map_r025_to_gis1to10kmR2_trbilin.20240903.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r025_traave.20240903.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r025_traave.20240903.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_traave.20240403.nc cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_trbilin.20240403.nc diff --git a/components/elm/cime_config/config_compsets.xml b/components/elm/cime_config/config_compsets.xml index 5eed24e49da6..4afbdfbf3a3a 100644 --- a/components/elm/cime_config/config_compsets.xml +++ b/components/elm/cime_config/config_compsets.xml @@ -972,6 +972,11 @@ 2000_DATM%QIA_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV + + IGERA5ELM_MLI + 2000_DATM%ERA5_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV + + I1PTELM 2000_DATM%1PT_ELM%SP_SICE_SOCN_MOSART_SGLC_SWAV From 26cb157e61f35ecfaa8e2f859b2065a09be38dbc Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Tue, 10 Sep 2024 12:14:31 -0500 Subject: [PATCH 684/904] Add some r025 files used by ERA5r025_r025_IcoswISC30E3r5_gis1to10r02 --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index e76a5647b4e7..cc1bd512e69a 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -613,6 +613,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/glcmaskdata_0.9x1.25_60S.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc +lnd/clm2/griddata/glcmaskdata_r025_GIS_AIS.20240910.nc lnd/clm2/griddata/glcmaskdata_r0125_GIS_AIS.210407.nc lnd/clm2/griddata/glcmaskdata_ne30pg2_GIS_AIS.210407.nc @@ -623,6 +624,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/topodata_48x96_USGS_070110.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c200609.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c211019.nc +lnd/clm2/surfdata_map/surfdata_0.25x0.25_simyr1850_c240125.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1850_c190730.nc lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr1850_c210402.nc From 026ff137c209f2e811c9a31026f23e765026b718 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 10 Sep 2024 13:09:36 -0500 Subject: [PATCH 685/904] Reduce test lengths Reduce the simulation time for tests from defaults following what is used in e3sm_integration for similar cases. --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 8 ++++---- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index f5c3ffcebf94..b113710f46f3 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -31,10 +31,10 @@ jobs: fail-fast: false matrix: test: - - SMS_D_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu - - ERS_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu.eam-wcprod_F2010 - - SMS_D_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu - - ERS_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod + - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu + - ERS_Ld5_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu.eam-wcprod_F2010 + - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu + - ERS_Ld5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index 178b9a6a4429..ca21646382b8 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -20,8 +20,8 @@ jobs: fail-fast: false matrix: test: - - SMS_D_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - - ERS_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 + - SMS_D_Ld1_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu + - ERS_Ld3_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 container: image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 From f38fe0e46ab0a225a126f793e5fbcee910dd7a78 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 15:56:02 -0500 Subject: [PATCH 686/904] big flow error compute rof lnd interaction only if rof_c2_lnd is true we were computing an unnecessary map always even if rof_c2_lnd is false (for most coupled cases it is true) --- driver-moab/main/prep_lnd_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 44ae5490e2c6..0272e835ea03 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -222,7 +222,6 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln call seq_map_init_rcfile(mapper_Fr2l, rof(1), lnd(1), & 'seq_maps.rc','rof2lnd_fmapname:','rof2lnd_fmaptype:',samegrid_lr, & string='mapper_Fr2l initialization',esmf_map=esmf_map_flag) - end if ! symmetric of l2r, from prep_rof #ifdef HAVE_MOAB ! Call moab intx only if land and river are init in moab @@ -381,6 +380,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln end if ! if ((mbrxid .ge. 0) .and. (mblxid .ge. 0)) ! endif HAVE_MOAB #endif + end if call shr_sys_flush(logunit) if (atm_c2_lnd) then From 8bd4d09450caf2f6f35892c4076f9e7db270774f Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 10 Sep 2024 15:58:30 -0500 Subject: [PATCH 687/904] Adjust PEs for MPAS dev-tests on Anvil --- .../testmods_dirs/config_pes_tests.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cime_config/testmods_dirs/config_pes_tests.xml b/cime_config/testmods_dirs/config_pes_tests.xml index 080e496d205b..1ac88ac5fe6f 100644 --- a/cime_config/testmods_dirs/config_pes_tests.xml +++ b/cime_config/testmods_dirs/config_pes_tests.xml @@ -37,6 +37,8 @@ -4 -4 -4 + -4 + -4 -4 @@ -203,6 +205,23 @@ + + + + tests+anvil: --compset GMPAS-JRA1p5-DIB-PISMF, 8 nodes + + -8 + -8 + -8 + -8 + -8 + -8 + -8 + -8 + + + + From 1892887ede89b000b3165ea5f76a05c5e1a3dbe2 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 16:15:26 -0500 Subject: [PATCH 688/904] MPI group needs to be retrieved before use in case rof c2 lnd, we were not retrieving the the MPI group needed for comm graph computation, for same grid path we were retrieving it for intx only how come we never had samegrid land rof before ? --- driver-moab/main/prep_rof_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index 59cc61c58890..a6a9184aefb5 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -327,6 +327,7 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) write(logunit,*) subname,' error in defining tags for seq_flds_a2x_fields on rof cpl' call shr_sys_abort(subname//' ERROR in defining tags for seq_flds_a2x_fields on rof cpl') endif + call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) if (samegrid_lr) then ! the same mesh , lnd and rof use the same dofs, but restricted ! we do not compute intersection, so we will have to just send data from lnd to rof and viceversa, by GLOBAL_ID matching @@ -363,7 +364,6 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) ! we also need to compute the comm graph for the second hop, from the lnd on coupler to the ! lnd for the intx lnd-rof context (coverage) ! - call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) type1 = 3 ! land is FV now on coupler side type2 = 3; From e4b86b6e707aa57589e4361001f1540ca4b8542b Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 18:35:51 -0500 Subject: [PATCH 689/904] reviews from Vijay --- .../data_comps/dlnd/src/dlnd_comp_mod.F90 | 4 +- driver-moab/main/prep_lnd_mod.F90 | 8 +-- driver-moab/main/prep_ocn_mod.F90 | 66 +++++++++---------- driver-moab/main/prep_rof_mod.F90 | 32 ++++----- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index 7768b2dfe815..4d76c0fe1a1f 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -148,10 +148,10 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & #ifdef HAVE_MOAB character*400 tagname real(R8) latv, lonv - integer iv, tagindex, ilat, ilon !, arrsize, nfields ! ierr is already defined as integer above + integer iv, tagindex, ilat, ilon real(R8), allocatable, target :: data(:) integer(IN), pointer :: idata(:) ! temporary - real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary + real(R8), dimension(:), allocatable :: moab_vert_coords ! temporary #ifdef MOABDEBUG character*100 outfile, wopts #endif diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 0272e835ea03..7bdb12376a3d 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -1,6 +1,6 @@ module prep_lnd_mod - use shr_kind_mod , only: r8 => SHR_KIND_R8 + use shr_kind_mod , only: R8 => SHR_KIND_R8 use shr_kind_mod , only: cs => SHR_KIND_CS use shr_kind_mod , only: cl => SHR_KIND_CL use shr_kind_mod , only: cxx => SHR_KIND_CXX @@ -162,7 +162,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln integer nrflds ! number of rof fields projected on land integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 #endif character(*), parameter :: subname = '(prep_lnd_init)' @@ -369,7 +369,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mblxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on land' @@ -693,7 +693,7 @@ subroutine prep_lnd_mrg_moab (infodata) #endif #ifdef MOABCOMP character(CXX) :: tagname, mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! diff --git a/driver-moab/main/prep_ocn_mod.F90 b/driver-moab/main/prep_ocn_mod.F90 index 44ec0c6a3c7a..b13cce62f95d 100644 --- a/driver-moab/main/prep_ocn_mod.F90 +++ b/driver-moab/main/prep_ocn_mod.F90 @@ -1,6 +1,6 @@ module prep_ocn_mod - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: CX => shr_kind_CX, CXX => shr_kind_CXX @@ -137,7 +137,7 @@ module prep_ocn_mod integer , target :: x2oacc_ox_cnt ! x2oacc_ox: number of time samples accumulated ! accumulation variables for moab data - real (kind=r8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array + real (kind=R8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array integer , target :: x2oacc_om_cnt ! x2oacc_ox: number of time samples accumulated, in moab array integer :: arrSize_x2o_om ! this will be a module variable, size moabLocal_size * nof @@ -154,20 +154,20 @@ module prep_ocn_mod !================================================================================================ - real (kind=r8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them + real (kind=R8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them ! they were init with ! character(*),parameter :: fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2o_om (:,:) - real (kind=r8) , allocatable, private :: a2x_om (:,:) - real (kind=r8) , allocatable, private :: i2x_om (:,:) - real (kind=r8) , allocatable, private :: r2x_om (:,:) - real (kind=r8) , allocatable, private :: xao_om (:,:) + real (kind=R8) , allocatable, private :: x2o_om (:,:) + real (kind=R8) , allocatable, private :: a2x_om (:,:) + real (kind=R8) , allocatable, private :: i2x_om (:,:) + real (kind=R8) , allocatable, private :: r2x_om (:,:) + real (kind=R8) , allocatable, private :: xao_om (:,:) ! this will be constructed first time, and be used to copy fields for shared indices ! between xao and x2o character(CXX) :: shared_fields_xao_x2o ! will need some array to hold the data for copying - real(r8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize + real(R8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize integer :: size_of_shared_values logical :: iamin_CPLALLICEID ! pe associated with CPLALLICEID @@ -268,7 +268,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags integer noflds ! used for number of fields in allocating moab accumulated array x2oacc_om - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 !--------------------------------------------------------------- @@ -786,7 +786,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mboxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on ocn' @@ -1095,7 +1095,7 @@ subroutine prep_ocn_mrg(infodata, fractions_ox, xao_ox, timer_mrg) ! ! Local Variables integer :: eii, ewi, egi, eoi, eai, eri, exi, efi, emi - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor type(mct_avect), pointer :: x2o_ox integer :: cnt character(*), parameter :: subname = '(prep_ocn_mrg)' @@ -1177,7 +1177,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) !--------------------------------------------------------------- - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor ! will build x2o_om , similar to x2o_ox ! no averages, just one ocn instance @@ -1187,11 +1187,11 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) integer :: kof,kif integer :: lsize, arrsize ! for double arrays integer , save :: noflds,naflds,niflds,nrflds,nxflds! ,ngflds,nwflds, no glacier or wave model - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -1289,7 +1289,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) #endif #ifdef MOABCOMP character(CXX) :: mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1345,7 +1345,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) allocate(a2x_om (lsize, naflds)) allocate(i2x_om (lsize, niflds)) allocate(r2x_om (lsize, nrflds)) - r2x_om = 0._r8 ! should we zero out all of them ? + r2x_om = 0._R8 ! should we zero out all of them ? allocate(xao_om (lsize, nxflds)) ! allocate fractions too ! use the fraclist fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' @@ -1769,7 +1769,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) ! fo_kif_ifrac(n) ! fractions_o%rAttr(kif,n) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -1777,7 +1777,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifracr = fractions_om(n,kir) ! fo_kir_ifrad(n) ! fractions_o%rAttr(kir,n) afracr = fractions_om(n,kor) ! fo_kor_ofrad(n) ! fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -1913,7 +1913,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) !fo_kif_ifrac(n) ! fractions_o%rAttr(kif) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2046,7 +2046,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa !----------------------------------------------------------------------- ! ! Arguments - real(r8) , intent(in) :: flux_epbalfact + real(R8) , intent(in) :: flux_epbalfact type(mct_aVect), intent(in) :: a2x_o type(mct_aVect), intent(in) :: i2x_o type(mct_aVect), intent(in) :: r2x_o @@ -2061,11 +2061,11 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa integer :: kof,kif integer :: lsize integer :: noflds,naflds,niflds,nrflds,nwflds,nxflds,ngflds - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -2584,7 +2584,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2592,7 +2592,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifracr = fractions_o%rAttr(kir,n) afracr = fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -2738,7 +2738,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -3068,7 +3068,7 @@ function prep_ocn_get_mapper_Sw2o() prep_ocn_get_mapper_Sw2o => mapper_Sw2o end function prep_ocn_get_mapper_Sw2o function prep_ocn_get_x2oacc_om() - real(r8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om + real(R8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om prep_ocn_get_x2oacc_om => x2oacc_om end function prep_ocn_get_x2oacc_om function prep_ocn_get_x2oacc_om_cnt() diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index a6a9184aefb5..83a8e331e916 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -1,7 +1,7 @@ module prep_rof_mod #include "shr_assert.h" - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: cxx => SHR_KIND_CXX @@ -112,22 +112,22 @@ module prep_rof_mod ! accumulation variables over moab fields character(CXX) :: sharedFieldsLndRof ! used in moab to define l2racc_lm - real (kind=r8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes - real (kind=r8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module + real (kind=R8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes + real (kind=R8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module integer , target :: l2racc_lm_cnt ! l2racc_lm: number of time samples accumulated integer :: nfields_sh_lr ! number of fields in sharedFieldsLndRof integer :: lsize_lm ! size of land in moab, local character(CXX) :: sharedFieldsAtmRof ! used in moab to define a2racc_am - real (kind=r8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes - real (kind=r8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module + real (kind=R8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes + real (kind=R8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module integer , target :: a2racc_am_cnt ! a2racc_am: number of time samples accumulated integer :: nfields_sh_ar ! number of fields in sharedFieldsAtmRof integer :: lsize_am ! size of atm in moab, local character(CXX) :: sharedFieldsOcnRof ! used in moab to define o2racc_om - real (kind=r8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes - real (kind=r8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof + real (kind=R8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes + real (kind=R8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof integer , target :: o2racc_om_cnt ! o2racc_om: number of time samples accumulated integer :: nfields_sh_or ! number of fields in sharedFieldsOcnRof integer :: lsize_om ! size of ocn in moab, local @@ -145,12 +145,12 @@ module prep_rof_mod logical :: samegrid_al ! samegrid atm and lnd ! moab stuff - real (kind=r8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them + real (kind=R8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them ! they were init with ! character(*),parameter :: fraclist_r = 'lfrac:lfrin:rfrac' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2r_rm (:,:) ! result of merge - real (kind=r8) , allocatable, private :: a2x_rm (:,:) - real (kind=r8) , allocatable, private :: l2x_rm (:,:) + real (kind=R8) , allocatable, private :: x2r_rm (:,:) ! result of merge + real (kind=R8) , allocatable, private :: a2x_rm (:,:) + real (kind=R8) , allocatable, private :: l2x_rm (:,:) !================================================================================================ @@ -1223,7 +1223,7 @@ subroutine prep_rof_merge(l2x_r, a2x_r, fractions_r, x2r_r, cime_model,o2x_r) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1529,7 +1529,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1548,7 +1548,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) character*32 :: outfile, wopts, lnum #endif #ifdef MOABCOMP - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1995,7 +1995,7 @@ function prep_rof_get_o2racc_om_cnt() end function prep_rof_get_o2racc_om_cnt function prep_rof_get_o2racc_om() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om prep_rof_get_o2racc_om => o2racc_om end function prep_rof_get_o2racc_om @@ -2011,7 +2011,7 @@ function prep_rof_get_l2racc_lm_cnt() end function prep_rof_get_l2racc_lm_cnt function prep_rof_get_l2racc_lm() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm prep_rof_get_l2racc_lm => l2racc_lm end function prep_rof_get_l2racc_lm From 726807322cd42c0d12737c7b5e14a4bb4f5a1194 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 10 Sep 2024 20:34:19 -0500 Subject: [PATCH 690/904] EAMxx: Improve rrtmgp frequency err msg in buildnml.py --- components/eamxx/cime_config/eamxx_buildnml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 121506f7eff3..3e8c4abe9e95 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -181,7 +181,8 @@ def perform_consistency_checks(case, xml): pass elif rest_opt in ["nsteps", "nstep"]: expect (rest_n % rad_freq == 0, - "rrtmgp::rad_frequency incompatible with restart frequency.\n" + f"rrtmgp::rad_frequency ({rad_freq} steps) incompatible with " + f"restart frequency ({rest_n} steps).\n" " Please, ensure restart happens on a step when rad is ON") elif rest_opt in ["nseconds", "nsecond", "nminutes", "nminute", "nhours", "nhour"]: if rest_opt in ["nseconds", "nsecond"]: From 85df9de1fe0ad764722e04164005134448f98f1f Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 10 Sep 2024 21:23:41 -0500 Subject: [PATCH 691/904] EAMxx: In planar geometry case, don't scale 'lat' and 'lon'. --- .../src/dynamics/homme/interface/dyn_grid_mod.F90 | 11 +++++++++-- .../src/dynamics/homme/interface/phys_grid_mod.F90 | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 index cf8d955fe902..3460feab2fe2 100644 --- a/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 @@ -51,6 +51,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) use edge_mod_base, only: edgeVpack_nlyr, edgeVunpack_nlyr use kinds, only: real_kind, int_kind use dof_mod, only: genLocalDof + use control_mod, only: geometry ! ! Inputs ! @@ -62,6 +63,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) real(kind=real_kind), allocatable :: el_cg_gids (:,:,:) ! Homme's bex stuff only works with reals integer(kind=int_kind), allocatable :: el_dg_gids (:,:,:) ! Homme's getLocalDof might not work with c_int integer :: idof, ip,jp, ie, icol + logical :: is_sphere ! Get the gids allocate(el_cg_gids(np,np,nelemd)) @@ -78,6 +80,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) call edgeVpack_nlyr(edge,elem(ie)%desc,el_cg_gids(:,:,ie),1,0,1) enddo call bndry_exchangeV(par,edge) + is_sphere = trim(geometry) /= 'plane' do ie=1,nelemd call edgeVunpack_nlyr(edge,elem(ie)%desc,el_cg_gids(:,:,ie),1,0,1) elgids(ie) = elem(ie)%GlobalId @@ -86,8 +89,12 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) idof = (ie-1)*16+(jp-1)*4+ip cg_gids(idof) = INT(el_cg_gids(ip,jp,ie),kind=c_int) dg_gids(idof) = INT(el_dg_gids(ip,jp,ie),kind=c_int) - lat(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lat * 180.0_c_double/pi - lon(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lon * 180.0_c_double/pi + lat(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lat + lon(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lon + if (is_sphere) then + lat(ip,jp,ie) = lat(ip,jp,ie) * 180.0_c_double/pi + lon(ip,jp,ie) = lon(ip,jp,ie) * 180.0_c_double/pi + end if elgpgp(1,idof) = ie-1 elgpgp(2,idof) = jp-1 elgpgp(3,idof) = ip-1 diff --git a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 index 4d928c800b03..cc1194aed216 100644 --- a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 @@ -325,6 +325,7 @@ end function get_num_global_columns subroutine get_my_phys_data (gids, lat, lon, area, pg_type) use homme_context_mod, only: iam use shr_const_mod, only: pi=>SHR_CONST_PI + use control_mod, only: geometry ! ! Input(s) ! @@ -336,6 +337,7 @@ subroutine get_my_phys_data (gids, lat, lon, area, pg_type) ! integer :: pgN, load_bal, idof, ndofs type(pg_specs_t), pointer :: pgs + logical :: is_sphere ! Possible values for pg_type: ! 0 : physics grid on GLL nodes @@ -364,10 +366,15 @@ subroutine get_my_phys_data (gids, lat, lon, area, pg_type) ! TODO: when you enable twin columns, you'll have to manually ! do the search, since you can't just grab the offset-ed entries ndofs = get_num_local_columns (pgN) + is_sphere = trim(geometry) /= 'plane' do idof=1,ndofs gids(idof) = pgs%g_dofs(pgs%g_dofs_offsets(iam+1) + idof) - lat(idof) = pgs%g_lat (pgs%g_dofs_offsets(iam+1) + idof) * 180.0_c_double / pi - lon(idof) = pgs%g_lon (pgs%g_dofs_offsets(iam+1) + idof) * 180.0_c_double / pi + lat(idof) = pgs%g_lat (pgs%g_dofs_offsets(iam+1) + idof) + lon(idof) = pgs%g_lon (pgs%g_dofs_offsets(iam+1) + idof) + if (is_sphere) then + lat(idof) = lat(idof) * 180.0_c_double / pi + lon(idof) = lon(idof) * 180.0_c_double / pi + end if area(idof) = pgs%g_area(pgs%g_dofs_offsets(iam+1) + idof) enddo From 73e37fab4cbf63d4ef16759bb02328f9b090798a Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 11 Sep 2024 08:28:27 -0700 Subject: [PATCH 692/904] update compy intel compiler --- cime_config/machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 5eaae75f2043..ff6f9ac37597 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3912,7 +3912,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss gcc/8.1.0 - intel/19.0.5 + intel/20.0.0 pgi/19.10 @@ -3921,7 +3921,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss mvapich2/2.3.1 - intelmpi/2019u4 + intelmpi/2020 intelmpi/2019u3 From f08004483cae52fb217a539ff1186c33add80e3f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 11 Sep 2024 10:23:10 -0600 Subject: [PATCH 693/904] EAMxx: make rrtmgp call find_package for kokkos If an existing install of Kokkos is used, the imported target has local scope, so following calls to cmake's target_link_libraries won't recognize Kokkos if the first call to find_package was in a subdir. In EAMxx, that's the case (it's found while parsing the ekat subdir). However, subsequent calls to find_package are virtually no-op, so it doesn't hurt to call it again --- components/eamxx/src/physics/rrtmgp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index 0d42d61fa662..1d3df9b4d2e4 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -127,6 +127,7 @@ yakl_process_target(rrtmgp) # NOTE: cannot use 'PUBLIC' in target_link_libraries, # since yakl_process_target already used it # with the "plain" signature +find_package(Kokkos REQUIRED) target_link_libraries(rrtmgp yakl Kokkos::kokkos) target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL From 57f549be17e95ef5b1ab5e1218fcd11547608044 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 11 Sep 2024 12:08:35 -0700 Subject: [PATCH 694/904] Enables output of land fraction used in dust mobilization Adds a new member variable (`lnd_frc_mbl_patch`) to `dust_type` to allow output of land fraction sued in dust mobilization. This new variable is added to ELM history file and is inactive by default. [BFB] --- components/elm/src/biogeochem/DUSTMod.F90 | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/components/elm/src/biogeochem/DUSTMod.F90 b/components/elm/src/biogeochem/DUSTMod.F90 index ba24f2596697..a6ee6c6c11f8 100644 --- a/components/elm/src/biogeochem/DUSTMod.F90 +++ b/components/elm/src/biogeochem/DUSTMod.F90 @@ -69,6 +69,7 @@ module DUSTMod real(r8), pointer, private :: vlc_trb_2_patch (:) ! turbulent deposition velocity 2(m/s) real(r8), pointer, private :: vlc_trb_3_patch (:) ! turbulent deposition velocity 3(m/s) real(r8), pointer, private :: vlc_trb_4_patch (:) ! turbulent deposition velocity 4(m/s) + real(r8), pointer, private :: lnd_frc_mbl_patch (:) ! land fraction for dust mobilization (-) real(r8), pointer, private :: mbl_bsn_fct_col (:) ! basin factor contains @@ -121,7 +122,8 @@ subroutine InitAllocate(this, bounds) allocate(this%vlc_trb_2_patch (begp:endp)) ; this%vlc_trb_2_patch (:) = nan allocate(this%vlc_trb_3_patch (begp:endp)) ; this%vlc_trb_3_patch (:) = nan allocate(this%vlc_trb_4_patch (begp:endp)) ; this%vlc_trb_4_patch (:) = nan - allocate(this%mbl_bsn_fct_col (begc:endc)) ; this%mbl_bsn_fct_col (:) = nan + allocate(this%lnd_frc_mbl_patch (begp:endp)) ; this%lnd_frc_mbl_patch (:) = spval + allocate(this%mbl_bsn_fct_col (begc:endc)) ; this%mbl_bsn_fct_col (:) = nan end subroutine InitAllocate @@ -167,6 +169,11 @@ subroutine InitHistory(this, bounds) avgflag='A', long_name='turbulent deposition velocity 4', & ptr_patch=this%vlc_trb_4_patch, default='inactive') + this%lnd_frc_mbl_patch(begp:endp) = spval + call hist_addfld1d (fname='LND_FRC_DUST_MBL', units='-', & + avgflag='A', long_name='land fraction for dust mobilization', & + ptr_patch=this%lnd_frc_mbl_patch, default='inactive') + end subroutine InitHistory !----------------------------------------------------------------------- @@ -239,7 +246,6 @@ subroutine DustEmission (bounds, & real(r8) :: Cd ! [dimless] The dust emission coefficient, which depends on ! the soil's standardized threshold friction speed -YF real(r8) :: wnd_frc_slt - real(r8) :: lnd_frc_mbl(bounds%begp:bounds%endp) real(r8) :: bd real(r8) :: gwc_sfc real(r8) :: ttlai(bounds%begp:bounds%endp) @@ -287,6 +293,7 @@ subroutine DustEmission (bounds, & u10 => frictionvel_vars%u10_patch , & ! Input: [real(r8) (:) ] 10-m wind (m/s) (created for dust model) mbl_bsn_fct => dust_vars%mbl_bsn_fct_col , & ! Input: [real(r8) (:) ] basin factor + lnd_frc_mbl => dust_vars%lnd_frc_mbl_patch , & ! Output: [real(r8) (:) ] land fraction for dust mobilization flx_mss_vrt_dst => dust_vars%flx_mss_vrt_dst_patch , & ! Output: [real(r8) (:,:) ] surface dust emission (kg/m**2/s) flx_mss_vrt_dst_tot => dust_vars%flx_mss_vrt_dst_tot_patch & ! Output: [real(r8) (:) ] total dust flux back to atmosphere (pft) ) From 2faa92748a137729e3269cb399b2fafe258588fb Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Fri, 12 Jul 2024 13:53:00 -0400 Subject: [PATCH 695/904] Added downscaling code for coupler bypass --- .../elm/src/cpl/lnd_downscale_atm_forcing.F90 | 526 ++++++++++++++++++ components/elm/src/cpl/lnd_import_export.F90 | 90 +-- components/elm/src/main/atm2lndType.F90 | 5 + 3 files changed, 584 insertions(+), 37 deletions(-) diff --git a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 index 99b03a2e25ae..8c003bbf5e90 100644 --- a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 +++ b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 @@ -35,10 +35,13 @@ module lnd_downscale_atm_forcing ! ! !PUBLIC MEMBER FUNCTIONS: public :: downscale_atm_forcing_to_topounit ! Calls downscaling subroutines of forcing fields from gridcell to topounit + public :: downscale_atm_forcing_to_topounit_cpl_bypass ! Calls downscaling subroutines of forcing fields from gridcell to topounit, for use within the CPL_BYPASS code ! ! !PRIVATE MEMBER FUNCTIONS: private :: downscale_atm_state_to_topounit ! Downscale atmosperic state fields from gridcell to topounit + private :: downscale_atm_state_to_topounit_cpl_bypass ! Downscale atmosperic state fields from gridcell to topounit, for cpl_bypass code private :: downscale_longwave_to_topounit ! Downscale longwave radiation field from gridcell to topounit + private :: downscale_longwave_to_topounit_cpl_bypass ! Downscale longwave radiation field from gridcell to topounit, for cpl_bypass code private :: downscale_precip_to_topounit_FNM ! Downscale precipitation field from gridcell to topounit using Froude number method (FNM) private :: downscale_precip_to_topounit_ERMM ! Downscale precipitation field from gridcell to topounit using elevation ration with maximum elevation method (ERMM) private :: build_normalization ! Compute normalization factors so that downscaled fields are conservative @@ -787,4 +790,527 @@ subroutine build_normalization(orig_field, sum_field, sum_wts, norms) end subroutine build_normalization + subroutine downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm_vars) + ! + ! !DESCRIPTION: + ! Downscale fields from gridcell to topounit + ! + ! Downscaling is done over topounits if the number of topounits > 1. + ! + ! !USES: + use elm_time_manager, only : get_nstep + use elm_varcon , only : rair, cpair, grav, lapse_glcmec + use elm_varcon , only : glcmec_rain_snow_threshold, o2_molar_const + use shr_const_mod , only : SHR_CONST_TKFRZ + use landunit_varcon , only : istice_mec + use elm_varctl , only : glcmec_downscale_rain_snow_convert + use domainMod , only : ldomain + use QsatMod , only : Qsat + use FrictionVelocityMod, only: atm_gustiness + ! + ! !ARGUMENTS: + integer , intent(in) :: g + real(r8) , intent(in) :: x2l(:,:) + type(atm2lnd_type) , intent(in) :: atm2atm_vars + type(lnd2atm_type) , intent(in) :: lnd2atm_vars + + ! + ! !LOCAL VARIABLES: + integer :: t, l, c, fc, t2 ! indices + integer :: clo, cc + integer :: numt_pg ! Number of topounits per grid + + ! temporaries for topo downscaling + real(r8) :: rain_g, snow_g + real(r8) :: mxElv ! Maximum elevation value per grid + real(r8) :: uovern_t ! Froude Number + real(r8) :: grdElv ! Grid elevation + real(r8) :: topoElv ! Topounit elevation + real(r8) :: max_tpuElv ! Maximum topounit elevation for calculating elevation range + real(r8) :: min_tpuElv ! Minimum topounit elevation for calculating elevation range + real(r8) :: crnt_temp_t ! Current downscaled topounit temperature + real(r8) :: temp_r ! Temporary topounit rainfall + real(r8) :: temp_s ! Temporary topounit snowfall + real(r8) :: t_th ! Temperature threshold for snowfall + real(r8) :: Ta_th1 ! Temperature at 0.5 Celsius in K 273.65 + real(r8) :: Ta_th2 ! Temperature at 2.0 Celsius in K 275.15 + real(r8) :: Ta_th3 ! Temperature at 2.5 Celsius in K 275.65 + real(r8) :: tmp_Snow_frc ! Current snow fraction + + real(r8) :: e + real(r8) :: qvsat + + real(r8) :: sum_qbot_g ! weighted sum of column-level lwrad + real(r8) :: sum_wtsq_g ! sum of weights that contribute to sum_lwrad_g + real(r8) :: qbot_norm_g ! normalization factors + real(r8) :: sum_lwrad_g ! weighted sum of column-level lwrad + real(r8) :: sum_wtslw_g ! sum of weights that contribute to sum_lwrad_g + real(r8) :: lwrad_norm_g ! normalization factors + real(r8) :: esatw ! saturation vapor pressure over water (Pa) + real(r8) :: esati ! saturation vapor pressure over ice (Pa) + real(r8) :: a0,a1,a2,a3,a4,a5,a6 ! coefficients for esat over water + real(r8) :: b0,b1,b2,b3,b4,b5,b6 ! coefficients for esat over ice + real(r8) :: tdc, temp ! Kelvins to Celcius function and its input + real(r8) :: vp ! water vapor pressure (Pa) + + real(r8), allocatable :: deltaRain(:) ! Deviation of subgrid rain from grid rain + real(r8), allocatable :: deltaSnow(:) ! Deviation of subgrid snow from grid snow + real(r8) :: deltaR ! Temporary deltaRain + real(r8) :: deltaS ! Temporary deltaSnow + real(r8) :: sum_of_hrise ! Sum of height rise of air parcel of all subgrids of a grid + real(r8) :: hrise ! Temporary height rise + real(r8) :: elvrnge ! Elevation range between lowest and highest tpu + real(r8) :: ave_elv + integer :: elv_flag ! Elevation flag to trac grids with +ve grid elevation and -ve tpu elevation + + integer :: uaflag = 0 + integer :: precip_dwn = 0 ! Used to turn on/off the downscaling of precipitation 0 = on; 1 = off + integer :: other_forcing_dwn = 0 ! Used to turn on/off the downscaling of other forcing 0 = on; 1 = off + + character(len=*), parameter :: subname = 'downscale_atm_forcing_to_topounit' + !---------------------------------------------------------------------------------------- + + ! Constants to compute vapor pressure + parameter (a0=6.107799961_r8 , a1=4.436518521e-01_r8, & + a2=1.428945805e-02_r8, a3=2.650648471e-04_r8, & + a4=3.031240396e-06_r8, a5=2.034080948e-08_r8, & + a6=6.136820929e-11_r8) + + parameter (b0=6.109177956_r8 , b1=5.034698970e-01_r8, & + b2=1.886013408e-02_r8, b3=4.176223716e-04_r8, & + b4=5.824720280e-06_r8, b5=4.838803174e-08_r8, & + b6=1.838826904e-10_r8) + + ! + ! function declarations + ! + tdc(temp) = min( 50._r8, max(-50._r8,(temp-SHR_CONST_TKFRZ)) ) ! Taken from lnd_import_export.F90 + esatw(temp) = 100._r8*(a0+temp*(a1+temp*(a2+temp*(a3+temp*(a4+temp*(a5+temp*a6)))))) ! Taken from lnd_import_export.F90 + esati(temp) = 100._r8*(b0+temp*(b1+temp*(b2+temp*(b3+temp*(b4+temp*(b5+temp*b6)))))) ! Taken from lnd_import_export.F90 + !----------------------------------------------------------------------- + ! Get required inputs + numt_pg = grc_pp%ntopounits(g) ! Number of topounits per grid + grdElv = grc_pp%elevation(g) ! Grid level sfc elevation + mxElv = grc_pp%MaxElevation(g) ! Maximum src elevation per grid obtained from the highest elevation topounit + uovern_t = atm2lnd_vars%forc_uovern(g) ! Froude Number + + snow_g = atm2lnd_vars%forc_snow_not_downscaled_grc(g) + rain_g = atm2lnd_vars%forc_rain_not_downscaled_grc(g) + + sum_qbot_g = 0._r8 + sum_wtsq_g = 0._r8 + sum_lwrad_g = 0._r8 + sum_wtslw_g = 0._r8 + + sum_of_hrise = 0._r8 + t_th = 273.15_r8 ! Freezing temperature in K + Ta_th1 = 273.65_r8 ! Lowest threshold for snow calculation + Ta_th2 = 275.15_r8 ! Middle threshold for rain/snow partitioning + Ta_th3 = 275.65_r8 ! Highest threshold for rain/snow partitioning + tmp_Snow_frc = 0._r8 ! Snow fraction + elv_flag = 0 + elvrnge = 0._r8 + ave_elv = 0._r8 + max_tpuElv = 0._r8 + min_tpuElv = 0._r8 + if (numt_pg > 1) then !downscaling is done only if a grid has more than 1 topounits + if (precip_downscaling_method == 'FNM') then + allocate(deltaRain(numt_pg)) + deltaRain(:) = 0._r8 + allocate(deltaSnow(numt_pg)) + deltaSnow(:) = 0._r8 + hrise = 0._r8 + end if + + ! calculate elevation range and track grids with +ve elevation but have -ve tpu elevation + max_tpuElv = -100000._r8 + min_tpuElv = 100000._r8 + do t = grc_pp%topi(g), grc_pp%topf(g) ! Check occurence of grid elevation is +ve while tpu elevation is -ve + topoElv = top_pp%elevation(t) + if (topoElv > max_tpuElv) then + max_tpuElv = topoElv + end if + if (topoElv < min_tpuElv) then + min_tpuElv = topoElv + end if + end do + elvrnge = max_tpuElv - min_tpuElv + + do t = grc_pp%topi(g), grc_pp%topf(g) + t2 = t - grc_pp%topi(g) + 1 + topoElv = top_pp%elevation(t) ! Topounit sfc elevation + + ! Downscale precipitation + if (mxElv == 0._r8 .or. precip_dwn == 1) then ! avoid dividing by 0 + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + else + if (precip_downscaling_method == 'FNM') then + call downscale_precip_to_topounit_FNM(mxElv,uovern_t,grdElv,topoElv,rain_g,snow_g,deltaR,deltaS,hrise) !Use FNM method + deltaRain(t2) = deltaR + deltaSnow(t2) = deltaS + sum_of_hrise = sum_of_hrise + hrise + else + call downscale_precip_to_topounit_ERMM(t,mxElv,grdElv,topoElv,rain_g, snow_g,elv_flag,elvrnge) ! Use ERMM method + end if + end if + + ! Downscale other fluxes + if (other_forcing_dwn == 1) then ! flag to turn on or off downscaling of other forcing + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + top_af%lwrad(t) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + ! Update top_as + top_as%tbot(t) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(t) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(t) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(t) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + else + call downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars, lnd2atm_vars, uaflag) + call downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars, lnd2atm_vars, uaflag) + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + sum_qbot_g = sum_qbot_g + top_pp%wtgcell(t)*top_as%qbot(t) + sum_wtsq_g = sum_wtsq_g + top_pp%wtgcell(t) + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + + ! Keep track of the gridcell-level weighted sum for later normalization. + ! + ! This gridcell-level weighted sum just includes points for which we do the + ! downscaling (e.g., glc_mec points). Thus the contributing weights + ! generally do not add to 1. So to do the normalization properly, we also + ! need to keep track of the weights that have contributed to this sum. + sum_lwrad_g = sum_lwrad_g + top_pp%wtgcell(t)*top_af%lwrad(t) + sum_wtslw_g = sum_wtslw_g + top_pp%wtgcell(t) + end if + end do + if (precip_downscaling_method == 'FNM') then + do t = grc_pp%topi(g), grc_pp%topf(g) + t2 = t - grc_pp%topi(g) + 1 + if (mxElv == 0.) then ! avoid dividing by 0 + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + else + top_af%rain(t) = rain_g + (deltaRain(t2) - (rain_g/mxElv)*(sum_of_hrise/numt_pg)) + top_af%snow(t) = snow_g + (deltaSnow(t2) - (snow_g/mxElv)*(sum_of_hrise/numt_pg)) + end if + + end do + deallocate(deltaRain) + deallocate(deltaSnow) + end if + + ! Precipitation partitioning using simple method following Jordan (1991) + do t = grc_pp%topi(g), grc_pp%topf(g) + crnt_temp_t = top_as%tbot(t) + if (crnt_temp_t > Ta_th3) then ! No snow or all rain + temp_r = top_af%rain(t) + top_af%snow(t) + temp_s = 0._r8 + else if (crnt_temp_t >= Ta_th2 .and. crnt_temp_t <= Ta_th3) then + temp_s = (top_af%snow(t) + top_af%rain(t))*0.6_r8 ! 0.6 fraction of the total precip is snow + temp_r = (top_af%snow(t) + top_af%rain(t)) - temp_s + else if (crnt_temp_t < Ta_th2 .and. crnt_temp_t > Ta_th1) then + tmp_Snow_frc = (crnt_temp_t-Ta_th2)*(0.4_r8/(Ta_th1-Ta_th2))+0.6_r8 ! Snow fraction value 1-0.6 = 0.4 snowfrc is 1 at t = Ta_th1 + temp_s = (top_af%snow(t) + top_af%rain(t))*tmp_Snow_frc ! 0.6 is snow fraction at t = Ta_th2 + temp_r = (top_af%snow(t) + top_af%rain(t)) - temp_s + else ! crnt_temp_t <= Ta_th1 ==> all snow + temp_s = top_af%snow(t) + top_af%rain(t) + temp_r = 0._r8 + + end if + + top_af%rain(t) = temp_r + top_af%snow(t) = temp_s + + end do + + else !grid has a single topounit + ! update top_af using grid level values + t = grc_pp%topi(g) + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + top_af%lwrad(t) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + + ! Update top_as + top_as%tbot(t) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(t) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(t) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(t) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + if (atm_gustiness) then + top_as%windbot(t) = sqrt(top_as%windbot(t)**2 + top_as%ugust(t)**2) + end if + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + + end if + + if (numt_pg > 1) then + + ! Normalize forc_qbot to conserve energy + + call build_normalization(orig_field=atm2lnd_vars%forc_q_not_downscaled_grc(g), & + sum_field=sum_qbot_g, sum_wts=sum_wtsq_g, norms=qbot_norm_g) + + do t = grc_pp%topi(g), grc_pp%topf(g) + top_as%qbot(t) = top_as%qbot(t) * qbot_norm_g + + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + end do + + + ! Normalize forc_lwrad_c(c) to conserve energy + + call build_normalization(orig_field=atm2lnd_vars%forc_lwrad_not_downscaled_grc(g), & + sum_field=sum_lwrad_g, sum_wts=sum_wtslw_g, norms=lwrad_norm_g) + + do t = grc_pp%topi(g), grc_pp%topf(g) + top_af%lwrad(t) = top_af%lwrad(t) * lwrad_norm_g + end do + + end if + + end subroutine downscale_atm_forcing_to_topounit_cpl_bypass + + !----------------------------------------------------------------------- + ! Downscale other atmospheric state variables + !----------------------------------------------------------------------- + subroutine downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + ! + ! !DESCRIPTION: + ! Downscale atmospheric forcing fields from gridcell to topounit + ! + ! Downscaling is done over topounits. + ! + ! !USES: + use elm_time_manager, only : get_nstep + use elm_varcon , only : rair, cpair, grav, lapse_glcmec + use elm_varcon , only : glcmec_rain_snow_threshold + use landunit_varcon , only : istice_mec + use elm_varctl , only : glcmec_downscale_rain_snow_convert + use domainMod , only : ldomain + use QsatMod , only : Qsat + ! + ! !ARGUMENTS: + integer , intent(in) :: g + integer , intent(in) :: t + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + ! + ! !LOCAL VARIABLES: + integer :: l, c, fc ! indices + integer :: clo, cc + integer :: nstep + + ! temporaries for topo downscaling + real(r8) :: hsurf_g,hsurf_t,Hbot + real(r8) :: zbot_g, tbot_g, pbot_g, thbot_g, qbot_g, qs_g, es_g + real(r8) :: zbot_t, tbot_t, pbot_t, thbot_t, qbot_t, qs_t, es_t + real(r8) :: egcm_t, rhos_t + real(r8) :: dum1, dum2 + + character(len=*), parameter :: subname = 'downscale_atm_state_to_topounit' + !----------------------------------------------------------------------- + + nstep = get_nstep() + + ! Downscale forc_t, forc_th, forc_q, forc_pbot, and forc_rho to columns. + ! For glacier_mec columns the downscaling is based on surface elevation. + ! For other columns the downscaling is a simple copy (above). + + ! This is a simple downscaling procedure + ! Note that forc_hgt, forc_u, and forc_v are not downscaled. + + hsurf_g = grc_pp%elevation(g) ! gridcell sfc elevation + hsurf_t = top_pp%elevation(t) ! topounit sfc elevation + tbot_g = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! atm sfc temp + thbot_g = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! atm sfc pot temp + qbot_g = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! atm sfc spec humid + pbot_g = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! atm sfc pressure + zbot_g = atm2lnd_vars%forc_hgt_grc(g) ! atm ref height + + zbot_t = zbot_g + tbot_t = tbot_g-lapse_glcmec*(hsurf_t-hsurf_g) ! sfc temp for column + + Hbot = rair*0.5_r8*(tbot_g+tbot_t)/grav ! scale ht at avg temp + pbot_t = pbot_g*exp(-(hsurf_t-hsurf_g)/Hbot) ! column sfc press + + ! Derivation of potential temperature calculation: + ! + ! The textbook definition would be: + ! thbot_c = tbot_c * (p0/pbot_c)^(rair/cpair) + ! + ! Note that pressure is related to scale height as: + ! pbot_c = p0 * exp(-zbot_c/H) + ! + ! Using Hbot in place of H, we get: + ! pbot_c = p0 * exp(-zbot_c/Hbot) + ! + ! Plugging this in to the textbook definition, then manipulating, we get: + ! thbot_c = tbot_c * (p0/(p0*exp(-zbot_c/Hbot)))^(rair/cpair) + ! = tbot_c * (1/exp(-zbot_c/Hbot))^(rair/cpair) + ! = tbot_c * (exp(zbot_c/Hbot))^(rair/cpair) + ! = tbot_c * exp((zbot_c/Hbot) * (rair/cpair)) + + thbot_t= tbot_t*exp((zbot_t/Hbot)*(rair/cpair)) ! pot temp calc + + call Qsat(tbot_g,pbot_g,es_g,dum1,qs_g,dum2) + call Qsat(tbot_t,pbot_t,es_t,dum1,qs_t,dum2) + + qbot_t = qbot_g*(qs_t/qs_g) + egcm_t = qbot_t*pbot_t/(0.622_r8+0.378_r8*qbot_t) + rhos_t = (pbot_t-0.378_r8*egcm_t) / (rair*tbot_t) + + top_as%tbot(t) = tbot_t + top_as%thbot(t) = thbot_t + top_as%qbot(t) = qbot_t + top_as%pbot(t) = pbot_t + +! call check_downscale_consistency(bounds, atm2lnd_vars) + + end subroutine downscale_atm_state_to_topounit_cpl_bypass + + !------------------------------------------------------- + ! Downscale longwave radiation place holder + subroutine downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + + ! !DESCRIPTION: + ! Downscale longwave radiation from gridcell to column + ! Must be done AFTER temperature downscaling + + ! !USES: + use elm_time_manager, only : get_nstep + use domainMod , only : ldomain + use landunit_varcon , only : istice_mec + use elm_varcon , only : lapse_glcmec + use elm_varctl , only : glcmec_downscale_longwave + + ! !ARGUMENTS: + integer , intent(in) :: g + integer , intent(in) :: t + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + + ! !LOCAL VARIABLES: + integer :: c,l,fc ! indices + integer :: nstep + real(r8) :: hsurf_t ! column-level elevation (m) + real(r8) :: hsurf_g ! gridcell-level elevation (m) + + real(r8) :: tair_g ! original gridcell mean air temperature + real(r8) :: tair_t ! downscaled topounit air temperature + real(r8) :: tsfc_g ! original gridcell surface temperature + real(r8) :: tsfc_t ! downscaled topounit surface temperature + real(r8) :: lwrad_g ! original gridcell mean LW radiation + real(r8) :: lwrad_t ! downscaled topounit LW radiation + real(r8) :: newsum_lwrad_g ! weighted sum of column-level lwrad after normalization + + character(len=*), parameter :: subname = 'downscale_longwave_to_topounit' + !----------------------------------------------------------------------- + + nstep = get_nstep() + + ! Do the downscaling + hsurf_g = grc_pp%elevation(g) + hsurf_t = top_pp%elevation(t) + + ! Here we assume that deltaLW = (dLW/dT)*(dT/dz)*deltaz + ! We get dLW/dT = 4*eps*sigma*T^3 = 4*LW/T from the Stefan-Boltzmann law, + ! evaluated at the mean temp. + ! We assume the same temperature lapse rate as above. + + tair_g = x2l(index_x2l_Sa_tbot,i) + tair_t = top_as%tbot(t) + lwrad_g = x2l(index_x2l_Faxa_lwdn,i) + top_af%lwrad(t) = lwrad_g - & + 4.0_r8 * lwrad_g/(0.5_r8*(tair_t+tair_g)) * & + lapse_glcmec * (hsurf_t - hsurf_g) + + end subroutine downscale_longwave_to_topounit_cpl_bypass + end module lnd_downscale_atm_forcing diff --git a/components/elm/src/cpl/lnd_import_export.F90 b/components/elm/src/cpl/lnd_import_export.F90 index ad81846196d0..27c01e668184 100644 --- a/components/elm/src/cpl/lnd_import_export.F90 +++ b/components/elm/src/cpl/lnd_import_export.F90 @@ -1030,52 +1030,68 @@ subroutine lnd_import( bounds, x2l, atm2lnd_vars, glc2lnd_vars, lnd2atm_vars) end if end if - !set the topounit-level atmospheric state and flux forcings (bypass mode) + ! Adding topographic downscaling capability within CPL_BYPASS code block + ! PET, 7/12/2024 + if (use_atm_downscaling_to_topunit) then + atm2lnd_vars%forc_uovern = x2l(index_x2l_Sa_uovern,i) + atm2lnd_vars%forc_rain_not_downscaled_grc = forc_rainc + forc_rainl + atm2lnd_vars%forc_snow_not_downscaled_grc = forc_snowc + forc_snowl + + if(atm_gustiness) then + call endrun("Error: atm_gustiness not yet supported with multiple topounits (in CPL_BYPASS)") + end if + do topo = grc_pp%topi(g) , grc_pp%topf(g) + top_as%ugust(topo) = 0._r8 + end do + + call downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm_vars) + else + do topo = grc_pp%topi(g), grc_pp%topf(g) + top_as%tbot(topo) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(topo) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(topo) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(topo) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(topo) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(topo) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(topo) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + top_as%windbot(topo) = sqrt(top_as%ubot(topo)**2 + top_as%vbot(topo)**2) + ! Relative humidity (percent) + if (top_as%tbot(topo) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(topo))) + else + e = esati(tdc(top_as%tbot(topo))) + end if + qsat = 0.622_r8*e / (top_as%pbot(topo) - 0.378_r8*e) + top_as%rhbot(topo) = 100.0_r8*(top_as%qbot(topo) / qsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(topo) = o2_molar_const * top_as%pbot(topo) + ! air density (kg/m**3) - uses a temporary calculation of water vapor pressure (Pa) + vp = top_as%qbot(topo) * top_as%pbot(topo) / (0.622_r8 + 0.378_r8 * top_as%qbot(topo)) + top_as%rhobot(topo) = (top_as%pbot(topo) - 0.378_r8 * vp) / (rair * top_as%tbot(topo)) + top_af%rain(topo) = forc_rainc + forc_rainl ! sum of convective and large-scale rain + top_af%snow(topo) = forc_snowc + forc_snowl ! sum of convective and large-scale snow + top_af%solad(topo,2) = atm2lnd_vars%forc_solad_grc(g,2) ! forc_sollxy Atm flux W/m^2 + top_af%solad(topo,1) = atm2lnd_vars%forc_solad_grc(g,1) ! forc_solsxy Atm flux W/m^2 + top_af%solai(topo,2) = atm2lnd_vars%forc_solai_grc(g,2) ! forc_solldxy Atm flux W/m^2 + top_af%solai(topo,1) = atm2lnd_vars%forc_solai_grc(g,1) ! forc_solsdxy Atm flux W/m^2 + top_af%lwrad(topo) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) ! flwdsxy Atm flux W/m^2 + ! derived flux forcings + top_af%solar(topo) = top_af%solad(topo,2) + top_af%solad(topo,1) + & + top_af%solai(topo,2) + top_af%solai(topo,1) + end do + end if + + !set the topounit-level atmospheric variables that are not handled in downscaling code do topo = grc_pp%topi(g), grc_pp%topf(g) ! first, all the state forcings - top_as%tbot(topo) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K - top_as%thbot(topo) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K - top_as%pbot(topo) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa - top_as%qbot(topo) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg - top_as%ubot(topo) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s - top_as%vbot(topo) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s - if (implicit_stress) then + if (implicit_stress) then top_as%wsresp(topo) = 0._r8 ! Atm state m/s/Pa top_as%tau_est(topo) = 0._r8 ! Atm state Pa end if top_as%ugust(topo) = 0._r8 ! Atm state m/s - top_as%zbot(topo) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m - ! assign the state forcing fields derived from other inputs - ! Horizontal windspeed (m/s) - top_as%windbot(topo) = sqrt(top_as%ubot(topo)**2 + top_as%vbot(topo)**2) if (atm_gustiness) then top_as%windbot(topo) = sqrt(top_as%windbot(topo)**2 + top_as%ugust(topo)**2) end if - ! Relative humidity (percent) - if (top_as%tbot(topo) > SHR_CONST_TKFRZ) then - e = esatw(tdc(top_as%tbot(topo))) - else - e = esati(tdc(top_as%tbot(topo))) - end if - qsat = 0.622_r8*e / (top_as%pbot(topo) - 0.378_r8*e) - top_as%rhbot(topo) = 100.0_r8*(top_as%qbot(topo) / qsat) - ! partial pressure of oxygen (Pa) - top_as%po2bot(topo) = o2_molar_const * top_as%pbot(topo) - ! air density (kg/m**3) - uses a temporary calculation of water vapor pressure (Pa) - vp = top_as%qbot(topo) * top_as%pbot(topo) / (0.622_r8 + 0.378_r8 * top_as%qbot(topo)) - top_as%rhobot(topo) = (top_as%pbot(topo) - 0.378_r8 * vp) / (rair * top_as%tbot(topo)) - - ! second, all the flux forcings - top_af%rain(topo) = forc_rainc + forc_rainl ! sum of convective and large-scale rain - top_af%snow(topo) = forc_snowc + forc_snowl ! sum of convective and large-scale snow - top_af%solad(topo,2) = atm2lnd_vars%forc_solad_grc(g,2) ! forc_sollxy Atm flux W/m^2 - top_af%solad(topo,1) = atm2lnd_vars%forc_solad_grc(g,1) ! forc_solsxy Atm flux W/m^2 - top_af%solai(topo,2) = atm2lnd_vars%forc_solai_grc(g,2) ! forc_solldxy Atm flux W/m^2 - top_af%solai(topo,1) = atm2lnd_vars%forc_solai_grc(g,1) ! forc_solsdxy Atm flux W/m^2 - top_af%lwrad(topo) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) ! flwdsxy Atm flux W/m^2 - ! derived flux forcings - top_af%solar(topo) = top_af%solad(topo,2) + top_af%solad(topo,1) + & - top_af%solai(topo,2) + top_af%solai(topo,1) end do !----------------------------------------------------------------------------------------------------- diff --git a/components/elm/src/main/atm2lndType.F90 b/components/elm/src/main/atm2lndType.F90 index a00140ab2dd6..82c96b971e0c 100644 --- a/components/elm/src/main/atm2lndType.F90 +++ b/components/elm/src/main/atm2lndType.F90 @@ -146,6 +146,10 @@ module atm2lndType real(r8) , pointer :: t_mo_patch (:) => null() ! patch 30-day average temperature (Kelvin) real(r8) , pointer :: t_mo_min_patch (:) => null() ! patch annual min of t_mo (Kelvin) + ! Needed for FNM precip downscaling, when used within CPL_BYPASS + real(r8), pointer :: forc_uovern (:) => null() ! Froude number (dimensionless) + + contains procedure, public :: Init @@ -310,6 +314,7 @@ subroutine InitAllocate(this, bounds) allocate(this%forc_ndep_nitr_grc (begg:endg)) ; this%forc_ndep_nitr_grc (:) = ival allocate(this%forc_soilph_grc (begg:endg)) ; this%forc_soilph_grc (:) = ival end if + allocate(this%forc_uovern (begg:endg)) ; this%uovern (:) = ival end subroutine InitAllocate From 41884332cfdcb2781c2f0850e845feb116d994a6 Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Fri, 12 Jul 2024 18:08:59 -0400 Subject: [PATCH 696/904] Bug fixes for coupler bypass downscaling --- components/elm/src/cpl/lnd_downscale_atm_forcing.F90 | 11 +++++------ components/elm/src/main/atm2lndType.F90 | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 index 8c003bbf5e90..3b69158f49eb 100644 --- a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 +++ b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 @@ -810,8 +810,7 @@ subroutine downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm ! ! !ARGUMENTS: integer , intent(in) :: g - real(r8) , intent(in) :: x2l(:,:) - type(atm2lnd_type) , intent(in) :: atm2atm_vars + type(atm2lnd_type) , intent(in) :: atm2lnd_vars type(lnd2atm_type) , intent(in) :: lnd2atm_vars ! @@ -995,8 +994,8 @@ subroutine downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & top_af%solai(t,2) + top_af%solai(t,1) else - call downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars, lnd2atm_vars, uaflag) - call downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars, lnd2atm_vars, uaflag) + call downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + call downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m @@ -1304,9 +1303,9 @@ subroutine downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) ! evaluated at the mean temp. ! We assume the same temperature lapse rate as above. - tair_g = x2l(index_x2l_Sa_tbot,i) + tair_g = atm2lnd_vars%forc_t_not_downscaled_grc(g) tair_t = top_as%tbot(t) - lwrad_g = x2l(index_x2l_Faxa_lwdn,i) + lwrad_g = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) top_af%lwrad(t) = lwrad_g - & 4.0_r8 * lwrad_g/(0.5_r8*(tair_t+tair_g)) * & lapse_glcmec * (hsurf_t - hsurf_g) diff --git a/components/elm/src/main/atm2lndType.F90 b/components/elm/src/main/atm2lndType.F90 index 82c96b971e0c..f2facc776224 100644 --- a/components/elm/src/main/atm2lndType.F90 +++ b/components/elm/src/main/atm2lndType.F90 @@ -314,7 +314,7 @@ subroutine InitAllocate(this, bounds) allocate(this%forc_ndep_nitr_grc (begg:endg)) ; this%forc_ndep_nitr_grc (:) = ival allocate(this%forc_soilph_grc (begg:endg)) ; this%forc_soilph_grc (:) = ival end if - allocate(this%forc_uovern (begg:endg)) ; this%uovern (:) = ival + allocate(this%forc_uovern (begg:endg)) ; this%forc_uovern (:) = ival end subroutine InitAllocate From db237930afb0b619464f7d8c0a967e9bdd5853ba Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Wed, 6 Mar 2024 13:37:03 -0700 Subject: [PATCH 697/904] initial commit of adding shrub bending parameter --- components/elm/src/biogeochem/VegStructUpdateMod.F90 | 4 +++- .../elm/src/data_types/VegetationPropertiesType.F90 | 2 ++ components/elm/src/main/pftvarcon.F90 | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 5943da355a04..a8352ba2672b 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -88,6 +88,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & z0mr => veg_vp%z0mr , & ! Input: [real(r8) (:) ] ratio of momentum roughness length to canopy top height (-) displar => veg_vp%displar , & ! Input: [real(r8) (:) ] ratio of displacement height to canopy top height (-) dwood => veg_vp%dwood , & ! Input: [real(r8) (:) ] density of wood (gC/m^3) + bend_parm => veg_vp%bend_parm , & ! Input: [real(r8) (:) ] shrub bending parameter after Sturm et al. 2005 (-) snow_depth => col_ws%snow_depth , & ! Input: [real(r8) (:) ] snow height (m) @@ -249,7 +250,8 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! Wang and Zeng, 2007. if (woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) - fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) + ! fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) ! Wang and Zeng, 2007 + fb = 1._r8 - ol / bend_parm(p)*max(1.e-06_r8, htop(p)-hbot(p)) ! Liston and Heimstra, 2011 else fb = 1._r8 - max(min(snow_depth(c),0.2_r8),0._r8)/0.2_r8 ! 0.2m is assumed !depth of snow required for complete burial of grasses diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 12f9a7b22ffa..f2de380578df 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -151,6 +151,8 @@ module VegetationPropertiesType real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 + !NGEE Arctic + real(r8), pointer :: bend_parm(:) => null() ! shrub bending parameter contains procedure, public :: Init => veg_vp_init diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index fc5bfd54e7d3..ccdb5afb6574 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -311,6 +311,8 @@ module pftvarcon real(r8), allocatable :: gcbc_q(:) !effectiveness of surface cover in reducing runoff-driven erosion real(r8), allocatable :: gcbr_p(:) !effectiveness of roots in reducing rainfall-driven erosion real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion + ! NGEE Arctic shrub bending + real(r8), allocatable :: bend_parm(:) ! parameter that describes the "stiffness" of shrub branches (Sturm et al. 2005, Liston and Hiemstra 2011) ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. @@ -648,6 +650,8 @@ subroutine pftconrd allocate( needleleaf (0:mxpft) ) allocate( nfixer (0:mxpft) ) + ! NGEE arctic + allocate( bend_parm (0:mxpft) ) ! Set specific vegetation type values @@ -1076,6 +1080,10 @@ subroutine pftconrd end do end if + ! new NGEE parameters: + call ncd_io('bend_parm', bend_parm, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) bend_parm(:) = 1._r8 ! set to 1 if not on file - by default: no change. + ! NOTE: the following 5 PFT flags/options are addtions to 'woody', 'stress_decid', 'season_decid', ! 'evergreen', and 'crop', 'percop'. For default ELM, will be hard-coded; while for user-defined From f08448244f59e4b61e77a89b1f8ffe8e692a4d95 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Sun, 31 Mar 2024 14:34:25 -0600 Subject: [PATCH 698/904] Update snow burial of vegetation parameterization Enables two major changes to the parameterization that calculates LAI burial by snow: a) pulls stocking density and taper parameters that influence vegetation height from hard-wired values in biogeochem/VegStructUpdateMod.F90 to allow to be specified on the surface file (per main/pftvarcon) and b) updates the LAI burial by snow rate from Wang and Zeng, 2007 (linear burial) to form proposed by Liston and Hiemstra, 2011 (linear or non-linear) burial. The Liston and Heimstra (2011) formulation introduces two new parameters: 1) bendresist, which specifies how resistant branches are to bending under snow loading (1 = no bending, lim->0 branches bend very rapidly) and 2) vegshape, which specifies how the shape of the vegetation crown affects how rapidly it is buried by snow (1 = paraboloid; 2 = hemispheroid). These new parameters can also be specified on surface file. --- .../elm/src/biogeochem/VegStructUpdateMod.F90 | 38 +- .../data_types/VegetationPropertiesType.F90 | 25 +- components/elm/src/main/pftvarcon.F90 | 372 +++++++++--------- 3 files changed, 229 insertions(+), 206 deletions(-) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index a8352ba2672b..7c56fc6b58c0 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -40,6 +40,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & use pftvarcon , only : noveg, woody, iscft, crop use pftvarcon , only : ncorn, ncornirrig, ztopmx, laimx use pftvarcon , only : nmiscanthus, nmiscanthusirrig, nswitchgrass, nswitchgrassirrig + use pftvarcon , only : bendresist, vegshape, stocking, taper use elm_time_manager , only : get_rad_step_size use elm_varctl , only : spinup_state, spinup_mortality_factor ! @@ -58,8 +59,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! !LOCAL VARIABLES: integer :: p,c,g ! indices integer :: fp ! lake filter indices - real(r8) :: taper ! ratio of height:radius_breast_height (tree allometry) - real(r8) :: stocking ! #stems / ha (stocking density) real(r8) :: ol ! thickness of canopy layer covered by snow (m) real(r8) :: fb ! fraction of canopy layer covered by snow real(r8) :: tlai_old ! for use in Zeng tsai formula @@ -88,7 +87,9 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & z0mr => veg_vp%z0mr , & ! Input: [real(r8) (:) ] ratio of momentum roughness length to canopy top height (-) displar => veg_vp%displar , & ! Input: [real(r8) (:) ] ratio of displacement height to canopy top height (-) dwood => veg_vp%dwood , & ! Input: [real(r8) (:) ] density of wood (gC/m^3) - bend_parm => veg_vp%bend_parm , & ! Input: [real(r8) (:) ] shrub bending parameter after Sturm et al. 2005 (-) + bendresist => veg_vp%bendresist , & ! Input: [real(r8) (:) ] resistance to bending under snow loading Sturm et al. 2005 (-) [0,1] + vegshape => veg_vp%vegshape , & ! Input: [real(r8) (:) ] vegetation shape for calculating snow buried fraction only (-) (Liston and Heimstra, 2011) + stocking => veg_vp%stocking , & ! Input: [real(r8) (:) ] Stocking density [stems / hectare] snow_depth => col_ws%snow_depth , & ! Input: [real(r8) (:) ] snow height (m) @@ -114,10 +115,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & dt = real( get_rad_step_size(), r8 ) - ! constant allometric parameters - taper = 200._r8 - stocking = 1000._r8 - ! convert from stems/ha -> stems/m^2 stocking = stocking / 10000._r8 @@ -161,24 +158,16 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & if (woody(ivt(p)) >= 1.0_r8) then - ! trees and shrubs - - ! if shrubs have a squat taper - if (woody(ivt(p)) == 2.0_r8) then - taper = 10._r8 - ! otherwise have a tall taper - else - taper = 200._r8 - end if - - ! trees and shrubs for now have a very simple allometry, with hard-wired - ! stem taper (height:radius) and hard-wired stocking density (#individuals/area) + ! trees and shrubs currently have a very simple allometry, based on + ! stem taper (height:radius) and stocking density (#individuals/area) + ! taper and stocking density can be set as input variables now to + ! change from default values set in pftvarcon.F90 if (spinup_state >= 1) then - htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper * taper)/ & - (SHR_CONST_PI * stocking * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper(p) * taper(p))/ & + (SHR_CONST_PI * stocking(p) * dwood(ivt(p))))**(1._r8/3._r8) else - htop(p) = ((3._r8 * deadstemc(p) * taper * taper)/ & - (SHR_CONST_PI * stocking * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * taper(p) * taper(p))/ & + (SHR_CONST_PI * stocking(p) * dwood(ivt(p))))**(1._r8/3._r8) end if ! Peter Thornton, 5/3/2004 @@ -250,8 +239,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! Wang and Zeng, 2007. if (woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) - ! fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) ! Wang and Zeng, 2007 - fb = 1._r8 - ol / bend_parm(p)*max(1.e-06_r8, htop(p)-hbot(p)) ! Liston and Heimstra, 2011 + fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(p) * (htop(p)-hbot(p)))) ** vegshape(p) else fb = 1._r8 - max(min(snow_depth(c),0.2_r8),0._r8)/0.2_r8 ! 0.2m is assumed !depth of snow required for complete burial of grasses diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index f2de380578df..2f06ea900329 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -115,7 +115,7 @@ module VegetationPropertiesType real(r8), pointer :: lamda_ptase => null()! critical value that incur biochemical production real(r8), pointer :: i_vc(:) => null() ! intercept of photosynthesis vcmax ~ leaf n content regression model real(r8), pointer :: s_vc(:) => null() ! slope of photosynthesis vcmax ~ leaf n content regression model - real(r8), pointer :: nsc_rtime(:) => null() ! non-structural carbon residence time + real(r8), pointer :: nsc_rtime(:) => null() ! non-structural carbon residence time real(r8), pointer :: pinit_beta1(:) => null() ! shaping parameter for P initialization real(r8), pointer :: pinit_beta2(:) => null() ! shaping parameter for P initialization real(r8), pointer :: alpha_nfix(:) => null() ! fraction of fixed N goes directly to plant @@ -151,8 +151,11 @@ module VegetationPropertiesType real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 - !NGEE Arctic - real(r8), pointer :: bend_parm(:) => null() ! shrub bending parameter + ! NGEE Arctic snow-vegetation interactions + real(r8), pointer :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011) + real(r8), pointer :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) + real(r8), pointer :: stocking(:) ! stocking density for pft (stems / hectare) + real(r8), pointer :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) contains procedure, public :: Init => veg_vp_init @@ -190,8 +193,10 @@ subroutine veg_vp_init(this) use pftvarcon , only : fnr, act25, kcha, koha, cpha, vcmaxha, jmaxha, tpuha use pftvarcon , only : lmrha, vcmaxhd, jmaxhd, tpuhd, lmrse, qe, theta_cj use pftvarcon , only : bbbopt, mbbopt, nstor, br_xr, tc_stress, lmrhd - ! new properties for flexible PFT + ! new properties for flexible PFT (NGEE Arctic IM4) use pftvarcon , only : climatezone, nonvascular, graminoid, iscft,needleleaf, nfixer + ! snow/vegetation interactions (NGEE Arctic IM3) + use pftvarcon , only : bendresist, stocking, vegshape, taper ! class (vegetation_properties_type) :: this @@ -324,6 +329,11 @@ subroutine veg_vp_init(this) allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval ! ----------------------------------------------------------------------------------------------------------- + ! NGEE Arctic show-vegetation interactions + allocate(this%bendresist(0:numpft)) ; this%bendresist(:) =spval + allocate(this%vegshape(0:numpft)) ; this%vegshape(:) =spval + allocate(this%stocking(0:numpft)) ; this%stocking(:) =spval + allocate(this%taper(0:numpft)) ; this%taper(:) =spval do m = 0,numpft @@ -474,6 +484,13 @@ subroutine veg_vp_init(this) this%lamda_ptase = lamda_ptase this%tc_stress = tc_stress + ! NGEE Arctic - snow/vegetation interactions + do m = 0, numpft ! RPF - move up to earlier pft loops? + this%bendresist(m) = bendresist(m) + this%vegshape(m) = vegshape(m) + this%stocking(m) = stocking(m) + this%taper(m) = taper(m) + end do end subroutine veg_vp_init end module VegetationPropertiesType diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index ccdb5afb6574..e06b5aba1953 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -29,7 +29,7 @@ module pftvarcon ! ! Vegetation type constants ! - integer :: noveg !value for not vegetated + integer :: noveg !value for not vegetated integer :: ndllf_evr_tmp_tree !value for Needleleaf evergreen temperate tree integer :: ndllf_evr_brl_tree !value for Needleleaf evergreen boreal tree integer :: ndllf_dcd_brl_tree !value for Needleleaf deciduous boreal tree @@ -76,7 +76,7 @@ module pftvarcon integer :: nrtubers !value for root tubers, rain fed (rf) integer :: nrtubersirrig !value for root tubers, irrigated (ir) integer :: nsugarcane !value for sugarcane, rain fed (rf) - integer :: nsugarcaneirrig !value for sugarcane, irrigated (ir) + integer :: nsugarcaneirrig !value for sugarcane, irrigated (ir) integer :: nmiscanthus !value for miscanthus, rain fed (rf) integer :: nmiscanthusirrig !value for miscanthus, irrigated (ir) integer :: nswitchgrass !value for switchgrass, rain fed (rf) @@ -84,8 +84,8 @@ module pftvarcon integer :: npoplar !value for poplar, rain fed (rf) integer :: npoplarirrig !value for poplar, irrigated (ir) integer :: nwillow !value for willow, rain fed (rf) - integer :: nwillowirrig !value for willow, irrigated (ir) - + integer :: nwillowirrig !value for willow, irrigated (ir) + ! Number of crop functional types actually used in the model. This includes each CFT for ! which is_pft_known_to_model is true. Note that this includes irrigated crops even if ! irrigation is turned off in this run: it just excludes crop types that aren't handled @@ -171,7 +171,7 @@ module pftvarcon real(r8), allocatable :: minplanttemp(:) !mininum planting temperature used in Phenology (K) real(r8), allocatable :: senestemp(:) !senescence temperature for perennial crops used in Phenology (K) real(r8), allocatable :: min_days_senes(:) !minimum leaf age to allow for leaf senescence - real(r8), allocatable :: froot_leaf(:) !allocation parameter: new fine root C per new leaf C (gC/gC) + real(r8), allocatable :: froot_leaf(:) !allocation parameter: new fine root C per new leaf C (gC/gC) real(r8), allocatable :: stem_leaf(:) !allocation parameter: new stem c per new leaf C (gC/gC) real(r8), allocatable :: croot_stem(:) !allocation parameter: new coarse root C per new stem C (gC/gC) real(r8), allocatable :: flivewd(:) !allocation parameter: fraction of new wood that is live (phloem and ray parenchyma) (no units) @@ -215,7 +215,7 @@ module pftvarcon real(r8), allocatable :: fyield(:) !fraction of grain that is actually harvested real(r8), allocatable :: root_dmx(:) !maximum root depth - integer, parameter :: pftname_len = 40 ! max length of pftname + integer, parameter :: pftname_len = 40 ! max length of pftname character(len=:), allocatable :: pftname(:) !PFT description real(r8), parameter :: reinickerp = 1.6_r8 !parameter in allometric equation @@ -276,7 +276,7 @@ module pftvarcon real(r8), allocatable :: deadwdcp_obs_flex(:,:) !upper and lower range of dead wood (xylem and heartwood) C:P (gC/gP) ! Photosynthesis parameters real(r8), allocatable :: fnr(:) !fraction of nitrogen in RuBisCO - real(r8), allocatable :: act25(:) + real(r8), allocatable :: act25(:) real(r8), allocatable :: kcha(:) !Activation energy for kc real(r8), allocatable :: koha(:) !Activation energy for ko real(r8), allocatable :: cpha(:) !Activation energy for cp @@ -293,7 +293,7 @@ module pftvarcon real(r8), allocatable :: theta_cj(:) ! real(r8), allocatable :: bbbopt(:) !Ball-Berry stomatal conductance intercept real(r8), allocatable :: mbbopt(:) !Ball-Berry stomatal conductance slope - real(r8), allocatable :: nstor(:) !Nitrogen storage pool timescale + real(r8), allocatable :: nstor(:) !Nitrogen storage pool timescale real(r8), allocatable :: br_xr(:) !Base rate for excess respiration real(r8) :: tc_stress !Critial temperature for moisture stress real(r8), allocatable :: vcmax_np1(:) !vcmax~np relationship coefficient @@ -311,8 +311,12 @@ module pftvarcon real(r8), allocatable :: gcbc_q(:) !effectiveness of surface cover in reducing runoff-driven erosion real(r8), allocatable :: gcbr_p(:) !effectiveness of roots in reducing rainfall-driven erosion real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion - ! NGEE Arctic shrub bending - real(r8), allocatable :: bend_parm(:) ! parameter that describes the "stiffness" of shrub branches (Sturm et al. 2005, Liston and Hiemstra 2011) + + ! NGEE Arctic snow-vegetation interactions + real(r8), allocatable :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011) + real(r8), allocatable :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) + real(r8), allocatable :: stocking(:) ! stocking density for pft (stems / hectare) + real(r8), allocatable :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. @@ -381,7 +385,7 @@ subroutine pftconrd ! and finally crops, ending with soybean ! DO NOT CHANGE THE ORDER -- WITHOUT MODIFYING OTHER PARTS OF THE CODE WHERE THE ORDER MATTERS! ! - character(len=pftname_len) :: expected_pftnames(0:mxpft) + character(len=pftname_len) :: expected_pftnames(0:mxpft) !----------------------------------------------------------------------- expected_pftnames( 0) = 'not_vegetated ' @@ -467,82 +471,82 @@ subroutine pftconrd allocate( rootb_par (0:mxpft) ) allocate( crop (0:mxpft) ) allocate( percrop (0:mxpft) ) - allocate( irrigated (0:mxpft) ) - allocate( smpso (0:mxpft) ) - allocate( smpsc (0:mxpft) ) - allocate( fnitr (0:mxpft) ) - allocate( slatop (0:mxpft) ) - allocate( dsladlai (0:mxpft) ) + allocate( irrigated (0:mxpft) ) + allocate( smpso (0:mxpft) ) + allocate( smpsc (0:mxpft) ) + allocate( fnitr (0:mxpft) ) + allocate( slatop (0:mxpft) ) + allocate( dsladlai (0:mxpft) ) allocate( leafcn (0:mxpft) ) - allocate( flnr (0:mxpft) ) - allocate( woody (0:mxpft) ) - allocate( lflitcn (0:mxpft) ) - allocate( frootcn (0:mxpft) ) - allocate( livewdcn (0:mxpft) ) - allocate( deadwdcn (0:mxpft) ) - - ! add phosphorus - allocate( leafcp (0:mxpft) ) - allocate( lflitcp (0:mxpft) ) - allocate( frootcp (0:mxpft) ) - allocate( livewdcp (0:mxpft) ) - allocate( deadwdcp (0:mxpft) ) - - allocate( grperc (0:mxpft) ) - allocate( grpnow (0:mxpft) ) + allocate( flnr (0:mxpft) ) + allocate( woody (0:mxpft) ) + allocate( lflitcn (0:mxpft) ) + allocate( frootcn (0:mxpft) ) + allocate( livewdcn (0:mxpft) ) + allocate( deadwdcn (0:mxpft) ) + + ! add phosphorus + allocate( leafcp (0:mxpft) ) + allocate( lflitcp (0:mxpft) ) + allocate( frootcp (0:mxpft) ) + allocate( livewdcp (0:mxpft) ) + allocate( deadwdcp (0:mxpft) ) + + allocate( grperc (0:mxpft) ) + allocate( grpnow (0:mxpft) ) allocate( rootprof_beta (0:mxpft) ) allocate( mergetoelmpft (0:mxpft) ) allocate( is_pft_known_to_model (0:mxpft) ) - allocate( graincn (0:mxpft) ) - allocate( graincp (0:mxpft) ) - allocate( mxtmp (0:mxpft) ) - allocate( baset (0:mxpft) ) - allocate( declfact (0:mxpft) ) - allocate( bfact (0:mxpft) ) - allocate( aleaff (0:mxpft) ) - allocate( arootf (0:mxpft) ) - allocate( astemf (0:mxpft) ) - allocate( arooti (0:mxpft) ) - allocate( fleafi (0:mxpft) ) - allocate( allconsl (0:mxpft) ) - allocate( allconss (0:mxpft) ) - allocate( ztopmx (0:mxpft) ) - allocate( laimx (0:mxpft) ) - allocate( gddmin (0:mxpft) ) - allocate( hybgdd (0:mxpft) ) - allocate( lfemerg (0:mxpft) ) - allocate( grnfill (0:mxpft) ) - allocate( mxmat (0:mxpft) ) + allocate( graincn (0:mxpft) ) + allocate( graincp (0:mxpft) ) + allocate( mxtmp (0:mxpft) ) + allocate( baset (0:mxpft) ) + allocate( declfact (0:mxpft) ) + allocate( bfact (0:mxpft) ) + allocate( aleaff (0:mxpft) ) + allocate( arootf (0:mxpft) ) + allocate( astemf (0:mxpft) ) + allocate( arooti (0:mxpft) ) + allocate( fleafi (0:mxpft) ) + allocate( allconsl (0:mxpft) ) + allocate( allconss (0:mxpft) ) + allocate( ztopmx (0:mxpft) ) + allocate( laimx (0:mxpft) ) + allocate( gddmin (0:mxpft) ) + allocate( hybgdd (0:mxpft) ) + allocate( lfemerg (0:mxpft) ) + allocate( grnfill (0:mxpft) ) + allocate( mxmat (0:mxpft) ) allocate( mnNHplantdate (0:mxpft) ) allocate( mxNHplantdate (0:mxpft) ) allocate( mnSHplantdate (0:mxpft) ) allocate( mxSHplantdate (0:mxpft) ) - allocate( planttemp (0:mxpft) ) - allocate( minplanttemp (0:mxpft) ) + allocate( planttemp (0:mxpft) ) + allocate( minplanttemp (0:mxpft) ) allocate( senestemp (0:mxpft) ) allocate( min_days_senes (0:mxpft) ) - allocate( froot_leaf (0:mxpft) ) - allocate( stem_leaf (0:mxpft) ) - allocate( croot_stem (0:mxpft) ) - allocate( flivewd (0:mxpft) ) - allocate( fcur (0:mxpft) ) - allocate( lf_flab (0:mxpft) ) - allocate( lf_fcel (0:mxpft) ) - allocate( lf_flig (0:mxpft) ) - allocate( fr_flab (0:mxpft) ) - allocate( fr_fcel (0:mxpft) ) - allocate( fr_flig (0:mxpft) ) - allocate( leaf_long (0:mxpft) ) + allocate( froot_leaf (0:mxpft) ) + allocate( stem_leaf (0:mxpft) ) + allocate( croot_stem (0:mxpft) ) + allocate( flivewd (0:mxpft) ) + allocate( fcur (0:mxpft) ) + allocate( lf_flab (0:mxpft) ) + allocate( lf_fcel (0:mxpft) ) + allocate( lf_flig (0:mxpft) ) + allocate( fr_flab (0:mxpft) ) + allocate( fr_fcel (0:mxpft) ) + allocate( fr_flig (0:mxpft) ) + allocate( leaf_long (0:mxpft) ) allocate( froot_long (0:mxpft) ) - allocate( evergreen (0:mxpft) ) - allocate( stress_decid (0:mxpft) ) - allocate( season_decid (0:mxpft) ) - allocate( pconv (0:mxpft) ) - allocate( pprod10 (0:mxpft) ) - allocate( pprod100 (0:mxpft) ) - allocate( pprodharv10 (0:mxpft) ) + allocate( evergreen (0:mxpft) ) + allocate( stress_decid (0:mxpft) ) + allocate( season_decid (0:mxpft) ) + allocate( pconv (0:mxpft) ) + allocate( pprod10 (0:mxpft) ) + allocate( pprod100 (0:mxpft) ) + allocate( pprodharv10 (0:mxpft) ) allocate( cc_leaf (0:mxpft) ) allocate( cc_lstem (0:mxpft) ) allocate( cc_dstem (0:mxpft) ) @@ -557,12 +561,12 @@ subroutine pftconrd allocate( fsr_pft (0:mxpft) ) allocate( fd_pft (0:mxpft) ) allocate( manunitro (0:mxpft) ) - allocate( fleafcn (0:mxpft) ) - allocate( ffrootcn (0:mxpft) ) + allocate( fleafcn (0:mxpft) ) + allocate( ffrootcn (0:mxpft) ) allocate( fstemcn (0:mxpft) ) allocate( presharv (0:mxpft) ) allocate( convfact (0:mxpft) ) - allocate( fyield (0:mxpft) ) + allocate( fyield (0:mxpft) ) allocate( root_dmx (0:mxpft) ) if (use_crop) then @@ -575,14 +579,14 @@ subroutine pftconrd allocate( VMAX_PLANT_NO3(0:mxpft) ) allocate( VMAX_PLANT_P(0:mxpft) ) allocate( VMAX_MINSURF_P_vr(1:nlevdecomp_full,0:nsoilorder)) - allocate( KM_PLANT_NH4(0:mxpft) ) + allocate( KM_PLANT_NH4(0:mxpft) ) allocate( KM_PLANT_NO3(0:mxpft) ) allocate( KM_PLANT_P(0:mxpft) ) allocate( KM_MINSURF_P_vr(1:nlevdecomp_full,0:nsoilorder)) allocate( decompmicc_patch_vr (1:nlevdecomp_full,0:mxpft)) allocate( VMAX_PTASE(0:mxpft)) - allocate( i_vc (0:mxpft) ) - allocate( s_vc (0:mxpft) ) + allocate( i_vc (0:mxpft) ) + allocate( s_vc (0:mxpft) ) allocate( nsc_rtime (0:mxpft) ) allocate( pinit_beta1 (0:nsoilorder)) allocate( pinit_beta2 (0:nsoilorder)) @@ -595,22 +599,22 @@ subroutine pftconrd allocate( VMAX_NFIX (0:mxpft) ) allocate( KM_NFIX (0:mxpft) ) ! new stoichiometry - allocate( leafcn_obs (0:mxpft) ) - allocate( frootcn_obs (0:mxpft) ) + allocate( leafcn_obs (0:mxpft) ) + allocate( frootcn_obs (0:mxpft) ) allocate( livewdcn_obs (0:mxpft) ) - allocate( deadwdcn_obs (0:mxpft) ) - allocate( leafcp_obs (0:mxpft) ) - allocate( frootcp_obs (0:mxpft) ) + allocate( deadwdcn_obs (0:mxpft) ) + allocate( leafcp_obs (0:mxpft) ) + allocate( frootcp_obs (0:mxpft) ) allocate( livewdcp_obs (0:mxpft) ) - allocate( deadwdcp_obs (0:mxpft) ) - allocate( leafcn_obs_flex (0:mxpft,1:2) ) - allocate( frootcn_obs_flex (0:mxpft,1:2) ) + allocate( deadwdcp_obs (0:mxpft) ) + allocate( leafcn_obs_flex (0:mxpft,1:2) ) + allocate( frootcn_obs_flex (0:mxpft,1:2) ) allocate( livewdcn_obs_flex (0:mxpft,1:2) ) - allocate( deadwdcn_obs_flex (0:mxpft,1:2) ) - allocate( leafcp_obs_flex (0:mxpft,1:2) ) - allocate( frootcp_obs_flex (0:mxpft,1:2) ) + allocate( deadwdcn_obs_flex (0:mxpft,1:2) ) + allocate( leafcp_obs_flex (0:mxpft,1:2) ) + allocate( frootcp_obs_flex (0:mxpft,1:2) ) allocate( livewdcp_obs_flex (0:mxpft,1:2) ) - allocate( deadwdcp_obs_flex (0:mxpft,1:2) ) + allocate( deadwdcp_obs_flex (0:mxpft,1:2) ) allocate( vcmax_np1 (0:mxpft) ) allocate( vcmax_np2 (0:mxpft) ) allocate( vcmax_np3 (0:mxpft) ) @@ -650,12 +654,16 @@ subroutine pftconrd allocate( needleleaf (0:mxpft) ) allocate( nfixer (0:mxpft) ) - ! NGEE arctic - allocate( bend_parm (0:mxpft) ) + ! NGEE arctic snow-vegetation interactions + allocate( bendresist (0:mxpft) ) + allocate( vegshape (0:mxpft) ) + allocate( stocking (0:mxpft) ) + allocate( taper (0:mxpft) ) + ! Set specific vegetation type values - call ncd_io('pftname',pftname, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('pftname',pftname, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) ! checking if using standard pft-names and default properties @@ -756,16 +764,16 @@ subroutine pftconrd if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) call ncd_io('fr_flab',fr_flab, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fr_fcel',fr_fcel, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('fr_fcel',fr_fcel, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fr_flig',fr_flig, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('fr_flig',fr_flig, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('leaf_long',leaf_long, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('leaf_long',leaf_long, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) call ncd_io('froot_long',froot_long, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv) froot_long = leaf_long !if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('evergreen',evergreen, 'read', ncid, readvar=readv, posNOTonfile=.true.) + call ncd_io('evergreen',evergreen, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) call ncd_io('stress_decid',stress_decid, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) @@ -805,95 +813,95 @@ subroutine pftconrd call ncd_io('rootprof_beta',rootprof_beta, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) end if - call ncd_io('pconv',pconv, 'read', ncid, readvar=readv) + call ncd_io('pconv',pconv, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('pprod10',pprod10, 'read', ncid, readvar=readv) + call ncd_io('pprod10',pprod10, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('pprodharv10',pprodharv10, 'read', ncid, readvar=readv) + call ncd_io('pprodharv10',pprodharv10, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('pprod100',pprod100, 'read', ncid, readvar=readv) + call ncd_io('pprod100',pprod100, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('graincn',graincn, 'read', ncid, readvar=readv) + call ncd_io('graincn',graincn, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('graincp',graincp, 'read', ncid, readvar=readv) + call ncd_io('graincp',graincp, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('mxtmp',mxtmp, 'read', ncid, readvar=readv) + call ncd_io('mxtmp',mxtmp, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('baset',baset, 'read', ncid, readvar=readv) + call ncd_io('baset',baset, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('declfact',declfact, 'read', ncid, readvar=readv) + call ncd_io('declfact',declfact, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('bfact',bfact, 'read', ncid, readvar=readv) + call ncd_io('bfact',bfact, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('aleaff',aleaff, 'read', ncid, readvar=readv) + call ncd_io('aleaff',aleaff, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('arootf',arootf, 'read', ncid, readvar=readv) + call ncd_io('arootf',arootf, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('astemf',astemf, 'read', ncid, readvar=readv) + call ncd_io('astemf',astemf, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('arooti',arooti, 'read', ncid, readvar=readv) + call ncd_io('arooti',arooti, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fleafi',fleafi, 'read', ncid, readvar=readv) + call ncd_io('fleafi',fleafi, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('allconsl',allconsl, 'read', ncid, readvar=readv) + call ncd_io('allconsl',allconsl, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('allconss',allconss, 'read', ncid, readvar=readv) + call ncd_io('allconss',allconss, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('crop',crop, 'read', ncid, readvar=readv) + call ncd_io('crop',crop, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('irrigated',irrigated, 'read', ncid, readvar=readv) + call ncd_io('irrigated',irrigated, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('ztopmx',ztopmx, 'read', ncid, readvar=readv) + call ncd_io('ztopmx',ztopmx, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('laimx',laimx, 'read', ncid, readvar=readv) + call ncd_io('laimx',laimx, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('gddmin',gddmin, 'read', ncid, readvar=readv) + call ncd_io('gddmin',gddmin, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('hybgdd',hybgdd, 'read', ncid, readvar=readv) + call ncd_io('hybgdd',hybgdd, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('lfemerg',lfemerg, 'read', ncid, readvar=readv) + call ncd_io('lfemerg',lfemerg, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('grnfill',grnfill, 'read', ncid, readvar=readv) + call ncd_io('grnfill',grnfill, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('mxmat',mxmat, 'read', ncid, readvar=readv) + call ncd_io('mxmat',mxmat, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('cc_leaf', cc_leaf, 'read', ncid, readvar=readv) + call ncd_io('cc_leaf', cc_leaf, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('cc_lstem',cc_lstem, 'read', ncid, readvar=readv) + call ncd_io('cc_lstem',cc_lstem, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('cc_dstem',cc_dstem, 'read', ncid, readvar=readv) + call ncd_io('cc_dstem',cc_dstem, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('cc_other',cc_other, 'read', ncid, readvar=readv) + call ncd_io('cc_other',cc_other, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_leaf', fm_leaf, 'read', ncid, readvar=readv) + call ncd_io('fm_leaf', fm_leaf, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_lstem',fm_lstem, 'read', ncid, readvar=readv) + call ncd_io('fm_lstem',fm_lstem, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_dstem',fm_dstem, 'read', ncid, readvar=readv) + call ncd_io('fm_dstem',fm_dstem, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_other',fm_other, 'read', ncid, readvar=readv) + call ncd_io('fm_other',fm_other, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_root', fm_root, 'read', ncid, readvar=readv) + call ncd_io('fm_root', fm_root, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_lroot',fm_lroot, 'read', ncid, readvar=readv) + call ncd_io('fm_lroot',fm_lroot, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fm_droot',fm_droot, 'read', ncid, readvar=readv) + call ncd_io('fm_droot',fm_droot, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fsr_pft', fsr_pft, 'read', ncid, readvar=readv) + call ncd_io('fsr_pft', fsr_pft, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('fd_pft', fd_pft, 'read', ncid, readvar=readv) + call ncd_io('fd_pft', fd_pft, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('planting_temp',planttemp, 'read', ncid, readvar=readv) + call ncd_io('planting_temp',planttemp, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('min_planting_temp',minplanttemp, 'read', ncid, readvar=readv) + call ncd_io('min_planting_temp',minplanttemp, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('min_NH_planting_date',mnNHplantdate, 'read', ncid, readvar=readv) + call ncd_io('min_NH_planting_date',mnNHplantdate, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('min_SH_planting_date',mnSHplantdate, 'read', ncid, readvar=readv) + call ncd_io('min_SH_planting_date',mnSHplantdate, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('max_NH_planting_date',mxNHplantdate, 'read', ncid, readvar=readv) + call ncd_io('max_NH_planting_date',mxNHplantdate, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) - call ncd_io('max_SH_planting_date',mxSHplantdate, 'read', ncid, readvar=readv) + call ncd_io('max_SH_planting_date',mxSHplantdate, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft data'//errMsg(__FILE__, __LINE__)) if (nu_com .ne. 'RD' ) then @@ -901,40 +909,40 @@ subroutine pftconrd ! These are soil parameters and used for both FATES and big leaf ELM call ncd_io('VMAX_MINSURF_P_vr',VMAX_MINSURF_P_vr, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in soil order VMAX_MINSURF_P_vr'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_MINSURF_P_vr',KM_MINSURF_P_vr, 'read', ncid, readvar=readv) + call ncd_io('KM_MINSURF_P_vr',KM_MINSURF_P_vr, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in soil order KM_MINSURF_P_vr'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_DECOMP_NH4',KM_DECOMP_NH4, 'read', ncid, readvar=readv) + call ncd_io('KM_DECOMP_NH4',KM_DECOMP_NH4, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_DECOMP_NH4'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_DECOMP_NO3',KM_DECOMP_NO3, 'read', ncid, readvar=readv) + call ncd_io('KM_DECOMP_NO3',KM_DECOMP_NO3, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_DECOMP_NO3'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_DECOMP_P',KM_DECOMP_P, 'read', ncid, readvar=readv) + call ncd_io('KM_DECOMP_P',KM_DECOMP_P, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_DECOMP_P'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_NIT',KM_NIT, 'read', ncid, readvar=readv) + call ncd_io('KM_NIT',KM_NIT, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_NIT'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_DEN',KM_DEN, 'read', ncid, readvar=readv) + call ncd_io('KM_DEN',KM_DEN, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_DEN'//errMsg(__FILE__, __LINE__)) call ncd_io('pinit_beta1',pinit_beta1, 'read', ncid, readvar=readv) if ( .not. readv ) pinit_beta1(:) = 0.5_r8 call ncd_io('pinit_beta2',pinit_beta2, 'read', ncid, readvar=readv) if ( .not. readv ) pinit_beta2(:) = 0.1_r8 - + if(.not.use_fates) then - - call ncd_io('VMAX_PLANT_NH4',VMAX_PLANT_NH4, 'read', ncid, readvar=readv) + + call ncd_io('VMAX_PLANT_NH4',VMAX_PLANT_NH4, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft VMAX_PLANT_NH4'//errMsg(__FILE__, __LINE__)) - call ncd_io('VMAX_PLANT_NO3',VMAX_PLANT_NO3, 'read', ncid, readvar=readv) + call ncd_io('VMAX_PLANT_NO3',VMAX_PLANT_NO3, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft VMAX_PLANT_NO3'//errMsg(__FILE__, __LINE__)) - call ncd_io('VMAX_PLANT_P',VMAX_PLANT_P, 'read', ncid, readvar=readv) + call ncd_io('VMAX_PLANT_P',VMAX_PLANT_P, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft VMAX_PLANT_P'//errMsg(__FILE__, __LINE__)) - - call ncd_io('KM_PLANT_NH4',KM_PLANT_NH4, 'read', ncid, readvar=readv) + + call ncd_io('KM_PLANT_NH4',KM_PLANT_NH4, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft KM_PLANT_NH4'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_PLANT_NO3',KM_PLANT_NO3, 'read', ncid, readvar=readv) + call ncd_io('KM_PLANT_NO3',KM_PLANT_NO3, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft KM_PLANT_NO3'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_PLANT_P',KM_PLANT_P, 'read', ncid, readvar=readv) + call ncd_io('KM_PLANT_P',KM_PLANT_P, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft KM_PLANT_P'//errMsg(__FILE__, __LINE__)) - - call ncd_io('decompmicc_patch_vr',decompmicc_patch_vr, 'read', ncid, readvar=readv) + + call ncd_io('decompmicc_patch_vr',decompmicc_patch_vr, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in pft decompmicc_patch_vr'//errMsg(__FILE__, __LINE__)) call ncd_io('alpha_nfix',alpha_nfix, 'read', ncid, readvar=readv) if ( .not. readv ) alpha_nfix(:)=0._r8 @@ -948,19 +956,19 @@ subroutine pftconrd if ( .not. readv ) ccost_ptase(:)=0._r8 call ncd_io('ncost_ptase',ncost_ptase, 'read', ncid, readvar=readv) if ( .not. readv ) ncost_ptase(:)=0._r8 - call ncd_io('VMAX_NFIX',VMAX_NFIX, 'read', ncid, readvar=readv) + call ncd_io('VMAX_NFIX',VMAX_NFIX, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in VMAX_NFIX'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_NFIX',KM_NFIX, 'read', ncid, readvar=readv) + call ncd_io('KM_NFIX',KM_NFIX, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_NFIX'//errMsg(__FILE__, __LINE__)) - call ncd_io('VMAX_PTASE',VMAX_PTASE, 'read', ncid, readvar=readv) + call ncd_io('VMAX_PTASE',VMAX_PTASE, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in VMAX_PTASE'//errMsg(__FILE__, __LINE__)) - call ncd_io('KM_PTASE',KM_PTASE, 'read', ncid, readvar=readv) + call ncd_io('KM_PTASE',KM_PTASE, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in KM_PTASE'//errMsg(__FILE__, __LINE__)) - call ncd_io('lamda_ptase',lamda_ptase, 'read', ncid, readvar=readv) + call ncd_io('lamda_ptase',lamda_ptase, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in lamda_ptase'//errMsg(__FILE__, __LINE__)) - call ncd_io('i_vc',i_vc, 'read', ncid, readvar=readv) + call ncd_io('i_vc',i_vc, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in i_vc'//errMsg(__FILE__, __LINE__)) - call ncd_io('s_vc',s_vc, 'read', ncid, readvar=readv) + call ncd_io('s_vc',s_vc, 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=' ERROR: error in reading in s_vc'//errMsg(__FILE__, __LINE__)) call ncd_io('nsc_rtime',nsc_rtime, 'read', ncid, readvar=readv) if ( .not. readv ) nsc_rtime(:) = 1.0_r8 @@ -1022,13 +1030,13 @@ subroutine pftconrd call ncd_io('fnr', fnr, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('act25', act25, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) + if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('kcha', kcha, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) + if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('koha', koha, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) + if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('cpha', cpha, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) + if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('vcmaxha', vcmaxha, 'read', ncid, readvar=readv, posNOTonfile=.true.) if ( .not. readv) call endrun(msg='ERROR: error in reading in pft data'//errMsg(__FILE__,__LINE__)) call ncd_io('jmaxha', jmaxha, 'read', ncid, readvar=readv, posNOTonfile=.true.) @@ -1080,10 +1088,20 @@ subroutine pftconrd end do end if - ! new NGEE parameters: - call ncd_io('bend_parm', bend_parm, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if (.not. readv ) bend_parm(:) = 1._r8 ! set to 1 if not on file - by default: no change. - + ! NGEE-Arctic snow-vegetation parameters + call ncd_io('bendresist', bendresist, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) bendresist(:) = 1._r8 + call ncd_io('vegshape', vegshape, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) vegshape(:) = 1._r8 + call ncd_io('stocking', stocking, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) stocking(:) = 1000._r8 + call ncd_io('taper', taper, 'read', ncid, readvar=readv, posNOTonfile=.true.) + if (.not. readv ) then + taper(:) = 200._r8 + do i = 0, npft - 1 + if (woody(i) == 2) taper(i) = 10.0_r8 ! shrubs + end do + end if ! NOTE: the following 5 PFT flags/options are addtions to 'woody', 'stress_decid', 'season_decid', ! 'evergreen', and 'crop', 'percop'. For default ELM, will be hard-coded; while for user-defined @@ -1162,7 +1180,7 @@ subroutine pftconrd do i = 0, mxpft if(.not. use_crop .and. i > mxpft_nc) EXIT ! exit the do loop - + ! (FATES-INTERF) Later, depending on how the team plans to structure the crop model ! or other modules that co-exist while FATES is on, we may want to preserve these pft definitions ! on non-fates columns. For now, they are incompatible, and this check is warranted (rgk 04-2017) From f9799c62ed603a7949659b9b2e68f11433b38cfd Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Fri, 29 Mar 2024 08:16:37 -0600 Subject: [PATCH 699/904] Update ci-compile-test.yml --- .github/workflows/ci-compile-test.yml | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ci-compile-test.yml diff --git a/.github/workflows/ci-compile-test.yml b/.github/workflows/ci-compile-test.yml new file mode 100644 index 000000000000..90e592630c6e --- /dev/null +++ b/.github/workflows/ci-compile-test.yml @@ -0,0 +1,56 @@ +name: CI +on: + push: + branches: + - '**' +jobs: + container-test-job: + runs-on: ubuntu-latest + container: + image: rfiorella/e3sm-dev:latest + steps: + - name: Check out the NGEE-Arctic-E3SM repo + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + submodules: true + - name: Setup cime and input datafiles + run: | + mkdir ~/.cime + cp /home/e3smuser/.cime/* /github/home/.cime/ + git clone https://github.com/rfiorella/pt-e3sm-inputdata /home/e3smuser/inputdata + cd /home/e3smuser/inputdata/lnd/clm2/firedata + tar -zxvf clmforc.Li_2012_hdm_0.5x0.5_AVHRR_simyr1850-2010_c130401.nc.tar.gz + cd /home/e3smuser/inputdata/atm/datm7/NASA_LIS/ + tar -zxvf clmforc.Li_2012_climo1995-2011.T62.lnfm_c130327.nc.tar.gz + tar -zxvf clmforc.Li_2012_climo1995-2011.T62.lnfm_Total_c140423.nc.tar.gz + ls -v /home/e3smuser/inputdata/atm/datm7/NASA_LIS/ + - name: create new case and setup + run: | + cd cime/scripts && ./create_newcase --mach docker --res ELM_USRDAT --compset ICB1850CNPRDCTCBC --case ~/output/ci_test \ + && cd ~/output/ci_test + ./xmlchange MOSART_MODE=NULL,DOUT_S=FALSE,DIN_LOC_ROOT=/home/e3smuser/inputdata + ./xmlchange DIN_LOC_ROOT_CLMFORC=/home/e3smuser/inputdata/atm/datm7 + ./xmlchange ELM_USRDAT_NAME=1x1pt_AK-TLG + ./xmlchange ATM_DOMAIN_PATH=/home/e3smuser/inputdata/share/domains/domain.clm + ./xmlchange LND_DOMAIN_PATH=/home/e3smuser/inputdata/share/domains/domain.clm + ./xmlchange ATM_DOMAIN_FILE=domain.lnd.1x1pt_teller-GRID_navy.nc + ./xmlchange LND_DOMAIN_FILE=domain.lnd.1x1pt_teller-GRID_navy.nc + ./xmlchange PIO_TYPENAME=netcdf,NTASKS=1,NTHRDS=1 + ./xmlchange STOP_OPTION=nyears,STOP_N=5 + ./xmlchange BATCH_SYSTEM=none + echo "&clm_inparm" >> user_nl_elm + echo " do_budgets = .false." >> user_nl_elm + echo " use_nofire = .true." >> user_nl_elm + echo " metdata_type = 'gswp3'" >> user_nl_elm + echo " metdata_bypass = '/home/e3smuser/inputdata/atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v2.c180716_NGEE-Grid/cpl_bypass_teller-Grid'" >> user_nl_elm + echo " fsurdat = '/home/e3smuser/inputdata/lnd/clm2/surfdata_map/surfdata_1x1pt_teller-GRID_simyr1850_c360x720_c171002.nc'" >> user_nl_elm + echo " stream_fldfilename_popdens = '/home/e3smuser/inputdata/lnd/clm2/firedata/clmforc.Li_2012_hdm_0.5x0.5_AVHRR_simyr1850-2010_c130401.nc'" >> user_nl_elm + ./case.setup + # add coupler bypass flag to cmake macros. + echo 'string(APPEND CPPDEFS " -DCPL_BYPASS")' >> cmake_macros/universal.cmake + ./case.build || cat /home/e3smuser/output/ci_test/bld/e3sm.bldlog.* + ./case.submit --no-batch + cat /home/e3smuser/output/ci_test/run/e3sm.log.* + cat /home/e3smuser/output/ci_test/run/lnd.log.* + From 7a2dd9c7c0ee073197f86a3ba72e8400e0ec492d Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Tue, 2 Apr 2024 07:17:15 -0600 Subject: [PATCH 700/904] hard-code pft numbers in pftvarcon for default taper values Temporary change, pending merge of IM4 updates. IM4 removed a lot of the hard-coded tests that looked for particular pfts by their assigned number. this required modifying the helper function "woody" to be a non-binary (i.e., now 0 = herbaceous, 1 = woody tree, 2 = woody shrub), but this updated function isn't available to IM3 currently. --- components/elm/src/main/pftvarcon.F90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index e06b5aba1953..e433d1f305a5 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -1099,7 +1099,9 @@ subroutine pftconrd if (.not. readv ) then taper(:) = 200._r8 do i = 0, npft - 1 - if (woody(i) == 2) taper(i) = 10.0_r8 ! shrubs + ! RPF 240331 - need to revisit this section when integrating with IM4, + ! to make consistent with attempt to remove hard-coded pft numbers. + if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) taper(i) = 10.0_r8 ! shrubs end do end if From 702f73c691f521fffebd84b93b661b9e33e9a944 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Fri, 14 Jun 2024 17:57:58 -0600 Subject: [PATCH 701/904] add bounds checking on bendresist and vegshape parameters --- .../elm/src/biogeochem/VegStructUpdateMod.F90 | 13 +++++-------- components/elm/src/main/pftvarcon.F90 | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 7c56fc6b58c0..868c417ff5f2 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -115,9 +115,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & dt = real( get_rad_step_size(), r8 ) - ! convert from stems/ha -> stems/m^2 - stocking = stocking / 10000._r8 - ! patch loop do fp = 1,num_soilp p = filter_soilp(fp) @@ -163,11 +160,11 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! taper and stocking density can be set as input variables now to ! change from default values set in pftvarcon.F90 if (spinup_state >= 1) then - htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper(p) * taper(p))/ & - (SHR_CONST_PI * stocking(p) * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper(ivt(p)) * taper(ivt(p)))/ & + (SHR_CONST_PI * stocking(ivt(p)) * dwood(ivt(p))))**(1._r8/3._r8) else - htop(p) = ((3._r8 * deadstemc(p) * taper(p) * taper(p))/ & - (SHR_CONST_PI * stocking(p) * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * taper(ivt(p)) * taper(ivt(p)))/ & + (SHR_CONST_PI * stocking(ivt(p)) * dwood(ivt(p))))**(1._r8/3._r8) end if ! Peter Thornton, 5/3/2004 @@ -239,7 +236,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! Wang and Zeng, 2007. if (woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) - fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(p) * (htop(p)-hbot(p)))) ** vegshape(p) + fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(ivt(p)) * (htop(p)-hbot(p)))) ** vegshape(ivt(p)) else fb = 1._r8 - max(min(snow_depth(c),0.2_r8),0._r8)/0.2_r8 ! 0.2m is assumed !depth of snow required for complete burial of grasses diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index e433d1f305a5..046bd880c0f0 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -1093,12 +1093,25 @@ subroutine pftconrd if (.not. readv ) bendresist(:) = 1._r8 call ncd_io('vegshape', vegshape, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) vegshape(:) = 1._r8 + ! check validity + do i = 0, npft -1 + if (bendresist(i) .gt. 1.0_r8 .or. bendresist(i) .le. 0._r8) then + call endrun(msg="Non-physical selection of bendresist parameter, set between 0 and 1"//errMsg(__FILE__, __LINE__)) + end if + if (vegshape(i) .gt. 2.0_r8 .or. vegshape(i) .le. 0_r8) then + call endrun(msg="Non-physical selection of vegshape parameter, set between 0 and 2"//errMsg(__FILE__, __LINE__)) + end if + end do call ncd_io('stocking', stocking, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) stocking(:) = 1000._r8 + ! convert from stems/ha -> stems/m2 + do i = 0, npft + stocking(i) = stocking(i) / 10000._r8 + end do call ncd_io('taper', taper, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) then taper(:) = 200._r8 - do i = 0, npft - 1 + do i = 0, npft ! RPF 240331 - need to revisit this section when integrating with IM4, ! to make consistent with attempt to remove hard-coded pft numbers. if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) taper(i) = 10.0_r8 ! shrubs From 20bb484b9a491ae68e9fc5ccf00687417b29068f Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Thu, 22 Aug 2024 09:39:47 -0600 Subject: [PATCH 702/904] Rearrange taper assignment in pftvarcon The taper array was being initialized incorrectly, leading to developer test failures. This moves the definition down further to be able to initialize correctly. --- components/elm/src/main/pftvarcon.F90 | 31 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index 046bd880c0f0..df732315d933 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -313,10 +313,11 @@ module pftvarcon real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion ! NGEE Arctic snow-vegetation interactions - real(r8), allocatable :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011) + real(r8), allocatable :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011; Sturm et al. 2005) real(r8), allocatable :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) real(r8), allocatable :: stocking(:) ! stocking density for pft (stems / hectare) real(r8), allocatable :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) + logical :: taper_defaults ! set flag to use taper defaults if not on params file (necessary as import and set values are in different places) ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. @@ -1094,7 +1095,7 @@ subroutine pftconrd call ncd_io('vegshape', vegshape, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) vegshape(:) = 1._r8 ! check validity - do i = 0, npft -1 + do i = 0, mxpft if (bendresist(i) .gt. 1.0_r8 .or. bendresist(i) .le. 0._r8) then call endrun(msg="Non-physical selection of bendresist parameter, set between 0 and 1"//errMsg(__FILE__, __LINE__)) end if @@ -1103,19 +1104,12 @@ subroutine pftconrd end if end do call ncd_io('stocking', stocking, 'read', ncid, readvar=readv, posNOTonfile=.true.) - if (.not. readv ) stocking(:) = 1000._r8 - ! convert from stems/ha -> stems/m2 - do i = 0, npft - stocking(i) = stocking(i) / 10000._r8 - end do + if (.not. readv ) stocking(:) = 0.1_r8 ! convert previous default of 1000 stems/ha to stems/m2 as had been done in VegStructUpdateMod.F90 + taper_defaults = .false. call ncd_io('taper', taper, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) then - taper(:) = 200._r8 - do i = 0, npft - ! RPF 240331 - need to revisit this section when integrating with IM4, - ! to make consistent with attempt to remove hard-coded pft numbers. - if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) taper(i) = 10.0_r8 ! shrubs - end do + taper(:) = 200._r8 ! pftnames not set to integers yet, so reassign further down. + taper_defaults = .true. end if ! NOTE: the following 5 PFT flags/options are addtions to 'woody', 'stress_decid', 'season_decid', @@ -1548,6 +1542,17 @@ subroutine pftconrd end do end if + ! reassign taper values for shrubs - RPF + if (taper_defaults) then + do i = 0, mxpft + if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) then + taper(i) = 10._r8 ! shrubs + else + taper(i) = 200._r8 + endif + end do + end if + if (masterproc) then write(iulog,*) 'Successfully read PFT physiological data' write(iulog,*) From e979a2cb9bcf93db43dba21aad8c76beed1c50d9 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Thu, 22 Aug 2024 18:59:16 -0600 Subject: [PATCH 703/904] Add IM3 ERS developer test to e3sm_land_developer Adds a new exact restart test that uses new parameters in the param file for snow/vegetation interactions. --- cime_config/tests.py | 1 + .../testmods_dirs/elm/snowveg_arctic/shell_commands | 10 ++++++++++ .../testmods_dirs/elm/snowveg_arctic/user_nl_elm | 8 ++++++++ 3 files changed, 19 insertions(+) create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b83974..d4c10f126b23 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -93,6 +93,7 @@ "SMS_Ly2_P1x1.1x1_smallvilleIA.IELMCNCROP.elm-fan", "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-snowveg_arctic", "ERS.r05_r05.IELM.elm-lnd_rof_2way", "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands new file mode 100644 index 000000000000..396d8cea8d66 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands @@ -0,0 +1,10 @@ +./xmlchange LND_DOMAIN_FILE=domain_42_FLUXNETSITES_simyr1850_c170912.nc +./xmlchange ATM_DOMAIN_FILE=domain_42_FLUXNETSITES_simyr1850_c170912.nc +./xmlchange LND_DOMAIN_PATH="\$DIN_LOC_ROOT/share/domains/domain.clm" +./xmlchange ATM_DOMAIN_PATH="\$DIN_LOC_ROOT/share/domains/domain.clm" +./xmlchange DATM_MODE=CLM1PT +./xmlchange DATM_CLMNCEP_YR_START=2000 +./xmlchange DATM_CLMNCEP_YR_END=2000 +./xmlchange ELM_USRDAT_NAME=42_FLUXNETSITES +./xmlchange NTASKS=1 +./xmlchange NTHRDS=1 diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm new file mode 100644 index 000000000000..bd07678a1345 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm @@ -0,0 +1,8 @@ +fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/surfdata_42_FLUXNETSITES_simyr1850_c170912.nc' +! this test has a parameter file with four new parameters defined for NGEE Arctic IM3 +! meant to improve snow-vegetation interactions. +! bendresist -> varies between 0 and 1, represents how much vegetation bends under snow loading +! vegshape -> suggested value 1 (parabolic), but 2 also used previously (Liston and Heimstra, 2011) +! taper -> deadstem height/radius ratio, moved from hardcoded in VegStructUpdateMod to pftvarcon +! stocking -> individual density on landscape (plants/m2), moved from hardcoded in VegStructUpdateMod to pftvarcon +paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_ngeea-im3_c240822.nc' From f0c93cf0b51ec029f7c35bde688ec4894e8f102f Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Thu, 22 Aug 2024 21:59:09 -0600 Subject: [PATCH 704/904] Fix new IM3 paramter values in pftvarcon fixes an issue where using less than the maximum pfts set new parameters with non-physical values --- components/elm/src/main/pftvarcon.F90 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index df732315d933..c6d4ab470bb0 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -1095,11 +1095,13 @@ subroutine pftconrd call ncd_io('vegshape', vegshape, 'read', ncid, readvar=readv, posNOTonfile=.true.) if (.not. readv ) vegshape(:) = 1._r8 ! check validity - do i = 0, mxpft + do i = 0, npft-1 if (bendresist(i) .gt. 1.0_r8 .or. bendresist(i) .le. 0._r8) then + write(iulog,*) "bendresist(i) is:", i, bendresist(i), npft call endrun(msg="Non-physical selection of bendresist parameter, set between 0 and 1"//errMsg(__FILE__, __LINE__)) end if - if (vegshape(i) .gt. 2.0_r8 .or. vegshape(i) .le. 0_r8) then + if (vegshape(i) .gt. 2.0_r8 .or. vegshape(i) .le. 0._r8) then + write(iulog,*) "vegshape(i) is:", i, vegshape(i), npft call endrun(msg="Non-physical selection of vegshape parameter, set between 0 and 2"//errMsg(__FILE__, __LINE__)) end if end do @@ -1544,7 +1546,7 @@ subroutine pftconrd ! reassign taper values for shrubs - RPF if (taper_defaults) then - do i = 0, mxpft + do i = 0, npft-1 if (i >= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) then taper(i) = 10._r8 ! shrubs else From 0f02df7bef612dda1933aeba3b09fb2d0f797505 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Tue, 27 Aug 2024 18:18:12 -0600 Subject: [PATCH 705/904] remove local ci test --- .github/workflows/ci-compile-test.yml | 56 --------------------------- 1 file changed, 56 deletions(-) delete mode 100644 .github/workflows/ci-compile-test.yml diff --git a/.github/workflows/ci-compile-test.yml b/.github/workflows/ci-compile-test.yml deleted file mode 100644 index 90e592630c6e..000000000000 --- a/.github/workflows/ci-compile-test.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: CI -on: - push: - branches: - - '**' -jobs: - container-test-job: - runs-on: ubuntu-latest - container: - image: rfiorella/e3sm-dev:latest - steps: - - name: Check out the NGEE-Arctic-E3SM repo - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - submodules: true - - name: Setup cime and input datafiles - run: | - mkdir ~/.cime - cp /home/e3smuser/.cime/* /github/home/.cime/ - git clone https://github.com/rfiorella/pt-e3sm-inputdata /home/e3smuser/inputdata - cd /home/e3smuser/inputdata/lnd/clm2/firedata - tar -zxvf clmforc.Li_2012_hdm_0.5x0.5_AVHRR_simyr1850-2010_c130401.nc.tar.gz - cd /home/e3smuser/inputdata/atm/datm7/NASA_LIS/ - tar -zxvf clmforc.Li_2012_climo1995-2011.T62.lnfm_c130327.nc.tar.gz - tar -zxvf clmforc.Li_2012_climo1995-2011.T62.lnfm_Total_c140423.nc.tar.gz - ls -v /home/e3smuser/inputdata/atm/datm7/NASA_LIS/ - - name: create new case and setup - run: | - cd cime/scripts && ./create_newcase --mach docker --res ELM_USRDAT --compset ICB1850CNPRDCTCBC --case ~/output/ci_test \ - && cd ~/output/ci_test - ./xmlchange MOSART_MODE=NULL,DOUT_S=FALSE,DIN_LOC_ROOT=/home/e3smuser/inputdata - ./xmlchange DIN_LOC_ROOT_CLMFORC=/home/e3smuser/inputdata/atm/datm7 - ./xmlchange ELM_USRDAT_NAME=1x1pt_AK-TLG - ./xmlchange ATM_DOMAIN_PATH=/home/e3smuser/inputdata/share/domains/domain.clm - ./xmlchange LND_DOMAIN_PATH=/home/e3smuser/inputdata/share/domains/domain.clm - ./xmlchange ATM_DOMAIN_FILE=domain.lnd.1x1pt_teller-GRID_navy.nc - ./xmlchange LND_DOMAIN_FILE=domain.lnd.1x1pt_teller-GRID_navy.nc - ./xmlchange PIO_TYPENAME=netcdf,NTASKS=1,NTHRDS=1 - ./xmlchange STOP_OPTION=nyears,STOP_N=5 - ./xmlchange BATCH_SYSTEM=none - echo "&clm_inparm" >> user_nl_elm - echo " do_budgets = .false." >> user_nl_elm - echo " use_nofire = .true." >> user_nl_elm - echo " metdata_type = 'gswp3'" >> user_nl_elm - echo " metdata_bypass = '/home/e3smuser/inputdata/atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v2.c180716_NGEE-Grid/cpl_bypass_teller-Grid'" >> user_nl_elm - echo " fsurdat = '/home/e3smuser/inputdata/lnd/clm2/surfdata_map/surfdata_1x1pt_teller-GRID_simyr1850_c360x720_c171002.nc'" >> user_nl_elm - echo " stream_fldfilename_popdens = '/home/e3smuser/inputdata/lnd/clm2/firedata/clmforc.Li_2012_hdm_0.5x0.5_AVHRR_simyr1850-2010_c130401.nc'" >> user_nl_elm - ./case.setup - # add coupler bypass flag to cmake macros. - echo 'string(APPEND CPPDEFS " -DCPL_BYPASS")' >> cmake_macros/universal.cmake - ./case.build || cat /home/e3smuser/output/ci_test/bld/e3sm.bldlog.* - ./case.submit --no-batch - cat /home/e3smuser/output/ci_test/run/e3sm.log.* - cat /home/e3smuser/output/ci_test/run/lnd.log.* - From 01ac9bdd8420c86eda51544c87f447d5db05d4c5 Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Tue, 3 Sep 2024 10:23:15 -0600 Subject: [PATCH 706/904] Update comment to correctly reflect references in VegStructUpdateMod --- components/elm/src/biogeochem/VegStructUpdateMod.F90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 868c417ff5f2..74e135f6f0ea 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -233,8 +233,10 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! adjust lai and sai for burying by snow. ! snow burial fraction for short vegetation (e.g. grasses) as in - ! Wang and Zeng, 2007. - if (woody(ivt(p)) >= 1.0_r8 ) then + ! Wang and Zeng et al 2007. + ! Taller vegetation (trees and shrubs) have been updated to use formulation similar to + ! Sturm et al. 2005; Liston and Hiemstra, 2011; and Belke-Brea et al. 2020 + if ( woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(ivt(p)) * (htop(p)-hbot(p)))) ** vegshape(ivt(p)) else From 3734c0fb2553621d30ee1f4cb8b8ab893ddb975a Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Fri, 6 Sep 2024 15:40:36 -0600 Subject: [PATCH 707/904] use woody instead of hardcoding to pft number --- components/elm/src/main/pftvarcon.F90 | 36 +++++++++++++-------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index c6d4ab470bb0..66c2adda90f8 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -459,18 +459,18 @@ subroutine pftconrd ! but cannot be less, i.e. there cannot be more pfts in the surface file than there are set of pft parameters, if running the non-crop version of model. if (npft= nbrdlf_evr_shrub .and. i <= nbrdlf_dcd_brl_shrub) then + if (woody(i) == 2._r8) then taper(i) = 10._r8 ! shrubs - else + else if (woody(i) == 1._r8) then taper(i) = 200._r8 - endif + end if end do end if From 18fa418c5e3d132de7967c75fad67ffe8f75a56e Mon Sep 17 00:00:00 2001 From: Rich Fiorella Date: Fri, 6 Sep 2024 15:57:46 -0600 Subject: [PATCH 708/904] update woody comment description to reflect IM4 changes --- .../elm/src/biogeochem/AllocationMod.F90 | 276 +++++++++--------- .../src/biogeochem/CNAllocationBetrMod.F90 | 12 +- .../src/biogeochem/CNGapMortalityBeTRMod.F90 | 8 +- .../src/biogeochem/CNNStateUpdate1BeTRMod.F90 | 2 +- .../elm/src/biogeochem/CNPhenologyBeTRMod.F90 | 8 +- .../src/biogeochem/CarbonStateUpdate1Mod.F90 | 2 +- .../elm/src/biogeochem/GrowthRespMod.F90 | 4 +- .../elm/src/biogeochem/MaintenanceRespMod.F90 | 6 +- .../biogeochem/NitrogenStateUpdate1Mod.F90 | 10 +- .../src/biogeochem/PhenologyFluxLimitMod.F90 | 14 +- .../elm/src/biogeochem/PhenologyMod.F90 | 32 +- .../biogeochem/PhosphorusStateUpdate1Mod.F90 | 12 +- .../data_types/VegetationPropertiesType.F90 | 4 +- 13 files changed, 195 insertions(+), 195 deletions(-) diff --git a/components/elm/src/biogeochem/AllocationMod.F90 b/components/elm/src/biogeochem/AllocationMod.F90 index 141194397fb0..5c315fc1769b 100644 --- a/components/elm/src/biogeochem/AllocationMod.F90 +++ b/components/elm/src/biogeochem/AllocationMod.F90 @@ -36,11 +36,11 @@ module AllocationMod use elm_varctl , only : NFIX_PTASE_plant use ELMFatesInterfaceMod , only : hlm_fates_interface_type use elm_varctl , only: iulog - use elm_varctl , only : carbon_only - use elm_varctl , only : carbonnitrogen_only + use elm_varctl , only : carbon_only + use elm_varctl , only : carbonnitrogen_only use elm_varctl , only : carbonphosphorus_only use shr_infnan_mod , only: nan => shr_infnan_nan, assignment(=) - + ! implicit none save @@ -114,23 +114,23 @@ module AllocationMod ! to toggle and update which processes are active. ! This will get set to false ! after ad_carbon_only is complete. - - + + logical :: crop_supln = .false. !Prognostic crop receives supplemental Nitrogen - + real(r8), allocatable,target :: veg_rootc_bigleaf(:,:) ! column-level fine-root biomas kgc/m3 integer, pointer :: ft_index_bigleaf(:) ! array holding the pft index of each competitor ! ECA parameters - ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance - real(r8), parameter :: e_plant_scalar = 0.0000125_r8 - - ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance - real(r8), parameter :: e_decomp_scalar = 0.05_r8 + ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance + real(r8), parameter :: e_plant_scalar = 0.0000125_r8 + + ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance + real(r8), parameter :: e_decomp_scalar = 0.05_r8 !$acc declare create(e_decomp_scalar) !$acc declare create(e_plant_scalar) - + !$acc declare copyin(crop_supln) !----------------------------------------------------------------------- @@ -213,8 +213,8 @@ subroutine AllocationInit ( bounds, elm_fates) use elm_time_manager, only: get_step_size use elm_varpar , only: crop_prog use elm_varctl , only: iulog - use elm_varctl , only : carbon_only - use elm_varctl , only : carbonnitrogen_only + use elm_varctl , only : carbon_only + use elm_varctl , only : carbonnitrogen_only use elm_varctl , only : carbonphosphorus_only @@ -234,9 +234,9 @@ subroutine AllocationInit ( bounds, elm_fates) integer :: max_comps ! maximum number of possible plant competitors ! elm big-leaf: number of pfts/patches ! fates: number of cohorts in the column - - + + !----------------------------------------------------------------------- if ( crop_prog )then @@ -253,7 +253,7 @@ subroutine AllocationInit ( bounds, elm_fates) end if end if - + ! set time steps dt = real( get_step_size(), r8 ) @@ -374,7 +374,7 @@ subroutine EvaluateSupplStatus() end subroutine EvaluateSupplStatus !------------------------------------------------------------------------------------------------- - + subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & photosyns_vars, crop_vars, canopystate_vars, cnstate_vars, dt, yr) ! PHASE-1 of Allocation: loop over patches to assess the total plant N demand and P demand @@ -430,7 +430,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -971,7 +971,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Fractional uptake profiles, that are proportional to root density real(r8):: nuptake_prof(bounds%begc:bounds%endc,1:nlevdecomp) real(r8):: puptake_prof(bounds%begc:bounds%endc,1:nlevdecomp) - integer, allocatable :: filter_pcomp(:) ! this is a plant competitor map for FATES/ELM-BL w/ ECA + integer, allocatable :: filter_pcomp(:) ! this is a plant competitor map for FATES/ELM-BL w/ ECA real(r8), allocatable,target :: plant_nh4demand_vr_fates(:,:) ! nh4 demand per competitor per soil layer real(r8), allocatable,target :: plant_no3demand_vr_fates(:,:) ! no3 demand per competitor per soil layer real(r8), allocatable,target :: plant_pdemand_vr_fates(:,:) ! p demand per competitor per soil layer @@ -990,7 +990,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & real(r8):: cp_stoich_var=0.4 ! variability of CP ratio - + !----------------------------------------------------------------------- associate( & @@ -1152,7 +1152,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & n_pcomp = elm_fates%fates(ci)%bc_out(s)%num_plant_comps pci = 1 pcf = n_pcomp - + if( nu_com.eq.'RD') then ! Overwrite the column level demands, since fates plants are all sharing @@ -1160,13 +1160,13 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! to scale up to column plant_ndemand_col(c) = 0._r8 plant_pdemand_col(c) = 0._r8 - + ! We fill the vertically resolved array to simplify some jointly used code do j = 1, nlevdecomp col_plant_ndemand_vr(c,j) = 0._r8 col_plant_pdemand_vr(c,j) = 0._r8 - + do f = 1,n_pcomp ft = elm_fates%fates(ci)%bc_out(s)%ft_index(f) @@ -1184,10 +1184,10 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! [gN/m2/s] plant_ndemand_col(c) = plant_ndemand_col(c) + col_plant_ndemand_vr(c,j)*dzsoi_decomp(j) plant_pdemand_col(c) = plant_pdemand_col(c) + col_plant_pdemand_vr(c,j)*dzsoi_decomp(j) - + end do - + else !(ECA) do f = 1,n_pcomp @@ -1195,7 +1195,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do veg_rootc_ptr => elm_fates%fates(ci)%bc_out(s)%veg_rootc - ft_index_ptr => elm_fates%fates(ci)%bc_out(s)%ft_index ! Should be + ft_index_ptr => elm_fates%fates(ci)%bc_out(s)%ft_index ! Should be decompmicc(:) = elm_fates%fates(ci)%bc_out(s)%decompmicc(:) ! Should be (nlevdecomp) cn_scalar_runmean_ptr => elm_fates%fates(ci)%bc_out(s)%cn_scalar ! This is 1.0 @@ -1216,7 +1216,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & pci = col_pp%pfti(c) ! Initial plant competitor index pcf = col_pp%pftf(c) ! Final plant competitor index - + if (nu_com .eq. 'RD') then do j = 1, nlevdecomp @@ -1225,7 +1225,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do else - + f = 0 decompmicc(:) = 0._r8 do p = col_pp%pfti(c), col_pp%pftf(c) @@ -1246,7 +1246,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end if end do n_pcomp = f - + ft_index_ptr => ft_index_bigleaf veg_rootc_ptr => veg_rootc_bigleaf @@ -1263,7 +1263,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & (leafcn(ivt(p)) - leafcn(ivt(p))*(1- cn_stoich_var)),0.0_r8),1.0_r8) end do end if - + km_nh4_ptr => km_plant_nh4 vmax_nh4_ptr => vmax_plant_nh4 cn_scalar_runmean_ptr => cn_scalar_runmean @@ -1298,14 +1298,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! (1) add nitrogen and phosphorus competition ! (2) nitrogen and phosphorus uptake is based on root kinetics ! (3) no second pass nutrient uptake for plants - ! ============================================================= - + ! ============================================================= + if (nu_com .eq. 'RD') then ! Estimate actual allocation rates via Relative Demand ! approach (RD) - + call NAllocationRD(col_plant_ndemand_vr(c,:), & ! IN potential_immob_vr(c,:), & ! IN AllocParamsInst%compet_plant_nh4, & ! IN @@ -1332,19 +1332,19 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Estimate actual allocation rates via Capacitance Aquisition ! approach (ECA/CA) - + call NAllocationECAMIC(pci,dt, & ! IN bd(c,:), & ! IN h2osoi_vol(c,:), & ! IN t_scalar(c,:), & ! IN - n_pcomp, & ! IN + n_pcomp, & ! IN filter_pcomp(1:n_pcomp), & ! IN veg_rootc_ptr(pci:pcf,:), & ! IN ft_index_ptr(pci:pcf), & ! IN cn_scalar_runmean_ptr(pci:pcf), & ! IN decompmicc, & ! IN smin_nh4_vr(c,:), & ! IN - nu_com, & ! IN + nu_com, & ! IN km_nh4_ptr, & ! IN vmax_nh4_ptr, & ! IN km_decomp_nh4, & ! IN @@ -1389,7 +1389,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! NO3 flux demands. supplement_to_sminn_vr(c,j) = 0._r8 if (carbon_only .or. carbonphosphorus_only) then - + if ( fpi_no3_vr(j) + fpi_nh4_vr(j) < 1._r8 ) then fpi_vr(c,j) = 1._r8 fpi_nh4_vr(j) = 1.0_r8 - fpi_no3_vr(j) @@ -1409,9 +1409,9 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & supplement_to_sminn_vr(c,j) = supplement_to_sminn_vr(c,j) + col_plant_ndemand_vr(c,j) smin_nh4_to_plant_vr(c,j) = col_plant_ndemand_vr(c,j) - smin_no3_to_plant_vr(c,j) end if - - + + end if ! sum up nitrogen limitation to decomposition @@ -1422,14 +1422,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_vr(c,j) = actual_immob_no3_vr(c,j) + actual_immob_nh4_vr(c,j) end do - + ! Starting resolving P limitation !!! ! ============================================================= if (nu_com .eq. 'RD') then ! Relative Demand (RD) - + call PAllocationRD(col_plant_pdemand_vr(c,:), & ! IN potential_immob_p_vr(c,:), & ! IN solutionp_vr(c,:), & ! IN @@ -1438,38 +1438,38 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_p_vr(c,:), & ! OUT sminp_to_plant_vr(c,:), & ! OUT supplement_to_sminp_vr(c,:)) ! OUT - + else call PAllocationECAMIC(pci,dt, & ! IN h2osoi_vol(c,:), & ! IN - t_scalar(c,:), & ! IN - gross_pmin_vr(c,:), & ! IN - potential_immob_p_vr(c,:), & ! IN - biochem_pmin_vr_col(c,:), & ! IN - primp_to_labilep_vr_col(c,:), & ! IN - pdep_to_sminp(c), & ! IN - pdep_prof(c,:), & ! IN - vmax_minsurf_p_vr(isoilorder(c),:), & ! IN - km_minsurf_p_vr(isoilorder(c),:), & ! IN - solutionp_vr(c,:), & ! IN - nu_com, & ! IN - n_pcomp, & ! IN + t_scalar(c,:), & ! IN + gross_pmin_vr(c,:), & ! IN + potential_immob_p_vr(c,:), & ! IN + biochem_pmin_vr_col(c,:), & ! IN + primp_to_labilep_vr_col(c,:), & ! IN + pdep_to_sminp(c), & ! IN + pdep_prof(c,:), & ! IN + vmax_minsurf_p_vr(isoilorder(c),:), & ! IN + km_minsurf_p_vr(isoilorder(c),:), & ! IN + solutionp_vr(c,:), & ! IN + nu_com, & ! IN + n_pcomp, & ! IN filter_pcomp(1:n_pcomp), & ! IN - veg_rootc_ptr(pci:pcf,:), & ! IN - ft_index_ptr(pci:pcf), & ! IN - decompmicc, & ! IN - cp_scalar_runmean_ptr(pci:pcf), & ! IN - km_p_ptr(:), & ! IN - vmax_p_ptr(:), & ! IN - km_decomp_p, & ! IN - labilep_vr(c,:), & ! IN + veg_rootc_ptr(pci:pcf,:), & ! IN + ft_index_ptr(pci:pcf), & ! IN + decompmicc, & ! IN + cp_scalar_runmean_ptr(pci:pcf), & ! IN + km_p_ptr(:), & ! IN + vmax_p_ptr(:), & ! IN + km_decomp_p, & ! IN + labilep_vr(c,:), & ! IN plant_pdemand_vr_ptr(pci:pcf,:), & ! INOUT - col_plant_pdemand_vr(c,:), & ! OUT + col_plant_pdemand_vr(c,:), & ! OUT adsorb_to_labilep_vr(c,:), & ! OUT fpi_p_vr(c,:), & ! OUT actual_immob_p_vr(c,:), & ! OUT - sminp_to_plant_vr(c,:), & ! OUT + sminp_to_plant_vr(c,:), & ! OUT desorb_to_solutionp_vr(c,:), & ! OUT supplement_to_sminp_vr(c,:)) ! OUT @@ -1482,13 +1482,13 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & supplement_to_sminp_vr(c,j) = col_plant_pdemand_vr(c,j) end do end if - + end if ! end of P competition ! resolving N limitation vs. P limitation for decomposition ! update (1) actual immobilization for N and P (2) sminn_to_plant and sminp_to_plant ! We only resolve co-limitations when are supplementing neither element - + np_bothactive: if ( .not.carbon_only .and. & .not.carbonphosphorus_only .and. & .not.carbonnitrogen_only ) then @@ -1593,7 +1593,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_vr(c,j) = potential_immob_vr(c,j) * fpi_p_vr(c,j) end do end if - + ! sum up plant N/P uptake at column level and patch level ! sum up N fluxes to plant after initial competition sminn_to_plant(c) = 0._r8 @@ -1602,7 +1602,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & sminn_to_plant(c) = sminn_to_plant(c) + sminn_to_plant_vr(c,j) * dzsoi_decomp(j) sminp_to_plant(c) = sminp_to_plant(c) + sminp_to_plant_vr(c,j) * dzsoi_decomp(j) end do - + ! update column plant N/P demand, pft level plant NP uptake for ECA and MIC mode eca_filter: if (nu_com .eq. 'ECA' .or. nu_com .eq. 'MIC') then @@ -1648,14 +1648,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end if end do end if - + end do end if eca_filter end do col_loop - + if ((nu_com .eq. 'ECA' .or. nu_com .eq. 'MIC')) then deallocate(filter_pcomp) if(.not.use_fates)then @@ -1750,7 +1750,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Set the FATES N and P uptake fluxes - + if(use_fates)then do fc=1,num_soilc c = filter_soilc(fc) @@ -1771,15 +1771,15 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & (elm_fates%fates(ci)%bc_pconst%vmax_nh4(ft)+elm_fates%fates(ci)%bc_pconst%vmax_no3(ft)) * & dzsoi_decomp(j) end do - + do j = 1,nlevdecomp - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & smin_nh4_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (ndemand/plant_ndemand_col(c)) - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & smin_no3_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (ndemand/plant_ndemand_col(c)) @@ -1787,12 +1787,12 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do end do end if - + if( plant_pdemand_col(c)>tiny(plant_pdemand_col(c)) ) then do f = 1,n_pcomp ft = elm_fates%fates(ci)%bc_out(s)%ft_index(f) - + pdemand=0._r8 do j = 1,nlevdecomp ! [gP/m2/s] @@ -1800,14 +1800,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & elm_fates%fates(ci)%bc_pconst%vmax_p(ft) * & dzsoi_decomp(j) end do - + do j = 1,nlevdecomp ! [gP/m2/step] - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & sminp_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (pdemand/plant_pdemand_col(c)) - + end do end do end if @@ -1817,21 +1817,21 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & do f = 1,n_pcomp do j = 1,nlevdecomp - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & plant_nh4demand_vr_fates(f,j) * fpg_nh4_vr(c,j) * dzsoi_decomp(j) * dt - - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & + + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & plant_no3demand_vr_fates(f,j) * fpg_no3_vr(c,j) * dzsoi_decomp(j) * dt - - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & + + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & (plant_pdemand_vr_fates(f,j) * fpg_p_vr(c,j)) * dzsoi_decomp(j) * dt - + end do end do - + end if end do @@ -1840,7 +1840,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & deallocate(plant_no3demand_vr_fates) deallocate(plant_pdemand_vr_fates) end if - + end if ! if(use_fates) end associate @@ -1907,7 +1907,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -2959,7 +2959,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & end associate end subroutine Allocation3_PlantCNPAlloc - + ! ====================================================================================== subroutine NAllocationECAMIC(pci,dt, & ! IN @@ -2976,14 +2976,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN nu_com, & ! IN km_nh4_plant, & ! IN (pft) vmax_nh4_plant, & ! IN (pft) - km_decomp_nh4, & ! IN + km_decomp_nh4, & ! IN potential_immob_vr, & ! IN (j) plant_nh4demand_vr, & ! INOUT (i,j) col_plant_nh4demand_vr, & ! OUT (j) fpi_nh4_vr, & ! OUT (j) actual_immob_nh4_vr, & ! OUT (j) smin_nh4_to_plant_vr, & ! OUT (j) - smin_no3_vr, & ! IN (j) + smin_no3_vr, & ! IN (j) km_no3_plant, & ! IN (pft) vmax_no3_plant, & ! IN (pft) km_decomp_no3, & ! IN (j) @@ -3004,12 +3004,12 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ! kinetics following Zhu et al., 2016 DOI: 10.1002/2016JG003554 ! ------------------------------------------------------------------------------------ use elm_varpar , only: nlevdecomp - + integer, intent(in) :: pci ! First index of plant comp arrays real(r8), intent(in) :: dt ! Time step duration [s] real(r8), intent(in) :: bd(:) ! Bulk density of dry soil material [kg m-3] real(r8), intent(in) :: h2osoi_vol(:) ! Vol. Soil Water in each layer [m3] - real(r8), intent(in) :: t_scalar(:) ! fraction by which decomposition is limited by temperature + real(r8), intent(in) :: t_scalar(:) ! fraction by which decomposition is limited by temperature integer, intent(in) :: n_pcomp ! number of plant competitors integer, intent(in) :: filter_pcomp(:) ! plant competition filter real(r8), intent(in) :: veg_rootc(pci:,:) ! total fine-root biomass of each competitor [gC/m3] @@ -3065,7 +3065,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN integer :: i,ip ! loop index for competitors integer :: ft ! loop index for pfts - ! 2.76 consider soil adsorption effect on [NH4+] availability, + ! 2.76 consider soil adsorption effect on [NH4+] availability, ! based on Zhu et al., 2016 DOI: 10.1002/2016JG003554 real(r8), parameter :: adsorp_nh4_eff = 2.76_r8 @@ -3073,14 +3073,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN do j = 1, nlevdecomp - ! Plant, microbial decomposers compete for NH4. Thus loop over each + ! Plant, microbial decomposers compete for NH4. Thus loop over each ! plant competitor in this competitive space (column). - ! Calculate competition coefficients for N/P, first need to convert - ! concentration to per soil water based + ! Calculate competition coefficients for N/P, first need to convert + ! concentration to per soil water based ! concentration of mineralized nutrient, per soil water solution_conc = smin_nh4_vr(j) / (bd(j)*adsorp_nh4_eff*m3_per_liter + h2osoi_vol(j)) - + e_km = 0._r8 do i = 1, n_pcomp ip = filter_pcomp(i) @@ -3089,14 +3089,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN end do e_km = e_km + e_decomp_scalar*decompmicc(j)*(1._r8/km_decomp_nh4 + 1._r8/km_nit) - + do i = 1, n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_conc / & + compet_plant(i) = solution_conc / & ( km_nh4_plant(ft) * (1._r8 + solution_conc/km_nh4_plant(ft) + e_km)) end do - + compet_decomp = solution_conc / (km_decomp_nh4 * (1._r8 + solution_conc/km_decomp_nh4 + e_km)) compet_nit = solution_conc / (km_nit * (1._r8 + solution_conc/km_nit + e_km)) @@ -3110,7 +3110,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ip = filter_pcomp(i) ft = ft_index(ip) - ! This is the demand per m3 of the column (not patch) + ! This is the demand per m3 of the column (not patch) ! (for native ELM divide through by the patch weight to get per m3 of patch) plant_nh4demand_vr(ip,j) = max(0._r8,vmax_nh4_plant(ft) * veg_rootc(ip,j) * & cn_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) @@ -3130,7 +3130,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN else ! 'MIC' mode - sum_nh4_demand_scaled = potential_immob_vr(j)*compet_decomp + & + sum_nh4_demand_scaled = potential_immob_vr(j)*compet_decomp + & pot_f_nit_vr(j)*compet_nit end if @@ -3199,10 +3199,10 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN do i = 1, n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_conc / & + compet_plant(i) = solution_conc / & ( km_no3_plant(ft) * (1._r8 + solution_conc/km_no3_plant(ft) + e_km)) end do - + compet_decomp = solution_conc / (km_decomp_no3 * (1._r8 + solution_conc/km_decomp_no3 + e_km)) compet_denit = solution_conc / (km_den * (1._r8 + solution_conc/km_den + e_km)) @@ -3215,7 +3215,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ip = filter_pcomp(i) ft = ft_index(ip) - ! This is the demand per m3 of the column (not patch) + ! This is the demand per m3 of the column (not patch) ! (for native ELM divide through by the patch weight to get per m3 of patch) plant_no3demand_vr(ip,j) = max(0._r8,vmax_no3_plant(ft) * veg_rootc(ip,j) * & cn_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) @@ -3245,7 +3245,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN smin_no3_to_plant_vr(j) = col_plant_no3demand_vr(j) f_denit_vr(j) = pot_f_denit_vr(j) - else + else ! NO3 availability can not satisfy the sum of immobilization, denitrification, and ! plant growth demands, so these three demands compete for available @@ -3284,34 +3284,34 @@ end subroutine NAllocationECAMIC subroutine PAllocationECAMIC(pci, & dt, & - h2osoi_vol, & - t_scalar, & - gross_pmin_vr, & - potential_immob_p_vr, & - biochem_pmin_vr_col, & - primp_to_labilep_vr_col, & - pdep_to_sminp, & - pdep_prof, & + h2osoi_vol, & + t_scalar, & + gross_pmin_vr, & + potential_immob_p_vr, & + biochem_pmin_vr_col, & + primp_to_labilep_vr_col, & + pdep_to_sminp, & + pdep_prof, & vmax_minsurf_p_vr, & km_minsurf_p_vr, & solutionp_vr, & nu_com, & n_pcomp, & - filter_pcomp, & - veg_rootc, & + filter_pcomp, & + veg_rootc, & ft_index, & decompmicc, & - cp_scalar_runmean, & + cp_scalar_runmean, & km_plant_p, & vmax_plant_p, & - km_decomp_p, & + km_decomp_p, & labilep_vr, & - plant_pdemand_vr_patch, & - col_plant_pdemand_vr, & + plant_pdemand_vr_patch, & + col_plant_pdemand_vr, & adsorb_to_labilep_vr, & fpi_p_vr, & actual_immob_p_vr, & - sminp_to_plant_vr, & + sminp_to_plant_vr, & desorb_to_solutionp_vr, & supplement_to_sminp_vr) @@ -3370,7 +3370,7 @@ subroutine PAllocationECAMIC(pci, & ! plant P uptake, microbial P uptake/release ! secondary P desorption is assumed to go into solution P pool - do j = 1, nlevdecomp + do j = 1, nlevdecomp ! plant, microbial decomposer, mineral surface compete for P ! loop over each pft within the same column @@ -3390,27 +3390,27 @@ subroutine PAllocationECAMIC(pci, & do i = 1,n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_pconc / & + compet_plant(i) = solution_pconc / & (km_plant_p(ft)*(1._r8 + solution_pconc/km_plant_p(ft) + e_km_p)) end do - + compet_decomp_p = solution_pconc / & (km_decomp_p * (1._r8 + solution_pconc/km_decomp_p + e_km_p)) - compet_minsurf_p = solution_pconc/ & + compet_minsurf_p = solution_pconc/ & (km_minsurf_p_vr(j) * (1._r8 + solution_pconc/km_minsurf_p_vr(j) + e_km_p)) col_plant_pdemand_vr(j) = 0._r8 do i = 1,n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - plant_pdemand_vr_patch(ip,j) = max(0._r8,vmax_plant_p(ft) * veg_rootc(ip,j) * & + plant_pdemand_vr_patch(ip,j) = max(0._r8,vmax_plant_p(ft) * veg_rootc(ip,j) * & cp_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) col_plant_pdemand_vr(j) = col_plant_pdemand_vr(j) + plant_pdemand_vr_patch(ip,j) end do ! potential adsorption rate without plant and microbial interaction - ! including weathering, deposition, phosphatase, mineralization, + ! including weathering, deposition, phosphatase, mineralization, ! immobilization, plant uptake dsolutionp_dt = gross_pmin_vr(j) -potential_immob_p_vr(j) - & col_plant_pdemand_vr(j) + biochem_pmin_vr_col(j) + & @@ -3477,7 +3477,7 @@ subroutine PAllocationECAMIC(pci, & if (nu_com .eq. 'MIC') sminp_to_plant_vr(j) = min(max( 0._r8, & (solutionp_vr(j)/dt) - actual_immob_p_vr(j) - adsorb_to_labilep_vr(j) ), & - col_plant_pdemand_vr(j)) + col_plant_pdemand_vr(j)) end if end do @@ -3488,7 +3488,7 @@ end subroutine PAllocationECAMIC subroutine NAllocationRD(col_plant_ndemand_vr, &! IN (j) potential_immob_vr, & ! IN (j) - compet_plants_nh4, & ! IN + compet_plants_nh4, & ! IN compet_decomp_nh4, & ! IN dt, & ! IN smin_nh4_vr, & ! IN (j) @@ -3641,7 +3641,7 @@ end subroutine NAllocationRD ! ====================================================================================== - subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN + subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN potential_immob_p_vr, & ! IN (j) solutionp_vr, & ! IN (j) dt, & ! IN @@ -3684,7 +3684,7 @@ subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN actual_immob_p_vr(j) = potential_immob_p_vr(j) sminp_to_plant_vr(j) = col_plant_pdemand_vr(j) supplement_to_sminp_vr(j) = sum_pdemand - (solutionp_vr(j)/dt) - + else ! P availability can not satisfy the sum of immobilization and ! plant growth demands, so these two demands compete for @@ -3702,7 +3702,7 @@ subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN fpi_p_vr(j) = 0.0_r8 end if - sminp_to_plant_vr(j) = max( 0._r8,(solutionp_vr(j)/dt) - actual_immob_p_vr(j) ) + sminp_to_plant_vr(j) = max( 0._r8,(solutionp_vr(j)/dt) - actual_immob_p_vr(j) ) end if end do @@ -3749,7 +3749,7 @@ subroutine calc_nuptake_prof(bounds, num_soilc, filter_soilc, cnstate_vars, nupt c = filter_soilc(fc) sminn_vr_loc(c,j) = smin_no3_vr(c,j) + smin_nh4_vr(c,j) - + if(use_pflotran .and. pf_cmode) then sminn_tot(c) = sminn_tot(c) + sminn_vr_loc(c,j) * dzsoi_decomp(j) & *(nfixation_prof(c,j)*dzsoi_decomp(j)) ! weighted by froot fractions in annual max. active layers diff --git a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 index 8cfc342bb404..32f9fc6eb23b 100644 --- a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 +++ b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 @@ -26,13 +26,13 @@ module CNAllocationBeTRMod use PhotosynthesisType , only : photosyns_type use CropType , only : crop_type use VegetationPropertiesType, only : veg_vp - use LandunitType , only : lun_pp + use LandunitType , only : lun_pp use ColumnType , only : col_pp - use ColumnDataType , only : col_cf, col_ns, col_nf, col_ps, col_pf + use ColumnDataType , only : col_cf, col_ns, col_nf, col_ps, col_pf use VegetationType , only : veg_pp use VegetationDataType , only : veg_cs, veg_ns, veg_nf, veg_ps, veg_pf - use VegetationDataType , only : veg_cf, c13_veg_cf, c14_veg_cf - + use VegetationDataType , only : veg_cf, c13_veg_cf, c14_veg_cf + ! bgc interface & pflotran module switches use elm_varctl , only : nu_com use SoilStatetype , only : soilstate_type @@ -359,7 +359,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -1167,7 +1167,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type ! - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) diff --git a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 index 3bbbf1db1f60..ed8811f61f89 100644 --- a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 @@ -18,9 +18,9 @@ module CNGapMortalityBeTRMod use ColumnType , only : col_pp use ColumnDataType , only : col_cf, col_nf, col_pf use VegetationPropertiesType , only : veg_vp - use VegetationType , only : veg_pp - use VegetationDataType , only : veg_cs, veg_cf, veg_ns, veg_nf - use VegetationDataType , only : veg_ps, veg_pf + use VegetationType , only : veg_pp + use VegetationDataType , only : veg_cs, veg_cf, veg_ns, veg_nf + use VegetationDataType , only : veg_ps, veg_pf use PhosphorusFluxType , only : phosphorusflux_type use PhosphorusStateType , only : phosphorusstate_type @@ -121,7 +121,7 @@ subroutine CNGapMortality (& associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody & ! Input: [real(r8) (:) ] binary flag for woody lifeform + woody => veg_vp%woody & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) ) ! set the mortality rate based on annual rate diff --git a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 index 465c92f82b4c..b785f52549e7 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 @@ -59,7 +59,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step diff --git a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 index ac3b58538e8f..066dcfb62c82 100644 --- a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 @@ -525,7 +525,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & prev_dayl => grc_pp%prev_dayl , & ! Input: [real(r8) (:) ] daylength from previous time step (s) season_decid => veg_vp%season_decid , & ! Input: [real(r8) (:) ] binary flag for seasonal-deciduous leaf habit (0 or 1) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) (-nlevsno+1:nlevgrnd) @@ -875,7 +875,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & dayl => grc_pp%dayl , & ! Input: [real(r8) (:) ] daylength (s) leaf_long => veg_vp%leaf_long , & ! Input: [real(r8) (:) ] leaf longevity (yrs) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) stress_decid => veg_vp%stress_decid , & ! Input: [real(r8) (:) ] binary flag for stress-deciduous leaf habit (0 or 1) soilpsi => soilstate_vars%soilpsi_col , & ! Input: [real(r8) (:,:) ] soil water potential in each soil layer (MPa) @@ -2295,7 +2295,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) onset_flag => cnstate_vars%onset_flag_patch , & ! Input: [real(r8) (:) ] onset flag onset_counter => cnstate_vars%onset_counter_patch , & ! Input: [real(r8) (:) ] onset days counter @@ -2880,7 +2880,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) livewdcn => veg_vp%livewdcn , & ! Input: [real(r8) (:) ] live wood (phloem and ray parenchyma) C:N (gC/gN) deadwdcn => veg_vp%deadwdcn , & ! Input: [real(r8) (:) ] dead wood (xylem and heartwood) C:N (gC/gN) diff --git a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 index 1eb7f64d025a..5133575abd30 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 @@ -200,7 +200,7 @@ subroutine CarbonStateUpdate1(bounds, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step harvdate => crop_vars%harvdate_patch & ! Input: [integer (:) ] harvest date diff --git a/components/elm/src/biogeochem/GrowthRespMod.F90 b/components/elm/src/biogeochem/GrowthRespMod.F90 index cb7ad522265f..cc6707d723b0 100644 --- a/components/elm/src/biogeochem/GrowthRespMod.F90 +++ b/components/elm/src/biogeochem/GrowthRespMod.F90 @@ -51,7 +51,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) associate( & ivt => veg_pp%itype , & ! Input: [integer (:)] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:)] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:)] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cpool_to_leafc => veg_cf%cpool_to_leafc , & ! Input: [real(r8) (:)] cpool_to_leafc_storage => veg_cf%cpool_to_leafc_storage , & ! Input: [real(r8) (:)] @@ -98,7 +98,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) ) ! Loop through patches - ! start pft loop + ! start pft loop do fp = 1,num_soilp p = filter_soilp(fp) diff --git a/components/elm/src/biogeochem/MaintenanceRespMod.F90 b/components/elm/src/biogeochem/MaintenanceRespMod.F90 index 54005275b296..829e5c742a06 100644 --- a/components/elm/src/biogeochem/MaintenanceRespMod.F90 +++ b/components/elm/src/biogeochem/MaintenanceRespMod.F90 @@ -70,7 +70,7 @@ subroutine readMaintenanceRespParams ( ncid ) call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun(msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) br_mr_Inst = tempr - + end subroutine readMaintenanceRespParams !----------------------------------------------------------------------- @@ -107,7 +107,7 @@ subroutine MaintenanceResp(bounds, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] patch vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) br_xr => veg_vp%br_xr , & ! Input: [real(r8) (:) ] base rate for excess respiration frac_veg_nosno => canopystate_vars%frac_veg_nosno_patch , & ! Input: [integer (:) ] fraction of vegetation not covered by snow (0 OR 1) [-] laisun => canopystate_vars%laisun_patch , & ! Input: [real(r8) (:) ] sunlit projected leaf area index @@ -160,7 +160,7 @@ subroutine MaintenanceResp(bounds, & ! calculate temperature corrections for each soil layer, for use in ! estimating fine root maintenance respiration with depth tcsoi(c,j) = Q10**((t_soisno(c,j)-SHR_CONST_TKFRZ - 20.0_r8)/10.0_r8) - + end do end do diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 index c1b2bfc5d65a..f95b9a258cb1 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 @@ -130,7 +130,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step @@ -147,7 +147,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp do j = 1, nlevdecomp do fc = 1,num_soilc c = filter_soilc(fc) - + ! N deposition and fixation (put all into NH4 pool) col_ns%smin_nh4_vr(c,j) = col_ns%smin_nh4_vr(c,j) + col_nf%ndep_to_sminn(c)*dt * ndep_prof(c,j) col_ns%smin_nh4_vr(c,j) = col_ns%smin_nh4_vr(c,j) + col_nf%nfix_to_sminn(c)*dt * nfixation_prof(c,j) @@ -219,7 +219,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp end do end if end do - + do j = 1, nlevdecomp ! column loop do fc = 1,num_soilc @@ -254,8 +254,8 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp end do ! end of column loop end do - - endif !end if is_active_betr_bgc + + endif !end if is_active_betr_bgc ! forest fertilization call get_curr_date(kyr, kmo, kda, mcsec) diff --git a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 index 87b9637d5512..d80b844e6009 100644 --- a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 +++ b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 @@ -397,7 +397,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, -1._r8, f_gresp_storage_to_xfer , s_gresp_storage,nelms) !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_carbon_d, nelms, f_gresp_storage_to_xfer) - + !initialize stoichiometric relationship between carbon production flux and corresponding state varaibles call spm_list_init(spm_list, 1._r8, f_cpool_to_leafc , s_leafc, nelms) call spm_list_insert(spm_list, 1._r8, f_leafc_xfer_to_leafc , s_leafc, nelms) @@ -432,7 +432,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, 1._r8, f_gresp_storage_to_xfer , s_gresp_xfer,nelms) call spm_list_insert(spm_list, 1._r8, f_cpool_to_gresp_storage , s_gresp_storage, nelms) - + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_carbon_p, nelms, f_gresp_storage_to_xfer) !initialize stoichiometry relationship between nutrient consumption and corresponding state variables @@ -474,7 +474,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, -1._r8, f_grainn_to_food , s_grainn, nelms) call spm_list_insert(spm_list, -1._r8, f_grainn_xfer_to_grainn , s_grainn_xfer,nelms) call spm_list_insert(spm_list, -1._r8, f_retransn_to_npool , s_retransn, nelms) - !turn the list into sparse matrix form + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_nutrient_d, nelms, f_supplement_to_plantn) !initialize stoichiometry relationship between nutrient production and corresponding state variables call spm_list_init(spm_list, 1._r8, f_retransn_to_npool , s_npool, nelms) @@ -512,7 +512,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, 1._r8, f_frootn_to_retransn , s_retransn, nelms) call spm_list_insert(spm_list, 1._r8, f_livestemn_to_retransn , s_retransn, nelms) call spm_list_insert(spm_list, 1._r8, f_livecrootn_to_retransn , s_retransn, nelms) - !turn the list into sparse matrix form + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_nutrient_p, nelms, f_supplement_to_plantn) end subroutine InitPhenoFluxLimiter !--------------------------------------------------------------------------- @@ -613,7 +613,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& real(r8) :: ar_p associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) harvdate => crop_vars%harvdate_patch & ! Input: [integer (:) ] harvest date ) ! set time steps @@ -847,7 +847,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& real(r8) :: dt associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) nf => veg_nf , & ns => veg_ns & ) @@ -1031,7 +1031,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) pf => veg_pf , & ps => veg_ps & ) diff --git a/components/elm/src/biogeochem/PhenologyMod.F90 b/components/elm/src/biogeochem/PhenologyMod.F90 index 08e9b94da524..c46e251b60cd 100644 --- a/components/elm/src/biogeochem/PhenologyMod.F90 +++ b/components/elm/src/biogeochem/PhenologyMod.F90 @@ -163,7 +163,7 @@ subroutine readPhenolParams ( ncid ) allocate(PhenolParamsInst%lwtop ) ! ! read in parameters - ! + ! tString='crit_dayl' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun( msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) @@ -175,7 +175,7 @@ subroutine readPhenolParams ( ncid ) else tString='crit_dayl_stress' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) - if ( .not. readv ) then + if ( .not. readv ) then PhenolParamsInst%crit_dayl_stress = secspqtrday else PhenolParamsInst%crit_dayl_stress = tempr @@ -238,7 +238,7 @@ subroutine readPhenolParams ( ncid ) tString='lwtop_ann' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun( msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) - PhenolParamsInst%lwtop=tempr + PhenolParamsInst%lwtop=tempr !!!!========== Update to device ========= !!! !$acc update device(PhenolParamsInst%crit_dayl, & @@ -601,7 +601,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) prev_dayl => grc_pp%prev_dayl , & ! Input: [real(r8) (:) ] daylength from previous time step (s) season_decid => veg_vp%season_decid , & ! Input: [real(r8) (:) ] binary flag for seasonal-deciduous leaf habit (0 or 1) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) (-nlevsno+1:nlevgrnd) @@ -946,7 +946,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & leaf_long => veg_vp%leaf_long , & ! Input: [real(r8) (:) ] leaf longevity (yrs) froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) stress_decid => veg_vp%stress_decid , & ! Input: [real(r8) (:) ] binary flag for stress-deciduous leaf habit (0 or 1) soilpsi => soilstate_vars%soilpsi_col , & ! Input: [real(r8) (:,:) ] soil water potential in each soil layer (MPa) @@ -1445,7 +1445,7 @@ subroutine CropPhenology(num_pcropp, filter_pcropp, & froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) leafcn => veg_vp%leafcn , & ! Input: [real(r8) (:) ] leaf C:N (gC/gN) - manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) + manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) t_ref2m_min => veg_es%t_ref2m_min , & ! Input: [real(r8) (:) ] daily minimum of average 2 m height surface air temperature (K) t10 => veg_es%t_a10 , & ! Input: [real(r8) (:) ] 10-day running mean of the 2 m temperature (K) a5tmin => veg_es%t_a5min , & ! Input: [real(r8) (:) ] 5-day running mean of min 2-m temperature @@ -1988,7 +1988,7 @@ subroutine PerennialCropPhenology(num_ppercropp, filter_ppercropp, & froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) leafcn => veg_vp%leafcn , & ! Input: [real(r8) (:) ] leaf C:N (gC/gN) leafcp => veg_vp%leafcp , & ! Input: [real(r8) (:) ] leaf C:P (gC/gP) - manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) + manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) t10 => veg_es%t_a10 , & ! Input: [real(r8) (:) ] 10-day running mean of the 2 m temperature (K) a10tmin => veg_es%t_a10min , & ! Input: [real(r8) (:) ] 10-day running mean of min 2-m temperature fertnitro => crop_vars%fertnitro_patch , & ! Input: [real(r8) (:) ] max fertilizer to be applied in total (kgN/m2) @@ -2016,8 +2016,8 @@ subroutine PerennialCropPhenology(num_ppercropp, filter_ppercropp, & crop_seedc_to_leaf => veg_cf%crop_seedc_to_leaf , & ! Output: [real(r8) (:) ] (gC/m2/s) seed source to PFT-level - synthfert => veg_nf%synthfert , & ! Output: [real(r8) (:) ] (gN/m2/s) fertilizer applied each timestep - manure => veg_nf%manure , & ! Output: [real(r8) (:) ] (gN/m2/s) manure applied each timestep + synthfert => veg_nf%synthfert , & ! Output: [real(r8) (:) ] (gN/m2/s) fertilizer applied each timestep + manure => veg_nf%manure , & ! Output: [real(r8) (:) ] (gN/m2/s) manure applied each timestep fert_p => veg_pf%fert_p , & ! Output: [real(r8) (:) ] (gP/m2/s) phosphorus fertilizer applied each timestep fert_counter => veg_nf%fert_counter , & ! Output: [real(r8) (:) ] >0 fertilize; <=0 not (seconds) @@ -2550,22 +2550,22 @@ subroutine CropPlantDate (num_soilp, filter_soilp, num_pcropp, filter_pcropp, & xt(p,kmo) = xt(p,kmo) + t_ref2m(p) * fracday/ndaypm(kmo) ! monthly average temperature xp(p,kmo) = xp(p,kmo) + (forc_rain(t)+forc_snow(t))*dt ! monthly average precipitation - ! calculate the potential evapotranspiration + ! calculate the potential evapotranspiration netrad = fsa(p) + eflx_lwrad_net(p) ! moved this here because it is calculated too late call calculate_eto(t_ref2m(p), netrad, eflx_soil_grnd(p), forc_pbot(t), forc_rh(t), forc_wind(t), dt, ETout) ! monthly ETo ETo(p,kmo) = ETo(p,kmo) + ETout - + ! calculate the P:PET for each month - if ( abs(ETo(p,kmo)) > 0._r8) then + if ( abs(ETo(p,kmo)) > 0._r8) then p2ETo(p,kmo) = xp(p,kmo)/ETo(p,kmo) else ! P:PET is undefined. ! Setting to a fill value ( 'spval' ) would - ! require nested if statements due to + ! require nested if statements due to ! the weighting of previous years (i.e., p2ETo and prev_p2ETo_bar ) ! So, set to zero for simplicity. p2ETo(p,kmo) = 0._r8 - end if + end if if (nyrs_crop_active(p) == 0) then ! for the first year, use last years values prev_xt_bar(p,kmo) = xt(p,kmo) @@ -2617,7 +2617,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) onset_flag => cnstate_vars%onset_flag_patch , & ! Input: [real(r8) (:) ] onset flag onset_counter => cnstate_vars%onset_counter_patch , & ! Input: [real(r8) (:) ] onset days counter @@ -3277,7 +3277,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp) associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) livewdcn => veg_vp%livewdcn , & ! Input: [real(r8) (:) ] live wood (phloem and ray parenchyma) C:N (gC/gN) deadwdcn => veg_vp%deadwdcn , & ! Input: [real(r8) (:) ] dead wood (xylem and heartwood) C:N (gC/gN) diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 index 6e1111169221..9297ab41e1bf 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 @@ -75,7 +75,7 @@ subroutine PhosphorusStateUpdateDynPatch(bounds, num_soilc_with_inactive,& c = filter_soilc_with_inactive(fc) col_ps%prod10p(c) = col_ps%prod10p(c) + col_pf%dwt_prod10p_gain(c)*dt col_ps%prod100p(c) = col_ps%prod100p(c) + col_pf%dwt_prod100p_gain(c)*dt - col_ps%prod1p(c) = col_ps%prod1p(c) + col_pf%dwt_crop_productp_gain(c)*dt + col_ps%prod1p(c) = col_ps%prod1p(c) + col_pf%dwt_crop_productp_gain(c)*dt do j = 1,nlevdecomp @@ -125,7 +125,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step @@ -147,18 +147,18 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi do j = 1, nlevdecomp do fc = 1,num_soilc c = filter_soilc(fc) - + ! plant to litter fluxes ! phenology and dynamic landcover fluxes col_pf%decomp_ppools_sourcesink(c,j,i_met_lit) = & col_pf%phenology_p_to_litr_met_p(c,j) * dt - + col_pf%decomp_ppools_sourcesink(c,j,i_cel_lit) = & col_pf%phenology_p_to_litr_cel_p(c,j) * dt - + col_pf%decomp_ppools_sourcesink(c,j,i_lig_lit) = & col_pf%phenology_p_to_litr_lig_p(c,j) * dt - + end do end do end if diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 2f06ea900329..d077bea74b0a 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -43,7 +43,7 @@ module VegetationPropertiesType real(r8), pointer :: dsladlai (:) => null() ! dSLA/dLAI, projected area basis [m^2/gC] real(r8), pointer :: leafcn (:) => null() ! leaf C:N (gC/gN) real(r8), pointer :: flnr (:) => null() ! fraction of leaf N in the Rubisco enzyme (gN Rubisco / gN leaf) - real(r8), pointer :: woody (:) => null() ! binary flag for woody lifeform (1=woody, 0=not woody) + real(r8), pointer :: woody (:) => null() woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) real(r8), pointer :: lflitcn (:) => null() ! leaf litter C:N (gC/gN) real(r8), pointer :: frootcn (:) => null() ! fine root C:N (gC/gN) real(r8), pointer :: livewdcn (:) => null() ! live wood (phloem and ray parenchyma) C:N (gC/gN) @@ -329,7 +329,7 @@ subroutine veg_vp_init(this) allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval ! ----------------------------------------------------------------------------------------------------------- - ! NGEE Arctic show-vegetation interactions + ! NGEE Arctic snow-vegetation interactions allocate(this%bendresist(0:numpft)) ; this%bendresist(:) =spval allocate(this%vegshape(0:numpft)) ; this%vegshape(:) =spval allocate(this%stocking(0:numpft)) ; this%stocking(:) =spval From f5ee59a81d41bb2506a268e0bb54afed19505b6d Mon Sep 17 00:00:00 2001 From: Peter Thornton Date: Wed, 11 Sep 2024 15:41:27 -0400 Subject: [PATCH 709/904] Fix syntax error, missing comment marker --- components/elm/src/data_types/VegetationPropertiesType.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index d077bea74b0a..d62167301cd4 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -43,7 +43,7 @@ module VegetationPropertiesType real(r8), pointer :: dsladlai (:) => null() ! dSLA/dLAI, projected area basis [m^2/gC] real(r8), pointer :: leafcn (:) => null() ! leaf C:N (gC/gN) real(r8), pointer :: flnr (:) => null() ! fraction of leaf N in the Rubisco enzyme (gN Rubisco / gN leaf) - real(r8), pointer :: woody (:) => null() woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) + real(r8), pointer :: woody (:) => null() ! woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) real(r8), pointer :: lflitcn (:) => null() ! leaf litter C:N (gC/gN) real(r8), pointer :: frootcn (:) => null() ! fine root C:N (gC/gN) real(r8), pointer :: livewdcn (:) => null() ! live wood (phloem and ray parenchyma) C:N (gC/gN) From f094149ea4e74cda23d72fddfea782abf0977e5d Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 11 Sep 2024 17:16:28 -0400 Subject: [PATCH 710/904] update ghci container to restore gh/ci --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 6 +----- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 6 +----- cime_config/machines/config_machines.xml | 16 ++++++++-------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index b113710f46f3..1f3f63d4f09f 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -36,7 +36,7 @@ jobs: - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu - ERS_Ld5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod container: - image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 steps: - @@ -49,10 +49,6 @@ jobs: name: CIME working-directory: cime/scripts run: | - mkdir -p $HOME/projects/e3sm/cesm-inputdata/atm/cam/physprops/ - wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/physprops/p3_lookup_table_1.dat-v4.1.2 - mv p3_lookup_table_1.dat-v4.1.2 $HOME/projects/e3sm/cesm-inputdata/atm/cam/physprops/ - export USER=test ./create_test ${{ matrix.test }} --wait --debug - name: Artifacts diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index ca21646382b8..5f064bcad7f7 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -23,7 +23,7 @@ jobs: - SMS_D_Ld1_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - ERS_Ld3_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 container: - image: ghcr.io/mahf708/e3sm-imgs:v0.0.9 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 steps: - @@ -36,10 +36,6 @@ jobs: name: CIME working-directory: cime/scripts run: | - mkdir -p $HOME/projects/e3sm/cesm-inputdata/atm/cam/physprops/ - wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/physprops/p3_lookup_table_1.dat-v4.1.2 - mv p3_lookup_table_1.dat-v4.1.2 $HOME/projects/e3sm/cesm-inputdata/atm/cam/physprops/ - export USER=test ./create_test ${{ matrix.test }} --wait --debug - name: Artifacts diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 30508f8a6307..daff3ed90386 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1715,16 +1715,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - OCI-based container 2.0 + OCI-based container ghci-oci LINUX gnu mpich - $ENV{HOME}/projects/e3sm/scratch - $ENV{HOME}/projects/e3sm/cesm-inputdata - $ENV{HOME}/projects/e3sm/ptclm-data - $ENV{HOME}/projects/e3sm/scratch/archive/$CASE - $ENV{HOME}/projects/e3sm/baselines/$COMPILER + /projects/e3sm/scratch + /projects/e3sm/inputdata + /projects/e3sm/ptclm-data + /projects/e3sm/scratch/archive/$CASE + /projects/e3sm/baselines/$COMPILER /usr/local/packages/bin/cprnc make 4 @@ -1740,8 +1740,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - $ENV{HOME}/projects/e3sm/scratch/$CASE/run - $ENV{HOME}/projects/e3sm/scratch/$CASE/bld + /projects/e3sm/scratch/$CASE/run + /projects/e3sm/scratch/$CASE/bld $SRCROOT From 20d044b8e29939163657b1eb039c3b36643baee7 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 11 Sep 2024 17:51:33 -0400 Subject: [PATCH 711/904] adjust artifacts to reflect recent edits --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 8 ++++---- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 1f3f63d4f09f..6c9ee1ab114c 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -57,7 +57,7 @@ jobs: with: name: ${{ matrix.test }} path: | - ~/projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log - ~/projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* - ~/projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* - ~/projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out + /projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log + /projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* + /projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* + /projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index 5f064bcad7f7..48c367c8f625 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -44,7 +44,7 @@ jobs: with: name: ${{ matrix.test }} path: | - ~/projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log - ~/projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* - ~/projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* - ~/projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out + /projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log + /projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* + /projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* + /projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out From 17aa536cc4540d52bc8a23f6ca86cf435f59d2d3 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 11 Sep 2024 15:52:34 -0600 Subject: [PATCH 712/904] EAMxx: look for kokkos pkg only if it's not a target --- components/eamxx/src/physics/rrtmgp/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index 1d3df9b4d2e4..a8e47384552a 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -127,7 +127,9 @@ yakl_process_target(rrtmgp) # NOTE: cannot use 'PUBLIC' in target_link_libraries, # since yakl_process_target already used it # with the "plain" signature -find_package(Kokkos REQUIRED) +if (NOT TARGET Kokkos::kokkos) + find_package(Kokkos REQUIRED) +endif () target_link_libraries(rrtmgp yakl Kokkos::kokkos) target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL From f21d4617fae7727c2ddfd691a598305b22a2ff54 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 11 Sep 2024 17:15:23 -0500 Subject: [PATCH 713/904] Updates the long_name --- components/elm/src/biogeochem/DUSTMod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/biogeochem/DUSTMod.F90 b/components/elm/src/biogeochem/DUSTMod.F90 index a6ee6c6c11f8..f53e970423be 100644 --- a/components/elm/src/biogeochem/DUSTMod.F90 +++ b/components/elm/src/biogeochem/DUSTMod.F90 @@ -171,7 +171,7 @@ subroutine InitHistory(this, bounds) this%lnd_frc_mbl_patch(begp:endp) = spval call hist_addfld1d (fname='LND_FRC_DUST_MBL', units='-', & - avgflag='A', long_name='land fraction for dust mobilization', & + avgflag='A', long_name='bare soil fraction for land fraction for dust mobilization', & ptr_patch=this%lnd_frc_mbl_patch, default='inactive') end subroutine InitHistory From 059f6e3def8692f81daeb05ebdf9b87054d90b1a Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Thu, 12 Sep 2024 12:50:30 -0500 Subject: [PATCH 714/904] Reset sim_year_range to 1850-2015 in namelist_defaults for flanduse_timeseries --- .../bld/namelist_files/namelist_defaults.xml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index e66ebfa637ba..beeae7858595 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -485,48 +485,48 @@ lnd/clm2/surfdata_map/surfdata_conusx4v1_simyr2000_c160503.nc dataset (arbitrarily, RCP8.5) for the historical period, because all of the RCP datasets contain the 1850-2000 period as well as the future period. --> -lnd/clm2/surfdata_map/landuse.timeseries_360x720cru_hist_simyr1850-2015_c180220.nc -lnd/clm2/surfdata_map/landuse.timeseries_0.9x1.25_hist_simyr1850-2015_c180306.nc -lnd/clm2/surfdata_map/landuse.timeseries_1.9x2.5_hist_simyr1850-2015_c180306.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_10x15_rcp8.5_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_48x96_rcp8.5_simyr1850-2100_c130524.nc lnd/clm2/surfdata_map/landuse.timeseries_antarcticax4v1pg2_hist_simyr1850-2015_c210131.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_1x1_tropicAtl_hist_simyr1850-2005_c130627.nc -lnd/clm2/surfdata_map/landuse.timeseries_1x1_brazil_rcp8.5_simyr1850-2100_c140610.nc -lnd/clm2/surfdata_map/landuse.timeseries_ne30np4_hist_simyr1850-2015_c180306.nc - lnd/clm2/surfdata_map/landuse.timeseries_ne30np4.pg2_hist_simyr1850-2015_c210113.nc - lnd/clm2/surfdata_map/landuse.timeseries_ne4pg2_hist_simyr1850-2015_c210722.nc - lnd/clm2/surfdata_map/landuse.timeseries_ne0np4_northamericax4v1.pg2_hist_simyr1850-2015_c211015.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_ne16np4_hist_simyr1850-2005_c160803.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_ne11np4_hist_simyr1850-2005_c160803.nc lnd/clm2/surfdata_map/landuse.timeseries_ne1024pg2_historical_simyr1990-2014_c240109.nc lnd/clm2/surfdata_map/landuse.timeseries_ne256pg2_hist_simyr1850-2015_c240131.nc -lnd/clm2/surfdata_map/landuse.timeseries_0.125x0.125_hist_simyr1850-2015_c191004.nc -lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_HIST_simyr1850-2015_c211019.nc -lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_hist_simyr1850-2015_c191004.nc From 66ce874b14ed691d12d61933cd2689f3c89f7c45 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Thu, 12 Sep 2024 14:49:24 -0500 Subject: [PATCH 715/904] Add LND_FRC_DUST_MBL to elm.h0 for wcprod mods --- cime_config/testmods_dirs/allactive/wcprod/user_nl_elm | 2 +- cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm | 2 +- .../testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm | 2 +- .../testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm | 2 +- cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm | 2 +- cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm | 2 +- cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm | 2 +- .../cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm | 2 +- .../testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm | 2 +- .../testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cime_config/testmods_dirs/allactive/wcprod/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/cime_config/testmods_dirs/allactive/wcprod/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprod/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprod_1850/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprod_1850_1pctCO2/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprod_1850_4xCO2/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprodrrm/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprodrrm_1850/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm b/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm index 7bfdbac5c84b..e2ac4d5d1ab7 100644 --- a/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm +++ b/cime_config/testmods_dirs/allactive/wcprodssp/user_nl_elm @@ -38,7 +38,7 @@ 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' - hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' + hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F2010/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm index cd1adab77404..9224c77e0f0a 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/wcprod_F20TR/user_nl_elm @@ -33,7 +33,7 @@ hist_fexcl1 = 'AGWDNPP','ALTMAX_LASTYEAR','AVAIL_RETRANSP','AVAILC','BAF_CROP', 'SOIL4N_vr','SOIL4P_TNDNCY_VERT_TRANS','SOIL4P_TO_SMINP','SOIL4P_vr','SOLUTIONP_vr', 'TCS_MONTH_BEGIN','TCS_MONTH_END','TOTCOLCH4','water_scalar','WF', 'wlim_m','WOODC_LOSS','WTGQ' -hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC' +hist_fincl1 = 'SNOWDP','COL_FIRE_CLOSS','NPOOL','PPOOL','TOTPRODC','LND_FRC_DUST_MBL' hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' hist_mfilt = 1,365 hist_nhtfrq = -24,-24 From f30afb2f1db294c3ddda8e9dcb7bd9ffdba1a3fb Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 10 Sep 2024 19:04:55 +0000 Subject: [PATCH 716/904] switch linker back to F --- components/homme/cmake/HommeMacros.cmake | 8 +------- components/homme/cmake/machineFiles/chrysalis-bfb.cmake | 2 -- components/homme/cmake/machineFiles/chrysalis.cmake | 2 -- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 5610947cb299..6d073dbbe83b 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,13 +112,7 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) - - if(SUNSPOT_MACHINE) - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) - else() - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) - endif() - + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index fa1f1ac545c5..b9f0d41a0606 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -17,8 +17,6 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") -SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") - # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index 97bc682c9546..68ff76ec8082 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -17,8 +17,6 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") -SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") - # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") From a4c907fea9039e608d04799ce725de2a8f10eb1a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 12 Sep 2024 18:21:11 -0500 Subject: [PATCH 717/904] more reviews --- components/data_comps/dlnd/src/dlnd_comp_mod.F90 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index 4d76c0fe1a1f..fbd6e35cf87d 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -105,10 +105,13 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & ! !DESCRIPTION: initialize dlnd model #ifdef HAVE_MOAB - use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + use iMOAB, only: iMOAB_DefineTagStorage, & iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & - iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh + iMOAB_UpdateMeshInfo +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif #endif implicit none From b69636d543a21c7384e239178fcfc003e8d8c705 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 12 Sep 2024 17:45:36 -0700 Subject: [PATCH 718/904] Updates cprnc version to match the new compiler version --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index ff6f9ac37597..6f0a952ac6ca 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3865,7 +3865,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /compyfs/inputdata/atm/datm7 /compyfs/$USER/e3sm_scratch/archive/$CASE /compyfs/e3sm_baselines/$COMPILER - /compyfs/e3sm_baselines/cprnc/cprnc + /compyfs/e3sm_baselines/cprnc/cprnc.intel.v20.0.04/cprnc 8 e3sm_integration 4 From d5dd94cb4fe8cd53fafb3b11b46f79d07e23ecf7 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Fri, 13 Sep 2024 09:02:25 -0500 Subject: [PATCH 719/904] Update sim_year_range in 20thC_transient.xml --- components/elm/bld/namelist_files/use_cases/20thC_transient.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/bld/namelist_files/use_cases/20thC_transient.xml b/components/elm/bld/namelist_files/use_cases/20thC_transient.xml index 86035ec6e081..1fe1f2860496 100644 --- a/components/elm/bld/namelist_files/use_cases/20thC_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/20thC_transient.xml @@ -9,7 +9,7 @@ 1850 -1850-2000 +1850-2015 arb_ic From 2c103c0e757ab099d82746e734dead767aa6742e Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 09:26:28 -0700 Subject: [PATCH 720/904] Allow interpinic to accept any netCDF filetype for output. Print warning and continue (instead of failing) when input has month dimension and output does not. --- .../elm/tools/interpinic/src/interpinic.F90 | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/components/elm/tools/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 index b75afd3a411d..0b8af6155775 100644 --- a/components/elm/tools/interpinic/src/interpinic.F90 +++ b/components/elm/tools/interpinic/src/interpinic.F90 @@ -146,9 +146,22 @@ subroutine interp_filei (fin, fout, cmdline) call check_ret (nf90_open(fin, NF90_NOWRITE, ncidi )) call check_ret (nf90_open(fout, NF90_NOWRITE, ncido )) call check_ret (nf_inq_format( ncido, ncformat )) - if ( ncformat /= NF_FORMAT_64BIT )then - write (6,*) 'error: output file is NOT in NetCDF large-file format!' - stop + ! 20240822 csz++ + ! Allow any format for output dataset + ! if ( ncformat /= NF_FORMAT_64BIT )then + ! write (6,*) 'error: output file is NOT in NetCDF large-file format!' + ! stop + ! 20240822 csz-- + if ( ncformat == NF_FORMAT_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_CLASSIC' + else if ( ncformat == NF_FORMAT_64BIT_OFFSET )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_OFFSET' + else if ( ncformat == NF_FORMAT_64BIT_DATA )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_DATA' + else if ( ncformat == NF_FORMAT_NETCDF4 )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4' + else if ( ncformat == NF_FORMAT_NETCDF4_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4_CLASSIC' end if call check_ret (nf90_inq_dimid(ncidi, "column", dimidcols )) @@ -214,13 +227,27 @@ subroutine interp_filei (fin, fout, cmdline) ret = nf90_inq_dimid(ncidi, "month", dimidmon) if (ret == NF90_NOERR) then call check_ret (nf90_inquire_dimension(ncidi, dimidmon, len=nlevmon)) - call check_ret (nf90_inq_dimid(ncido, "month", dimid )) - call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) - if (dimlen/=nlevmon) then - write (6,*) 'error: input and output nlevmon values disagree' - write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen - stop + ! 20240912 csz++ + ! Many restart files have "month" dimension in input dataset + ! It is only necessary that the output dataset contains "month" dimension + ! when a variable in the input dataset contains the "month" dimension + ! Otherwise, the "month" dimension will never be used + ! Warn rather than die when input has "month" and output does not + !call check_ret (nf90_inq_dimid(ncido, "month", dimid )) + ret = nf90_inq_dimid(ncido, "month", dimid ) + if ( ret == nf_ebaddim ) then + write (6,*) 'warning: input has "month" dimension and output does not' + write (6,*) 'warning: interpolation will fail if any input variable uses "month" dimension' + write (6,*) 'chill: many times the "month" dimension is superfluous so this might work...' + else + call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) + if (dimlen/=nlevmon) then + write (6,*) 'error: input and output nlevmon values disagree' + write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen + stop + end if end if + ! 20240912 csz-- else write (6,*) 'month dimension does NOT exist on the input dataset' dimidmon = -9999 @@ -321,7 +348,11 @@ subroutine interp_filei (fin, fout, cmdline) ! OK now, open the output file for writing ! call check_ret(nf90_close( ncido)) - call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) + ! 20240822 csz++ + ! Allow any format for output dataset + ! call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) + call check_ret (nf90_open(fout, NF90_WRITE, ncido )) + ! csz-- call addglobal (ncido, cmdline) @@ -1503,8 +1534,7 @@ subroutine addglobal (ncid, cmdline) character(len=10) :: time character(len= 5) :: zone character(len=18) :: datetime - character(len=256):: version = & - "$HeadURL: https://svn-ccsm-models.cgd.ucar.edu/clm2/trunk_tags/clm4_5_1_r085/models/lnd/clm/tools/clm4_5/interpinic/src/interpinic.F90 $" + character(len=256):: version = "" character(len=256) :: revision_id = "$Id: interpinic.F90 54953 2013-11-06 16:29:45Z sacks $" character(len=16) :: logname character(len=16) :: hostname From 4d1c5ea1a6af9249eb616ba408f6c4a0e36e293c Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 09:42:06 -0700 Subject: [PATCH 721/904] Place hexadecimal parameters within int() intrinsic in NaN module for modern compilers (per Xylar) --- .../tools/interpinic/src/shr_infnan_mod.F90 | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 index 638cad84d20e..31ffb1dff329 100644 --- a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 +++ b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 @@ -2,11 +2,11 @@ module shr_infnan_mod -!! Inf_NaN_Detection module +!! Inf_NaN_Detection module !! Copyright(c) 2003, Lahey Computer Systems, Inc. -!! Copies of this source code, or standalone compiled files +!! Copies of this source code, or standalone compiled files !! derived from this source may not be sold without permission -!! from Lahey Computers Systems. All or part of this module may be +!! from Lahey Computers Systems. All or part of this module may be !! freely incorporated into executable programs which are offered !! for sale. Otherwise, distribution of all or part of this file is !! permitted, provided this copyright notice and header are included. @@ -22,12 +22,12 @@ module shr_infnan_mod !! isneginf(x) - test for a negative "infinite" value !! !! Each function accepts a single or double precision real argument, and -!! returns a true or false value to indicate the presence of the value +!! returns a true or false value to indicate the presence of the value !! being tested for. If the argument is array valued, the function returns !! a conformable logical array, suitable for use with the ANY function, or !! as a logical mask. !! -!! Each function operates by transferring the bit pattern from a real +!! Each function operates by transferring the bit pattern from a real !! variable to an integer container. Unless testing for + or - infinity, !! the sign bit is cleared to zero. The value is exclusive ORed with !! the value being tested for. The integer result of the IEOR function is @@ -48,14 +48,14 @@ module shr_infnan_mod integer, parameter :: Double = selected_int_kind(precision(1.0_r8)) ! Single precision IEEE values - integer(Single), parameter :: sNaN = Z"7FC00000" - integer(Single), parameter :: sPosInf = Z"7F800000" - integer(Single), parameter :: sNegInf = Z"FF800000" + integer(Single), parameter :: sNaN = int(Z"7FC00000") + integer(Single), parameter :: sPosInf = int(Z"7F800000") + integer(Single), parameter :: sNegInf = int(Z"FF800000") ! Double precision IEEE values - integer(Double), parameter :: dNaN = Z"7FF8000000000000" - integer(Double), parameter :: dPosInf = Z"7FF0000000000000" - integer(Double), parameter :: dNegInf = Z"FFF0000000000000" + integer(Double), parameter :: dNaN = int(Z"7FF8000000000000") + integer(Double), parameter :: dPosInf = int(Z"7FF0000000000000") + integer(Double), parameter :: dNegInf = int(Z"FFF0000000000000") ! Locatation of single and double precision sign bit (Intel) ! Subtract one because bit numbering starts at zero @@ -84,22 +84,22 @@ module shr_infnan_mod module procedure sisnan module procedure disnan #endif - end interface + end interface interface shr_infnan_isinf module procedure sisinf module procedure disinf - end interface - + end interface + interface shr_infnan_isposinf module procedure sisposinf module procedure disposinf - end interface - + end interface + interface shr_infnan_isneginf module procedure sisneginf module procedure disneginf - end interface + end interface integer :: shr_sisnan @@ -107,7 +107,7 @@ module shr_infnan_mod integer :: shr_disnan external :: shr_disnan -contains +contains ! ! If FORTRAN intrinsic's exist use them @@ -134,7 +134,7 @@ elemental function sisnan(x) result(res) res = isnan(x) #endif - end function + end function ! Double precision test for NaN elemental function disnan(d) result(res) @@ -156,7 +156,7 @@ elemental function disnan(d) result(res) res = isnan(d) #endif - end function + end function ! ! Otherwise link to a C function call that either uses the C90 isnan function or a x != x check @@ -176,13 +176,13 @@ function c_sisnan_1D(x) result(res) real(r4), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_sisnan(x(i)) /= 0) end do end function c_sisnan_1D - + function c_sisnan_2D(x) result(res) real(r4), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -195,7 +195,7 @@ function c_sisnan_2D(x) result(res) end do end do end function c_sisnan_2D - + function c_sisnan_3D(x) result(res) real(r4), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -210,7 +210,7 @@ function c_sisnan_3D(x) result(res) end do end do end function c_sisnan_3D - + function c_sisnan_4D(x) result(res) real(r4), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -227,7 +227,7 @@ function c_sisnan_4D(x) result(res) end do end do end function c_sisnan_4D - + function c_sisnan_5D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -246,7 +246,7 @@ function c_sisnan_5D(x) result(res) end do end do end function c_sisnan_5D - + function c_sisnan_6D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -267,7 +267,7 @@ function c_sisnan_6D(x) result(res) end do end do end function c_sisnan_6D - + function c_sisnan_7D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -290,7 +290,7 @@ function c_sisnan_7D(x) result(res) end do end do end function c_sisnan_7D - + function c_disnan_scalar(x) result(res) real(r8), intent(in) :: x logical :: res @@ -302,13 +302,13 @@ function c_disnan_1D(x) result(res) real(r8), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_disnan(x(i)) /= 0) end do end function c_disnan_1D - + function c_disnan_2D(x) result(res) real(r8), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -321,7 +321,7 @@ function c_disnan_2D(x) result(res) end do end do end function c_disnan_2D - + function c_disnan_3D(x) result(res) real(r8), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -336,7 +336,7 @@ function c_disnan_3D(x) result(res) end do end do end function c_disnan_3D - + function c_disnan_4D(x) result(res) real(r8), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -353,7 +353,7 @@ function c_disnan_4D(x) result(res) end do end do end function c_disnan_4D - + function c_disnan_5D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -372,7 +372,7 @@ function c_disnan_5D(x) result(res) end do end do end function c_disnan_5D - + function c_disnan_6D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -393,7 +393,7 @@ function c_disnan_6D(x) result(res) end do end do end function c_disnan_6D - + function c_disnan_7D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -418,48 +418,48 @@ function c_disnan_7D(x) result(res) end function c_disnan_7D #endif - + ! Single precision test for Inf elemental function sisinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(ibclr(transfer(x,sPosInf),SPSB), sPosInf) == 0 - end function + end function ! Double precision test for Inf elemental function disinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(ibclr(transfer(d,dPosInf),DPSB), dPosInf) == 0 - end function - + end function + ! Single precision test for +Inf elemental function sisposinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sPosInf), sPosInf) == 0 - end function + end function ! Double precision test for +Inf elemental function disposinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dPosInf), dPosInf) == 0 - end function - + end function + ! Single precision test for -Inf elemental function sisneginf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sNegInf), sNegInf) == 0 - end function + end function ! Double precision test for -Inf elemental function disneginf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dNegInf), dNegInf) == 0 - end function + end function end module shr_infnan_mod From c11886bcd6e3056be6dd28ca577ea6f97bd807ec Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 11 Sep 2024 23:06:33 -0500 Subject: [PATCH 722/904] Log PIO buffer size limit for default case Logging the SCORPIO internal buffer cache size when the user lets the library choose the buffer size (the default case) --- share/util/shr_pio_mod.F90 | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/share/util/shr_pio_mod.F90 b/share/util/shr_pio_mod.F90 index 0935bce351a5..d500cbc680a2 100644 --- a/share/util/shr_pio_mod.F90 +++ b/share/util/shr_pio_mod.F90 @@ -173,17 +173,24 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia logical, intent(in) :: comp_iamin(:) character(len=*), intent(in) :: comp_name(:) integer, intent(in) :: comp_comm(:), comp_comm_iam(:) + integer(kind=pio_offset_kind) :: cur_buffer_size_limit=-1 integer :: i character(len=shr_kind_cl) :: nlfilename, cname integer :: ret character(*), parameter :: subName = '(shr_pio_init2) ' ! 0 is a valid value of pio_buffer_size_limit - if(pio_buffer_size_limit>=0) then + ! -1 is the value used by CIME to let the library choose the buffer limit + if(pio_buffer_size_limit>=-1) then + call pio_set_buffer_size_limit(pio_buffer_size_limit, prev_limit=cur_buffer_size_limit) if(comp_comm_iam(1)==0) then - write(shr_log_unit,*) 'Setting pio_buffer_size_limit : ',pio_buffer_size_limit + if(pio_buffer_size_limit >= 0) then + write(shr_log_unit,*) 'Set pio_buffer_size_limit to : ', pio_buffer_size_limit, ' (bytes)' + else + ! Default pio_buffer_size_limit + write(shr_log_unit,*) 'Using pio_buffer_size_limit (default): ', cur_buffer_size_limit, ' (bytes)' + end if end if - call pio_set_buffer_size_limit(pio_buffer_size_limit) endif if(pio_blocksize>0) then if(comp_comm_iam(1)==0) then From bf6b5482bbb3dbb05ed859217d950a1907e65e14 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 13 Sep 2024 13:26:59 -0500 Subject: [PATCH 723/904] add MOAB root for improv machine at LCRC --- cime_config/machines/config_machines.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 30508f8a6307..2308d3b9962d 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3025,6 +3025,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6 /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH} $SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi} + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/improv/gnu; else echo "$MOAB_ROOT"; fi} ^lockedfile From 25fcf92996d401b7c674c8a7f18abad1c798583e Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 13 Sep 2024 14:21:12 -0700 Subject: [PATCH 724/904] add newline at end of file --- components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl index e3d4433d4a8d..3a47a8bb2ff9 100644 --- a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl +++ b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -31,4 +31,5 @@ seq_flux_mct_albdif = 0.07 seq_flux_mct_albdir = 0.07 seq_flux_atmocn_minwind = 1 -constant_zenith_deg = 42.04 \ No newline at end of file +constant_zenith_deg = 42.04 + From a282adca3bb0b89d7d942307328227573b13c25e Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 13 Sep 2024 15:39:51 -0600 Subject: [PATCH 725/904] Sync lat/lon to host in rrtmgp interface for IOP lat/lon are used on both host and device in run_impl(), but when IOP is used we were changing the value without syncing to host. --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index e89e02fe301b..9942d612f9f5 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -59,12 +59,15 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ if (m_iop) { // For IOP runs, we need to use the lat/lon from the - // IOP files instead of the geometry data. + // IOP files instead of the geometry data. Deep copy + // to device and sync to host since both will be used. m_lat = m_grid->get_geometry_data("lat").clone(); m_lat.deep_copy(m_iop->get_params().get("target_latitude")); + m_lat.sync_to_host(); m_lon = m_grid->get_geometry_data("lon").clone(); m_lon.deep_copy(m_iop->get_params().get("target_longitude")); + m_lon.sync_to_host(); } else { m_lat = m_grid->get_geometry_data("lat"); m_lon = m_grid->get_geometry_data("lon"); From ef7aeca234bdd44b9bce518671e69489e13df6b6 Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 16:38:59 -0700 Subject: [PATCH 726/904] remove external qualifier from iargc definition --- components/elm/tools/interpinic/src/fmain.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/tools/interpinic/src/fmain.F90 b/components/elm/tools/interpinic/src/fmain.F90 index 0f55c12c67c9..a67e6e2627fb 100644 --- a/components/elm/tools/interpinic/src/fmain.F90 +++ b/components/elm/tools/interpinic/src/fmain.F90 @@ -14,7 +14,7 @@ program fmain character(len= 256) :: arg integer :: n !index integer :: nargs !number of arguments - integer, external :: iargc !number of arguments function + integer :: iargc !number of arguments function character(len=256) :: finidati !input initial dataset to read character(len=256) :: finidato !output initial dataset to create character(len=256) :: cmdline !input command line From ece56d136c5861288291243cae2f49e73414c250 Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 16:59:21 -0700 Subject: [PATCH 727/904] Remove commented-out code, CSZ markers, and dates last changed --- .../elm/tools/interpinic/src/interpinic.F90 | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/components/elm/tools/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 index 0b8af6155775..3d890a14b895 100644 --- a/components/elm/tools/interpinic/src/interpinic.F90 +++ b/components/elm/tools/interpinic/src/interpinic.F90 @@ -146,12 +146,9 @@ subroutine interp_filei (fin, fout, cmdline) call check_ret (nf90_open(fin, NF90_NOWRITE, ncidi )) call check_ret (nf90_open(fout, NF90_NOWRITE, ncido )) call check_ret (nf_inq_format( ncido, ncformat )) - ! 20240822 csz++ + ! Allow any format for output dataset - ! if ( ncformat /= NF_FORMAT_64BIT )then - ! write (6,*) 'error: output file is NOT in NetCDF large-file format!' - ! stop - ! 20240822 csz-- + if ( ncformat == NF_FORMAT_CLASSIC )then write (6,*) 'info: output file is NF_FORMAT_CLASSIC' else if ( ncformat == NF_FORMAT_64BIT_OFFSET )then @@ -227,13 +224,13 @@ subroutine interp_filei (fin, fout, cmdline) ret = nf90_inq_dimid(ncidi, "month", dimidmon) if (ret == NF90_NOERR) then call check_ret (nf90_inquire_dimension(ncidi, dimidmon, len=nlevmon)) - ! 20240912 csz++ + ! Many restart files have "month" dimension in input dataset ! It is only necessary that the output dataset contains "month" dimension ! when a variable in the input dataset contains the "month" dimension ! Otherwise, the "month" dimension will never be used ! Warn rather than die when input has "month" and output does not - !call check_ret (nf90_inq_dimid(ncido, "month", dimid )) + ret = nf90_inq_dimid(ncido, "month", dimid ) if ( ret == nf_ebaddim ) then write (6,*) 'warning: input has "month" dimension and output does not' @@ -247,7 +244,6 @@ subroutine interp_filei (fin, fout, cmdline) stop end if end if - ! 20240912 csz-- else write (6,*) 'month dimension does NOT exist on the input dataset' dimidmon = -9999 @@ -348,11 +344,9 @@ subroutine interp_filei (fin, fout, cmdline) ! OK now, open the output file for writing ! call check_ret(nf90_close( ncido)) - ! 20240822 csz++ + ! Allow any format for output dataset - ! call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) call check_ret (nf90_open(fout, NF90_WRITE, ncido )) - ! csz-- call addglobal (ncido, cmdline) From 2aadb974efacf339f7e76e6ce98f250da573aefc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:02:49 +0000 Subject: [PATCH 728/904] Bump DavidAnson/markdownlint-cli2-action from 16 to 17 Bumps [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action) from 16 to 17. - [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases) - [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/v16...v17) --- updated-dependencies: - dependency-name: DavidAnson/markdownlint-cli2-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-md-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 507c2c3745ea..424a871637b6 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -22,7 +22,7 @@ jobs: with: files: '**/*.md' separator: "," - - uses: DavidAnson/markdownlint-cli2-action@v16 + - uses: DavidAnson/markdownlint-cli2-action@v17 if: steps.changed-files.outputs.any_changed == 'true' with: config: 'docs/.markdownlint.json' From dfeefffa109c1136068eefce1881121f67951a4f Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Mon, 16 Sep 2024 08:31:31 -0700 Subject: [PATCH 729/904] clean up new mock-walker SST section to make it more compact and less redundant --- .../data_comps/docn/src/docn_comp_mod.F90 | 49 +++---------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index ac1278673656..43bac32bff73 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -1177,52 +1177,17 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) !------------------------------------------------------------------------------- ! RCEMIP phase 2 - Mock-Walker - - ! MW_295dT1p25 - mean SST = 295 K / dSST = 1.25 K - - if (sst_option == 11) then - mean_SST = 295 - TkFrz - delta_SST = 1.25 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT0p625 - mean SST = 300 K / dSST = 0.625 K - if (sst_option == 12) then - mean_SST = 300 - TkFrz - delta_SST = 0.625 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT1p25 - mean SST = 300 K / dSST = 1.25 K - if (sst_option == 13) then - mean_SST = 300 - TkFrz - delta_SST = 1.25 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT2p5 - mean SST = 300 K / dSST = 2.5 K - if (sst_option == 14) then - mean_SST = 300 - TkFrz - delta_SST = 2.5 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_305dT1p25 - mean SST = 305 K / dSST = 1.25 K - if (sst_option == 15) then - mean_SST = 305 - TkFrz - delta_SST = 1.25 + if (sst_option>=11 .and. sst_option<=15) then + if (sst_option==11) then; mean_SST = 295 - TkFrz; delta_SST = 1.250; end if ! MW_295dT1p25 + if (sst_option==12) then; mean_SST = 300 - TkFrz; delta_SST = 0.625; end if ! MW_300dT0p625 + if (sst_option==13) then; mean_SST = 300 - TkFrz; delta_SST = 1.250; end if ! MW_300dT1p25 + if (sst_option==14) then; mean_SST = 300 - TkFrz; delta_SST = 2.500; end if ! MW_300dT2p5 + if (sst_option==15) then; mean_SST = 305 - TkFrz; delta_SST = 1.250; end if ! MW_305dT1p25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if + !------------------------------------------------------------------------------- end subroutine prescribed_sst From 975a93feaa74891911aec8fe9ccd92b91d3fa18c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:27:44 +0000 Subject: [PATCH 730/904] Bump DavidAnson/markdownlint-cli2-action from 16 to 17 Bumps [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action) from 16 to 17. - [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases) - [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/v16...v17) --- updated-dependencies: - dependency-name: DavidAnson/markdownlint-cli2-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-md-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 507c2c3745ea..424a871637b6 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -22,7 +22,7 @@ jobs: with: files: '**/*.md' separator: "," - - uses: DavidAnson/markdownlint-cli2-action@v16 + - uses: DavidAnson/markdownlint-cli2-action@v17 if: steps.changed-files.outputs.any_changed == 'true' with: config: 'docs/.markdownlint.json' From 3cb0e38377340333859ec0e1df3b5d4d4a0ca52d Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Mon, 16 Sep 2024 13:02:05 -0400 Subject: [PATCH 731/904] Moved tests to e3sm_land_exeshare test suite --- cime_config/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index d4c10f126b23..25558ef8ad88 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -49,6 +49,9 @@ "ERS.f19_g16.I20TRGSWCNPECACNTBC.elm-eca_f19_g16_I20TRGSWCNPECACNTBC", "ERS.f19_g16.I20TRGSWCNPRDCTCBC.elm-ctc_f19_g16_I20TRGSWCNPRDCTCBC", "ERS.r05_r05.ICNPRDCTCBC.elm-cbudget", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-snowveg_arctic", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", ) }, @@ -93,10 +96,7 @@ "SMS_Ly2_P1x1.1x1_smallvilleIA.IELMCNCROP.elm-fan", "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-snowveg_arctic", "ERS.r05_r05.IELM.elm-lnd_rof_2way", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", "ERS.r05_r05.IELM.elm-V2_ELM_MOSART_features", "ERS.ELM_USRDAT.IELM.elm-surface_water_dynamics" ) From 9eb7a7fa562efc5b75bdb987d26792fbe3f8b698 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 16 Sep 2024 13:09:56 -0700 Subject: [PATCH 732/904] bug fix when defining gsize in dp mode --- share/streams/shr_dmodel_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/streams/shr_dmodel_mod.F90 b/share/streams/shr_dmodel_mod.F90 index eea74a66cdee..e1a7c39d783e 100644 --- a/share/streams/shr_dmodel_mod.F90 +++ b/share/streams/shr_dmodel_mod.F90 @@ -383,7 +383,7 @@ subroutine shr_dmodel_readgrid( gGrid, gsMap, nxgo, nygo, nzgo, filename, compid nxgo = scm_nx nygo = scm_ny nzgo = -1 - gsize = abs(scm_nx*scm_nx*nzgo) + gsize = abs(scm_nx*scm_ny*nzgo) else nxgo = nxg nygo = nyg From a2e91dc0e9b476e9f6a845fd3c1b9f7e3e2746d8 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 16 Sep 2024 15:14:56 -0600 Subject: [PATCH 733/904] Get mappy working again with RHEL9 --- cime_config/machines/config_machines.xml | 38 +++++++++++++----------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index fe1febd6ddcb..dcbd43ad6f08 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1861,31 +1861,32 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/env_modules_python.py - /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/perl - /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/sh - /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/csh - /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/libexec/lmod python - /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/libexec/lmod perl + /projects/sems/install/rhel9-x86_64/sems/lmod/lmod/8.7.24/gcc/11.4.1/base/lnirq74/lmod/lmod/init/env_modules_python.py + /projects/sems/install/rhel9-x86_64/sems/lmod/lmod/8.7.24/gcc/11.4.1/base/lnirq74/lmod/lmod/init/perl + /projects/sems/install/rhel9-x86_64/sems/lmod/lmod/8.7.24/gcc/11.4.1/base/lnirq74/lmod/lmod/init/sh + /projects/sems/install/rhel9-x86_64/sems/lmod/lmod/8.7.24/gcc/11.4.1/base/lnirq74/lmod/lmod/init/csh + /projects/sems/install/rhel9-x86_64/sems/lmod/lmod/8.7.24/gcc/11.4.1/base/lnirq74/lmod/lmod/libexec/lmod python + /projects/sems/install/rhel9-x86_64/sems/lmod/lmod/8.7.24/gcc/11.4.1/base/lnirq74/lmod/lmod/libexec/lmod perl module module - sems-archive-env - acme-env - sems-archive-git - acme-cmake/3.26.3 + sems-git/2.42.0 + sems-cmake/3.27.9 - acme-gcc/11.2.0 + sems-gcc/11.4.0 + sems-openblas - acme-netcdf/4.4.1/exo_acme - acme-pfunit/3.2.8/base + sems-netcdf-c-serial/4.9.2 - acme-openmpi/4.1.4 - acme-netcdf/4.7.4/acme + sems-openmpi/4.1.6 + sems-netcdf-c/4.9.2 + sems-netcdf-cxx/4.2 + sems-netcdf-fortran/4.6.1 + sems-parallel-netcdf/1.12.3 $CIME_OUTPUT_ROOT/$CASE/run @@ -1894,11 +1895,14 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss 0 - $ENV{SEMS_NETCDF_ROOT} + $ENV{NETCDF_C_ROOT} + $ENV{NETCDF_FORTRAN_ROOT} + $ENV{PARALLEL_NETCDF_ROOT} + $ENV{OPENBLAS_ROOT} 64M spread threads - Generic + OpenBLAS 4000MB From a56bd4e2efdc2e14b975d2efe8272484be8f2a9e Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 16 Sep 2024 17:47:30 -0400 Subject: [PATCH 734/904] fix outdated path of output yaml files --- .../testdefs/testmods_dirs/eamxx/prod/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands index 435556401a2b..a5ad8d6c668e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands @@ -61,7 +61,7 @@ else fi # set the output yaml files -output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/ -maxdepth 1 -type f) +output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/ -maxdepth 1 -type f) for file in ${output_yaml_files[@]}; do # if the word "coarse" is in the file name, do nothing if [[ "${file}" == *"_coarse.yaml" && "${hmapfile}" == "not-supported-yet" ]]; then From c194ee2d40bd7b50f66233b8367f78fb3468f428 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 16 Sep 2024 17:09:14 -0600 Subject: [PATCH 735/904] Remove member of class from parallel_for. --- .../eamxx_mam_constituent_fluxes_interface.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index c168c1ef0e39..440e6e3dbdca 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -274,20 +274,25 @@ void MAMConstituentFluxes::run_impl(const double dt) { // Compute vertical layer heights and updraft velocity. We need these to fully // populate dry_atm_, so that we can form a HAERO atmosphere object. HAERO // atmosphere object is used to for state%q like array. + // NOTE: We cannot pass a member of the interface class (in this case, MAMConstituentFluxes) + // inside a parallel_for. Instead, we must create a soft copy of each member. + const auto & wet_atm = wet_atm_; + const auto & dry_atm= dry_atm_; + auto lambda = KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { const int icol = team.league_rank(); // column index - compute_dry_mixing_ratios(team, wet_atm_, // in - dry_atm_, // out + compute_dry_mixing_ratios(team, wet_atm, // in + dry_atm, // out icol); // in team.team_barrier(); // vertical heights has to be computed after computing dry mixing ratios // for atmosphere compute_vertical_layer_heights(team, // in - dry_atm_, // out + dry_atm, // out icol); // in - compute_updraft_velocities(team, wet_atm_, // in - dry_atm_, // out + compute_updraft_velocities(team, wet_atm, // in + dry_atm, // out icol); // in }; // policy From 6a6415a80c6bf1342c19abe789bfcac4dbf50d50 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 16 Sep 2024 19:45:04 -0400 Subject: [PATCH 736/904] edit logic to accomodate eamxx specs --- cime_config/config_archive.xml | 2 +- .../testdefs/testmods_dirs/eamxx/prod/shell_commands | 3 +++ ..._native.yaml => eamxx_output.decadal.1dailyAVG_native.yaml} | 2 +- ..._native.yaml => eamxx_output.decadal.1dailyMAX_native.yaml} | 2 +- ..._native.yaml => eamxx_output.decadal.1dailyMIN_native.yaml} | 2 +- ...INST_arm.yaml => eamxx_output.decadal.1hourlyINST_arm.yaml} | 2 +- ...ative.yaml => eamxx_output.decadal.1hourlyINST_native.yaml} | 2 +- ...coarse.yaml => eamxx_output.decadal.3hourlyAVG_coarse.yaml} | 2 +- ...oarse.yaml => eamxx_output.decadal.3hourlyINST_coarse.yaml} | 2 +- ...coarse.yaml => eamxx_output.decadal.6hourlyAVG_coarse.yaml} | 2 +- ...oarse.yaml => eamxx_output.decadal.6hourlyINST_coarse.yaml} | 2 +- ...ative.yaml => eamxx_output.decadal.6hourlyINST_native.yaml} | 2 +- ...G_coarse.yaml => eamxx_output.decadal.dailyAVG_coarse.yaml} | 2 +- 13 files changed, 15 insertions(+), 12 deletions(-) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyAVG_native.yaml => eamxx_output.decadal.1dailyAVG_native.yaml} (81%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyMAX_native.yaml => eamxx_output.decadal.1dailyMAX_native.yaml} (82%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyMIN_native.yaml => eamxx_output.decadal.1dailyMIN_native.yaml} (80%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1hourlyINST_arm.yaml => eamxx_output.decadal.1hourlyINST_arm.yaml} (92%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1hourlyINST_native.yaml => eamxx_output.decadal.1hourlyINST_native.yaml} (85%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.3hourlyAVG_coarse.yaml => eamxx_output.decadal.3hourlyAVG_coarse.yaml} (96%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.3hourlyINST_coarse.yaml => eamxx_output.decadal.3hourlyINST_coarse.yaml} (96%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyAVG_coarse.yaml => eamxx_output.decadal.6hourlyAVG_coarse.yaml} (94%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyINST_coarse.yaml => eamxx_output.decadal.6hourlyINST_coarse.yaml} (91%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyINST_native.yaml => eamxx_output.decadal.6hourlyINST_native.yaml} (90%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.dailyAVG_coarse.yaml => eamxx_output.decadal.dailyAVG_coarse.yaml} (97%) diff --git a/cime_config/config_archive.xml b/cime_config/config_archive.xml index f343fce81ba0..8cb7a5305386 100644 --- a/cime_config/config_archive.xml +++ b/cime_config/config_archive.xml @@ -29,7 +29,7 @@ r\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* rhist\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* - hi\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d*\.\d{4}-\d{2}-\d{2}-\d{5}\.nc$ + .*\.h\.(?!rhist\.).*\.nc$ diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands index a5ad8d6c668e..2520f7b6b12b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands @@ -3,6 +3,7 @@ cime_root=$(./xmlquery --value CIMEROOT) input_data_dir=$(./xmlquery --value DIN_LOC_ROOT) atmchange=$cime_root/../components/eamxx/scripts/atmchange +case_name=$(./xmlquery --value CASE) # Change run length ./xmlchange RUN_STARTDATE="1994-10-01" @@ -82,6 +83,8 @@ for file in ${output_yaml_files[@]}; do sed -i "s|horiz_remap_file:.*_to_ne30.*|horiz_remap_file: ${hmapfile}|" ./$(basename ${file}) sed -i "s|horiz_remap_file:.*_to_DecadalSites.*|horiz_remap_file: ${armmapfile}|" ./$(basename ${file}) fi + # replace all filename prefixes so that st_archive works... + sed -i "s|eamxx_output.decadal|${case_name}.scream|" ./$(basename ${file}) done # TODO: diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml similarity index 81% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml index 4e1239b5c33c..181f841eeb81 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyAVG_native.h +filename_prefix: eamxx_output.decadal.1dailyAVG_native.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml similarity index 82% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml index a7d10efe0707..a8974c75b35e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMAX_native.h +filename_prefix: eamxx_output.decadal.1dailyMAX_native.h iotype: pnetcdf Averaging Type: Max Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml similarity index 80% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml index 653e194d278b..8d48a1bedf63 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMIN_native.h +filename_prefix: eamxx_output.decadal.1dailyMIN_native.h iotype: pnetcdf Averaging Type: Min Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml index 5bb07048aed6..52fc391ca4d9 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_arm.h +filename_prefix: eamxx_output.decadal.1hourlyINST_arm.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml similarity index 85% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml index 7a221e89f1c3..0aba4827ead7 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_native.h +filename_prefix: eamxx_output.decadal.1hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml index 665294c62273..d429b11ebd1f 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml index 42c649545088..a2faa1b971c1 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml index 5e4aaed07384..437142ba5599 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml similarity index 91% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml index e9e0f34d5e0f..bb83718a8eb0 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml similarity index 90% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml index bb7fd275abf4..c69dc4b2212b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_native.h +filename_prefix: eamxx_output.decadal.6hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml similarity index 97% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml index 7c1990a7b56e..2d1e6e7221ef 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.dailyAVG_coarse.h +filename_prefix: eamxx_output.decadal.dailyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 From 30ffb16650cfeeb54098495f630fd9698586d16c Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:02:49 +0000 Subject: [PATCH 737/904] switch to 16,8 pair --- components/homme/src/share/cxx/ExecSpaceDefs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index c9ca8a0ecd93..4f3d97135fea 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -176,7 +176,7 @@ team_num_threads_vectors_for_gpu ( num_vectors ); } #else - return std::make_pair(4,16); + return std::make_pair(16,8); #endif } From 112fe79e60eeaa4d7ca0d85a54f24640b6a974f6 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:03:44 +0000 Subject: [PATCH 738/904] adjust aurora flags --- components/homme/cmake/machineFiles/aurora-aot.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake index b6fe34a78d72..094ec8882784 100644 --- a/components/homme/cmake/machineFiles/aurora-aot.cmake +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -43,9 +43,9 @@ SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") #AOT flags SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") -SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-link-huge-device-code -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") -SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") From 0fcac3cc2906dee8a391f2d446bcf5286db38919 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:04:25 +0000 Subject: [PATCH 739/904] adopt ekat sycl changes, TeamVectorRange --- .../share/cxx/utilities/scream_tridiag.hpp | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp index e18bbc4e7e27..26221db39552 100644 --- a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp +++ b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp @@ -128,6 +128,10 @@ int get_thread_id_within_team_gpu (const TeamMember& team) { // Can't use team.team_rank() here because vector direction also uses physical // threads but TeamMember types don't expose that information. return blockDim.x * threadIdx.y + threadIdx.x; +#elif defined(__SYCL_DEVICE_ONLY__) + auto item = team.item(); + return static_cast(item.get_local_range(1) * item.get_local_id(0) + + item.get_local_id(1)); #else assert(0); return -1; @@ -138,6 +142,9 @@ template KOKKOS_FORCEINLINE_FUNCTION int get_team_nthr_gpu (const TeamMember& team) { #if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ return blockDim.x * blockDim.y; +#elif defined __SYCL_DEVICE_ONLY__ + auto item = team.item(); + return static_cast(item.get_local_range(0) * item.get_local_range(1)); #else assert(0); return -1; @@ -161,6 +168,16 @@ KOKKOS_FORCEINLINE_FUNCTION int get_team_nthr (const Kokkos::Impl::HIPTeamMember& team) { return get_team_nthr_gpu(team); } #endif // KOKKOS_ENABLE_HIP + +#ifdef KOKKOS_ENABLE_SYCL +KOKKOS_FORCEINLINE_FUNCTION +int get_thread_id_within_team (const Kokkos::Impl::SYCLTeamMember& team) +{ return get_thread_id_within_team_gpu(team); } +KOKKOS_FORCEINLINE_FUNCTION +int get_team_nthr (const Kokkos::Impl::SYCLTeamMember& team) +{ return get_team_nthr_gpu(team); } +#endif // KOKKOS_ENABLE_SYCL + template KOKKOS_INLINE_FUNCTION const T& min (const T& a, const T& b) { return a < b ? a : b; } @@ -634,7 +651,7 @@ void bfb (const TeamMember& team, const auto f = [&] (const int& j) { impl::bfb_thomas_solve(dl, d, du, Kokkos::subview(X , Kokkos::ALL(), j)); }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nrhs), f); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nrhs), f); } template @@ -664,7 +681,7 @@ void bfb (const TeamMember& team, subview(du, ALL(), j), subview(X , ALL(), j)); }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nrhs), f); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nrhs), f); } } // namespace tridiag From 312c442ac6018cecde4b9024f4b0ea6090d57017 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Mon, 16 Sep 2024 21:27:35 -0500 Subject: [PATCH 740/904] Revert "Merge remote-tracking branch 'liho/liho745/river/bug-fix-7792c63' (PR #6313)" This reverts commit 427b86040f21ce3e41cc99f7cc2aeff02267f17f, reversing changes made to cf32a25b559c71ba63208fc09adaae5daebe6412. --- components/mosart/src/riverroute/MOSART_physics_mod.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mosart/src/riverroute/MOSART_physics_mod.F90 b/components/mosart/src/riverroute/MOSART_physics_mod.F90 index cad978577cda..912014904e93 100644 --- a/components/mosart/src/riverroute/MOSART_physics_mod.F90 +++ b/components/mosart/src/riverroute/MOSART_physics_mod.F90 @@ -682,8 +682,8 @@ subroutine Euler ! check for negative channel storage if (negchan < -1.e-10) then - write(iulog,*) 'Error: Negative channel storage found! ',negchan - call shr_sys_abort('mosart: negative channel storage') + write(iulog,*) 'Warning: Negative channel storage found! ',negchan +! call shr_sys_abort('mosart: negative channel storage') endif TRunoff%flow = TRunoff%flow / Tctl%DLevelH2R TRunoff%erowm_regi(:,nt_nmud:nt_nsan) = TRunoff%erowm_regi(:,nt_nmud:nt_nsan) / Tctl%DLevelH2R From 43239f35bcf42b4c33ef713c1ee2fdc3a8699238 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 17 Sep 2024 11:31:00 -0600 Subject: [PATCH 741/904] Fixes for mpi-serial on mappy --- cime_config/machines/config_machines.xml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index dcbd43ad6f08..fedb4318b3eb 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1896,8 +1896,6 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{NETCDF_C_ROOT} - $ENV{NETCDF_FORTRAN_ROOT} - $ENV{PARALLEL_NETCDF_ROOT} $ENV{OPENBLAS_ROOT} 64M spread @@ -1905,6 +1903,14 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss OpenBLAS 4000MB + + /home/jgfouca/packages/netcdf-fortran-4.6.1-bin + /home/jgfouca/packages/netcdf-fortran-4.6.1-bin:$ENV{LD_LIBRARY_PATH} + + + $ENV{NETCDF_FORTRAN_ROOT} + $ENV{PARALLEL_NETCDF_ROOT} + @@ -3497,7 +3503,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss 104 104 48 - FALSE + FALSE mpiexec @@ -4290,7 +4296,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - Chicoma GPU nodes at LANL IC. Each GPU node has single + Chicoma GPU nodes at LANL IC. Each GPU node has single AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' ch-fe* Linux From 3e6959e0260e962bf402825108cb76286a32be6b Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 18 Sep 2024 07:17:31 -0700 Subject: [PATCH 742/904] Fixes a typo --- components/elm/docs/user-guide/interpinic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md index 63f8c30b03da..5127aeeb94a9 100644 --- a/components/elm/docs/user-guide/interpinic.md +++ b/components/elm/docs/user-guide/interpinic.md @@ -15,7 +15,7 @@ The steps involved in creating a new IC files are as follows: The notes below provide an example of creating 1850 ELM IC file for the NARRM grid using E3SM v3 LR piControl from year = 0101. These notes are provided for Chrysalis. -### 1. Identification of the input ELM IC file +## 1. Identification of the input ELM IC file The identified input land condition file for this case is the following: @@ -39,7 +39,7 @@ elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c`date "+ ``` # Load relevant modules cd -eval $(./cime/CIME/Tools/get_case_env/get_case_env) +eval $(./cime/CIME/Tools/get_case_env) # change directory cd components/elm/tools/interpinic/src From ff40c9cff08029ec29b3a98f974f74357dab61aa Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 18 Sep 2024 07:32:45 -0700 Subject: [PATCH 743/904] Adds notes about using mksurfdata_map --- components/elm/docs/user-guide/index.md | 8 ++- .../elm/docs/user-guide/surface_dataset.md | 49 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 components/elm/docs/user-guide/surface_dataset.md diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index d33278ce7617..bfab59d92b1f 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -77,7 +77,11 @@ Using the above-mentioned settings: The [FATES User's Guide section on namelist options](https://fates-users-guide.readthedocs.io/en/latest/user/Namelist-Options-and-Run-Time-Modes.html) provides guidance on enabling these different FATES run modes. -## Generate land initial condition +## Create land surface dataset + +A new surface dataset for ELM is generated using `mksurfdata_map` and the notes about it are available [here](surface_dataset.md) -Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). +## Generate land initial condition + +Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). \ No newline at end of file diff --git a/components/elm/docs/user-guide/surface_dataset.md b/components/elm/docs/user-guide/surface_dataset.md new file mode 100644 index 000000000000..c513d2d1f59f --- /dev/null +++ b/components/elm/docs/user-guide/surface_dataset.md @@ -0,0 +1,49 @@ +# Creating an ELM surface dataset + +The notes describe the steps in creating an ELM surface dataset at 0.5x0.5 resolution for 1950 on Perlmutter. + +1. Load the appropriate modules. + +``` +cd +eval $(./cime/CIME/Tools/get_case_env) +``` + +2. Compile `mksurfdata_map`. + +``` +cd components/elm/tools/mksurfdata_map/src/ + +make clean +export USER_LDFLAGS="-L$NETCDF_DIR/lib -lnetcdf -lnetcdff -lnetcdf_intel" +export USER_LDFLAGS=$USER_LDFLAGS" -L$HDF5_DIR/lib -lhdf5 -lhdf5_fortran -lhdf5_hl_intel -lhdf5hl_fortran_intel" + +USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 +``` + +3. Build the namelist. This steps assumes that the resolution for which the new surface dataset is being created is a supported resolution. +If the surface dataset is being created for an unsupported resolution, 16 mapping files will have to be created to map the raw datasets +onto this unsupported resolution. + +``` +cd ../ + +RES=0.5x0.5 +YR=1950 +DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata + +./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT +mv namelist namelist.$RES.$YR +``` + +4. Run `mksurfdata_map` via an interactive job. + +``` +salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account e3sm + +RES=0.5x0.5 +YR=1950 +NAMELIST=namelist.$RES.$YR + +srun -n 1 ./mksurfdata_map < $NAMELIST +``` \ No newline at end of file From f61c6f8000a9418c7e5e9f2f6976d4bbb8352a89 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 18 Sep 2024 08:14:05 -0700 Subject: [PATCH 744/904] Few fixes --- components/elm/docs/user-guide/index.md | 3 +-- components/elm/docs/user-guide/interpinic.md | 12 +++++------ .../elm/docs/user-guide/surface_dataset.md | 20 ++++++++++--------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index bfab59d92b1f..43fe53d66552 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -81,7 +81,6 @@ provides guidance on enabling these different FATES run modes. A new surface dataset for ELM is generated using `mksurfdata_map` and the notes about it are available [here](surface_dataset.md) - ## Generate land initial condition -Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). \ No newline at end of file +Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md index 5127aeeb94a9..fcb897834f22 100644 --- a/components/elm/docs/user-guide/interpinic.md +++ b/components/elm/docs/user-guide/interpinic.md @@ -19,7 +19,7 @@ The notes below provide an example of creating 1850 ELM IC file for the NARRM gr The identified input land condition file for this case is the following: -``` +```bash /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc ``` @@ -27,7 +27,7 @@ The identified input land condition file for this case is the following: Using an existing NARRM land IC and making a copy of it -``` +```bash cd components/elm/tools/interpinic cp /lcrc/group/e3sm/data/inputdata/lnd/clm2/initdata_map/elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1870-01-01-00000.c20240704.nc \ @@ -36,7 +36,7 @@ elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c`date "+ ## 3. Compiling `interpinic` -``` +```bash # Load relevant modules cd eval $(./cime/CIME/Tools/get_case_env) @@ -47,14 +47,13 @@ cd components/elm/tools/interpinic/src export USER_LDFLAGS="-L$NETCDF_C_DIR/lib -lnetcdf -L$NETCDF_F_DIR/lib -lnetcdff -L$HDF5_DIR/lib -lhdf5" USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 - ``` ## 4. Run `interpinic` -The `interpinic ` can then be run via the following batch job (e.g., `remap.r025_RRSwISC6to18E3r4.1850.batch`) to generate the initial condition. +The `interpinic` can then be run via the following batch job (e.g., `remap.r025_RRSwISC6to18E3r4.1850.batch`) to generate the initial condition. -``` +```bash >cat remap.r025_RRSwISC6to18E3r4.1850.batch #!/bin/sh @@ -76,4 +75,3 @@ srun -n 1 ./interpinic \ -i /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc \ -o elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c20240903.nc ``` - diff --git a/components/elm/docs/user-guide/surface_dataset.md b/components/elm/docs/user-guide/surface_dataset.md index c513d2d1f59f..04d5b00f63ce 100644 --- a/components/elm/docs/user-guide/surface_dataset.md +++ b/components/elm/docs/user-guide/surface_dataset.md @@ -2,16 +2,16 @@ The notes describe the steps in creating an ELM surface dataset at 0.5x0.5 resolution for 1950 on Perlmutter. -1. Load the appropriate modules. +## 1. Load the appropriate modules -``` +```bash cd eval $(./cime/CIME/Tools/get_case_env) ``` -2. Compile `mksurfdata_map`. +## 2. Compile `mksurfdata_map` -``` +```bash cd components/elm/tools/mksurfdata_map/src/ make clean @@ -21,11 +21,13 @@ export USER_LDFLAGS=$USER_LDFLAGS" -L$HDF5_DIR/lib -lhdf5 -lhdf5_fortran -lhdf5_ USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 ``` -3. Build the namelist. This steps assumes that the resolution for which the new surface dataset is being created is a supported resolution. +## Build the namelist + +This steps assumes that the resolution for which the new surface dataset is being created is a supported resolution. If the surface dataset is being created for an unsupported resolution, 16 mapping files will have to be created to map the raw datasets onto this unsupported resolution. -``` +```bash cd ../ RES=0.5x0.5 @@ -36,9 +38,9 @@ DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata mv namelist namelist.$RES.$YR ``` -4. Run `mksurfdata_map` via an interactive job. +## Run `mksurfdata_map` via an interactive job -``` +```bash salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account e3sm RES=0.5x0.5 @@ -46,4 +48,4 @@ YR=1950 NAMELIST=namelist.$RES.$YR srun -n 1 ./mksurfdata_map < $NAMELIST -``` \ No newline at end of file +``` From ba4dea01f7b51316e3ed2bee9478ed50a4f4db4a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 18 Sep 2024 09:24:59 -0600 Subject: [PATCH 745/904] EAMxx: add checks on permissions for input files at buildnml time --- .../eamxx/cime_config/eamxx_buildnml.py | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 3e8c4abe9e95..14c9cf76bea8 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -4,7 +4,7 @@ Used by buildnml. See buildnml for documetation. """ -import os, sys, re +import os, sys, re, pwd, grp, stat, getpass from collections import OrderedDict import xml.etree.ElementTree as ET @@ -953,7 +953,37 @@ def create_input_data_list_file(case,caseroot): # Only add files whose full path starts with the CIME's input data location if file_path.startswith(din_loc_root): fd.write("scream_dl_input_{} = {}\n".format(idx, file_path)) - + if os.path.exists(file_path): + if os.path.isdir(file_path): + raise IsADirectoryError(f"Input file '{file_path}' is a directory, not a regular file.") + if not os.path.isfile(file_path): + raise OSError(f"Input file '{file_path}' exists but is not a regular file.") + if not os.access(file_path,os.R_OK): + try: + file_stat = os.stat(file_path) + + # Get owner and group names + owner = pwd.getpwuid(file_stat.st_uid).pw_name + group = grp.getgrgid(file_stat.st_gid).gr_name + + # Get file permissions + permissions = stat.filemode(file_stat.st_mode) + + except Exception as e: + raise RuntimeError(f"Error retrieving file info for '{file_path}': {e}") + + curr_user = getpass.getuser() + user_info = pwd.getpwnam(curr_user) + group_ids = os.getgrouplist(curr_user, user_info.pw_gid) + curr_groups = [grp.getgrgid(gid).gr_name for gid in group_ids] + + raise PermissionError ("Input file exists but it is not readable for current user\n" + f" - file name: {file_path}\n" + f" - file owner: {owner}\n" + f" - file group: {group}\n" + f" - permissions: {permissions}\n" + f" - current user: {curr_user}\n" + f" - current user groups: {curr_groups}\n") ############################################################################### def do_cime_vars_on_yaml_output_files(case, caseroot): From 644019ac981589be630435b115d93842ebe86489 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Wed, 18 Sep 2024 14:35:10 -0500 Subject: [PATCH 746/904] minor changes to improve clarity --- components/ww3/cime_config/buildlib_cmake | 25 ++++++++++------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/components/ww3/cime_config/buildlib_cmake b/components/ww3/cime_config/buildlib_cmake index 9df52bc35c67..42d92b2f0ad4 100755 --- a/components/ww3/cime_config/buildlib_cmake +++ b/components/ww3/cime_config/buildlib_cmake @@ -45,21 +45,18 @@ def buildlib(bldroot, installpath, case): modeldir = "{}/WW3/model".format(repodir) builddir = "{}/wav".format(exeroot) - # TODO: these work dirs will have to be changed to live in the binary/build area. - # Achieving this will probably require a significant refactor of the ww3 infrastructure. - # Doing this stuff in-source not only clutters the repo, but also introduces potential race - # conditions if we were to try to build multiple ww3 cases simultaneously. - bindir1 = "{}/bin".format(modeldir) - bindir = "{}/wav/bin".format(exeroot) - shutil.copytree(bindir1, bindir) - auxdir1 = "{}/aux".format(modeldir) - auxdir = "{}/wav/aux".format(exeroot) - shutil.copytree(auxdir1, auxdir) - ftndir1 = "{}/ftn".format(modeldir) - ftndir = "{}/wav/ftn".format(exeroot) - shutil.copytree(ftndir1, ftndir) + # work dirs are placed in the binary/build area. + bindir_source = "{}/bin".format(modeldir) + bindir = "{}/bin".format(builddir) + shutil.copytree(bindir_source, bindir) + auxdir_source = "{}/aux".format(modeldir) + auxdir = "{}/aux".format(builddir) + shutil.copytree(auxdir_source, auxdir) + ftndir_source = "{}/ftn".format(modeldir) + ftndir = "{}/ftn".format(builddir) + shutil.copytree(ftndir_source, ftndir) - tmpdir = "{}/wav/tmp".format(exeroot) + tmpdir = "{}/tmp".format(builddir) # Run w3_setup to create wwatch3.env file env_file = os.path.join(bindir, "wwatch3.env") From b4c9aab266e2daf6481f2602c91b78bdb0d52a18 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 18 Sep 2024 14:04:43 -0600 Subject: [PATCH 747/904] EAMxx: add machine files to use with SNL-hosted containers for gh actions testing --- .../cmake/machine-files/ghci-snl-openmp.cmake | 9 +++++++++ .../eamxx/cmake/machine-files/ghci-snl.cmake | 15 +++++++++++++++ components/eamxx/scripts/machines_specs.py | 5 +++++ 3 files changed, 29 insertions(+) create mode 100644 components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake create mode 100644 components/eamxx/cmake/machine-files/ghci-snl.cmake diff --git a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake new file mode 100644 index 000000000000..0d604cbb3b28 --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake @@ -0,0 +1,9 @@ +# Common settings for our ghci images +include(${CMAKE_CURRENT_LIST_DIR}/ghci.cmake) + +# Set SCREAM_MACHINE +set(SCREAM_MACHINE ghci-openmp CACHE STRING "") + +# Set OpenMP backend +set(EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) diff --git a/components/eamxx/cmake/machine-files/ghci-snl.cmake b/components/eamxx/cmake/machine-files/ghci-snl.cmake new file mode 100644 index 000000000000..f8115aec966b --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-snl.cmake @@ -0,0 +1,15 @@ +# Set Fortran flags +set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "Fortran compiler flags" FORCE) + +# Set the path to SCREAM input data +set(SCREAM_INPUT_ROOT /projects/e3sm/inputdata CACHE PATH "Path to SCREAM input data" FORCE) + +# Set the path to BLAS/LAPACK libraries +set(BLAS_LIBRARIES "/spack-installs/netlib-lapack/3.11.0/gcc/12.3.0/base/65x6uge/lib64/libblas.so" CACHE STRING "Path to BLAS library" FORCE) +set(LAPACK_LIBRARIES "/spack-installs/netlib-lapack/3.11.0/gcc/12.3.0/base/65x6uge/lib64/liblapack.so" CACHE STRING "Path to LAPACK library" FORCE) + +# Let's catch usage of code deprecated in Kokkos 4 +option (Kokkos_ENABLE_DEPRECATED_CODE_4 "" OFF) + +# We need to manage resources to spread across available cores/gpus +option (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES "" ON) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 337f8b66d222..713f49200fbe 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -93,6 +93,11 @@ "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), + "ghci-snl-openmp" : ([], + ["mpicxx","mpifort","mpicc"], + "", + "/projects/e3sm/baselines/scream/master-baselines" + ), } if pathlib.Path("~/.cime/scream_mach_specs.py").expanduser().is_file(): # pylint: disable=no-member From 03158e3e8fddfb860ea1818acea9986c81746f7a Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Wed, 18 Sep 2024 16:16:13 -0500 Subject: [PATCH 748/904] Updating CRYO1850* compsets to v3 ELM settings --- cime_config/allactive/config_compsets.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index e66f1e473d85..b17bfd7c7ca9 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -383,12 +383,12 @@ CRYO1850 - 1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYO1850-4xCO2 - 1850SOI_EAM%CMIP6-4xCO2_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6-4xCO2_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV @@ -398,7 +398,7 @@ CRYO1850-DISMF - 1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV From 6fbecc13ce1906f9d3162b1dedeb295c80769d1e Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 18 Sep 2024 17:20:50 -0500 Subject: [PATCH 749/904] Add support for new JRA forcing through year 2023 --- .../data_comps/datm/cime_config/config_component.xml | 10 +++++++--- .../data_comps/drof/cime_config/config_component.xml | 7 ++++++- components/mpas-ocean/cime_config/config_compsets.xml | 5 +++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/components/data_comps/datm/cime_config/config_component.xml b/components/data_comps/datm/cime_config/config_component.xml index 9145bdf4b16d..5b884ecd1ad1 100644 --- a/components/data_comps/datm/cime_config/config_component.xml +++ b/components/data_comps/datm/cime_config/config_component.xml @@ -10,7 +10,7 @@ This file may have atm desc entries. --> - Data driven ATM + Data driven ATM QIAN data set QIAN with water isotopes CRUNCEP data set @@ -28,8 +28,9 @@ COREv2 normal year forcing COREv2 interannual forcing interannual JRA55 forcing - interannual JRA55 forcing, v1.5, through 2020 interannual JRA55 forcing, v1.4, through 2018 + interannual JRA55 forcing, v1.5, through 2020 + interannual JRA55 forcing, v1.5, through 2023 JRA55 Repeat Year Forcing v1.3 1984-1985 JRA55 Repeat Year Forcing v1.3 1990-1991 JRA55 Repeat Year Forcing v1.3 2003-2004 @@ -63,8 +64,9 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). CORE2_NYF CORE2_IAF CORE_IAF_JRA - IAF_JRA_1p5 CORE_IAF_JRA_1p4_2018 + IAF_JRA_1p5 + IAF_JRA_1p5 CORE_RYF8485_JRA CORE_RYF9091_JRA CORE_RYF0304_JRA @@ -391,6 +393,7 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). $DATM_CLMNCEP_YR_START 1 1 + 1 $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START @@ -488,6 +491,7 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). 2003 2016 2020 + 2023 1979 1979 diff --git a/components/data_comps/drof/cime_config/config_component.xml b/components/data_comps/drof/cime_config/config_component.xml index b5e0a8071fab..aede006caf26 100644 --- a/components/data_comps/drof/cime_config/config_component.xml +++ b/components/data_comps/drof/cime_config/config_component.xml @@ -13,7 +13,7 @@ --> - Data runoff model + Data runoff model NULL mode COREv2 normal year forcing: COREv2 normal year forcing: @@ -24,6 +24,7 @@ COREv2 interannual year forcing: COREv2 interannual year forcing: CPLHIST mode: + JRA55 interannual forcing, v1.5, through 2023 JRA55 interannual forcing, v1.5, through 2020 JRA55 interannual forcing, v1.5, through 2020, no rofi or rofl around AIS JRA55 interannual forcing, v1.4, through 2018 @@ -61,6 +62,7 @@ DIATREN_IAF_AIS55_RX1 CPLHIST IAF_JRA + IAF_JRA_1p5 IAF_JRA_1p5 IAF_JRA_1p5_AIS0ROF IAF_JRA_1p4_2018 @@ -165,6 +167,7 @@ 1 1 1 + 1 run_component_drof env_run.xml @@ -179,6 +182,7 @@ 1958 1958 1958 + 1958 run_component_drof env_run.xml @@ -193,6 +197,7 @@ 2016 2018 2020 + 2023 run_component_drof env_run.xml diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index be36f0103472..52ad20a5ab7c 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -67,6 +67,11 @@ 2000_DATM%JRA-1p5_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5_SGLC_SWAV + + GMPAS-JRA1p5-2023 + 2000_DATM%JRA-1p5-2023_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5-2023_SGLC_SWAV + + GMPAS-OECO-JRA1p4 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%OECODATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV From 07577b6ce66b3cafdbe698bc998ab68e506b3a6a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 18 Sep 2024 17:20:39 -0600 Subject: [PATCH 750/904] EAMxx: fix ghci-snl-openmp mach file --- components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake index 0d604cbb3b28..3139580e9ee0 100644 --- a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake +++ b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake @@ -1,8 +1,8 @@ # Common settings for our ghci images -include(${CMAKE_CURRENT_LIST_DIR}/ghci.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/ghci-snl.cmake) # Set SCREAM_MACHINE -set(SCREAM_MACHINE ghci-openmp CACHE STRING "") +set(SCREAM_MACHINE ghci-snl-openmp CACHE STRING "") # Set OpenMP backend set(EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) From 67a5f9aca7e8e532894af791612d92689a0fd5a4 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Sep 2024 09:56:10 -0600 Subject: [PATCH 751/904] Everything working now --- cime_config/machines/config_machines.xml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index fedb4318b3eb..5d997dcb5d65 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1880,9 +1880,10 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss sems-netcdf-c-serial/4.9.2 + sems-netcdf-fortran-serial/4.6.1 - sems-openmpi/4.1.6 + sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 @@ -1896,6 +1897,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{NETCDF_C_ROOT} + $ENV{NETCDF_FORTRAN_ROOT} $ENV{OPENBLAS_ROOT} 64M spread @@ -1903,12 +1905,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss OpenBLAS 4000MB - - /home/jgfouca/packages/netcdf-fortran-4.6.1-bin - /home/jgfouca/packages/netcdf-fortran-4.6.1-bin:$ENV{LD_LIBRARY_PATH} - - $ENV{NETCDF_FORTRAN_ROOT} $ENV{PARALLEL_NETCDF_ROOT} From a8763c20197cee06943a527eba000a6e0b93fb34 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 19 Sep 2024 08:58:52 -0700 Subject: [PATCH 752/904] Cleanup to fix issues from earlier PR causing sporadic test failures --- .../bld/namelist_files/namelist_defaults_mali.xml | 2 +- components/mpas-albany-landice/cime_config/buildnml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml index 0450eb44f116..8dc305533ca3 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml @@ -18,7 +18,7 @@ 'fo' -'none' +'fo' 3 0.25 .false. diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 9489b6dfa8fd..3b38ed39c438 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -280,7 +280,9 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + if mali_use_albany: + lines.append(' ') + lines.append('') lines.append('') lines.append(' - -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc - - - @@ -2056,76 +2049,83 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/northamericax4v1pg2/map_0.5x0.5_GSDTG2000_to_northamericax4v1pg2_nomask_aave_da_c210112.nc - + -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_AVHRR_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_MODIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.9x1.25_GRDC_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_10x10min_IGBPmergeICESatGIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_10x10min_nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_360x720cru_cruncep_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_GLOBE-Gardner_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_LandScan2004_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_MODIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_USGS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_IGBP-GSDP_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_ISRIC-WISE_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_GSDTG2000_to_0.125x0.125_nomask_aave_da_c190725.nc - + -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_AVHRR_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.9x1.25_GRDC_to_0.25x0.25_nomask_aave_da_c240124.nc lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_IGBPmergeICESatGIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.25x0.25_nomask_aave_da_c240123.nc lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720cru_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_LandScan2004_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_USGS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_IGBP-GSDP_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_ISRIC-WISE_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_GSDTG2000_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.1x0.1_nomask_to_0.25x0.25_nomask_aave_da_c240308.nc + lnd/clm2/mappingdata/maps/antarcticax4v1/map_0.5x0.5_AVHRR_to_antarcticax4v1_nomask_aave_da_c210130.nc diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 7063d7a11054..d36bf5565ce3 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -1502,7 +1502,7 @@ If TRUE, irrigation will be active (find surface datasets with active irrigation +"1000,850,1100,1350,1600,1850,1855,1865,1875,1885,1895,1905,1915,1925,1935,1945,1950,1955,1965,1975,1980,1985,1995,2000,2005,2010,2015,2025,2035,2045,2055,2065,2075,2085,2095,2100,2105"> Year to simulate and to provide datasets for (such as surface datasets, initial conditions, aerosol-deposition, Nitrogen deposition rates etc.) A sim_year of 1000 corresponds to data used for testing only, NOT corresponding to any real datasets. A sim_year greater than 2005 corresponds to rcp scenario data From 1204d21fdfe4c3af43ddce44ce4ac945de7e458d Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Fri, 20 Sep 2024 11:04:58 -0700 Subject: [PATCH 765/904] Adds notes on creating surface dataset with 10 glacier layers --- .../elm/docs/user-guide/surface_dataset.md | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/components/elm/docs/user-guide/surface_dataset.md b/components/elm/docs/user-guide/surface_dataset.md index 04d5b00f63ce..5adaad100d6c 100644 --- a/components/elm/docs/user-guide/surface_dataset.md +++ b/components/elm/docs/user-guide/surface_dataset.md @@ -23,9 +23,9 @@ USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --included ## Build the namelist -This steps assumes that the resolution for which the new surface dataset is being created is a supported resolution. +This step assumes that the resolution for which the new surface dataset is being created is a supported resolution. If the surface dataset is being created for an unsupported resolution, 16 mapping files will have to be created to map the raw datasets -onto this unsupported resolution. +onto this unsupported resolution. The `namelist` file with default number of glaciers (equal to zero) can be generated as: ```bash cd ../ @@ -35,7 +35,16 @@ YR=1950 DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata ./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT -mv namelist namelist.$RES.$YR +``` + +An example of generating the namelist for 0.25 deg (`r025`) resolution for 1980 with 10 glacier layers is as follows: + +```bash +RES=r025 +YR=1980 +DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata + +./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT -glc_nec 10 ``` ## Run `mksurfdata_map` via an interactive job @@ -43,9 +52,5 @@ mv namelist namelist.$RES.$YR ```bash salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account e3sm -RES=0.5x0.5 -YR=1950 -NAMELIST=namelist.$RES.$YR - -srun -n 1 ./mksurfdata_map < $NAMELIST +srun -n 1 ./mksurfdata_map < namelist ``` From 227f736b3b82d51b8d750514a977bf2c72cfe34e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 19 Sep 2024 20:00:32 -0700 Subject: [PATCH 766/904] Updates lc machines Signed-off-by: Jason Boutte --- .../machines/cmake_macros/intel_dane.cmake | 6 ++ .../machines/cmake_macros/intel_ruby.cmake | 6 ++ cime_config/machines/config_batch.xml | 36 +++------ cime_config/machines/config_machines.xml | 76 ++++++++++++------- 4 files changed, 71 insertions(+), 53 deletions(-) diff --git a/cime_config/machines/cmake_macros/intel_dane.cmake b/cime_config/machines/cmake_macros/intel_dane.cmake index 8091325c6cec..ef25a97b300e 100644 --- a/cime_config/machines/cmake_macros/intel_dane.cmake +++ b/cime_config/machines/cmake_macros/intel_dane.cmake @@ -1,4 +1,10 @@ string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") + +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck/lib") + set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/cmake_macros/intel_ruby.cmake b/cime_config/machines/cmake_macros/intel_ruby.cmake index 8091325c6cec..e874bfb7eaf6 100644 --- a/cime_config/machines/cmake_macros/intel_ruby.cmake +++ b/cime_config/machines/cmake_macros/intel_ruby.cmake @@ -1,4 +1,10 @@ string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") + +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5/lib") + set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 13cfdfb337b6..923e497367f7 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -237,33 +237,17 @@ - - squeue - sbatch - scancel - #SBATCH - (\d+)$ - --dependency=afterok:jobid - --dependency=afterany:jobid - : - %H:%M:%S - --mail-user - --mail-type - none, all, begin, end, fail - - --export=ALL - -p {{ job_queue }} - -J {{ job_id }} - -N {{ num_nodes }} - -n {{ total_tasks }} - -t {{ job_wallclock_time }} - -o {{ job_id }}.out - -e {{ job_id }}.err - -A {{ project }} - + + + pbatch + pdebug + + + + - pbatch - pdebug + pbatch + pdebug diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index ff0c583fc93a..249cfdfc5a88 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3065,9 +3065,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 56 56 @@ -3076,8 +3076,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -3089,24 +3097,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion + cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 - cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core + mvapich2/2.3.7-ll7cmqm + hdf5/1.10.7-ewjpbjd + netcdf-c/4.4.1.1-vaxofek + netcdf-fortran/4.4.4-3pzbx2u + parallel-netcdf/1.11.0-tzgdala $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2 + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5 @@ -3121,9 +3132,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 224 112 @@ -3132,8 +3143,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -3145,24 +3164,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core + mvapich2/2.3.7-27jao34 + hdf5/1.10.7-766kapa + netcdf-c/4.4.1.1-2uznnlw + netcdf-fortran/4.4.4-itpstyo + parallel-netcdf/1.11.0-26sxm4m $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck From af5bc64e3faa6f5905cfd4e7659bf1f7eccee1c6 Mon Sep 17 00:00:00 2001 From: iulian Date: Fri, 20 Sep 2024 15:49:27 -0700 Subject: [PATCH 767/904] add pm-gpu MOAB root tested on perlmutter compiled with "gnugpu" compiler environment --- cime_config/machines/config_machines.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index eeb80756bb8c..b27bc0ce19b6 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -441,6 +441,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} From 7d2259d531e2c089e013b48d29ea0a1f01a3fae8 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 23 Sep 2024 14:02:13 -0700 Subject: [PATCH 768/904] update fates default parameter file with new ngee-a shrubs --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index beeae7858595..7d2390d267b1 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -134,7 +134,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.36.0.0_12pft_c240517.nc +lnd/clm2/paramdata/fates_params_api.36.1.0_12pft_c240926.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc From af5d1e33c7fa0073e36677ae4d5b41ea1a8fe0ec Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Tue, 24 Sep 2024 12:35:51 -0500 Subject: [PATCH 769/904] Updates to bgc registry and history fields -Corrects units/descriptions of bgc fields in registry -Defines column integrated and cell average bgc diagnostics -Updates streams build to include new diagnostics for bgc runs BFB --- components/mpas-seaice/cime_config/buildnml | 57 +- components/mpas-seaice/src/Registry.xml | 758 +++++++++++++----- .../src/shared/mpas_seaice_icepack.F | 424 +++++++++- 3 files changed, 1015 insertions(+), 224 deletions(-) diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index c38ca08a31da..d43fbdc151c2 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -924,22 +924,41 @@ def buildnml(case, caseroot, compname): lines.append(' ') if ice_bgc == 'ice_bgc': - lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -999,15 +1018,35 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') diff --git a/components/mpas-seaice/src/Registry.xml b/components/mpas-seaice/src/Registry.xml index 7c842ca2f3c4..78830161c62d 100644 --- a/components/mpas-seaice/src/Registry.xml +++ b/components/mpas-seaice/src/Registry.xml @@ -866,7 +866,7 @@ icepack_name="tr_bgc_PON" /> @@ -948,77 +948,77 @@ possible_values="positive real number" icepack_name="frazil_scav" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1473,32 +1474,32 @@ possible_values="-1 = entirely in the mobile phase; 0 = retention dominated; 1 = release dominated; 0.5 = equal but rapid exchange; 2 = equal but slow exchange" icepack_name="zaerotype_dust4" /> - - - - - - - - - - - - - - - - - - - - + + @@ -2701,17 +2705,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -2885,11 +2889,13 @@ @@ -3422,241 +3474,325 @@ + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -5887,6 +6242,7 @@ diff --git a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F index 36899b4b10d7..fa3b28c92c8f 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F @@ -3923,7 +3923,11 @@ subroutine column_biogeochemistry(domain) abortFlag = icepack_warnings_aborted() call get_cice_tracer_array_category(block, ciceTracerObject, & - tracerArrayCategory, iCell, setGetPhysicsTracers, setGetBGCTracers) + tracerArrayCategory, iCell, setGetPhysicsTracers, setGetBGCTracers) + + if (config_use_vertical_tracers) & + call define_total_biogeochemistry_array_cell(block, ciceTracerObject, totalVerticalBiologyIce(:,iCell), & + totalVerticalBiologySnow(:,iCell), iCell) if (checkCarbon) then call seaice_total_carbon_content_category(block,totalCarbonCatFinal(:,iCell),iceAreaCategory(1,:,:),iceVolumeCategory(1,:,:),iCell) @@ -8648,10 +8652,13 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace nzAerosols type(MPAS_pool_type), pointer :: & - tracers_aggregate + tracers_aggregate, & + biogeochemistry real(kind=RKIND), dimension(:), pointer :: & - brineFractionCell + brineFractionCell, & + carbonToNitrogenRatioAlgae, & + carbonToNitrogenRatioDON real(kind=RKIND), dimension(:,:), pointer :: & skeletalAlgaeConcCell, & @@ -8699,14 +8706,21 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalParticulateIronIceCell, & verticalDissolvedIronIceCell, & verticalAerosolsIceCell, & - verticalAerosolsSnowCell + verticalAerosolsSnowCell, & + verticalDOCLabileIceCell, & + verticalAlgaeTotalCarbonIceCell, & + verticalBCTotalIceCell, & + verticalDustTotalIceCell, & + verticalBCTotalSnowCell, & + verticalDustTotalSnowCell integer :: & iBioTracers, & iBioCount, & iLayers, & iIceCount, & - iSnowCount + iSnowCount, & + nAeroType call MPAS_pool_get_config(block % configs, "config_use_skeletal_biochemistry", config_use_skeletal_biochemistry) call MPAS_pool_get_config(block % configs, "config_use_vertical_biochemistry", config_use_vertical_biochemistry) @@ -8736,6 +8750,7 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace call MPAS_pool_get_dimension(block % dimensions, "nDissolvedIron", nDissolvedIron) call MPAS_pool_get_subpool(block % structs, "tracers_aggregate", tracers_aggregate) + call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistry) call MPAS_pool_get_array(tracers_aggregate, "skeletalAlgaeConcCell", skeletalAlgaeConcCell) call MPAS_pool_get_array(tracers_aggregate, "skeletalDOCConcCell", skeletalDOCConcCell) @@ -8784,6 +8799,14 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace call MPAS_pool_get_array(tracers_aggregate, "verticalAerosolsSnowCell", verticalAerosolsSnowCell) call MPAS_pool_get_array(tracers_aggregate, "verticalSalinityCell", verticalSalinityCell) call MPAS_pool_get_array(tracers_aggregate, "brineFractionCell", brineFractionCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalAlgaeTotalCarbonIceCell", verticalAlgaeTotalCarbonIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDOCLabileIceCell", verticalDOCLabileIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalBCTotalIceCell", verticalBCTotalIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDustTotalIceCell", verticalDustTotalIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDustTotalSnowCell", verticalDustTotalSnowCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalBCTotalSnowCell", verticalBCTotalSnowCell) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioAlgae", carbonToNitrogenRatioAlgae) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioDON", carbonToNitrogenRatioDON) ! biogeochemistry ! brine height fraction @@ -8866,12 +8889,15 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace ! algal nitrogen do iBioTracers = 1, nAlgae iIceCount = (iBioTracers-1)*nBioLayersP1 + verticalAlgaeTotalCarbonIceCell(:,iCell) = 0.0_RKIND do iLayers = 1,nBioLayersP1 iBiocount = iBiocount + 1 verticalAlgaeConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_algaeConc(iBioTracers)+iLayers-1) verticalAlgaeIceCell(iLayers+iIceCount,iCell) = verticalAlgaeConcCell(iBioCount,iCell) + verticalAlgaeTotalCarbonIceCell(iLayers,iCell) = verticalAlgaeTotalCarbonIceCell(iLayers,iCell) + & + carbonToNitrogenRatioAlgae(iBioTracers) * verticalAlgaeConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBiocount = iBiocount + 1 @@ -8894,6 +8920,7 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace if (config_use_carbon) then iBioCount = 0 + verticalDOCLabileIceCell(:,iCell) = 0.0_RKIND ! DOC do iBioTracers = 1, nDOC @@ -8904,6 +8931,8 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalDOCConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_DOCConc(iBioTracers) + iLayers-1) verticalDOCIceCell(iLayers+iIceCount,iCell) = verticalDOCConcCell(iBioCount,iCell) + verticalDOCLabileIceCell(iLayers, iCell) = verticalDOCLabileIceCell(iLayers, iCell) + & + verticalDOCConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -8942,6 +8971,8 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalDONConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_DONConc(iBioTracers) + iLayers-1) verticalDONIceCell(iLayers+iIceCount,iCell) = verticalDONConcCell(iBioCount,iCell) + verticalDOCLabileIceCell(iLayers, iCell) = verticalDOCLabileIceCell(iLayers, iCell) + & + verticalDONConcCell(iBioCount,iCell) * carbonToNitrogenRatioDON(iBioTracers) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -9058,15 +9089,25 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace ! black carbon and dust aerosols if (config_use_zaerosols) then iBioCount = 0 + verticalDustTotalIceCell(:,iCell) = 0.0_RKIND + verticalBCTotalIceCell(:,iCell) = 0.0_RKIND + verticalDustTotalSnowCell(:,iCell) = 0.0_RKIND + verticalBCTotalSnowCell(:,iCell) = 0.0_RKIND + do iBioTracers = 1, nzAerosols iIceCount = (iBioTracers-1)*nBioLayersP1 iSnowCount = (iBioTracers-1)*2 - + nAeroType = 1 + if (iBioTracers .gt. 2) nAeroType = 0 do iLayers = 1,nBioLayersP1 iBioCount = iBioCount + 1 verticalAerosolsConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_verticalAerosolsConc(iBioTracers)+iLayers-1) verticalAerosolsIceCell(iLayers+iIceCount,iCell) = verticalAerosolsConcCell(iBioCount,iCell) + verticalBCTotalIceCell(iLayers, iCell) = verticalBCTotalIceCell(iLayers, iCell) + nAeroType * & + verticalAerosolsConcCell(iBioCount,iCell) + verticalDustTotalIceCell(iLayers, iCell) = verticalDustTotalIceCell(iLayers, iCell) + (1-nAeroType) * & + verticalAerosolsConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -9074,6 +9115,10 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace tracerArrayCell(tracerObject % index_verticalAerosolsConc(iBioTracers)+iLayers-1) verticalAerosolsSnowCell(iLayers-nBioLayersP1+iSnowCount,iCell) = & verticalAerosolsConcCell(iBioCount,iCell) + verticalBCTotalSnowCell(iLayers-nBioLayersP1, iCell) = verticalBCTotalSnowCell(iLayers-nBioLayersP1, iCell) + & + nAeroType * verticalAerosolsConcCell(iBioCount,iCell) + verticalDustTotalSnowCell(iLayers-nBioLayersP1,iCell) = verticalDustTotalSnowCell(iLayers-nBioLayersP1,iCell) + & + (1-nAeroType) * verticalAerosolsConcCell(iBioCount,iCell) enddo enddo endif @@ -9088,6 +9133,274 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace end subroutine get_cice_biogeochemistry_tracer_array_cell +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! define_total_biogeochemistry_array_cell +! +!> \brief +!> \author Nicole Jeffery, LANL +!> \date 16rd September 2024 +!> \details +!> +! Defines total ice and snow arrays for biogeochemistry and aerosols diagnostics +! +!----------------------------------------------------------------------- + + subroutine define_total_biogeochemistry_array_cell(block, tracerObject, totalVerticalBioIceCell, & + totalVerticalBioSnowCell, iCell) + + type(block_type), intent(in) :: & + block + + type(ciceTracerObjectType), intent(in) :: & + tracerObject + + real(kind=RKIND), dimension(:), intent(in):: & + totalVerticalBioIceCell, & + totalVerticalBioSnowCell + + integer, intent(in) :: & + iCell + + logical, pointer :: & + config_use_vertical_biochemistry, & + config_use_nitrate, & + config_use_carbon, & + config_use_ammonium, & + config_use_silicate, & + config_use_DMS, & + config_use_nonreactive, & + config_use_humics, & + config_use_DON, & + config_use_iron, & + config_use_zaerosols + + integer, pointer :: & + nAlgae, & + nDOC, & + nDIC, & + nDON, & + nParticulateIron, & + nDissolvedIron, & + nzAerosols + + type(MPAS_pool_type), pointer :: & + biogeochemistry + + real(kind=RKIND), dimension(:), pointer :: & + totalVerticalDiatomIce, & + totalVerticalSmallPlanktonIce, & + totalVerticalPhaeocystisIce, & + totalVerticalNitrateIce, & + totalVerticalPolysaccharidsIce, & + totalVerticalLipidsIce, & + totalVerticalDICIce, & + totalVerticalAmmoniumIce, & + totalVerticalSilicateIce, & + totalVerticalDMSPpIce, & + totalVerticalDMSPdIce, & + totalVerticalDMSIce, & + totalVerticalNonreactiveIce, & + totalVerticalHumicsIce, & + totalVerticalProteinsIce, & + totalVerticalDissolvedIronIce, & + totalVerticalParticulateIronIce, & + totalVerticalDustIce, & + totalVerticalDustSnow, & + totalVerticalBC1Ice, & + totalVerticalBC1Snow, & + totalVerticalBC2Ice, & + totalVerticalBC2Snow, & + totalVerticalBCIce, & + totalVerticalBCSnow, & + totalVerticalDissolvedIronSnow, & + totalVerticalAlgaeCarbonIce, & + totalVerticalDOCLabileIce, & + carbonToNitrogenRatioAlgae, & + carbonToNitrogenRatioDON + + call MPAS_pool_get_config(block % configs, "config_use_vertical_biochemistry", config_use_vertical_biochemistry) + call MPAS_pool_get_config(block % configs, "config_use_nitrate", config_use_nitrate) + call MPAS_pool_get_config(block % configs, "config_use_carbon", config_use_carbon) + call MPAS_pool_get_config(block % configs, "config_use_ammonium",config_use_ammonium) + call MPAS_pool_get_config(block % configs, "config_use_silicate",config_use_silicate) + call MPAS_pool_get_config(block % configs, "config_use_DMS",config_use_DMS) + call MPAS_pool_get_config(block % configs, "config_use_nonreactive",config_use_nonreactive) + call MPAS_pool_get_config(block % configs, "config_use_humics",config_use_humics) + call MPAS_pool_get_config(block % configs, "config_use_DON",config_use_DON) + call MPAS_pool_get_config(block % configs, "config_use_iron",config_use_iron) + call MPAS_pool_get_config(block % configs, "config_use_zaerosols",config_use_zaerosols) + + call MPAS_pool_get_dimension(block % dimensions, "nzAerosols", nzAerosols) + call MPAS_pool_get_dimension(block % dimensions, "nAlgae", nAlgae) + call MPAS_pool_get_dimension(block % dimensions, "nDOC", nDOC) + call MPAS_pool_get_dimension(block % dimensions, "nDIC", nDIC) + call MPAS_pool_get_dimension(block % dimensions, "nDON", nDON) + call MPAS_pool_get_dimension(block % dimensions, "nParticulateIron", nParticulateIron) + call MPAS_pool_get_dimension(block % dimensions, "nDissolvedIron", nDissolvedIron) + + call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistry) + + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDiatomIce", totalVerticalDiatomIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalSmallPlanktonIce", totalVerticalSmallPlanktonIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalPhaeocystisIce", totalVerticalPhaeocystisIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalPolysaccharidsIce", totalVerticalPolysaccharidsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalLipidsIce", totalVerticalLipidsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDICIce", totalVerticalDICIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalProteinsIce", totalVerticalProteinsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalNitrateIce", totalVerticalNitrateIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalSilicateIce", totalVerticalSilicateIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalAmmoniumIce", totalVerticalAmmoniumIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSIce", totalVerticalDMSIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSPpIce", totalVerticalDMSPpIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSPdIce", totalVerticalDMSPdIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalNonreactiveIce", totalVerticalNonreactiveIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalHumicsIce", totalVerticalHumicsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalParticulateIronIce", totalVerticalParticulateIronIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDissolvedIronIce", totalVerticalDissolvedIronIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDissolvedIronSnow", totalVerticalDissolvedIronSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDustIce", totalVerticalDustIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDustSnow", totalVerticalDustSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC1Ice", totalVerticalBC1Ice) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC1Snow", totalVerticalBC1Snow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC2Ice", totalVerticalBC2Ice) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC2Snow", totalVerticalBC2Snow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBCIce", totalVerticalBCIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBCSnow", totalVerticalBCSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioAlgae", carbonToNitrogenRatioAlgae) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioDON", carbonToNitrogenRatioDON) + + ! biogeochemistry + + if (config_use_vertical_biochemistry) then + + ! algal nitrogen + totalVerticalDiatomIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(1)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalDiatomIce(iCell) * carbonToNitrogenRatioAlgae(1) + + SELECT CASE (nAlgae) + CASE (2) + totalVerticalSmallPlanktonIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(2)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalAlgaeCarbonIce(iCell) + & + totalVerticalSmallPlanktonIce(iCell) * carbonToNitrogenRatioAlgae(2) + CASE (3) + totalVerticalSmallPlanktonIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(2)) + totalVerticalPhaeocystisIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(3)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalAlgaeCarbonIce(iCell) + & + totalVerticalSmallPlanktonIce(iCell) * carbonToNitrogenRatioAlgae(2) + & + totalVerticalPhaeocystisIce(iCell) * carbonToNitrogenRatioAlgae(3) + END SELECT + end if + + ! nitrate + if (config_use_nitrate) & + totalVerticalNitrateIce(iCell) = totalVerticalBioIceCell(tracerObject % index_nitrateConcLayer) + + if (config_use_carbon) then + + ! DOC + totalVerticalDOCLabileIce(iCell) = 0.0_RKIND + SELECT CASE (nDOC) + CASE (1) + totalVerticalPolysaccharidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalPolysaccharidsIce(iCell) + CASE (2) + totalVerticalPolysaccharidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalLipidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalPolysaccharidsIce(iCell) + totalVerticalLipidsIce(iCell) + END SELECT + + ! DIC + SELECT CASE (nDIC) + CASE (1) + totalVerticalDICIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DICConcLayer(1)) + END SELECT + end if + + if (config_use_DON) then + + ! DON + SELECT CASE (nDON) + CASE (1) + totalVerticalProteinsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DONConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalDOCLabileIce(iCell) + & + totalVerticalProteinsIce(iCell)* carbonToNitrogenRatioDON(1) + END SELECT + end if + + ! ammonium + if (config_use_ammonium) & + totalVerticalAmmoniumIce(iCell) = totalVerticalBioIceCell(tracerObject % index_ammoniumConcLayer) + + ! silicate + if (config_use_silicate) & + totalVerticalSilicateIce(iCell) = totalVerticalBioIceCell(tracerObject % index_silicateConcLayer) + + ! DMS, DMSPp, DMSPd + if (config_use_DMS) then + totalVerticalDMSIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSConcLayer) + totalVerticalDMSPpIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSPpConcLayer) + totalVerticalDMSPdIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSPdConcLayer) + end if + + ! nonreactive + if (config_use_nonreactive) & + totalVerticalNonreactiveIce(iCell) = totalVerticalBioIceCell(tracerObject % index_nonreactiveConcLayer) + + ! humic material (refractory DOC) + if (config_use_humics) then + totalVerticalHumicsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_humicsConcLayer) + end if + + if (config_use_iron) then + + ! ParticulateIron + SELECT CASE (nParticulateIron) + CASE (1) + totalVerticalParticulateIronIce(iCell) = totalVerticalBioIceCell(tracerObject % index_particulateIronConcLayer(1)) + END SELECT + + ! DissolvedIron + SELECT CASE (nDissolvedIron) + CASE (1) + totalVerticalDissolvedIronIce(iCell) = totalVerticalBioIceCell(tracerObject % index_dissolvedIronConcLayer(1)) + totalVerticalDissolvedIronSnow(iCell) = totalVerticalBioSnowCell(tracerObject % index_dissolvedIronConcLayer(1)) + END SELECT + + end if + + ! black carbon and dust aerosols + if (config_use_zaerosols) then + + SELECT CASE (nzAerosols) + CASE (1) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + CASE (2) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC2Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBC2Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBC2Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + totalVerticalBC2Snow(iCell) + CASE (3) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC2Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBC2Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalDustIce(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(3)) + totalVerticalDustSnow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(3)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBC2Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + totalVerticalBC2Snow(iCell) + END SELECT + end if + + end subroutine define_total_biogeochemistry_array_cell + !----------------------------------------------------------------------- ! Init CICE parameters !----------------------------------------------------------------------- @@ -14699,7 +15012,35 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) ! zSalinityFlux, & !echmod deprecate ! zSalinityGDFlux, & !echmod deprecate totalChlorophyll, & - totalCarbonContentCell + totalCarbonContentCell, & + totalVerticalDiatomIce, & + totalVerticalSmallPlanktonIce, & + totalVerticalPhaeocystisIce, & + totalVerticalNitrateIce, & + totalVerticalPolysaccharidsIce, & + totalVerticalLipidsIce, & + totalVerticalDICIce, & + totalVerticalAmmoniumIce, & + totalVerticalSilicateIce, & + totalVerticalDMSPpIce, & + totalVerticalDMSPdIce, & + totalVerticalDMSIce, & + totalVerticalNonreactiveIce, & + totalVerticalHumicsIce, & + totalVerticalProteinsIce, & + totalVerticalDissolvedIronIce, & + totalVerticalParticulateIronIce, & + totalVerticalDustIce, & + totalVerticalDustSnow, & + totalVerticalBC1Ice, & + totalVerticalBC1Snow, & + totalVerticalBC2Ice, & + totalVerticalBC2Snow, & + totalVerticalBCIce, & + totalVerticalBCSnow, & + totalVerticalAlgaeCarbonIce, & + totalVerticalDOCLabileIce, & + totalVerticalDissolvedIronSnow real(kind=RKIND), dimension(:,:), pointer :: & oceanBioFluxes, & @@ -14747,13 +15088,12 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridSalinityIceCell", bgridSalinityIceCell) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridPorosityIceCell", bgridPorosityIceCell) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridTemperatureIceCell", bgridTemperatureIceCell) - - primaryProduction = 0.0_RKIND - totalChlorophyll = 0.0_RKIND - netSpecificAlgalGrowthRate = 0.0_RKIND - bgridSalinityIceCell = 0.0_RKIND - bgridPorosityIceCell = 0.0_RKIND - bgridTemperatureIceCell = 0.0_RKIND + primaryProduction = 0.0_RKIND + totalChlorophyll = 0.0_RKIND + netSpecificAlgalGrowthRate = 0.0_RKIND + bgridSalinityIceCell = 0.0_RKIND + bgridPorosityIceCell = 0.0_RKIND + bgridTemperatureIceCell = 0.0_RKIND end if @@ -14764,6 +15104,34 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBiologyIce", totalVerticalBiologyIce) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBiologySnow", totalVerticalBiologySnow) call MPAS_pool_get_array(biogeochemistryPool, "totalCarbonContentCell", totalCarbonContentCell) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDiatomIce", totalVerticalDiatomIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalSmallPlanktonIce", totalVerticalSmallPlanktonIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalPhaeocystisIce", totalVerticalPhaeocystisIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalPolysaccharidsIce", totalVerticalPolysaccharidsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalLipidsIce", totalVerticalLipidsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDICIce", totalVerticalDICIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalProteinsIce", totalVerticalProteinsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalNitrateIce", totalVerticalNitrateIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalSilicateIce", totalVerticalSilicateIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAmmoniumIce", totalVerticalAmmoniumIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSIce", totalVerticalDMSIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSPpIce", totalVerticalDMSPpIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSPdIce", totalVerticalDMSPdIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalNonreactiveIce", totalVerticalNonreactiveIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalHumicsIce", totalVerticalHumicsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalParticulateIronIce", totalVerticalParticulateIronIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDissolvedIronIce", totalVerticalDissolvedIronIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDissolvedIronSnow", totalVerticalDissolvedIronSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDustIce", totalVerticalDustIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDustSnow", totalVerticalDustSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC1Ice", totalVerticalBC1Ice) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC1Snow", totalVerticalBC1Snow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC2Ice", totalVerticalBC2Ice) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC2Snow", totalVerticalBC2Snow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCIce", totalVerticalBCIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCSnow", totalVerticalBCSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) netBrineHeight = 0.0_RKIND @@ -14773,6 +15141,34 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBiologyIce = 0.0_RKIND totalVerticalBiologySnow = 0.0_RKIND totalCarbonContentCell = 0.0_RKIND + totalVerticalDiatomIce = 0.0_RKIND + totalVerticalSmallPlanktonIce = 0.0_RKIND + totalVerticalPhaeocystisIce = 0.0_RKIND + totalVerticalPolysaccharidsIce = 0.0_RKIND + totalVerticalLipidsIce = 0.0_RKIND + totalVerticalDICIce = 0.0_RKIND + totalVerticalProteinsIce = 0.0_RKIND + totalVerticalNitrateIce = 0.0_RKIND + totalVerticalSilicateIce = 0.0_RKIND + totalVerticalAmmoniumIce = 0.0_RKIND + totalVerticalDMSIce = 0.0_RKIND + totalVerticalDMSPpIce = 0.0_RKIND + totalVerticalDMSPdIce = 0.0_RKIND + totalVerticalNonreactiveIce = 0.0_RKIND + totalVerticalHumicsIce = 0.0_RKIND + totalVerticalParticulateIronIce = 0.0_RKIND + totalVerticalDissolvedIronIce = 0.0_RKIND + totalVerticalDissolvedIronSnow = 0.0_RKIND + totalVerticalBC1Ice = 0.0_RKIND + totalVerticalBC2Ice = 0.0_RKIND + totalVerticalDustIce = 0.0_RKIND + totalVerticalBC1Snow = 0.0_RKIND + totalVerticalBC2Snow = 0.0_RKIND + totalVerticalBCSnow = 0.0_RKIND + totalVerticalBCSnow = 0.0_RKIND + totalVerticalDustSnow = 0.0_RKIND + totalVerticalAlgaeCarbonIce = 0.0_RKIND + totalVerticalDOCLabileIce = 0.0_RKIND endif From 2413f42432a1e635117335c31d3bc7feef4f285d Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 24 Sep 2024 14:10:12 -0500 Subject: [PATCH 770/904] Hommexx: Temporarily work around some EAMxx-Hommexx incompatibilities. These are due to PR #6594 and break the E3SM-repo EAMxx build. Once the SCREAM and E3SM repos are unified, we can back out these workarounds. The workarounds are isolated to components/homme, keeping the commit separation of components/eamxx (SCREAM repo) and components/homme (E3SM repo) clean. --- .../homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp | 1 + components/homme/src/share/cxx/HommexxEnums.hpp | 6 ++++++ components/homme/src/share/cxx/SimulationParams.hpp | 1 + .../src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp | 1 + components/homme/src/theta-l_kokkos/prim_driver_mod.F90 | 6 ++++-- 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index b433a48c2abc..aa21f5c16f29 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -91,6 +91,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = disable_diagnostics; params.use_moisture = use_moisture; + params.moisture = params.use_moisture ? MoistDry::MOIST : MoistDry::DRY; //todo-repo-unification params.use_cpstar = use_cpstar; params.transport_alg = transport_alg; // SphereOperators parameters; preqx supports only the sphere. diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 06abbf35adbc..f41ed9d3651e 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -40,6 +40,12 @@ enum class ComparisonOp { // =================== Run parameters enums ====================== // +//todo-repo-unification Remove this enum in favor of bool +// SimulationParams::use_moisture once we change EAMxx to use +// use_moisture. Search "todo-repo-unification" for other bits of code to +// remove. +enum class MoistDry { MOIST, DRY }; + enum class ForcingAlg : int { FORCING_OFF =-1, FORCING_0 = 0, diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index 4f36962b16c3..9247f54a352c 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -24,6 +24,7 @@ struct SimulationParams TimeStepType time_step_type; bool use_moisture; + MoistDry moisture; //todo-repo-unification RemapAlg remap_alg; TestCase test_case; ForcingAlg ftype = ForcingAlg::FORCING_OFF; diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 40c4ae64dc98..e96b838f0961 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -114,6 +114,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = (bool)disable_diagnostics; params.use_moisture = (bool)use_moisture; + params.moisture = params.use_moisture ? MoistDry::MOIST : MoistDry::DRY; //todo-repo-unification params.use_cpstar = (bool)use_cpstar; params.transport_alg = transport_alg; params.theta_hydrostatic_mode = (bool)theta_hydrostatic_mode; diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index eae8544ca865..e7cc245a2bd7 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -353,12 +353,14 @@ subroutine prim_init_elements_views (elem) end subroutine prim_init_elements_views subroutine prim_init_kokkos_functors (allocate_buffer) - use iso_c_binding, only : c_int + !todo-repo-unification Remove the use of c_bool here. It's used in purely + ! F90 code. + use iso_c_binding, only : c_int, c_bool use theta_f2c_mod, only : init_functors_c, init_boundary_exchanges_c ! ! Optional Input ! - logical, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + logical(kind=c_bool), intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally integer(kind=c_int) :: ab ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, From 788eefa9b679060bbad516dd545da2c2a9fbbe7d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 24 Sep 2024 13:59:38 -0600 Subject: [PATCH 771/904] EAMxx: explicitly source the sems modules init script in mappy standalone testing env setup --- components/eamxx/scripts/machines_specs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 479d402e7090..af61c2bff5f9 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -28,7 +28,8 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q rhel8 -n 4 -gpu num=4", "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["module purge", "module load sems-cmake/3.27.9 sems-git/2.42.0 sems-gcc/11.4.0 sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 sems-parallel-netcdf/1.12.3 sems-openblas", + "mappy" : (["source /projects/sems/modulefiles/utils/sems-modules-init.sh", + "module purge", "module load sems-cmake/3.27.9 sems-git/2.42.0 sems-gcc/11.4.0 sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 sems-parallel-netcdf/1.12.3 sems-openblas", "export GATOR_INITIAL_MB=4000MB", ], ["mpicxx","mpifort","mpicc"], From b5c3c7f64d2334d5cf5015eb2c9c0e5d0e0a7a0f Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 23 Sep 2024 19:25:02 -0500 Subject: [PATCH 772/904] fix sat hist capability and add test Ressurecting a long-forgotten capability to output variable along ship/flight tracks, and adding a test to (weakly try to) ensure it doesn't break again. The code is rewritten such that it uses a different IO logic (more serial) and support is added for different types of output variables. Fixes #6543 [BFB] Note: I don't have full provenance from where the code edits are from. I recall attempting to fix this capability in July and August 2024, and I recall trying different versions of available CAM code to do so. But the final version isn't easily traceable (so it could be a mix of edits from CAM and debug edits to make it work in current EAM). For reference, here's the CAM code: https://github.com/ESCOMP/CAM/blob/cam_cesm2_1_rel/src/control/sat_hist.F90. I don't claim ownership of the code at all, and I am happy to give "authorship" to whomever claims it. --- cime_config/tests.py | 1 + .../testmods_dirs/eam/sathist_F2010/readme | 1 + .../eam/sathist_F2010/shell_commands | 3 + .../eam/sathist_F2010/user_nl_eam | 7 + components/eam/src/control/sat_hist.F90 | 261 ++++++++++++------ 5 files changed, 181 insertions(+), 92 deletions(-) create mode 100644 components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme create mode 100644 components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands create mode 100644 components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b83974..c3d7dc757409 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -207,6 +207,7 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", + "SMS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme new file mode 100644 index 000000000000..35f64eb4b9a1 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme @@ -0,0 +1 @@ +test for sat hist capability components/eam/src/control/sat_hist.F90 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands new file mode 100644 index 000000000000..0dc0f9cc72c6 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands @@ -0,0 +1,3 @@ +#!/bin/bash + +./xmlchange RUN_STARTDATE=2015-01-01 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam new file mode 100644 index 000000000000..146e2da835c5 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam @@ -0,0 +1,7 @@ +&satellite_options_nl + sathist_mfilt = 10000, + sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/sathist_master_19700410-20150419_c20150616.nc' + sathist_hfilename_spec = '%c.sathist_master_19700410-20150419_c20150616.%y-%m-%d-%s.nc' + sathist_nclosest = 1 + sathist_ntimestep = 1 + sathist_fincl = 'T', 'PS' diff --git a/components/eam/src/control/sat_hist.F90 b/components/eam/src/control/sat_hist.F90 index 64b978ac0f16..af8e330e557c 100644 --- a/components/eam/src/control/sat_hist.F90 +++ b/components/eam/src/control/sat_hist.F90 @@ -6,18 +6,20 @@ module sat_hist use perf_mod, only: t_startf, t_stopf - use shr_kind_mod, only: r8 => shr_kind_r8 + use shr_kind_mod, only: r4 => shr_kind_r4 + use shr_kind_mod, only: r8 => shr_kind_r8, cl=>shr_kind_cl use cam_logfile, only: iulog - use ppgrid, only: pcols, pver, begchunk, endchunk + use ppgrid, only: pcols, pver, pverp, begchunk, endchunk use cam_history_support, only: fieldname_lenp2, max_string_len, ptapes use spmd_utils, only: masterproc, iam use cam_abortutils, only: endrun - use pio, only: file_desc_t, iosystem_desc_t, iosystem_desc_t, var_desc_t, io_desc_t - use pio, only: pio_openfile, pio_redef, pio_enddef, pio_inq_dimid, pio_inq_varid, pio_seterrorhandling, pio_def_var + use pio, only: file_desc_t,iosystem_desc_t, var_desc_t, io_desc_t + use pio, only: pio_inq_dimid, pio_inq_varid + use pio, only: pio_seterrorhandling, pio_def_var use pio, only: pio_inq_dimlen, pio_get_att, pio_put_att, pio_get_var, pio_put_var, pio_write_darray - use pio, only: pio_real, pio_int, pio_double - use pio, only: PIO_WRITE,PIO_NOWRITE, PIO_NOERR, PIO_BCAST_ERROR, PIO_INTERNAL_ERROR, PIO_Rearr_box, PIO_GLOBAL + use pio, only: pio_real,pio_double + use pio, only: PIO_NOWRITE, PIO_NOERR, PIO_BCAST_ERROR, PIO_INTERNAL_ERROR, PIO_GLOBAL use spmd_utils, only: mpicom #ifdef SPMD use mpishorthand, only: mpichar, mpiint @@ -82,6 +84,7 @@ module sat_hist real(r8), parameter :: rad2deg = 180._r8/pi ! degrees per radian + contains !------------------------------------------------------------------------------- @@ -189,14 +192,13 @@ end subroutine sat_hist_readnl subroutine sat_hist_init use cam_pio_utils, only: cam_pio_openfile use ioFileMod, only: getfil - use spmd_utils, only: npes use time_manager, only: get_step_size use string_utils, only: to_lower, GLC implicit none character(len=max_string_len) :: locfn ! Local filename - integer :: ierr, dimid, i + integer :: ierr, dimid character(len=128) :: date_format @@ -406,17 +408,15 @@ end subroutine sat_hist_define !------------------------------------------------------------------------------- subroutine sat_hist_write( tape , nflds, nfils) - use ppgrid, only : pcols, begchunk, endchunk use phys_grid, only: phys_decomp use dyn_grid, only: dyn_decomp use cam_history_support, only : active_entry - use pio, only : pio_file_is_open - implicit none + use pio, only : pio_file_is_open, pio_syncfile type(active_entry) :: tape integer, intent(in) :: nflds integer, intent(inout) :: nfils - integer :: t, f, i, ncols, nocols + integer :: ncols, nocols integer :: ierr integer, allocatable :: col_ndxs(:) @@ -430,9 +430,13 @@ subroutine sat_hist_write( tape , nflds, nfils) real(r8),allocatable :: phs_dists(:) integer :: coldim - - integer :: io_type - logical :: has_dyn_flds + logical :: has_dyn_flds = .false. + logical :: has_phys_srf_flds = .false. + logical :: has_phys_lev_flds = .false. + logical :: has_phys_ilev_flds = .false. + logical :: has_dyn_srf_flds = .false. + logical :: has_dyn_lev_flds = .false. + logical :: has_dyn_ilev_flds = .false. if (.not.has_sat_hist) return @@ -456,13 +460,11 @@ subroutine sat_hist_write( tape , nflds, nfils) allocate( mlons(nocols) ) allocate( phs_dists(nocols) ) - has_dyn_flds = .false. - dyn_flds_loop: do f=1,nflds - if ( tape%hlist(f)%field%decomp_type == dyn_decomp ) then - has_dyn_flds = .true. - exit dyn_flds_loop - endif - enddo dyn_flds_loop + call scan_flds( tape, nflds & + , has_phys_srf_flds, has_phys_lev_flds, has_phys_ilev_flds & + , has_dyn_srf_flds, has_dyn_lev_flds, has_dyn_ilev_flds ) + + has_dyn_flds = has_dyn_srf_flds .or. has_dyn_lev_flds .or. has_dyn_ilev_flds call get_indices( obs_lats, obs_lons, ncols, nocols, has_dyn_flds, col_ndxs, chk_ndxs, & fdyn_ndxs, ldyn_ndxs, phs_owners, dyn_owners, mlats, mlons, phs_dists ) @@ -479,16 +481,35 @@ subroutine sat_hist_write( tape , nflds, nfils) call write_record_coord( tape, mlats(:), mlons(:), phs_dists(:), ncols, nfils ) - do f=1,nflds + ! dump columns of 2D fields + if (has_phys_srf_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, 1, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_srf_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, 1, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif - select case (tape%hlist(f)%field%decomp_type) - case (phys_decomp) - call dump_columns(tape%File, tape%hlist(f), nocols, nfils, col_ndxs(:), chk_ndxs(:), phs_owners(:) ) - case (dyn_decomp) - call dump_columns(tape%File, tape%hlist(f), nocols, nfils, fdyn_ndxs(:), ldyn_ndxs(:), dyn_owners(:) ) - end select + ! dump columns of 3D fields defined on mid pres levels + if (has_phys_lev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pver, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_lev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pver, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif - enddo + ! dump columns of 3D fields defined on interface pres levels + if (has_phys_ilev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pverp, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_ilev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pverp, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif deallocate( col_ndxs, chk_ndxs, fdyn_ndxs, ldyn_ndxs, phs_owners, dyn_owners ) deallocate( mlons, mlats, phs_dists ) @@ -501,92 +522,150 @@ subroutine sat_hist_write( tape , nflds, nfils) end subroutine sat_hist_write !------------------------------------------------------------------------------- - subroutine dump_columns( File, hitem, ncols, nfils, fdims, ldims, owners ) - use cam_history_support, only: field_info, hentry, hist_coords, fillvalue - use pio, only: pio_initdecomp, pio_freedecomp, pio_setframe, pio_iam_iotask, pio_setdebuglevel, pio_offset_kind + subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, owners, decomp ) + use cam_history_support, only: field_info, hentry, fillvalue + use pio, only: pio_setframe, pio_offset_kind + use spmd_utils, only: mpi_real4, mpi_real8, mpicom, mpi_sum type(File_desc_t),intent(inout) :: File - type(hentry), intent(in), target :: hitem + type(hentry), intent(in), target :: hitems(:) + integer, intent(in) :: nflds integer, intent(in) :: ncols + integer, intent(in) :: nlevs integer, intent(in) :: nfils integer, intent(in) :: fdims(:) integer, intent(in) :: ldims(:) integer, intent(in) :: owners(:) + integer, intent(in) :: decomp + type(field_info), pointer :: field type(var_desc_t) :: vardesc - type(iosystem_desc_t), pointer :: sat_iosystem - type(io_desc_t) :: iodesc - integer :: t, ierr, ndims - integer, allocatable :: dimlens(:) + integer :: ierr - real(r8), allocatable :: buf(:) - integer, allocatable :: dof(:) - integer :: i,k, cnt + real(r8) :: sbuf1d(ncols),rbuf1d(ncols) + real(r4) :: buf1d(ncols) + real(r8) :: sbuf2d(nlevs,ncols), rbuf2d(nlevs,ncols) + real(r4) :: buf2d(nlevs,ncols) + integer :: i,k,f, cnt call t_startf ('sat_hist::dump_columns') - sat_iosystem => File%iosystem - field => hitem%field - vardesc = hitem%varid(1) - - - ndims=1 - if(associated(field%mdims)) then - ndims = size(field%mdims)+1 - else if(field%numlev>1) then - ndims=2 - end if - allocate(dimlens(ndims)) - dimlens(ndims)=ncols - if(ndims>2) then - do i=1,ndims-1 - dimlens(i)=hist_coords(field%mdims(i))%dimsize - enddo - else if(field%numlev>1) then - dimlens(1) = field%numlev - end if - - - allocate( buf( product(dimlens) ) ) - allocate( dof( product(dimlens) ) ) + do f = 1,nflds + field => hitems(f)%field + + if (field%numlev==nlevs .and. field%decomp_type==decomp) then + vardesc = hitems(f)%varid(1) + + if (nlevs==1) then + sbuf1d = 0.0_r8 + rbuf1d = 0.0_r8 + do i=1,ncols + if ( iam == owners(i) ) then + sbuf1d(i) = hitems(f)%hbuf( fdims(i), 1, ldims(i) ) + endif + enddo + call mpi_allreduce(sbuf1d,rbuf1d,ncols,mpi_real8, mpi_sum, mpicom, ierr) + buf1d(:) = real(rbuf1d(:),r4) + ierr = pio_put_var(File, vardesc, (/nfils/),(/ncols/), buf1d(:)) + else + sbuf2d = 0.0_r8 + rbuf2d = 0.0_r8 + do i=1,ncols + if ( iam == owners(i) ) then + do k = 1,nlevs + sbuf2d(k,i) = hitems(f)%hbuf( fdims(i), k, ldims(i) ) + enddo + endif + enddo + call mpi_allreduce(sbuf2d,rbuf2d,ncols*nlevs,mpi_real8, mpi_sum, mpicom, ierr) + buf2d(:,:) = real(rbuf2d(:,:),r4) + ierr = pio_put_var(File, vardesc, (/1,nfils/),(/nlevs,ncols/), buf2d(:,:)) + endif - cnt = 0 - buf = fillvalue - dof = 0 + endif - do i = 1,ncols - do k = 1,field%numlev - cnt = cnt+1 - if ( iam == owners(i) ) then - buf(cnt) = hitem%hbuf( fdims(i), k, ldims(i) ) - dof(cnt) = cnt - endif - enddo enddo - call pio_setframe(File, vardesc, int(-1,kind=PIO_OFFSET_KIND)) - - call pio_initdecomp(sat_iosystem, pio_double, dimlens, dof, iodesc ) + call t_stopf ('sat_hist::dump_columns') - call pio_setframe(File, vardesc, int(nfils,kind=PIO_OFFSET_KIND)) + end subroutine dump_columns - call pio_write_darray(File, vardesc, iodesc, buf, ierr, fillval=fillvalue) +!------------------------------------------------------------------------------- +! scan the fields for possible different decompositions +!------------------------------------------------------------------------------- + subroutine scan_flds( tape, nflds & + , has_phys_srf_flds, has_phys_lev_flds, has_phys_ilev_flds & + , has_dyn_srf_flds, has_dyn_lev_flds, has_dyn_ilev_flds ) + use cam_history_support, only : active_entry + use phys_grid, only: phys_decomp + use dyn_grid, only: dyn_decomp - call pio_freedecomp(sat_iosystem, iodesc) + type(active_entry), intent(in) :: tape + integer, intent(in) :: nflds + logical, save :: flds_scanned + logical, intent(out) :: has_phys_srf_flds + logical, intent(out) :: has_phys_lev_flds + logical, intent(out) :: has_phys_ilev_flds + logical, intent(out) :: has_dyn_srf_flds + logical, intent(out) :: has_dyn_lev_flds + logical, intent(out) :: has_dyn_ilev_flds + + integer :: f + character(len=cl) :: msg1, msg2 + + if (flds_scanned) return + + do f = 1,nflds + if ( tape%hlist(f)%field%decomp_type == phys_decomp ) then + if ( tape%hlist(f)%field%numlev == 1 ) then + has_phys_srf_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pver ) then + has_phys_lev_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pverp ) then + has_phys_ilev_flds = .true. + else + call endrun('sat_hist::scan_flds numlev error : '//tape%hlist(f)%field%name) + endif + elseif ( tape%hlist(f)%field%decomp_type == dyn_decomp ) then + if ( tape%hlist(f)%field%numlev == 1 ) then + has_dyn_srf_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pver ) then + has_dyn_lev_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pverp ) then + has_dyn_ilev_flds = .true. + else + call endrun('sat_hist::scan_flds numlev error : '//tape%hlist(f)%field%name) + endif + else + call endrun('sat_hist::scan_flds decomp_type error : '//tape%hlist(f)%field%name) + endif - deallocate( buf ) - deallocate( dof ) - deallocate( dimlens ) + ! Check that the only "mdim" is the vertical coordinate. + if (has_phys_srf_flds .or. has_phys_lev_flds .or. has_phys_ilev_flds .or. & + has_dyn_srf_flds .or. has_dyn_lev_flds .or. has_dyn_ilev_flds) then + ! The mdims pointer is unassociated on a restart. The restart initialization + ! should be fixed rather than requiring the check to make sure it is associated. + if (associated(tape%hlist(f)%field%mdims)) then + if (size(tape%hlist(f)%field%mdims) > 1) then + msg1 = 'sat_hist::scan_flds mdims error :'//tape%hlist(f)%field%name + msg2 = trim(msg1)//' has mdims in addition to the vertical coordinate.'//& + new_line('a')//' This is not currently supported.' + write(iulog,*) msg2 + call endrun(msg1) + end if + end if + end if - call t_stopf ('sat_hist::dump_columns') + enddo - end subroutine dump_columns + flds_scanned = .true. + end subroutine scan_flds !------------------------------------------------------------------------------- !------------------------------------------------------------------------------- subroutine read_next_position( ncols ) - use time_manager, only: get_curr_date, get_prev_date + use time_manager, only: get_curr_date use time_manager, only: set_time_float_from_date implicit none @@ -660,7 +739,7 @@ end subroutine read_next_position !------------------------------------------------------------------------------- subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils ) - use time_manager, only: get_nstep, get_curr_date, get_curr_time + use time_manager, only: get_curr_date, get_curr_time use cam_history_support, only : active_entry implicit none type(active_entry), intent(inout) :: tape @@ -671,9 +750,8 @@ subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils real(r8), intent(in) :: mod_dists(ncols * sathist_nclosest) integer, intent(in) :: nfils - integer :: t, ierr, i + integer :: ierr, i integer :: yr, mon, day ! year, month, and day components of a date - integer :: nstep ! current timestep number integer :: ncdate ! current date in integer format [yyyymmdd] integer :: ncsec ! current time of day [seconds] integer :: ndcur ! day component of current time @@ -686,7 +764,6 @@ subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils call t_startf ('sat_hist::write_record_coord') - nstep = get_nstep() call get_curr_date(yr, mon, day, ncsec) ncdate = yr*10000 + mon*100 + day call get_curr_time(ndcur, nscur) From a2fb3e7af1b9bb39db6531b87e735240ae973c50 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Tue, 24 Sep 2024 11:11:50 -0500 Subject: [PATCH 773/904] Correct ocean conservation check settings --- .../namelist_defaults_mpaso.xml | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index fd2785616655..4f9b5d8fe4d3 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -1232,37 +1232,10 @@ .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. 'dt' 'conservationCheckOutput' .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. .true. 'conservationCheckRestart' From beca87f8cece8ad0b650ee4ee3587cd9e2b2f2b9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 25 Sep 2024 04:29:47 -0500 Subject: [PATCH 774/904] Change conservationCheckOutput to append clobber mode --- components/mpas-ocean/cime_config/buildnml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 8f5193c334be..1b413efbb863 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -1156,7 +1156,7 @@ def buildnml(case, caseroot, compname): lines.append(' filename_interval="00-01-00_00:00:00"') lines.append(' reference_time="01-01-01_00:00:00"') lines.append(' output_interval="00-00-01_00:00:00"') - lines.append(' clobber_mode="truncate"') + lines.append(' clobber_mode="append"') lines.append(' packages="conservationCheckAMPKG">') lines.append('') lines.append('') From ff5aa6f2732488223737315fa464f7ccc933628f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 25 Sep 2024 05:02:08 -0500 Subject: [PATCH 775/904] Add a stealth ERS test for conservation check AM --- cime_config/tests.py | 1 + .../testmods_dirs/mpaso/conservation_check/README | 8 ++++++++ .../testmods_dirs/mpaso/conservation_check/shell_commands | 2 ++ .../testmods_dirs/mpaso/conservation_check/user_nl_mpaso | 1 + 4 files changed, 12 insertions(+) create mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README create mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands create mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b83974..c24f24e22f72 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -266,6 +266,7 @@ "SMS_D_Ld1.T62_oQU240wLI.GMPAS-IAF-PISMF.mpaso-impl_top_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-harmonic_mean_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-upwind_advection", + "ERS_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", ) }, diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README new file mode 100644 index 000000000000..9e13f46ced04 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README @@ -0,0 +1,8 @@ +This testdef is used to test the conservation check analysis member, which +wasintroduced in MPAS-Ocean PR #4521 and has been made a stealth feature in +#6643. This test turns on the consevation check analysis member by setting: + + config_AM_conservationCheck_enable = .true. + +Then, it enables the use of MPAS-Ocean history files in testing by running +a sed command to modify env_archive.xml. \ No newline at end of file diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands new file mode 100644 index 000000000000..67789cfe69c7 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands @@ -0,0 +1,2 @@ +# include mpas-ocean outputs in testing +sed -i 's#compclass="ocn" exclude_testing="true"#compclass="ocn" exclude_testing="false"#g' env_archive.xml diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso new file mode 100644 index 000000000000..c57241a9fcf2 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso @@ -0,0 +1 @@ + config_AM_conservationCheck_enable = .true. From 663e88019b52561bb4d00963c78e05b114554272 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Wed, 25 Sep 2024 11:38:44 -0500 Subject: [PATCH 776/904] Update stealth test based on review --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index c24f24e22f72..c707f607690c 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -266,7 +266,7 @@ "SMS_D_Ld1.T62_oQU240wLI.GMPAS-IAF-PISMF.mpaso-impl_top_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-harmonic_mean_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-upwind_advection", - "ERS_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", + "ERS_Ld5_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", ) }, From a60ae936ccdefeab3be8801d46613549faf8ce3e Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 25 Sep 2024 11:54:05 -0500 Subject: [PATCH 777/904] add fixme note, handle error, and update test - Updated default filename spec - Handled error from put call - Added note to improve code - Updated test to ERS --- cime_config/tests.py | 2 +- .../eam/sathist_F2010/user_nl_eam | 2 +- components/eam/src/control/sat_hist.F90 | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index c3d7dc757409..d967ce311ef3 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -207,7 +207,7 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", - "SMS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", + "ERS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam index 146e2da835c5..bab3fc6f6192 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam @@ -1,7 +1,7 @@ &satellite_options_nl sathist_mfilt = 10000, sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/sathist_master_19700410-20150419_c20150616.nc' - sathist_hfilename_spec = '%c.sathist_master_19700410-20150419_c20150616.%y-%m-%d-%s.nc' + sathist_hfilename_spec = '%c.eam.h9.sathist.%y-%m-%d-%s.nc' sathist_nclosest = 1 sathist_ntimestep = 1 sathist_fincl = 'T', 'PS' diff --git a/components/eam/src/control/sat_hist.F90 b/components/eam/src/control/sat_hist.F90 index af8e330e557c..77b902babdfb 100644 --- a/components/eam/src/control/sat_hist.F90 +++ b/components/eam/src/control/sat_hist.F90 @@ -125,7 +125,7 @@ subroutine sat_hist_readnl(nlfile, hfilename_spec, mfilt, fincl, nhtfrq, avgflag ! set defaults sathist_track_infile = ' ' - sathist_hfilename_spec = '%c.cam' // trim(inst_suffix) // '.hs.%y-%m-%d-%s.nc' + sathist_hfilename_spec = '%c.eam.hs.' // trim(inst_suffix) // '.%y-%m-%d-%s.nc' sathist_fincl(:) = ' ' sathist_mfilt = 100000 sathist_nclosest = 1 @@ -522,6 +522,13 @@ subroutine sat_hist_write( tape , nflds, nfils) end subroutine sat_hist_write !------------------------------------------------------------------------------- +! FIXME extra work > +! dump_columns routine is doing unnecessary extra work serially +! this happens because there is an unneeded mpi_allreduce call +! and then the gathered data is written in a serial manner; this +! could be improved by avoiding the mpi_allreduce call, and then +! writing local data out using pio_write_darray, which is parallel +! FIXME extra work < subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, owners, decomp ) use cam_history_support, only: field_info, hentry, fillvalue use pio, only: pio_setframe, pio_offset_kind @@ -565,9 +572,14 @@ subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, sbuf1d(i) = hitems(f)%hbuf( fdims(i), 1, ldims(i) ) endif enddo + ! FIXME extra work: unnecessary mpi call, then serial write + ! FIXME extra work: can use pio_write_darray on local data instead call mpi_allreduce(sbuf1d,rbuf1d,ncols,mpi_real8, mpi_sum, mpicom, ierr) buf1d(:) = real(rbuf1d(:),r4) ierr = pio_put_var(File, vardesc, (/nfils/),(/ncols/), buf1d(:)) + if ( ierr /= PIO_NOERR ) then + call endrun('sat_hist::dump_columns: pio_put_var error') + endif else sbuf2d = 0.0_r8 rbuf2d = 0.0_r8 @@ -578,9 +590,14 @@ subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, enddo endif enddo + ! FIXME extra work: unnecessary mpi call, then serial write + ! FIXME extra work: can use pio_write_darray on local data instead call mpi_allreduce(sbuf2d,rbuf2d,ncols*nlevs,mpi_real8, mpi_sum, mpicom, ierr) buf2d(:,:) = real(rbuf2d(:,:),r4) ierr = pio_put_var(File, vardesc, (/1,nfils/),(/nlevs,ncols/), buf2d(:,:)) + if ( ierr /= PIO_NOERR ) then + call endrun('sat_hist::dump_columns: pio_put_var error') + endif endif endif From ed30c08cb84e478033799418fc0e089461f4582d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 11:11:23 -0600 Subject: [PATCH 778/904] EAMxx: change handling of multiple matches in atmchange/atmquery Use ANY:: in the node name string to allow multiple matches --- components/eamxx/scripts/atm_manip.py | 118 ++++++++++++++------------ components/eamxx/scripts/atmchange | 13 +-- 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index fbd8381e1727..75526676975c 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -14,7 +14,6 @@ from utils import expect, run_cmd_no_fail ATMCHANGE_SEP = "-ATMCHANGE_SEP-" -ATMCHANGE_ALL = "__ALL__" ATMCHANGE_BUFF_XML_NAME = "SCREAM_ATMCHANGE_BUFFER" ############################################################################### @@ -22,40 +21,32 @@ def apply_atm_procs_list_changes_from_buffer(case, xml): ############################################################################### atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) if atmchg_buffer: - atmchgs, atmchgs_all = unbuffer_changes(case) + atmchgs = unbuffer_changes(case) - expect (len(atmchgs)==len(atmchgs_all),"Failed to unbuffer changes from SCREAM_ATMCHANGE_BUFFER") - for chg, to_all in zip(atmchgs,atmchgs_all): + for chg in atmchgs: if "atm_procs_list" in chg: - expect (not to_all, "Makes no sense to change 'atm_procs_list' for all groups") - atm_config_chg_impl(xml, chg, all_matches=False) + atm_config_chg_impl(xml, chg) ############################################################################### def apply_non_atm_procs_list_changes_from_buffer(case, xml): ############################################################################### atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) if atmchg_buffer: - atmchgs, atmchgs_all = unbuffer_changes(case) + atmchgs = unbuffer_changes(case) - expect (len(atmchgs)==len(atmchgs_all),"Failed to unbuffer changes from SCREAM_ATMCHANGE_BUFFER") - for chg, to_all in zip(atmchgs,atmchgs_all): + for chg in atmchgs: if "atm_procs_list" not in chg: - atm_config_chg_impl(xml, chg, all_matches=to_all) + atm_config_chg_impl(xml, chg) ############################################################################### -def buffer_changes(changes, all_matches=False): +def buffer_changes(changes): ############################################################################### """ Take a list of raw changes and buffer them in the XML case settings. Raw changes are what goes to atm_config_chg_impl. """ # Commas confuse xmlchange and so need to be escaped. - if all_matches: - changes_temp = [c + ATMCHANGE_ALL for c in changes] - changes_str = ATMCHANGE_SEP.join(changes_temp).replace(",",r"\,") - else: - # changes_str += f"{ATMCHANGE_SEP}--all" - changes_str = ATMCHANGE_SEP.join(changes).replace(",",r"\,") + changes_str = ATMCHANGE_SEP.join(changes).replace(",",r"\,") run_cmd_no_fail(f"./xmlchange --append {ATMCHANGE_BUFF_XML_NAME}='{changes_str}{ATMCHANGE_SEP}'") @@ -63,18 +54,15 @@ def buffer_changes(changes, all_matches=False): def unbuffer_changes(case): ############################################################################### """ - From a case, get a list of raw changes. Returns (changes, all_matches_flag) + From a case, get and return a list of raw changes """ atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) atmchgs = [] - atmchgs_all = [] for item in atmchg_buffer.split(ATMCHANGE_SEP): if item.strip(): - atmchgs_all.append(ATMCHANGE_ALL in item) - atmchgs.append(item.replace(ATMCHANGE_ALL,"").replace(r"\,", ",").strip()) - - return atmchgs, atmchgs_all + atmchgs.append(item.replace(r"\,", ",").strip()) + return atmchgs ############################################################################### def reset_buffer(): @@ -116,20 +104,64 @@ def get_xml_nodes(xml_root, name): >>> [p.tag for p in get_parents(item, parent_map)] ['root', 'sub'] """ - expect("::::" not in name, f"Invalid xml node name format, '{name}' contains ::::") + expect('::::' not in name, + "Badly formatted node name: found '::::'") + + tokens = name.split("::") + expect (tokens[-1] != '', "Input query string ends with '::'. It should end with an actual node name") + if 'ANY' in tokens: + multiple_hits_ok = True + + # Check there's only ONE 'ANY' token + expect(tokens.count('ANY') == 1, "Invalid xml node name format, multiple 'ANY' tokens found.") + + # Split tokens list into two parts: before and after 'ANY' + before_any = tokens[:tokens.index('ANY')] + after_any = tokens[tokens.index('ANY') + 1:] + expect (after_any, "Input name should not end with ANY") + + # The case where name starts with ::ANY is delicate, since before_any=[''], and this + # trips the call to get_xml_nodes. Since ANY and ::ANY are conceptually the same, + # we set before_any=[] if before_any==[''] + if before_any == ['']: + before_any = [] + + # Call get_xml_nodes with '::'.join(before_any) to get the new root + new_root = get_xml_nodes(xml_root, '::'.join(before_any)) if before_any else [xml_root] + + # Reset xml_root to new_root for the next search + xml_root = new_root[0] + + # Use new_root to find all matches for whatever comes after 'ANY::' + name = '::'.join(after_any) + else: + multiple_hits_ok = False if name.startswith("::"): prefix = "./" # search immediate children only name = name[2:] else: - prefix = ".//" # search entire tree + prefix = ".//" # search entire tree + + # Handle case without ANY try: xpath_str = prefix + name.replace("::", "/") result = xml_root.findall(xpath_str) except SyntaxError as e: expect(False, f"Invalid syntax '{name}' -> {e}") + # Note: don't check that len(result)>0, since user may be ok with 0 matches + if not multiple_hits_ok and len(result)>1: + parent_map = create_parent_map(xml_root) + error_str = f"{name} is ambiguous. Use ANY in the node path to allow multiple matches. Matches:\n" + for node in result: + parents = get_parents(node, parent_map) + name = "::".join(e.tag for e in parents) + "::" + node.tag + error_str += " " + name + "\n" + + expect(False, error_str) + return result ############################################################################### @@ -303,7 +335,7 @@ def parse_change(change): return node_name,new_value,append_this ############################################################################### -def atm_config_chg_impl(xml_root, change, all_matches=False): +def atm_config_chg_impl(xml_root, change): ############################################################################### """ >>> xml = ''' @@ -361,7 +393,7 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): >>> ################ AMBIGUOUS CHANGE ####################### >>> atm_config_chg_impl(tree,'prop1=three') Traceback (most recent call last): - SystemExit: ERROR: prop1 is ambiguous (use --all to change all matches), matches: + SystemExit: ERROR: prop1 is ambiguous. Use ANY in the node path to allow multiple matches. Matches: root::prop1 root::sub::prop1 @@ -372,7 +404,7 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): False >>> atm_config_chg_impl(tree,'sub::prop1=one') True - >>> atm_config_chg_impl(tree,'prop1=three', all_matches=True) + >>> atm_config_chg_impl(tree,'ANY::prop1=three') True >>> [item.text for item in get_xml_nodes(tree,'prop1')] ['three', 'three'] @@ -413,16 +445,6 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): expect(len(matches) > 0, f"{node_name} did not match any items") - if len(matches) > 1 and not all_matches: - parent_map = create_parent_map(xml_root) - error_str = "" - for node in matches: - parents = get_parents(node, parent_map) - name = "::".join(e.tag for e in parents) + "::" + node.tag - error_str += " " + name + "\n" - - expect(False, f"{node_name} is ambiguous (use --all to change all matches), matches:\n{error_str}") - any_change = False for node in matches: any_change |= apply_change(xml_root, node, new_value, append_this) @@ -524,25 +546,11 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= expect (print_style in ["short","full"], f"Invalid print_style '{print_style}' for print_var. Use 'full' or 'short'.") - # Get the shortest unique repr of the var name - tokens = var.split("::") - if tokens[0]=='': - tokens.pop(0) - - while len(tokens)>1: - new_name = "::".join(tokens[1:]) - matches = get_xml_nodes(xml_root, new_name) - if len(matches) > 1: - break - else: - tokens.pop(0) - - # Get node, along with all its parents (which might be used for 'full' print style) + # Get matches matches = get_xml_nodes(xml_root,var) - expect(len(matches) == 1, f"Expected one match for {var}") - node = matches[0] - print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) + for node in matches: + print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) ############################################################################### def print_all_vars(xml_root,xml_node,parent_map,curr_namespace,full,dtype,value,valid_values,print_style,indent): diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index ff886ae83507..fab03a7993bd 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -31,7 +31,7 @@ def recreate_raw_xml_file(): create_raw_xml_file(case, caseroot) ############################################################################### -def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): +def atm_config_chg(changes, reset=False, buffer_only=False): ############################################################################### if not buffer_only: expect(os.path.exists("namelist_scream.xml"), @@ -66,7 +66,7 @@ def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): root = tree.getroot() for change in changes: - this_changed = atm_config_chg_impl(root, change, all_matches) + this_changed = atm_config_chg_impl(root, change) any_change |= this_changed @@ -74,7 +74,7 @@ def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): # NOTE: if a change is wrong (e.g., typo in param name), we are still buffering it. # We have no way of checking this, unfortunately. If you get an error that is # not just syntax, your best course of action is to run atmchange --reset. - buffer_changes(changes, all_matches=all_matches) + buffer_changes(changes) if not buffer_only: recreate_raw_xml_file() @@ -103,13 +103,6 @@ OR formatter_class=argparse.ArgumentDefaultsHelpFormatter ) - parser.add_argument( - "-a", "--all", - default=False, - dest="all_matches", - action="store_true", - help="Apply change to all entries matching the name" - ) parser.add_argument( "-r", "--reset", default=False, From b49df7116486cb6b36a200f20cbe2a3d642788da Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 11:43:04 -0600 Subject: [PATCH 779/904] EAMxx: adapt testmods to new atmchange syntax --- .../scream/internal_diagnostics_level/shell_commands | 2 +- .../testdefs/testmods_dirs/scream/small_kernels/shell_commands | 2 +- .../testmods_dirs/scream/small_kernels_p3/shell_commands | 2 +- .../testmods_dirs/scream/small_kernels_shoc/shell_commands | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands index cf3ca97f6dd3..f65ccdec105e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands @@ -1,2 +1,2 @@ -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b ./xmlchange POSTRUN_SCRIPT="$CIMEROOT/../components/eamxx/scripts/check-hashes-ers" diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands index e6773dce4199..a2678e6f00c0 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands @@ -1,2 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SMALL_KERNELS On' -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands index ea0e647fdcd3..a3ec62388f09 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands @@ -1,2 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_P3_SMALL_KERNELS On' -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands index da0429652975..ca520c108584 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands @@ -1,2 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SHOC_SMALL_KERNELS On' -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b From 25fa494d327a4b260317177ba1b4c2f6ea10f454 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 11:31:13 -0600 Subject: [PATCH 780/904] EAMxx: propagate atm_manip changes to cime-nml-tests and docs --- components/eamxx/docs/user/model_input.md | 6 +++--- components/eamxx/scripts/cime-nml-tests | 26 ++++++++++------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index a0076c02f89e..44fa8688fe0c 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -148,7 +148,7 @@ scoped name, which allows `atmchange` to uniquely identify the XML node to modif $ ./atmquery homme::number_of_subcycles namelist_defaults::atmosphere_processes::homme::number_of_subcycles: 1 $ ./atmchange number_of_subcycles=10 -ERROR: number_of_subcycles is ambiguous (use --all to change all matches), matches: +ERROR: internal_diagnostics_level is ambiguous. Use ANY in the node path to allow multiple matches. Matches: namelist_defaults::atmosphere_processes::number_of_subcycles namelist_defaults::atmosphere_processes::sc_import::number_of_subcycles namelist_defaults::atmosphere_processes::homme::number_of_subcycles @@ -167,7 +167,7 @@ $ ./atmquery homme::number_of_subcycles ``` In some cases, the user may be interested in changing _all_ nodes with a given name. In that case, -the `--all` flag can be used: +you can use 'ANY' as a node name: ```shell $ ./atmquery --grep number_of_subcycles @@ -183,7 +183,7 @@ $ ./atmquery --grep number_of_subcycles p3::number_of_subcycles: 1 rrtmgp::number_of_subcycles: 1 sc_export::number_of_subcycles: 1 -$ ./atmchange --all number_of_subcycles=3 +$ ./atmchange ANY::number_of_subcycles=3 Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. $ ./atmquery --grep number_of_subcycles atmosphere_processes::number_of_subcycles: 3 diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index b128121cf95f..529e10ece709 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -63,13 +63,13 @@ class TestBuildnml(unittest.TestCase): return cases[0] if len(cases) == 1 else cases ########################################################################### - def _get_values(self, case, name, value=None, expect_equal=True, all_matches=False): + def _get_values(self, case, name, value=None, expect_equal=True): ########################################################################### """ Queries a name, optionally checking if the value matches or does not match the argument. """ - if not all_matches: + if 'ANY' not in name: orig = run_cmd_assert_result(self, f"./atmquery {name} --value", from_dir=case) if value: if expect_equal: @@ -80,7 +80,7 @@ class TestBuildnml(unittest.TestCase): return [name] else: - output = run_cmd_assert_result(self, f"./atmquery {name} --grep", from_dir=case).splitlines() + output = run_cmd_assert_result(self, f"./atmquery {name}", from_dir=case).splitlines() names = [line.rsplit(": ", maxsplit=1)[0].strip() for line in output] values = [line.rsplit(": ", maxsplit=1)[1].strip() for line in output] @@ -96,17 +96,13 @@ class TestBuildnml(unittest.TestCase): ########################################################################### def _chg_atmconfig(self, changes, case, reset=False, expect_lost=None): ########################################################################### - changes = [(item[0], item[1], False) if len(item) == 2 else item for item in changes] - - expect_lost = reset if expect_lost is None else expect_lost + expect_lost = reset if expect_lost is None else expect_lost changes_unpacked = {} - for name, value, all_matches in changes: - all_matches_opt = "-a" if all_matches else "" - - names = self._get_values(case, name, value=value, expect_equal=False, all_matches=all_matches) + for name, value in changes: + names = self._get_values(case, name, value=value, expect_equal=False) - run_cmd_assert_result(self, f"./atmchange {all_matches_opt} {name}='{value}'", from_dir=case) + run_cmd_assert_result(self, f"./atmchange {name}='{value}'", from_dir=case) for item in names: changes_unpacked[item] = value @@ -288,21 +284,21 @@ class TestBuildnml(unittest.TestCase): def test_atmchanges_on_all_matches(self): ########################################################################### """ - Test that atmchange --all works + Test that atmchange works with ANY """ case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") - self._chg_atmconfig([("enable_precondition_checks", "false", True)], case) + self._chg_atmconfig([("ANY::enable_precondition_checks", "false")], case) ########################################################################### def test_atmchanges_on_all_matches_plus_spec(self): ########################################################################### """ - Test atmchange --all followed by an atmchange of one of them + Test atmchange with ANY followed by an atmchange of one of them """ case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") - self._chg_atmconfig([("enable_precondition_checks", "false", True), + self._chg_atmconfig([("ANY::enable_precondition_checks", "false"), ("p3::enable_precondition_checks", "true")], case) ########################################################################### From e46f04df79f2e275cbf0d307b48c768bcbb4dc39 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 11:56:23 -0600 Subject: [PATCH 781/904] EAMxx: keep -a/--all in atmchange as a deprecated option * Added example in help to show how to change all matches * Deprecated --all. If used, it will print an error and exit --- components/eamxx/scripts/atmchange | 16 ++++++++++++++++ components/eamxx/scripts/atmquery | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index fab03a7993bd..c45d7cd767a4 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -23,6 +23,13 @@ sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) from standard_script_setup import * # pylint: disable=wildcard-import from CIME.case import Case +class DeprecatedAllFlag(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + error_msg = f"Error: '{option_string}' has been deprecated and should not be used.\n" + error_msg += " If you want to change ALL matches for a given var, use 'ANY::$varname=value' syntax instead." + print(error_msg, file=sys.stderr) + sys.exit(1) + ############################################################################### def recreate_raw_xml_file(): ############################################################################### @@ -96,6 +103,9 @@ OR \033[1;32m# Change param foo to 'hi' (only works if foo is unambiguous)\033[0m > {0} foo=hi + \033[1;32m# Change all matches of param foo to 'hi'\033[0m + > {0} ANY::foo=hi + \033[1;32m# Change params foo to 'hi' and append 'there' to bar (only works if both are unambiguous)\033[0m > {0} foo=hi bar+=there """.format(pathlib.Path(args[0]).name), @@ -110,6 +120,12 @@ OR help="Forget all previous atmchanges", ) + parser.add_argument( + "-a", "--all", + action=DeprecatedAction, + help="This syntax is deprecated and should not be used. Use ANY:: scope instead." + ) + parser.add_argument( "-b", "--buffer-only", default=False, diff --git a/components/eamxx/scripts/atmquery b/components/eamxx/scripts/atmquery index f07ecefa94f1..1dbfbe5f5795 100755 --- a/components/eamxx/scripts/atmquery +++ b/components/eamxx/scripts/atmquery @@ -69,7 +69,7 @@ OR "--grep", default=False, action="store_true", - help="List all matching variables and their values.", + help="List all variables matching the input regex string and their values.", ) parser.add_argument( From 9ceaa5e9c56b78933c9600a248054f1dafa0a8fb Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 12:12:17 -0600 Subject: [PATCH 782/904] EAMxx: add example for "scoped" use of ANY in user docs --- components/eamxx/docs/user/model_input.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index 44fa8688fe0c..def86e25c80f 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -157,6 +157,7 @@ ERROR: internal_diagnostics_level is ambiguous. Use ANY in the node path to allo namelist_defaults::atmosphere_processes::physics::mac_aero_mic::tms::number_of_subcycles namelist_defaults::atmosphere_processes::physics::mac_aero_mic::shoc::number_of_subcycles namelist_defaults::atmosphere_processes::physics::mac_aero_mic::cldFraction::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::mac_aero_mic::spa::internal_diagnostics_level namelist_defaults::atmosphere_processes::physics::mac_aero_mic::p3::number_of_subcycles namelist_defaults::atmosphere_processes::physics::rrtmgp::number_of_subcycles namelist_defaults::atmosphere_processes::sc_export::number_of_subcycles @@ -199,6 +200,24 @@ $ ./atmquery --grep number_of_subcycles rrtmgp::number_of_subcycles: 3 sc_export::number_of_subcycles: 3 ``` +In addition, "ANY" can be used in a "scoped" string, to limit the set of matches: +```shell +$ ./atmchange mac_aero_mic::ANY::number_of_subcycles=1 +Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. +$ ./atmquery --grep number_of_subcycles + atmosphere_processes::number_of_subcycles: 3 + sc_import::number_of_subcycles: 3 + homme::number_of_subcycles: 3 + physics::number_of_subcycles: 3 + mac_aero_mic::number_of_subcycles: 1 + tms::number_of_subcycles: 1 + shoc::number_of_subcycles: 1 + cldFraction::number_of_subcycles: 1 + spa::number_of_subcycles: 1 + p3::number_of_subcycles: 1 + rrtmgp::number_of_subcycles: 3 + sc_export::number_of_subcycles: 3 +``` Since the XML file stores constraints on the parameter value (like its type or valid values), attempting to use the wrong type will cause an error: From de37080af685d0319b808d5b81ace840d6f3f327 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 25 Sep 2024 13:19:05 -0600 Subject: [PATCH 783/904] Revert Kokkos rrtmgp There are FAILs and big performance regressions --- components/eamxx/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 8d23ee67c432..5715e5be1745 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -212,8 +212,8 @@ endif() # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) -option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE) -option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE) if (SCREAM_RRTMGP_ENABLE_YAKL) add_definitions("-DRRTMGP_ENABLE_YAKL") endif() From 448f4f18f5b71c5cd694ce7764fcc3ea0a5bb9dd Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 25 Sep 2024 15:16:23 -0500 Subject: [PATCH 784/904] Fix sed command --- .../testmods_dirs/mpaso/conservation_check/shell_commands | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands index 67789cfe69c7..62c82542595f 100644 --- a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands @@ -1,2 +1,3 @@ # include mpas-ocean outputs in testing -sed -i 's#compclass="ocn" exclude_testing="true"#compclass="ocn" exclude_testing="false"#g' env_archive.xml +sed 's|compname="mpaso" exclude_testing="true"|compname="mpaso" exclude_testing="false"|' env_archive.xml > tmp_env_archive.xml +mv tmp_env_archive.xml env_archive.xml From bc518277cbe2034aa88cccf788b670462900877f Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 25 Sep 2024 13:31:33 -0700 Subject: [PATCH 785/904] remove DPxx dt_remap_factor setting --- components/eamxx/cime_config/namelist_defaults_scream.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 3d6a86556dee..57611fc3aa91 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -659,7 +659,6 @@ be lost if SCREAM_HACK_XML is not enabled. 2 False 2 - 1 1 6 0 From 6944e95cd34004d942d1345b32c44917e6738904 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Wed, 25 Sep 2024 15:49:47 -0500 Subject: [PATCH 786/904] Homme(xx): Flush the log before calling MPI_Abort in Homme's abortmp. This is an attempt to assure atm.log (EAM) and homme_atm.log (EAMxx) get full output when the dycore calls MPI_Abort. --- components/homme/src/share/parallel_mod.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/components/homme/src/share/parallel_mod.F90 b/components/homme/src/share/parallel_mod.F90 index 69016a8a0636..49ff71779a81 100644 --- a/components/homme/src/share/parallel_mod.F90 +++ b/components/homme/src/share/parallel_mod.F90 @@ -274,6 +274,7 @@ subroutine abortmp(string) #endif #endif character*(*) string + call flush(iulog) #ifdef CAM call endrun(string) #else From 0260e85a3afb3ad44ccb91dd87e6f27557b3c28c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 15:18:57 -0600 Subject: [PATCH 787/904] EAMxx: fix error in atmchange --- components/eamxx/scripts/atmchange | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index c45d7cd767a4..eb8de06a6931 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -122,7 +122,8 @@ OR parser.add_argument( "-a", "--all", - action=DeprecatedAction, + action=DeprecatedAllFlag, + default=argparse.SUPPRESS, help="This syntax is deprecated and should not be used. Use ANY:: scope instead." ) From fac8382406db337c15166585a75b28f79bf53adb Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 23 Sep 2024 12:01:53 -0400 Subject: [PATCH 788/904] add gh/ci eamxx standalone testing --- .github/workflows/eamxx-gh-ci-standalone.yml | 69 +++++++++++++++++++ components/eamxx/cmake/BuildCprnc.cmake | 62 ++++++++++------- .../eamxx/cmake/machine-files/ghci-oci.cmake | 13 ++++ components/eamxx/scripts/machines_specs.py | 4 ++ 4 files changed, 123 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/eamxx-gh-ci-standalone.yml diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml new file mode 100644 index 000000000000..fbbdf813581a --- /dev/null +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -0,0 +1,69 @@ +name: gh-standalone + +on: + pull_request: + branches: [ master ] + paths: + # first, yes to these + - '.github/workflows/eamxx-gh-ci-standalone.yml' + - 'cime_config/**' + - 'components/eam/**' + - 'components/eamxx/**' + - 'components/elm/**' + - 'driver-moab/**' + - 'driver-mct/**' + - 'components/homme/**' + # second, no to these + - '!components/eam/docs/**' + - '!components/eam/mkdocs.yml' + - '!components/eamxx/docs/**' + - '!components/eamxx/mkdocs.yml' + - '!components/elm/docs/**' + - '!components/elm/mkdocs.yml' + + workflow_dispatch: + +jobs: + + ci: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test: + # TODO: add opt, dbg, etc. here once we stabilize testing + # TODO: note that currently, there is a fail in atm_proc test + # TODO: components/eamxx/src/share/tests/atm_process_tests.cpp + # TODO: REQUIRE (dag.has_unmet_dependencies()); + # TODO: but only on some machines... + - sp + container: + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.2 + + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + show-progress: false + submodules: recursive + - + name: standalone + env: + SHELL: sh + run: | + # TODO: get rid of this extra line if we can? + git config --global safe.directory '*' + ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} + # TODO: add logging... + # - + # name: Artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ always() }} + # with: + # name: ${{ matrix.test }} + # path: | + # /projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log + # /projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* + # /projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* + # /projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out diff --git a/components/eamxx/cmake/BuildCprnc.cmake b/components/eamxx/cmake/BuildCprnc.cmake index 2f4f1f00a362..287956c5a9dd 100644 --- a/components/eamxx/cmake/BuildCprnc.cmake +++ b/components/eamxx/cmake/BuildCprnc.cmake @@ -8,32 +8,44 @@ include (EkatUtils) macro(BuildCprnc) - # Make sure this is built only once - if (NOT TARGET cprnc) - if (SCREAM_CIME_BUILD) - string (CONCAT MSG - "WARNING! By default, scream should not build tests in a CIME build,\n" - "and cprnc should only be built by scream in case tests are enabled.\n" - "If you explicitly requested tests to be on in a CIME build,\n" - "then you can discard this warning. Otherwise, please, contact developers.\n") - message("${MSG}") - endif() - set(BLDROOT ${PROJECT_BINARY_DIR}/externals/cprnc) - file(WRITE ${BLDROOT}/Macros.cmake - " - set(SCC ${CMAKE_C_COMPILER}) - set(SFC ${CMAKE_Fortran_COMPILER}) - set(FFLAGS \"${CMAKE_Fortran_FLAGS}\") - set(NETCDF_PATH ${NetCDF_Fortran_PATH}) - " - ) - set(SRC_ROOT ${SCREAM_BASE_DIR}/../..) - add_subdirectory(${SRC_ROOT}/cime/CIME/non_py/cprnc ${BLDROOT}) - EkatDisableAllWarning(cprnc) - - set(CPRNC_BINARY ${BLDROOT}/cprnc CACHE INTERNAL "") - + # TODO: handle this more carefully and more gracefully in the future + # TODO: For now, it is just a hack to get going... + # find cprnc defined in machine entries + set(CCSM_CPRNC $ENV{CCSM_CPRNC}) + if(EXISTS "${CCSM_CPRNC}") + message(STATUS "Path ${CCSM_CPRNC} exists, so we will use it") + set(CPRNC_BINARY ${CCSM_CPRNC} CACHE INTERNAL "") configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + else() + message(WARNING "Path ${CCSM_CPRNC} does not exist, so we will try to build it") + # Make sure this is built only once + if (NOT TARGET cprnc) + if (SCREAM_CIME_BUILD) + string (CONCAT MSG + "WARNING! By default, scream should not build tests in a CIME build,\n" + "and cprnc should only be built by scream in case tests are enabled.\n" + "If you explicitly requested tests to be on in a CIME build,\n" + "then you can discard this warning. Otherwise, please, contact developers.\n") + message("${MSG}") + endif() + set(BLDROOT ${PROJECT_BINARY_DIR}/externals/cprnc) + file(WRITE ${BLDROOT}/Macros.cmake + " + set(SCC ${CMAKE_C_COMPILER}) + set(SFC ${CMAKE_Fortran_COMPILER}) + set(FFLAGS \"${CMAKE_Fortran_FLAGS}\") + set(NETCDF_PATH ${NetCDF_Fortran_PATH}) + " + ) + set(SRC_ROOT ${SCREAM_BASE_DIR}/../..) + add_subdirectory(${SRC_ROOT}/cime/CIME/non_py/cprnc ${BLDROOT}) + EkatDisableAllWarning(cprnc) + + set(CPRNC_BINARY ${BLDROOT}/cprnc CACHE INTERNAL "") + + configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in + ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + endif() endif() endmacro() diff --git a/components/eamxx/cmake/machine-files/ghci-oci.cmake b/components/eamxx/cmake/machine-files/ghci-oci.cmake index 86a2fb1d5302..85eabbaa848a 100644 --- a/components/eamxx/cmake/machine-files/ghci-oci.cmake +++ b/components/eamxx/cmake/machine-files/ghci-oci.cmake @@ -1,2 +1,15 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() + +set(CMAKE_Fortran_FLAGS "-Wno-maybe-uninitialized -Wno-unused-dummy-argument -fallow-argument-mismatch" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-fvisibility-inlines-hidden -fmessage-length=0 -Wno-use-after-free -Wno-unused-variable -Wno-maybe-uninitialized" CACHE STRING "" FORCE) + +# TODO: figure out a better way to handle this, e.g., +# TODO: --map-by ppr:1:node:pe=1 doesn't work with mpich, +# TODO: but -map-by core:1:numa:hwthread=1 may work well? +# TODO: this will need to be handled in EKAT at some point +set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "-np") + +# TODO: hack in place to get eamxx to recognize CPRNC +# TODO: See note in BuildCprnc.cmake... +set(ENV{CCSM_CPRNC} "/usr/local/packages/bin/cprnc") diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 9536d415c0de..0b0567c3cce2 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,6 +82,10 @@ ["mpicxx","mpifort","mpicc"], "", ""), + "ghci-oci" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-oci_gnu)"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), From 89b7ade03312aac5aa8ece869cebc356919741c6 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 24 Sep 2024 11:42:17 -0400 Subject: [PATCH 789/904] fix pl semanitics --- components/eamxx/src/share/tests/atm_process_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/tests/atm_process_tests.cpp b/components/eamxx/src/share/tests/atm_process_tests.cpp index 01985c33c6d0..fce25cd28e21 100644 --- a/components/eamxx/src/share/tests/atm_process_tests.cpp +++ b/components/eamxx/src/share/tests/atm_process_tests.cpp @@ -443,7 +443,7 @@ TEST_CASE("atm_proc_dag", "") { using strvec_t = std::vector; auto params = create_test_params(); - auto p1 = params.sublist("BarBaz"); + auto& p1 = params.sublist("BarBaz"); // Make sure there's a missing piece (whatever Baz computes); p1.set("atm_procs_list",{"Bar"}); From cdaa5c4b8c4af6eeae2f11a487a185e96d7dbae6 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 24 Sep 2024 18:09:01 -0400 Subject: [PATCH 790/904] only test for eamxx, save logs --- .github/workflows/eamxx-gh-ci-standalone.yml | 36 ++++++-------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index fbbdf813581a..4a194a891062 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -6,20 +6,12 @@ on: paths: # first, yes to these - '.github/workflows/eamxx-gh-ci-standalone.yml' - - 'cime_config/**' - - 'components/eam/**' + - 'cime_config/machine/config_machines.xml' - 'components/eamxx/**' - - 'components/elm/**' - - 'driver-moab/**' - - 'driver-mct/**' - 'components/homme/**' # second, no to these - - '!components/eam/docs/**' - - '!components/eam/mkdocs.yml' - '!components/eamxx/docs/**' - '!components/eamxx/mkdocs.yml' - - '!components/elm/docs/**' - - '!components/elm/mkdocs.yml' workflow_dispatch: @@ -31,12 +23,8 @@ jobs: fail-fast: false matrix: test: - # TODO: add opt, dbg, etc. here once we stabilize testing - # TODO: note that currently, there is a fail in atm_proc test - # TODO: components/eamxx/src/share/tests/atm_process_tests.cpp - # TODO: REQUIRE (dag.has_unmet_dependencies()); - # TODO: but only on some machines... - sp + - opt container: image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.2 @@ -55,15 +43,11 @@ jobs: # TODO: get rid of this extra line if we can? git config --global safe.directory '*' ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} - # TODO: add logging... - # - - # name: Artifacts - # uses: actions/upload-artifact@v4 - # if: ${{ always() }} - # with: - # name: ${{ matrix.test }} - # path: | - # /projects/e3sm/scratch/${{ matrix.test }}*/TestStatus.log - # /projects/e3sm/scratch/${{ matrix.test }}*/bld/*.bldlog.* - # /projects/e3sm/scratch/${{ matrix.test }}*/run/*.log.* - # /projects/e3sm/scratch/${{ matrix.test }}*/run/*.cprnc.out + - + name: Artifacts + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: ${{ matrix.test }} + path: | + components/eamxx/ctest-build/*/Testing/Temporary/Last*.log From 2982d4af4f10b5a09628927bbfe54b58e8628062 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 26 Sep 2024 01:04:28 -0500 Subject: [PATCH 791/904] Update conservation_check test to remove sed command --- .../testdefs/testmods_dirs/mpaso/conservation_check/README | 6 ++++-- .../testmods_dirs/mpaso/conservation_check/shell_commands | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README index 9e13f46ced04..ef7ef254823b 100644 --- a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README @@ -4,5 +4,7 @@ wasintroduced in MPAS-Ocean PR #4521 and has been made a stealth feature in config_AM_conservationCheck_enable = .true. -Then, it enables the use of MPAS-Ocean history files in testing by running -a sed command to modify env_archive.xml. \ No newline at end of file +However, it shoudl be noted that MPAS-Ocean history files are not currently +included in E3SM testing so non-BFB results will not be detected unless one +manually changes to 'compname="mpaso" exclude_testing="false"' in the file +cime_config/config_archive.xml. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands deleted file mode 100644 index 62c82542595f..000000000000 --- a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands +++ /dev/null @@ -1,3 +0,0 @@ -# include mpas-ocean outputs in testing -sed 's|compname="mpaso" exclude_testing="true"|compname="mpaso" exclude_testing="false"|' env_archive.xml > tmp_env_archive.xml -mv tmp_env_archive.xml env_archive.xml From 67ab8bfec0492f8f1941bde87b8fc9f8fbb76ba7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 11:50:54 -0600 Subject: [PATCH 792/904] EAMxx: do not include the xml root in the list of parents Besides not adding much, when used in cime-nml-tests, as part of finding matches to atmchange, it messes up future searches --- components/eamxx/scripts/atm_manip.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index 75526676975c..d3421ef62573 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -454,7 +454,9 @@ def atm_config_chg_impl(xml_root, change): ############################################################################### def create_parent_map(root): ############################################################################### - return {c: p for p in root.iter() for c in p} + pmap = {c: p for p in root.iter() for c in p} + pmap[root] = None + return pmap ############################################################################### def get_parents(elem, parent_map): @@ -464,9 +466,11 @@ def get_parents(elem, parent_map): be the furthest ancestor, last item will be direct parent) """ results = [] - if elem in parent_map: + if elem in parent_map and parent_map[elem] is not None: parent = parent_map[elem] - results = get_parents(parent, parent_map) + [parent] + results = get_parents(parent, parent_map) + if parent_map[parent] is not None: + results.append(parent) return results From e7a3bdb65bbbdca56a559d31d35fd01db7fd3593 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 11:51:12 -0600 Subject: [PATCH 793/904] EAMxx: allow to print non-leaf XML nodes recursively in atmquery --- components/eamxx/scripts/atm_manip.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index d3421ef62573..d555c3bf1f4d 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -478,6 +478,14 @@ def get_parents(elem, parent_map): def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="invalid",indent=""): ############################################################################### + if len(node)>0: + print (f"{indent}{node.tag}:") + # This is not a leaf, so print all nested nodes + for child in node: + print_var_impl(child,parent_map,full,dtype,value,valid_values,'short',indent+" ") + + return + expect (print_style in ["short","full"], f"Invalid print_style '{print_style}' for print_var_impl. Use 'full' or 'short'.") @@ -597,6 +605,7 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, root::prop1: one sub::prop1: two """ + parent_map = create_parent_map(xml_root) if listall: print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") @@ -619,6 +628,6 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, else: for var in variables: - print_var(xml_root,parent_map,var,full,dtype,value,valid_values,"full"," ") + print_var(xml_root,parent_map,var,full,dtype,value,valid_values,"full"," ") return True From 87182bba27827f59b0bda3df94cdb6553d18a928 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 26 Sep 2024 20:03:55 +0200 Subject: [PATCH 794/904] Fix readme for the new test --- .../testdefs/testmods_dirs/mpaso/conservation_check/README | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README index ef7ef254823b..fb85b185fd47 100644 --- a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README @@ -4,7 +4,5 @@ wasintroduced in MPAS-Ocean PR #4521 and has been made a stealth feature in config_AM_conservationCheck_enable = .true. -However, it shoudl be noted that MPAS-Ocean history files are not currently -included in E3SM testing so non-BFB results will not be detected unless one -manually changes to 'compname="mpaso" exclude_testing="false"' in the file -cime_config/config_archive.xml. +However, it should be noted that MPAS-Ocean history files are not currently +included in E3SM testing so non-BFB results will not be detected. From e85bfe9237dc1a8e99fd1ed90ce6e237df4febe4 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 12:47:05 -0600 Subject: [PATCH 795/904] EAMxx: fix bug in cime-nml-tests --- components/eamxx/scripts/cime-nml-tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 529e10ece709..2ff960ca2712 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -253,7 +253,8 @@ class TestBuildnml(unittest.TestCase): Test that atmchanges via testmod are preserved """ def_mach_comp = \ - run_cmd_assert_result(self, "../CIME/Tools/list_e3sm_tests cime_tiny", from_dir=CIME_SCRIPTS_DIR).splitlines()[-1].split(".")[-1] + run_cmd_assert_result(self, "../CIME/Tools/list_e3sm_tests cime_tiny", from_dir=CIME_SCRIPTS_DIR).splitlines()[-2].split(".")[-1] + case = self._create_test(f"SMS.ne30_ne30.F2010-SCREAMv1.{def_mach_comp}.scream-scream_example_testmod_atmchange --no-build") # Check that the value match what's in the testmod From 1035a5095df2a647ea34fdfd38fc584cabde42f2 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 12:47:37 -0600 Subject: [PATCH 796/904] EAMxx: allow using ANY in atmquery as a leaf name --- components/eamxx/scripts/atm_manip.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index d555c3bf1f4d..c26eb3b3933e 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -118,7 +118,6 @@ def get_xml_nodes(xml_root, name): # Split tokens list into two parts: before and after 'ANY' before_any = tokens[:tokens.index('ANY')] after_any = tokens[tokens.index('ANY') + 1:] - expect (after_any, "Input name should not end with ANY") # The case where name starts with ::ANY is delicate, since before_any=[''], and this # trips the call to get_xml_nodes. Since ANY and ::ANY are conceptually the same, @@ -133,7 +132,7 @@ def get_xml_nodes(xml_root, name): xml_root = new_root[0] # Use new_root to find all matches for whatever comes after 'ANY::' - name = '::'.join(after_any) + name = '*' if not after_any else '::'.join(after_any) else: multiple_hits_ok = False From b5f97f8cf80e5d6be4603775657e56e0d56e658c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 12:51:03 -0600 Subject: [PATCH 797/904] EAMxx: changes for print style in atmquery Before we had two styles, 'short', and 'full'. The former only printed the leaf name, the latter printed the full scope. I changed those to 'node-name' and 'full-scope', and added a new style called 'parent-scope', which only adds one parent to the scope of the name. These types are used by default for: - node-name: when printing recursively - full-scope: when using --grep (unless the grep matches a non-leaf node, in which case the nested values are printed with node-name format) - parent-scope: the default when printing a parameter --- components/eamxx/scripts/atm_manip.py | 57 +++++++++++++-------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index c26eb3b3933e..41150d12f655 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -479,18 +479,21 @@ def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="in if len(node)>0: print (f"{indent}{node.tag}:") - # This is not a leaf, so print all nested nodes + # This is not a leaf, so print all nested nodes. for child in node: - print_var_impl(child,parent_map,full,dtype,value,valid_values,'short',indent+" ") - + # Since prints are nicely nested, use 'node-name' as print style + print_var_impl(child,{},full,dtype,value,valid_values,'node-name',indent+" ") return - expect (print_style in ["short","full"], + # print (f"printing leaf={node.tag}, len(parent_map)={len(parent_map)}") + expect (print_style in ["node-name","full-scope","parent-scope"], f"Invalid print_style '{print_style}' for print_var_impl. Use 'full' or 'short'.") - if print_style=="short": + if print_style=="node-name": # Just the inner most name name = node.tag + elif print_style=="parent-scope": + name = parent_map[node].tag + "::" + node.tag else: parents = get_parents(node, parent_map) name = "::".join(e.tag for e in parents) + "::" + node.tag @@ -538,24 +541,24 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= >>> tree = ET.fromstring(xml) >>> parent_map = create_parent_map(tree) >>> ################ Missing type data ####################### - >>> print_var(tree,parent_map,'::prop1',False,True,False,False,"short") + >>> print_var(tree,parent_map,'::prop1',False,True,False,False,"node-name") Traceback (most recent call last): SystemExit: ERROR: Error! Missing type information for prop1 - >>> print_var(tree,parent_map,'prop2',True,False,False,False,"short") + >>> print_var(tree,parent_map,'prop2',True,False,False,False,"node-name") prop2 value: 2 type: integer valid values: ['1', '2'] - >>> print_var(tree,parent_map,'prop2',False,True,False,False,"short") + >>> print_var(tree,parent_map,'prop2',False,True,False,False,"node-name") prop2: integer - >>> print_var(tree,parent_map,'prop2',False,False,True,False,"short") + >>> print_var(tree,parent_map,'prop2',False,False,True,False,"node-name") 2 - >>> print_var(tree,parent_map,'prop2',False,False,False,True,"short"," ") + >>> print_var(tree,parent_map,'prop2',False,False,False,True,"node-name"," ") prop2: ['1', '2'] """ - expect (print_style in ["short","full"], - f"Invalid print_style '{print_style}' for print_var. Use 'full' or 'short'.") + expect (print_style in ["node-name","full-scope","parent-scope"], + f"Invalid print_style '{print_style}' for print_var. Use 'node-name', 'full-scope', or 'parent-scope'.") # Get matches matches = get_xml_nodes(xml_root,var) @@ -563,20 +566,9 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= for node in matches: print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) -############################################################################### -def print_all_vars(xml_root,xml_node,parent_map,curr_namespace,full,dtype,value,valid_values,print_style,indent): -############################################################################### - - print (f"{indent}{xml_node.tag}") - for c in xml_node: - if len(c)>0: - print_all_vars(xml_root,c,parent_map,curr_namespace+c.tag+"::",full,dtype,value,valid_values,print_style,indent+" ") - else: - print_var(xml_root,parent_map,curr_namespace+c.tag,full,dtype,value,valid_values,print_style,indent+" ") - ############################################################################### def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, - dtype=False, valid_values=False, grep=False): + dtype=False, valid_values=False, grep=False,parent_map=None): ############################################################################### """ >>> xml = ''' @@ -605,28 +597,33 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, sub::prop1: two """ - parent_map = create_parent_map(xml_root) + if not parent_map: + parent_map = create_parent_map(xml_root) if listall: - print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") elif grep: for regex in variables: expect("::" not in regex, "query --grep does not support including parent info") var_re = re.compile(f'{regex}') if var_re.search(xml_root.tag): - print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") + if len(xml_root)>0: + parents = get_parents(xml_root,parent_map) + print (f"{'::'.join([p.tag for p in parents]) + '::' + xml_root.tag}:") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") else: for elem in xml_root: if len(elem)>0: - atm_query_impl(elem,variables,listall,full,value,dtype,valid_values,grep) + atm_query_impl(elem,variables,listall,full,value,dtype,valid_values,grep,parent_map) else: if var_re.search(elem.tag): nodes = get_xml_nodes(xml_root, "::"+elem.tag) expect(len(nodes) == 1, "No matches?") - print_var_impl(nodes[0],parent_map,full,dtype,value,valid_values,"full"," ") + print_var_impl(nodes[0],parent_map,full,dtype,value,valid_values,"full-scope"," ") else: for var in variables: - print_var(xml_root,parent_map,var,full,dtype,value,valid_values,"full"," ") + pmap = {} if var=='ANY' else parent_map + print_var(xml_root,pmap,var,full,dtype,value,valid_values,"parent-scope"," ") return True From 712d71fd7cc7eef3684b970f9321624d3df2b73c Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Thu, 26 Sep 2024 14:44:08 -0500 Subject: [PATCH 798/904] Zero aerosol diagnostics to prevent NaNs during bgc only runs BFB --- .../src/shared/mpas_seaice_icepack.F | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F index fa3b28c92c8f..9dc6f06ec342 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F @@ -15002,7 +15002,8 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) type(MPAS_pool_type), pointer :: & biogeochemistryPool, & - diagnostics_biogeochemistryPool + diagnostics_biogeochemistryPool, & + tracers_aggregatePool ! biogeochemistry real(kind=RKIND), dimension(:), pointer :: & @@ -15050,7 +15051,12 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBiologySnow, & bgridPorosityIceCell, & bgridSalinityIceCell, & - bgridTemperatureIceCell + bgridTemperatureIceCell, & + verticalBCTotalIceCell, & + verticalDustTotalIceCell, & + verticalBCTotalSnowCell, & + verticalDustTotalSnowCell + real(kind=RKIND), dimension(:,:,:), pointer :: & bioTracerShortwave @@ -15080,6 +15086,7 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistryPool) call MPAS_pool_get_subpool(block % structs, "diagnostics_biogeochemistry", diagnostics_biogeochemistryPool) + call MPAS_pool_get_subpool(block % structs, "tracers_aggregate", tracers_aggregatePool) if (config_use_vertical_tracers) then call MPAS_pool_get_array(biogeochemistryPool, "primaryProduction", primaryProduction) @@ -15132,6 +15139,10 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCSnow", totalVerticalBCSnow) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalBCTotalIceCell", verticalBCTotalIceCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalDustTotalIceCell", verticalDustTotalIceCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalDustTotalSnowCell", verticalDustTotalSnowCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalBCTotalSnowCell", verticalBCTotalSnowCell) netBrineHeight = 0.0_RKIND @@ -15165,10 +15176,14 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBC1Snow = 0.0_RKIND totalVerticalBC2Snow = 0.0_RKIND totalVerticalBCSnow = 0.0_RKIND - totalVerticalBCSnow = 0.0_RKIND + totalVerticalBCIce = 0.0_RKIND totalVerticalDustSnow = 0.0_RKIND totalVerticalAlgaeCarbonIce = 0.0_RKIND totalVerticalDOCLabileIce = 0.0_RKIND + verticalBCTotalIceCell = 0.0_RKIND + verticalDustTotalIceCell = 0.0_RKIND + verticalBCTotalSnowCell = 0.0_RKIND + verticalDustTotalSnowCell = 0.0_RKIND endif From fa3ff46c4a937ac59036308440dd2fa28c6185bd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 14:35:01 -0600 Subject: [PATCH 799/904] EAMxx: fix doctests and corner-cases in atm_manip --- .../eamxx/cime_config/eamxx_buildnml.py | 4 +- components/eamxx/scripts/atm_manip.py | 99 ++++++++++++++----- components/eamxx/scripts/cime-nml-tests | 31 +++--- 3 files changed, 92 insertions(+), 42 deletions(-) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 14c9cf76bea8..bf425c2279c7 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -126,7 +126,7 @@ def perform_consistency_checks(case, xml): >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':2, 'REST_OPTION':'nsteps'}) >>> perform_consistency_checks(case,xml) Traceback (most recent call last): - CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. + CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency (3 steps) incompatible with restart frequency (2 steps). Please, ensure restart happens on a step when rad is ON >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':10800, 'REST_OPTION':'nseconds'}) >>> perform_consistency_checks(case,xml) @@ -970,7 +970,7 @@ def create_input_data_list_file(case,caseroot): permissions = stat.filemode(file_stat.st_mode) except Exception as e: - raise RuntimeError(f"Error retrieving file info for '{file_path}': {e}") + raise RuntimeError(f"Error retrieving file info for '{file_path}': {e}") from e curr_user = getpass.getuser() user_info = pwd.getpwnam(curr_user) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index 41150d12f655..ae77db7e1826 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -89,11 +89,11 @@ def get_xml_nodes(xml_root, name): >>> ################ INVALID SYNTAX ####################### >>> get_xml_nodes(tree,'sub::::prop1') Traceback (most recent call last): - SystemExit: ERROR: Invalid xml node name format, 'sub::::prop1' contains :::: + SystemExit: ERROR: Invalid xml node name format, 'sub::::prop1' contains '::::' >>> ################ VALID USAGE ####################### >>> get_xml_nodes(tree,'invalid::prop1') [] - >>> [item.text for item in get_xml_nodes(tree,'prop1')] + >>> [item.text for item in get_xml_nodes(tree,'ANY::prop1')] ['one', 'two'] >>> [item.text for item in get_xml_nodes(tree,'::prop1')] ['one'] @@ -102,10 +102,10 @@ def get_xml_nodes(xml_root, name): >>> item = get_xml_nodes(tree,'prop2')[0] >>> parent_map = create_parent_map(tree) >>> [p.tag for p in get_parents(item, parent_map)] - ['root', 'sub'] + ['sub'] """ expect('::::' not in name, - "Badly formatted node name: found '::::'") + f"Invalid xml node name format, '{name}' contains '::::'") tokens = name.split("::") expect (tokens[-1] != '', "Input query string ends with '::'. It should end with an actual node name") @@ -156,7 +156,8 @@ def get_xml_nodes(xml_root, name): error_str = f"{name} is ambiguous. Use ANY in the node path to allow multiple matches. Matches:\n" for node in result: parents = get_parents(node, parent_map) - name = "::".join(e.tag for e in parents) + "::" + node.tag + name = "::".join(e.tag for e in parents) + name = node.tag if name=="" else name + "::" + node.tag error_str += " " + name + "\n" expect(False, error_str) @@ -393,8 +394,8 @@ def atm_config_chg_impl(xml_root, change): >>> atm_config_chg_impl(tree,'prop1=three') Traceback (most recent call last): SystemExit: ERROR: prop1 is ambiguous. Use ANY in the node path to allow multiple matches. Matches: - root::prop1 - root::sub::prop1 + prop1 + sub::prop1 >>> ################ VALID USAGE ####################### >>> atm_config_chg_impl(tree,'::prop1=two') @@ -405,7 +406,7 @@ def atm_config_chg_impl(xml_root, change): True >>> atm_config_chg_impl(tree,'ANY::prop1=three') True - >>> [item.text for item in get_xml_nodes(tree,'prop1')] + >>> [item.text for item in get_xml_nodes(tree,'ANY::prop1')] ['three', 'three'] >>> ################ TEST APPEND += ################# >>> atm_config_chg_impl(tree,'a+=4') @@ -465,14 +466,49 @@ def get_parents(elem, parent_map): be the furthest ancestor, last item will be direct parent) """ results = [] - if elem in parent_map and parent_map[elem] is not None: + if not is_root(elem,parent_map): parent = parent_map[elem] results = get_parents(parent, parent_map) - if parent_map[parent] is not None: + if not is_root(parent,parent_map): results.append(parent) return results +############################################################################### +def is_anchestor_of(parent,child,parent_map): +############################################################################### + """ + >>> xml = ''' + ... + ... one + ... + ... two + ... 2 + ... + ... + ... ''' + >>> import xml.etree.ElementTree as ET + >>> tree = ET.fromstring(xml) + >>> parent_map = create_parent_map(tree) + >>> sub = get_xml_nodes(tree,'sub')[0] + >>> sub_prop1 = get_xml_nodes(tree,'sub::prop1')[0] + >>> is_anchestor_of(sub,sub_prop1,parent_map) + True + >>> is_anchestor_of(sub_prop1,sub,parent_map) + False + """ + curr = child + while curr is not None: + if curr is parent: + return True + curr = parent_map[curr] + return False + +############################################################################### +def is_root (node,parent_map): +############################################################################### + return parent_map[node] is None + ############################################################################### def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="invalid",indent=""): ############################################################################### @@ -482,10 +518,9 @@ def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="in # This is not a leaf, so print all nested nodes. for child in node: # Since prints are nicely nested, use 'node-name' as print style - print_var_impl(child,{},full,dtype,value,valid_values,'node-name',indent+" ") + print_var_impl(child,{},full,dtype,value,valid_values,'node-name',indent+" ") return - # print (f"printing leaf={node.tag}, len(parent_map)={len(parent_map)}") expect (print_style in ["node-name","full-scope","parent-scope"], f"Invalid print_style '{print_style}' for print_var_impl. Use 'full' or 'short'.") @@ -493,10 +528,13 @@ def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="in # Just the inner most name name = node.tag elif print_style=="parent-scope": - name = parent_map[node].tag + "::" + node.tag + parent = parent_map[node] + name = node.tag if is_root(parent,parent_map) else parent.tag + "::" + node.tag else: parents = get_parents(node, parent_map) - name = "::".join(e.tag for e in parents) + "::" + node.tag + name = "::".join(e.tag for e in parents) + name += "::" if parents else "" + name += node.tag if full: expect ("type" in node.attrib.keys(), @@ -563,7 +601,19 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= # Get matches matches = get_xml_nodes(xml_root,var) - for node in matches: + # If ANY is in the var name, we may hit the case where one of the matches + # is a parent of another match. In this case, we want to get rid of + unique_matches = [] + for i in matches: + add_this = True + for j in matches: + if i is not j and is_anchestor_of(j,i,parent_map): + add_this = False + + if add_this: + unique_matches.append(i) + + for node in unique_matches: print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) ############################################################################### @@ -584,23 +634,22 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, >>> tree = ET.fromstring(xml) >>> vars = ['prop2','::prop1'] >>> success = atm_query_impl(tree, vars) - root::sub::prop2: 2 - root::prop1: one + sub::prop2: 2 + prop1: one >>> success = atm_query_impl(tree, [], listall=True, valid_values=True) - root + prop1: + sub: prop1: - sub - prop1: - prop2: ['1', '2'] + prop2: ['1', '2'] >>> success = atm_query_impl(tree,['prop1'], grep=True) - root::prop1: one + prop1: one sub::prop1: two """ if not parent_map: parent_map = create_parent_map(xml_root) if listall: - print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") elif grep: for regex in variables: @@ -610,7 +659,7 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, if len(xml_root)>0: parents = get_parents(xml_root,parent_map) print (f"{'::'.join([p.tag for p in parents]) + '::' + xml_root.tag}:") - print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") else: for elem in xml_root: if len(elem)>0: @@ -624,6 +673,6 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, else: for var in variables: pmap = {} if var=='ANY' else parent_map - print_var(xml_root,pmap,var,full,dtype,value,valid_values,"parent-scope"," ") + print_var(xml_root,pmap,var,full,dtype,value,valid_values,"parent-scope"," ") return True diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 2ff960ca2712..d5ed8e14f2e0 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -119,6 +119,7 @@ class TestBuildnml(unittest.TestCase): def setUp(self): ########################################################################### self._dirs_to_cleanup = [] + self._common_case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") ########################################################################### def tearDown(self): @@ -174,7 +175,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are not lost when eamxx setup is called """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("atm_log_level", "trace"), ("output_to_screen", "true")], case) @@ -192,7 +193,7 @@ class TestBuildnml(unittest.TestCase): """ Test that the append attribute for array-type params in namelist defaults works as expected """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Add testOnly proc self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) @@ -213,7 +214,7 @@ class TestBuildnml(unittest.TestCase): """ Test that var+=value syntax behaves as expected """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Append to an existing entry name = 'output_yaml_files' @@ -230,7 +231,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are lost when resetting """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("atm_log_level", "trace")], case, reset=True) @@ -240,7 +241,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are lost if SCREAM_HACK_XML=TRUE """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case run_cmd_assert_result(self, "./xmlchange SCREAM_HACK_XML=TRUE", from_dir=case) @@ -267,7 +268,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works when using 'namespace' syntax foo::bar """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("p3::enable_precondition_checks", "false")], case) @@ -277,7 +278,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works for array data """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("surf_mom_flux", "40.0,2.0")], case) @@ -287,7 +288,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works with ANY """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("ANY::enable_precondition_checks", "false")], case) @@ -297,7 +298,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchange with ANY followed by an atmchange of one of them """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("ANY::enable_precondition_checks", "false"), ("p3::enable_precondition_checks", "true")], case) @@ -308,7 +309,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) @@ -322,7 +323,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # modifying a procs list requires known processes or "_" pre/post suffixes stat, out, err = run_cmd ("./atmchange mac_aero_mic::atm_procs_list=shoc,cldfraction,spa,p3,spiderman", @@ -336,7 +337,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchange does not change buffer if syntax was wrong """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Attempting a bad change should not alter the content of SCREAM_ATMCHANGE_BUFFER old = run_cmd_no_fail ("./xmlquery --value SCREAM_ATMCHANGE_BUFFER",from_dir=case) @@ -353,7 +354,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchange errors out with invalid param names """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case stat, out, err = run_cmd ("./atmchange p3::non_existent=3",from_dir=case) expect (stat!=0,"Command './atmchange p3::non_existent=3' should have failed") @@ -364,7 +365,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm proc groups """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case out = run_cmd_no_fail ("./atmchange mac_aero_mic::atm_procs_list=shoc,_my_group_",from_dir=case) @@ -380,7 +381,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that remove atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa")], case) From c30f09b60957a6e41fa928d9a83a52929a500f75 Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Thu, 26 Sep 2024 16:30:46 -0500 Subject: [PATCH 800/904] rename ww3 obstructions/shadow files to include ".rtd." --- components/ww3/bld/build-namelist | 4 ++-- components/ww3/cime_config/buildnml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/ww3/bld/build-namelist b/components/ww3/bld/build-namelist index 314949671c4f..915e6b0bf2aa 100755 --- a/components/ww3/bld/build-namelist +++ b/components/ww3/bld/build-namelist @@ -398,8 +398,8 @@ if ($NML_TYPE eq "ww3_grid_nml") { add_default($nl, 'icedisp'); add_default($nl, 'rwndc'); - add_default($nl, 'uostfilelocal', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_local.${WAV_GRID}${WAV_SPEC}.in'"); - add_default($nl, 'uostfileshadow', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_shadow.${WAV_GRID}${WAV_SPEC}.in'"); + add_default($nl, 'uostfilelocal', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_local.${WAV_GRID}${WAV_SPEC}.rtd.in'"); + add_default($nl, 'uostfileshadow', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_shadow.${WAV_GRID}${WAV_SPEC}.rtd.in'"); } diff --git a/components/ww3/cime_config/buildnml b/components/ww3/cime_config/buildnml index 8e7cd2ea84ec..f550d0321888 100755 --- a/components/ww3/cime_config/buildnml +++ b/components/ww3/cime_config/buildnml @@ -66,8 +66,8 @@ def buildnml(case, caseroot, compname): input_list.write("mesh = {}/wav/ww3/{}.msh\n".format(din_loc_root,wav_grid)) input_list.write("stations = {}/wav/ww3/stations.txt\n".format(din_loc_root)) - input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.in\n".format(din_loc_root,wav_grid,wav_spec)) - input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.in\n".format(din_loc_root,wav_grid,wav_spec)) + input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) + input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) #-------------------------------------------------------------------- # Invoke ww3 build-namelist - output will go in $CASEBUILD/ww3conf From dd108ac849e64ad2efdc72419faa1aca23c595a3 Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Thu, 26 Sep 2024 17:39:50 -0400 Subject: [PATCH 801/904] Fix initialization of doubles --- components/elm/src/biogeochem/CH4Mod.F90 | 2 +- .../elm/src/biogeochem/VerticalProfileMod.F90 | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/components/elm/src/biogeochem/CH4Mod.F90 b/components/elm/src/biogeochem/CH4Mod.F90 index 3b3c1fa80611..a5cbd2f041c2 100644 --- a/components/elm/src/biogeochem/CH4Mod.F90 +++ b/components/elm/src/biogeochem/CH4Mod.F90 @@ -223,7 +223,7 @@ subroutine InitAllocate(this, bounds) ! Allocate module variables and data structures ! ! !USES: - use shr_infnan_mod, only: spval => shr_infnan_nan, assignment(=) + use shr_infnan_mod, only: nan => shr_infnan_nan, assignment(=) use elm_varpar , only: nlevgrnd ! ! !ARGUMENTS: diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index 3f5ced72f336..d5bd1d1c83fb 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -25,9 +25,9 @@ module VerticalProfileMod logical , public :: exponential_rooting_profile = .true. logical , public :: pftspecific_rootingprofile = .true. ! how steep profile is for root C inputs (1/ e-folding depth) (1/m) - real(r8), public :: rootprof_exp = 3. + real(r8), public :: rootprof_exp = 3._r8 ! how steep profile is for surface components (1/ e_folding depth) (1/m) - real(r8), public :: surfprof_exp = 10. + real(r8), public :: surfprof_exp = 10._r8 !----------------------------------------------------------------------- contains @@ -83,7 +83,7 @@ subroutine decomp_vertprofiles(bounds, & real(r8) :: ndep_prof_sum real(r8) :: nfixation_prof_sum real(r8) :: pdep_prof_sum - real(r8) :: delta = 1.e-10 + real(r8) :: delta = 1.e-10_r8 character(len=32) :: subname = 'decomp_vertprofiles' !----------------------------------------------------------------------- @@ -294,9 +294,9 @@ subroutine decomp_vertprofiles(bounds, & ! check to make sure integral of all profiles = 1. do fc = 1,num_soilc c = filter_soilc(fc) - ndep_prof_sum = 0. - nfixation_prof_sum = 0. - pdep_prof_sum = 0. + ndep_prof_sum = 0._r8 + nfixation_prof_sum = 0._r8 + pdep_prof_sum = 0._r8 do j = 1, nlevdecomp ndep_prof_sum = ndep_prof_sum + ndep_prof(c,j) * dzsoi_decomp(j) nfixation_prof_sum = nfixation_prof_sum + nfixation_prof(c,j) * dzsoi_decomp(j) @@ -324,10 +324,10 @@ subroutine decomp_vertprofiles(bounds, & do fp = 1,num_soilp p = filter_soilp(fp) - froot_prof_sum = 0. - croot_prof_sum = 0. - leaf_prof_sum = 0. - stem_prof_sum = 0. + froot_prof_sum = 0._r8 + croot_prof_sum = 0._r8 + leaf_prof_sum = 0._r8 + stem_prof_sum = 0._r8 do j = 1, nlevdecomp froot_prof_sum = froot_prof_sum + froot_prof(p,j) * dzsoi_decomp(j) croot_prof_sum = croot_prof_sum + croot_prof(p,j) * dzsoi_decomp(j) From 387c32722f6dbe72f8414a34b8388df671d6673a Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Thu, 26 Sep 2024 17:02:44 -0500 Subject: [PATCH 802/904] add tests to wav_developer test suite --- cime_config/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b83974..98d11caea88b 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -810,7 +810,10 @@ "e3sm_wav_developer" : { "time" : "0:45:00", "tests" : ( - "ERS.T62_oEC60to30v3_wQU225EC60to30.GMPAS-IAF-WW3", + "ERS.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "PEM.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "PET.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "SMS_D_Ln3.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", ) }, From e9c8bcf87933614d54cb1b83b55f9d4cf4db1ef3 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 26 Sep 2024 18:33:36 -0400 Subject: [PATCH 803/904] remove test from the e3sm integration test suite --- cime_config/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index d967ce311ef3..1cbf28b83974 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -207,7 +207,6 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", - "ERS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, From f4d95ed6ad7d81ebbb19dbeaf6be259de2ceac17 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 26 Sep 2024 18:18:23 -0500 Subject: [PATCH 804/904] Add missing omp private variables for data subglacial runoff --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index a2d6586a7d63..1777e1f6d1e9 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3540,6 +3540,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & !$omp parallel !$omp do schedule(runtime) & !$omp private(kmin, fracAbsorbed, fracAbsorbedRunoff, fracAbsorbedSubglacialRunoff, & + !$omp zTop, k, zBot, transmissionCoeffTop, transmissionCoeffBot, & !$omp sumSurfaceStressSquared, i, iEdge) do iCell = 1, nCells From 8c905eb15766b28ccbaf9359a0970bcba627c528 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Fri, 27 Sep 2024 11:57:47 -0500 Subject: [PATCH 805/904] Make bgc balance_check_tolerance a nml variable and increase the default --- components/elm/bld/namelist_files/namelist_definition.xml | 6 ++++++ .../elm/src/biogeochem/EcosystemBalanceCheckMod.F90 | 7 +++++-- components/elm/src/main/controlMod.F90 | 8 +++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 7063d7a11054..1ba599031709 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -139,6 +139,12 @@ If TRUE consider priority of plant to get a fraction of symbiotic N fixation and P phosphatase + +BGC balance check tolerance +Default 1.0e-7 hardwired + + Set date for beginning of adding temperature to atmospheric forcing diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index ffe1be5d909b..dab999478ebc 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -48,7 +48,8 @@ module EcosystemBalanceCheckMod implicit none save private - real(r8), parameter :: balance_check_tolerance = 1e-8_r8 + + real(r8), public :: balance_check_tolerance = 1e-7_r8 ! ! !PUBLIC MEMBER FUNCTIONS: public :: BeginColCBalance @@ -64,6 +65,8 @@ module EcosystemBalanceCheckMod public :: EndGridCBalanceAfterDynSubgridDriver public :: EndGridNBalanceAfterDynSubgridDriver public :: EndGridPBalanceAfterDynSubgridDriver + + !----------------------------------------------------------------------- contains @@ -498,7 +501,7 @@ subroutine ColNBalanceCheck(bounds, & ! here is '-' adjustment. It says that the adding to PF decomp n pools was less. end if - if (abs(col_errnb(c)) > 1e-8_r8) then + if (abs(col_errnb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 1d8f48d7cc7d..7775a59dbbf7 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -55,6 +55,7 @@ module controlMod use elm_varctl , only: const_climate_hist use elm_varctl , only: use_top_solar_rad use elm_varctl , only: snow_shape, snicar_atm_type, use_dust_snow_internal_mixing + use EcosystemBalanceCheckMod, only: balance_check_tolerance ! ! !PUBLIC TYPES: @@ -181,6 +182,10 @@ subroutine control_init( ) namelist /elm_inparm/ & NFIX_PTASE_plant + ! BGC balance check + namelist /elm_inparm/ & + balance_check_tolerance + ! For experimental manipulations namelist /elm_inparm/ & startdate_add_temperature @@ -325,7 +330,7 @@ subroutine control_init( ) namelist /elm_mosart/ & lnd_rof_coupling_nstep - + namelist /elm_inparm/ & snow_shape, snicar_atm_type, use_dust_snow_internal_mixing @@ -796,6 +801,7 @@ subroutine control_spmd() call mpi_bcast (forest_fert_exp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (ECA_Pconst_RGspin, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (NFIX_PTASE_plant, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (balance_check_tolerance, 1, MPI_REAL8, 0, mpicom, ier) call mpi_bcast (use_pheno_flux_limiter, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (startdate_add_temperature, 1, MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (startdate_add_co2, 1, MPI_CHARACTER, 0, mpicom, ier) From c1631a81ecad43dfaf4dec371aec14eb0b145ea7 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Fri, 27 Sep 2024 12:12:57 -0500 Subject: [PATCH 806/904] Remove extra blank lines --- components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index dab999478ebc..879bd6cc64d5 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -65,8 +65,6 @@ module EcosystemBalanceCheckMod public :: EndGridCBalanceAfterDynSubgridDriver public :: EndGridNBalanceAfterDynSubgridDriver public :: EndGridPBalanceAfterDynSubgridDriver - - !----------------------------------------------------------------------- contains From 9a66d99c9438922e1a6482f5eda7c9bab5038135 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 26 Sep 2024 19:24:23 -0700 Subject: [PATCH 807/904] reinstate integration test with smaller sat file --- cime_config/tests.py | 1 + .../testmods_dirs/eam/sathist_F2010/shell_commands | 2 +- .../testmods_dirs/eam/sathist_F2010/user_nl_eam | 2 +- components/eam/src/control/sat_hist.F90 | 10 ++++++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b83974..d967ce311ef3 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -207,6 +207,7 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", + "ERS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands index 0dc0f9cc72c6..92cb057059a4 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands @@ -1,3 +1,3 @@ #!/bin/bash -./xmlchange RUN_STARTDATE=2015-01-01 +./xmlchange RUN_STARTDATE=2018-01-01 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam index bab3fc6f6192..e9b723f13b88 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam @@ -1,6 +1,6 @@ &satellite_options_nl sathist_mfilt = 10000, - sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/sathist_master_19700410-20150419_c20150616.nc' + sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/satellite_profilelist_orcas_to_socrates_c190208.nc' sathist_hfilename_spec = '%c.eam.h9.sathist.%y-%m-%d-%s.nc' sathist_nclosest = 1 sathist_ntimestep = 1 diff --git a/components/eam/src/control/sat_hist.F90 b/components/eam/src/control/sat_hist.F90 index 77b902babdfb..17d2a92bc7bd 100644 --- a/components/eam/src/control/sat_hist.F90 +++ b/components/eam/src/control/sat_hist.F90 @@ -722,8 +722,14 @@ subroutine read_next_position( ncols ) call read_buffered_datetime( datetime, i ) - if ( datetime>begdatetime .and. beg_ndx<0 ) beg_ndx = i - if ( datetime>enddatetime ) exit bnds_loop + if (datetime > begdatetime .and. beg_ndx < 0) then + beg_ndx = i + end if + + if (datetime > enddatetime) then + exit bnds_loop + end if + end_ndx = i enddo bnds_loop From f46455b88ad7f60e70d4322915353e080f31a717 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Fri, 27 Sep 2024 15:23:24 -0500 Subject: [PATCH 808/904] Rename the nml variable as bgc_balance_check_tolerance --- components/elm/bld/namelist_files/namelist_definition.xml | 2 +- components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 | 2 ++ components/elm/src/main/controlMod.F90 | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 1ba599031709..959279120147 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -139,7 +139,7 @@ If TRUE consider priority of plant to get a fraction of symbiotic N fixation and P phosphatase - BGC balance check tolerance Default 1.0e-7 hardwired diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index 879bd6cc64d5..a2ad41bf6658 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -49,7 +49,9 @@ module EcosystemBalanceCheckMod save private + ! This corersponds to namelist variable bgc_balance_check_tolerance real(r8), public :: balance_check_tolerance = 1e-7_r8 + ! ! !PUBLIC MEMBER FUNCTIONS: public :: BeginColCBalance diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 7775a59dbbf7..dc7fa975dbb4 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -55,7 +55,7 @@ module controlMod use elm_varctl , only: const_climate_hist use elm_varctl , only: use_top_solar_rad use elm_varctl , only: snow_shape, snicar_atm_type, use_dust_snow_internal_mixing - use EcosystemBalanceCheckMod, only: balance_check_tolerance + use EcosystemBalanceCheckMod, only: bgc_balance_check_tolerance => balance_check_tolerance ! ! !PUBLIC TYPES: @@ -184,7 +184,7 @@ subroutine control_init( ) ! BGC balance check namelist /elm_inparm/ & - balance_check_tolerance + bgc_balance_check_tolerance ! For experimental manipulations namelist /elm_inparm/ & @@ -801,7 +801,7 @@ subroutine control_spmd() call mpi_bcast (forest_fert_exp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (ECA_Pconst_RGspin, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (NFIX_PTASE_plant, 1, MPI_LOGICAL, 0, mpicom, ier) - call mpi_bcast (balance_check_tolerance, 1, MPI_REAL8, 0, mpicom, ier) + call mpi_bcast (bgc_balance_check_tolerance, 1, MPI_REAL8, 0, mpicom, ier) call mpi_bcast (use_pheno_flux_limiter, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (startdate_add_temperature, 1, MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (startdate_add_co2, 1, MPI_CHARACTER, 0, mpicom, ier) From 39da2624d7320d55f095af72c89682aa583e7783 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 22 Aug 2023 10:09:08 +0200 Subject: [PATCH 809/904] Add SOwISC12to30E3r3 to MPAS-Ocean, MPAS-Seaice and ELM --- .../bld/namelist_files/namelist_definition.xml | 2 +- .../namelist_files/namelist_defaults_mpaso.xml | 18 ++++++++++++++++++ components/mpas-ocean/cime_config/buildnml | 14 ++++++++++++++ .../namelist_defaults_mpassi.xml | 4 ++++ components/mpas-seaice/cime_config/buildnml | 10 ++++++++++ 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 7063d7a11054..83e7fcf94906 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -1491,7 +1491,7 @@ Representative concentration pathway for future scenarios [radiative forcing at + valid_values="USGS,gx3v7,gx1v6,navy,test,tx0.1v2,tx1v1,T62,TL319,cruncep,oEC60to30v3,oEC60to30v3wLI,ECwISC30to60E1r2,EC30to60E2r2,WC14to60E2r3,WCAtl12to45E2r4,SOwISC12to60E2r4,ECwISC30to60E2r1,oRRS18to6,oRRS18to6v3,oRRS15to5,oARRM60to10,oARRM60to6,ARRM10to60E2r1,oQU480,oQU240,oQU240wLI,oQU120,oRRS30to10v3,oRRS30to10v3wLI,360x720cru,NLDASww3a,NLDAS,tx0.1v2,ICOS10,IcoswISC30E3r5,IcosXISC30E3r7,RRSwISC6to18E3r5,SOwISC12to30E3r3"> Land mask description diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 4f9b5d8fe4d3..3b93400f4e87 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -55,6 +55,7 @@ '00:02:00' '00:01:00' '00:05:00' +'00:10:00' 'split_explicit_ab2' 2 @@ -85,6 +86,7 @@ .true. .true. .true. +.true. -1.0 .false. 30.0e3 @@ -107,6 +109,7 @@ .true. .true. .true. +.true. 10.0 1000.0 1000.0 @@ -123,6 +126,7 @@ 77.0 38.5 100.0 +462.0 .false. 10.0 @@ -154,6 +158,7 @@ 5.46e07 6.83e06 3.2e09 +1.18e10 1.0 .false. 0.0 @@ -191,6 +196,7 @@ 'RossbyRadius' 'RossbyRadius' 'RossbyRadius' +'RossbyRadius' 20e3 30e3 30e3 @@ -222,6 +228,7 @@ 'N2_dependent' 'N2_dependent' 'N2_dependent' +'N2_dependent' 900.0 600.0 600.0 @@ -236,6 +243,7 @@ 600.0 600.0 600.0 +600.0 0.3 'constant' 300.0 @@ -249,6 +257,7 @@ 1.0 1.0 1.0 +1.0 0.13 1000.0 200.0 @@ -264,6 +273,7 @@ 'RossbyRadius' 'RossbyRadius' 'RossbyRadius' +'RossbyRadius' 20e3 30e3 30e3 @@ -416,6 +426,7 @@ 'pressure_only' 'pressure_only' 'pressure_only' +'pressure_only' 'Jenkins' .false. 10.0 @@ -435,6 +446,7 @@ 4.48e-3 4.48e-3 4.48e-3 +4.48e-3 1e-4 0.011 0.00295 @@ -448,6 +460,7 @@ 0.00295 0.00295 0.00295 +0.00295 3.1e-4 8.42e-5 8.42e-5 @@ -460,6 +473,7 @@ 8.42e-5 8.42e-5 8.42e-5 +8.42e-5 1.0 0.0 5e-2 @@ -492,6 +506,7 @@ 4.48e-3 4.48e-3 4.48e-3 +4.48e-3 1.0e-3 10.0 2.5e-3 @@ -579,6 +594,7 @@ '0000_00:00:02.5' '0000_00:00:01.25' '0000_00:00:05' +'0000_00:00:15' 2 .true. 2 @@ -626,6 +642,7 @@ .false. .false. .false. +.false. .false. .false. .false. @@ -1153,6 +1170,7 @@ .true. .true. .true. +.true. '0000-00-00_01:00:00' 'mocStreamfunctionOutput' .true. diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 1b413efbb863..819572f617c3 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -406,6 +406,20 @@ def buildnml(case, caseroot, compname): if ocn_ismf == 'data': data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' + elif ocn_grid == 'SOwISC12to30E3r3': + decomp_date = '20240705' + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240705.nc' + analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' + ic_date = '20240705' + ic_prefix = 'mpaso.SOwISC12to30E3r3' + if ocn_ic_mode == 'spunup': + logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") + logger.warning(" But no file available for this grid.") + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240705.n' + + #-------------------------------------------------------------------- # Set OCN_FORCING = datm_forced_restoring if restoring file is available #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml index d62797de4518..69294640ff67 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml @@ -31,6 +31,7 @@ 120.0 60.0 900.0 +1800.0 'noleap' '2000-01-01_00:00:00' 'none' @@ -87,6 +88,7 @@ 85.0 85.0 85.0 +85.0 75.0 85.0 85.0 @@ -102,6 +104,7 @@ -85.0 -85.0 -85.0 +-85.0 -85.0 -85.0 -85.0 @@ -170,6 +173,7 @@ 1 1 2 +1 true true 120 diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index c38ca08a31da..821b2d09d540 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -330,6 +330,16 @@ def buildnml(case, caseroot, compname): logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") + elif ice_grid == 'SOwISC12to30E3r3': + grid_date = '20240705' + grid_prefix = 'mpassi.SOwISC12to30E3r3' + decomp_date = '20240705' + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240705.nc' + if ice_ic_mode == 'spunup': + logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") + logger.warning(" But no file available for this grid.") + elif ice_grid == 'ICOS10': grid_date = '211015' grid_prefix = 'seaice.ICOS10' From 4ea7036711c4678361df326232b9e4fb6612dbc4 Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Fri, 9 Feb 2024 12:25:12 -0600 Subject: [PATCH 810/904] Adding M PE layout for chrysalis --- cime_config/allactive/config_pesall.xml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 3392ad71ca10..53e1dd2e3376 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -843,6 +843,29 @@ + + + + -compset WCYCL*/CRYO* -res SOwISC12to30E3r3* on 52 nodes pure-MPI, ~8.5 sypd + + 1350 + 128 + 128 + 1280 + 1920 + 1408 + + + 0 + 1280 + 1280 + 0 + 1408 + 0 + + + + From d47632e624dad3cfab3c01752e271b6668cf33d6 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 12 Jul 2024 01:27:16 -0500 Subject: [PATCH 811/904] Update file datestamps --- components/mpas-ocean/cime_config/buildnml | 8 ++++---- components/mpas-seaice/cime_config/buildnml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 819572f617c3..acc0d2b64b00 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -407,17 +407,17 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' elif ocn_grid == 'SOwISC12to30E3r3': - decomp_date = '20240705' + decomp_date = '20240718' decomp_prefix = 'partitions/mpas-o.graph.info.' - restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240705.nc' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240718.nc' analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' - ic_date = '20240705' + ic_date = '20240718' ic_prefix = 'mpaso.SOwISC12to30E3r3' if ocn_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") logger.warning(" But no file available for this grid.") if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240705.n' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240718.n' #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index 821b2d09d540..a53ee01324c5 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -331,11 +331,11 @@ def buildnml(case, caseroot, compname): logger.warning(" But no file available for this grid.") elif ice_grid == 'SOwISC12to30E3r3': - grid_date = '20240705' + grid_date = '20240718' grid_prefix = 'mpassi.SOwISC12to30E3r3' - decomp_date = '20240705' + decomp_date = '20240718' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240705.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240718.nc' if ice_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") From eb834f151fb1e13ee8da680136c230f7873d74bc Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 17 Jul 2024 13:33:10 -0500 Subject: [PATCH 812/904] Add mapping and domain files for SOwISC12to30E3r3 --- cime_config/config_grids.xml | 101 ++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 0f1c3512ec8a..6c70e7e9ba3a 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -416,6 +416,16 @@ RRSwISC6to18E3r5 + + T62 + T62 + SOwISC12to30E3r3 + rx1 + null + null + SOwISC12to30E3r3 + + TL319 TL319 @@ -656,6 +666,16 @@ RRSwISC6to18E3r5 + + TL319 + TL319 + SOwISC12to30E3r3 + JRA025 + null + null + SOwISC12to30E3r3 + + TL319 TL319 @@ -1390,6 +1410,16 @@ RRSwISC6to18E3r5 + + ne30np4.pg2 + ne30np4.pg2 + SOwISC12to30E3r3 + r05 + null + null + SOwISC12to30E3r3 + + ne0np4_northamericax4v1 r0125 @@ -2427,6 +2457,16 @@ RRSwISC6to18E3r5 + + ne30np4.pg2 + r05 + SOwISC12to30E3r3 + r05 + null + null + SOwISC12to30E3r3 + + ne30np4.pg2 r05 @@ -2738,6 +2778,7 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.T62_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240716.nc T62 is Gaussian grid: @@ -2796,6 +2837,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240716.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: @@ -2909,6 +2952,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240716.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -3227,6 +3272,13 @@ RRSwISC6to18E3r5 is a MPAS ocean grid generated with the jigsaw/compass process using a mesh density function that is roughly proportional to the Rossby radius of deformation, with 18 km gridcells at low and 6 km gridcells at high latitudes. Additionally, it has ocean in ice-shelf cavities: + + 807479 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240716.nc + SOwISC12to30E3r3 is a MPAS ocean grid generated with the jigsaw/compass process using XXXXX. Additionally, it has ocean in ice-shelf cavities: + + @@ -3263,6 +3315,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240716.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -3777,6 +3831,16 @@ cpl/gridmaps/ne30pg2/map_ne30pg2_to_RRSwISC6to18E3r5_trfvnp2.20240328.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240716.nc + + cpl/gridmaps/ne30pg3/map_ne30pg3_to_oEC60to30v3_mono.200331.nc cpl/gridmaps/ne30pg3/map_ne30pg3_to_oEC60to30v3_bilin.200331.nc @@ -4565,6 +4629,14 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_T62_traave.20240328.nc + + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240716.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240716.nc + + cpl/gridmaps/TL319/map_TL319_to_oQU240wLI_traave.20240509.nc cpl/gridmaps/TL319/map_TL319_to_oQU240wLI-nomask_trbilin.20240509.nc @@ -4717,6 +4789,14 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_TL319_traave.20240328.nc + + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240716.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240716.nc + + cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_aave.220124.nc cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_bilin.220124.nc @@ -5101,6 +5181,10 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_r05_traave.20240328.nc + + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240716.nc + + cpl/cpl6/map_EC30to60E2r2_to_r05_neareststod.220728.nc @@ -5216,6 +5300,11 @@ cpl/cpl6/map_rx1_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + + cpl/cpl6/map_JRA025_to_oQU240wLI_cstmnn.r150e300.20240516.nc cpl/cpl6/map_JRA025_to_oQU240wLI_cstmnn.r150e300.20240516.nc @@ -5311,6 +5400,11 @@ cpl/cpl6/map_JRA025_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + + cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc @@ -5402,10 +5496,15 @@ - cpl/cpl6/map_r05_to_RRSwISC6to18E3r5.cstmnn.r250e1250_58NS.20240328.nc + cpl/cpl6/map_r05_to_RRSwISC6to18E3r5_cstmnn.r250e1250_58NS.20240328.nc cpl/cpl6/map_r05_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + + cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc From 1f71f562a658709723794af1859c76e6971fe2e8 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 24 Jul 2024 15:47:58 -0500 Subject: [PATCH 813/904] Update mapping and domain files for the SOwISC12to30E3r3 mesh --- cime_config/config_grids.xml | 66 ++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 6c70e7e9ba3a..8eb10bd2ae3b 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -2778,7 +2778,7 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.T62_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240723.nc T62 is Gaussian grid: @@ -2837,8 +2837,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240716.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240723.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: @@ -2952,8 +2952,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240716.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240723.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -3273,9 +3273,9 @@ - 807479 + 807630 1 - $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240723.nc SOwISC12to30E3r3 is a MPAS ocean grid generated with the jigsaw/compass process using XXXXX. Additionally, it has ocean in ice-shelf cavities: @@ -3315,8 +3315,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240716.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240716.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240723.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -3832,13 +3832,13 @@ - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240716.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240716.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240716.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240716.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240716.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240723.nc @@ -4630,11 +4630,11 @@ - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240716.nc - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240716.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240723.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240723.nc @@ -4790,11 +4790,11 @@ - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240716.nc - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240716.nc - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240716.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240716.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240723.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240723.nc @@ -5182,7 +5182,7 @@ - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240716.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240723.nc @@ -5301,8 +5301,8 @@ - cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc - cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc @@ -5401,8 +5401,8 @@ - cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc - cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc @@ -5501,8 +5501,8 @@ - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240716.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc From 2c54f55f81fce411cb71765f360ea191e1fe226d Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 25 Jul 2024 01:49:10 -0500 Subject: [PATCH 814/904] Update datestamps (again) --- components/mpas-ocean/cime_config/buildnml | 8 ++++---- components/mpas-seaice/cime_config/buildnml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index acc0d2b64b00..57913c2cab29 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -407,17 +407,17 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' elif ocn_grid == 'SOwISC12to30E3r3': - decomp_date = '20240718' + decomp_date = '20240801' decomp_prefix = 'partitions/mpas-o.graph.info.' - restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240718.nc' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240801.nc' analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' - ic_date = '20240718' + ic_date = '20240801' ic_prefix = 'mpaso.SOwISC12to30E3r3' if ocn_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") logger.warning(" But no file available for this grid.") if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240718.n' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240801.n' #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index a53ee01324c5..ae91dea2250e 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -331,11 +331,11 @@ def buildnml(case, caseroot, compname): logger.warning(" But no file available for this grid.") elif ice_grid == 'SOwISC12to30E3r3': - grid_date = '20240718' + grid_date = '20240801' grid_prefix = 'mpassi.SOwISC12to30E3r3' - decomp_date = '20240718' + decomp_date = '20240801' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240718.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240801.nc' if ice_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") From 7aa9e1d4e7ae0488ac9d944b868a06076e43ebe5 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 8 Aug 2024 13:51:56 -0500 Subject: [PATCH 815/904] Update domain and mapping file datestamps (also again) --- cime_config/config_grids.xml | 64 ++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 8eb10bd2ae3b..b92fe082d7ea 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -2778,7 +2778,7 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.T62_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240808.nc T62 is Gaussian grid: @@ -2837,8 +2837,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240723.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: @@ -2952,8 +2952,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240723.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -3275,7 +3275,7 @@ 807630 1 - $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240808.nc SOwISC12to30E3r3 is a MPAS ocean grid generated with the jigsaw/compass process using XXXXX. Additionally, it has ocean in ice-shelf cavities: @@ -3315,8 +3315,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240723.nc - $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240723.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -3832,13 +3832,13 @@ - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240723.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240723.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240723.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240723.nc - cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240723.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240808.nc @@ -4630,11 +4630,11 @@ - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240723.nc - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc - cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240723.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240808.nc @@ -4790,11 +4790,11 @@ - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240723.nc - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240723.nc - cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240723.nc - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240723.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240808.nc @@ -5182,7 +5182,7 @@ - cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240723.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240808.nc @@ -5301,8 +5301,8 @@ - cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc - cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc @@ -5401,8 +5401,8 @@ - cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc - cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc @@ -5501,8 +5501,8 @@ - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240723.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc From a3bf76465a445097a0e816a0a24790e9038ac914 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 29 Aug 2024 08:43:18 -0500 Subject: [PATCH 816/904] Update ocn and ice datestamps (again, again) --- components/mpas-ocean/cime_config/buildnml | 8 ++++---- components/mpas-seaice/cime_config/buildnml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 57913c2cab29..d13857086194 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -407,17 +407,17 @@ def buildnml(case, caseroot, compname): data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' elif ocn_grid == 'SOwISC12to30E3r3': - decomp_date = '20240801' + decomp_date = '20240829' decomp_prefix = 'partitions/mpas-o.graph.info.' - restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240801.nc' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240829.nc' analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' - ic_date = '20240801' + ic_date = '20240829' ic_prefix = 'mpaso.SOwISC12to30E3r3' if ocn_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") logger.warning(" But no file available for this grid.") if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240801.n' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240829.n' #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index ae91dea2250e..2fee2d12b113 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -331,11 +331,11 @@ def buildnml(case, caseroot, compname): logger.warning(" But no file available for this grid.") elif ice_grid == 'SOwISC12to30E3r3': - grid_date = '20240801' + grid_date = '20240829' grid_prefix = 'mpassi.SOwISC12to30E3r3' - decomp_date = '20240801' + decomp_date = '20240829' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240801.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240829.nc' if ice_ic_mode == 'spunup': logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") From 327638728f727c19c795b51ed8a8477d66a3b252 Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Mon, 16 Sep 2024 10:57:13 -0500 Subject: [PATCH 817/904] Adding ocn/ice ICs for B-cases, fixing typo in dismf file extention --- components/mpas-ocean/cime_config/buildnml | 6 +++--- components/mpas-seaice/cime_config/buildnml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index d13857086194..e4b2f7f8b42e 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -414,10 +414,10 @@ def buildnml(case, caseroot, compname): ic_date = '20240829' ic_prefix = 'mpaso.SOwISC12to30E3r3' if ocn_ic_mode == 'spunup': - logger.warning("WARNING: The specified compset is requesting ocean ICs spunup from a G-case") - logger.warning(" But no file available for this grid.") + ic_date = '20240829' + ic_prefix = 'mpaso.SOwISC12to30E3r3.rstFromG-chrysalis' if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240829.n' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240829.nc' #-------------------------------------------------------------------- diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index 2fee2d12b113..471fbf9233f2 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -337,8 +337,8 @@ def buildnml(case, caseroot, compname): decomp_prefix = 'partitions/mpas-seaice.graph.info.' data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240829.nc' if ice_ic_mode == 'spunup': - logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") - logger.warning(" But no file available for this grid.") + grid_date = '20240829' + grid_prefix = 'mpassi.SOwISC12to30E3r3.rstFromG-chrysalis' elif ice_grid == 'ICOS10': grid_date = '211015' From 5a5e7f9c93e34059db8ea1f8007cf97886b30672 Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Fri, 27 Sep 2024 12:15:07 -0500 Subject: [PATCH 818/904] Adding default finidat for 1850 conditions with r05 land grid --- components/elm/bld/namelist_files/namelist_defaults.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index beeae7858595..cf3c7ef0b091 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -232,6 +232,13 @@ ic_tod="0" sim_year="1850" glc_nec="0" use_crop=".false." >lnd/clm2/initdata_map ic_tod="0" sim_year="2000" glc_nec="0" use_crop=".false." >lnd/clm2/initdata_map/clmi.ICRUCLM45SP.2000-01-01.ne240np4_tx0.1v2.162cd32_simyr2000_c170910.nc + + +lnd/clm2/initdata/elmi.v3-SORRM.ne30pg2_r05_SOwISC12to30E3r3.1850-01-01-00000.c20240923.nc + + + lnd/clm2/surfdata_map/surfdata_ne0np4_northamericax4v1.pg2_simyr1850_c210112_with_TOP.nc From d6bf9943ca314a693e2db46d5235e4c024ded28e Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 28 Sep 2024 15:38:48 -0400 Subject: [PATCH 819/904] try new slimmed container --- .github/workflows/eamxx-gh-ci-standalone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index 4a194a891062..f916384476aa 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -25,8 +25,10 @@ jobs: test: - sp - opt + - dbg + - cov container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.2 + image: ghcr.io/e3sm-project/containers-standalone-ghci:standalone-ghci-0.1.0 steps: - From da9951ac22463f0443b47daf3445d4a723a797e1 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 28 Sep 2024 16:15:11 -0400 Subject: [PATCH 820/904] don't run coverage test --- .github/workflows/eamxx-gh-ci-standalone.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index f916384476aa..ffb9afb02fe7 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -26,7 +26,6 @@ jobs: - sp - opt - dbg - - cov container: image: ghcr.io/e3sm-project/containers-standalone-ghci:standalone-ghci-0.1.0 From 0012b49af6326a8213b2f8b55d586e69856b1b5a Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 28 Sep 2024 18:42:11 -0500 Subject: [PATCH 821/904] add fpe test --- .github/workflows/eamxx-gh-ci-standalone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index ffb9afb02fe7..9fde89ed35bf 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -26,6 +26,7 @@ jobs: - sp - opt - dbg + - fpe container: image: ghcr.io/e3sm-project/containers-standalone-ghci:standalone-ghci-0.1.0 From b33da8c83090b4c7a468f5f04caa8c63e165e55c Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 28 Sep 2024 18:45:44 -0500 Subject: [PATCH 822/904] update cosp2 submod --- .gitmodules | 4 ++-- components/eam/src/physics/cosp2/external | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 69d56e140361..b36d90a00a42 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,8 +22,8 @@ branch = scorpio_classic [submodule "cosp2"] path = components/eam/src/physics/cosp2/external - url = git@github.com:CFMIP/COSPv2.0.git - branch = CESM_v2.1.4 + url = git@github.com:bartgol/COSPv2.0.git + branch = bartgol/fix-cosp_optical_inputs [submodule "cime"] path = cime url = git@github.com:ESMCI/cime.git diff --git a/components/eam/src/physics/cosp2/external b/components/eam/src/physics/cosp2/external index 9d910acba3e3..2deb41975faa 160000 --- a/components/eam/src/physics/cosp2/external +++ b/components/eam/src/physics/cosp2/external @@ -1 +1 @@ -Subproject commit 9d910acba3e3a3151de231184d4b109f65e28aee +Subproject commit 2deb41975faa4f5eacfc8b8f12b85acf6583d407 From fa54928852d3b6b8105c22c06bbe54ba97b09ba0 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 17:38:46 -0600 Subject: [PATCH 823/904] EAMxx: avoid allocating small views at runtime These were likely residues from when FieldLayout did not store device-friendly extents --- .../eamxx/src/share/field/field_impl.hpp | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 67753605864c..68f7fe6115a3 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -306,17 +306,10 @@ deep_copy_impl (const Field& src) { auto src_alloc_props = src.get_header().get_alloc_properties(); auto tgt_alloc_props = get_header().get_alloc_properties(); - // If a manual parallel_for is required (b/c of alloc sizes difference), - // we need to create extents (rather than just using the one in layout), - // since we don't know if we're running on host or device - using device_t = typename Field::get_device; - using exec_space = typename device_t::execution_space; - using RangePolicy = Kokkos::RangePolicy; - using extents_type = typename ekat::KokkosTypes::template view_1d; - extents_type ext ("",rank); - Kokkos::deep_copy(ext,layout.extents()); + using RangePolicy = typename KokkosTypes::RangePolicy; auto policy = RangePolicy(0,layout.size()); + auto ext = layout.extents(); switch (rank) { case 1: { @@ -324,14 +317,14 @@ deep_copy_impl (const Field& src) { auto v = get_view< ST*,HD>(); auto v_src = src.get_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { - v(idx) = v_src(idx); - }); + v(idx) = v_src(idx); + }); } else { auto v = get_strided_view< ST*,HD>(); auto v_src = src.get_strided_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { - v(idx) = v_src(idx); - }); + v(idx) = v_src(idx); + }); } } break; @@ -658,15 +651,10 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) " - x layout: " + x_l.to_string() + "\n" " - y layout: " + y_l.to_string() + "\n"); - using device_t = typename Field::get_device; - using exec_space = typename device_t::execution_space; - using RangePolicy = Kokkos::RangePolicy; + using RangePolicy = typename KokkosTypes::RangePolicy; + auto policy = RangePolicy(0,x_l.size()); - // Need to create extents (rather than just using the one in x_l), - // since we don't know if we're running on host or device - using extents_type = typename ekat::KokkosTypes::template view_1d; - extents_type ext ("",x_l.rank()); - Kokkos::deep_copy(ext,x_l.extents()); + auto ext = layout.extents(); auto policy = RangePolicy(0,x_l.size()); switch (x_l.rank()) { From cefaf92afc80be743bac613af5ca4ea5917ec691 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 19:49:52 -0600 Subject: [PATCH 824/904] EAMxx: always use output control specs for creating the rhist file name This avoid clashing of streams with same prefix, but different freq specs --- components/eamxx/src/share/io/scream_output_manager.cpp | 6 +++--- components/eamxx/src/share/io/scream_output_manager.hpp | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 0b738efe6483..b24b473d0b1d 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -400,7 +400,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) // Check if we need to open a new file if (not filespecs.is_open) { - filespecs.filename = compute_filename (control,filespecs,timestamp); + filespecs.filename = compute_filename (filespecs,timestamp); // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) setup_file(filespecs,control); } @@ -613,11 +613,11 @@ long long OutputManager::res_dep_memory_footprint () const { } std::string OutputManager:: -compute_filename (const IOControl& control, - const IOFileSpecs& file_specs, +compute_filename (const IOFileSpecs& file_specs, const util::TimeStamp& timestamp) const { auto filename = m_filename_prefix + file_specs.suffix(); + const auto& control = m_output_control; // Always add avg type and frequency info filename += "." + e2str(m_avg_type); diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index d25dd037c67f..4d7486380cfe 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -120,8 +120,7 @@ class OutputManager long long res_dep_memory_footprint () const; protected: - std::string compute_filename (const IOControl& control, - const IOFileSpecs& file_specs, + std::string compute_filename (const IOFileSpecs& file_specs, const util::TimeStamp& timestamp) const; void set_file_header(const IOFileSpecs& file_specs); From e540ae952b55dedb4a85bf3587b9c6a6faafeb73 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 19:57:28 -0600 Subject: [PATCH 825/904] EAMxx: fix how we find filename in rpointer * We must consider avg type and output freq specs, to avoid name clashing * Since rhist filename contains output control specs, no need to check that freq/freq_units/avg_type are unchanged upon restart: if rhist file is found, they are ok. --- .../eamxx/src/share/io/scream_io_utils.cpp | 77 +++++++++++-------- .../eamxx/src/share/io/scream_io_utils.hpp | 11 ++- .../src/share/io/scream_output_manager.cpp | 22 ++---- .../eamxx/src/share/io/tests/io_utils.cpp | 39 +++++++--- 4 files changed, 87 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 4a0beb813b76..9ff7a641958f 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -4,6 +4,7 @@ #include "share/util/scream_utils.hpp" #include +#include namespace scream { @@ -11,37 +12,46 @@ std::string find_filename_in_rpointer ( const std::string& filename_prefix, const bool model_restart, const ekat::Comm& comm, - const util::TimeStamp& run_t0) + const util::TimeStamp& run_t0, + const OutputAvgType avg_type, + const IOControl& control) { std::string filename; bool found = false; std::string content; std::string suffix = model_restart ? ".r." : ".rhist."; + std::string pattern_str = filename_prefix + suffix; + + // The AD will pass a default constructed control, since it doesn't know the values + // of REST_N/REST_OPTION used in the previous run. Also, model restart is *always* INSTANT. + if (model_restart) { + EKAT_REQUIRE_MSG (avg_type==OutputAvgType::Instant, + "Error! Model restart output should have INSTANT avg type.\n" + " - input avg_type: " + e2str(avg_type) + "\n"); + pattern_str += e2str(OutputAvgType::Instant) + R"(.n(step|sec|min|hour|day|month|year)s_x\d+)"; + } else { + EKAT_REQUIRE_MSG (control.output_enabled(), + "Error! When restarting an output stream, we need a valid IOControl structure.\n" + " - filename prefix: " + filename_prefix + "\n"); + pattern_str += e2str(avg_type) + "." + control.frequency_units + "_x" + std::to_string(control.frequency); + } + pattern_str += "." + run_t0.to_string() + ".nc"; + std::regex pattern (pattern_str); + if (comm.am_i_root()) { std::ifstream rpointer_file; + std::string line; rpointer_file.open("rpointer.atm"); - // If the timestamp is in the filename, then the filename ends with "S.nc", - // with S being the string representation of the timestamp - auto ts_len = run_t0.to_string().size(); - auto extract_ts = [&] (const std::string& line) -> util::TimeStamp { - auto min_size = ts_len+3; - if (line.size()>=min_size) { - auto ts_str = line.substr(line.size()-min_size,ts_len); - auto ts = util::str_to_time_stamp(ts_str); - return ts; - } else { - return util::TimeStamp(); - } - }; - - while ((rpointer_file >> line) and not found) { + while (std::getline(rpointer_file,line)) { content += line + "\n"; - found = line.find(filename_prefix+suffix) != std::string::npos && - extract_ts(line)==run_t0; - filename = line; + if (std::regex_match(line,pattern)) { + filename = line; + found = true; + break; + } } } @@ -52,18 +62,23 @@ std::string find_filename_in_rpointer ( if (not found) { broadcast_string(content,comm,comm.root_rank()); - // If the history restart file is not found, it must be because the last - // model restart step coincided with a model output step, in which case - // a restart history file is not written. - // If that's the case, *disable* output restart, by setting - // 'Restart'->'Perform Restart' = false - // in the input parameter list - EKAT_ERROR_MSG ( - "Error! Restart requested, but no restart file found in 'rpointer.atm'.\n" - " restart filename prefix: " + filename_prefix + "\n" - " restart file type: " + std::string(model_restart ? "model restart" : "history restart") + "\n" - " run t0 : " + run_t0.to_string() + "\n" - " rpointer content:\n" + content); + if (model_restart) { + EKAT_ERROR_MSG ( + "Error! Restart requested, but no model restart file found in 'rpointer.atm'.\n" + " model restart filename prefix: " + filename_prefix + "\n" + " run t0 : " + run_t0.to_string() + "\n" + " rpointer content:\n" + content + "\n\n"); + } else { + EKAT_ERROR_MSG ( + "Error! Restart requested, but no history restart file found in 'rpointer.atm'.\n" + " hist restart filename prefix: " + filename_prefix + "\n" + " run t0 : " + run_t0.to_string() + "\n" + " avg_type : " + e2str(avg_type) + "\n" + " output freq : " + std::to_string(control.frequency) + "\n" + " output freq units: " + control.frequency_units + "\n" + " rpointer content:\n" + content + "\n\n" + " Did you change output specs (avg type, freq, or freq units) across restart? If so, please, remember that it is not allowed.\n"); + } } // Have the root rank communicate the nc filename diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index efb2a4fd65bb..01bc46e1e601 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -1,6 +1,7 @@ #ifndef SCREAM_IO_UTILS_HPP #define SCREAM_IO_UTILS_HPP +#include "scream_io_control.hpp" #include "share/util/scream_time_stamp.hpp" #include @@ -59,11 +60,17 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } +// The AD will pass a default constructed control, since it doesn't know the values +// of REST_N/REST_OPTION used in the previous run +// Output streams MUST pass a valid control structure, cause we need to differentiate +// between, e.g., streams with same filename prefix, but different output freq specs std::string find_filename_in_rpointer ( - const std::string& casename, + const std::string& filename_prefix, const bool model_restart, const ekat::Comm& comm, - const util::TimeStamp& run_t0); + const util::TimeStamp& run_t0, + const OutputAvgType avg_type = OutputAvgType::Instant, + const IOControl& control = {}); struct LongNames { diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index b24b473d0b1d..7797a0f76bd9 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -171,7 +171,9 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, if (perform_history_restart) { using namespace scorpio; - auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0); + IOFileSpecs hist_restart_specs; + hist_restart_specs.ftype = FileType::HistoryRestart; + auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0,m_avg_type,m_output_control); scorpio::register_file(rhist_file,scorpio::Read); // From restart file, get the time of last write, as well as the current size of the avg sample @@ -196,22 +198,8 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // We do NOT allow changing output specs across restart. If you do want to change // any of these, you MUST start a new output stream (e.g., setting 'Perform Restart: false') - auto old_freq = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_frequency"); - EKAT_REQUIRE_MSG (old_freq == m_output_control.frequency, - "Error! Cannot change frequency when performing history restart.\n" - " - old freq: " << old_freq << "\n" - " - new freq: " << m_output_control.frequency << "\n"); - auto old_freq_units = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_frequency_units"); - EKAT_REQUIRE_MSG (old_freq_units == m_output_control.frequency_units, - "Error! Cannot change frequency units when performing history restart.\n" - " - old freq units: " << old_freq_units << "\n" - " - new freq units: " << m_output_control.frequency_units << "\n"); - auto old_avg_type = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_type"); - EKAT_REQUIRE_MSG (old_avg_type == e2str(m_avg_type), - "Error! Cannot change avg type when performing history restart.\n" - " - old avg type: " << old_avg_type + "\n" - " - new avg type: " << e2str(m_avg_type) << "\n"); - + // NOTE: we do not check that freq/freq_units/avg_type are not changed: since we used + // that info to find the correct rhist file, we already know that they match! auto old_storage_type = scorpio::get_attribute(rhist_file,"GLOBAL","file_max_storage_type"); EKAT_REQUIRE_MSG (old_storage_type == e2str(m_output_file_specs.storage.type), "Error! Cannot change file storage type when performing history restart.\n" diff --git a/components/eamxx/src/share/io/tests/io_utils.cpp b/components/eamxx/src/share/io/tests/io_utils.cpp index 77779307a720..b8187f31c3f4 100644 --- a/components/eamxx/src/share/io/tests/io_utils.cpp +++ b/components/eamxx/src/share/io/tests/io_utils.cpp @@ -9,6 +9,9 @@ TEST_CASE ("find_filename_in_rpointer") { using namespace scream; + constexpr auto AVG = OutputAvgType::Average; + constexpr auto INST = OutputAvgType::Instant; + ekat::Comm comm(MPI_COMM_WORLD); util::TimeStamp t0({2023,9,7},{12,0,0}); @@ -17,21 +20,33 @@ TEST_CASE ("find_filename_in_rpointer") { // Create a dummy rpointer std::ofstream rpointer ("rpointer.atm"); - rpointer << "foo.r." + t0.to_string() + ".nc\n"; - rpointer << "bar2.rhist." + t0.to_string() + ".nc\n"; - rpointer << "bar.rhist." + t0.to_string() + ".nc\n"; - rpointer.close(); + IOControl foo_c, bar_c, bar2_c; + foo_c.frequency = 3; foo_c.frequency_units = "nsteps"; + bar_c.frequency = 1; bar_c.frequency_units = "ndays"; + bar2_c.frequency = 6; bar2_c.frequency_units = "nhours"; - // Now test find_filename_in_rpointer with different inputs + std::string foo_fname = "foo.r.INSTANT.nsteps_x3." + t0.to_string() + ".nc"; + std::string bar_fname = "bar.rhist.AVERAGE.ndays_x1." + t0.to_string() + ".nc"; + std::string bar2_fname = "bar.rhist.AVERAGE.nhours_x6." + t0.to_string() + ".nc"; - REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0)); // wrong prefix - REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t1)); // wrong timestamp - REQUIRE_THROWS (find_filename_in_rpointer("bar",true, comm,t0)); // bar is not model restart - REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0)); // foo is model restart + rpointer << foo_fname<< "\n"; + rpointer << bar_fname<< "\n"; + rpointer << bar2_fname << "\n"; + rpointer.close(); - REQUIRE (find_filename_in_rpointer("bar", false,comm,t0)==("bar.rhist."+t0.to_string()+".nc")); - REQUIRE (find_filename_in_rpointer("bar2",false,comm,t0)==("bar2.rhist."+t0.to_string()+".nc")); - REQUIRE (find_filename_in_rpointer("foo", true, comm,t0)==("foo.r."+t0.to_string()+".nc")); + // Now test find_filename_in_rpointer with different inputs + REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0,AVG)); // missing control (needed for rhist files) + REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0,AVG,bar_c)); // wrong prefix + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t1,AVG,bar_c)); // wrong timestamp + REQUIRE_THROWS (find_filename_in_rpointer("bar",true, comm,t0,AVG,bar_c)); // bar is not model restart + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t0,INST,bar_c)); // wrong avg type + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t0,INST,bar2_c)); // wrong freq specs + REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0,INST,foo_c)); // foo is model restart + REQUIRE_THROWS (find_filename_in_rpointer("foo",true,comm,t0,AVG)); // model restart MUST be INSTANT + + REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar_c)==bar_fname); + REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar2_c)==bar2_fname); + REQUIRE (find_filename_in_rpointer("foo",true, comm,t0)==foo_fname); } TEST_CASE ("io_control") { From f8705dfa43cd16256b8f1bd51a644b996d71d497 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 9 Sep 2024 11:16:26 -0600 Subject: [PATCH 826/904] EAMxx: improve error message in case rhist file is not found --- components/eamxx/src/share/io/scream_io_utils.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 9ff7a641958f..3d79184bac9d 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -77,7 +77,11 @@ std::string find_filename_in_rpointer ( " output freq : " + std::to_string(control.frequency) + "\n" " output freq units: " + control.frequency_units + "\n" " rpointer content:\n" + content + "\n\n" - " Did you change output specs (avg type, freq, or freq units) across restart? If so, please, remember that it is not allowed.\n"); + " Did you change output specs (avg type, freq, or freq units) across restart? If so, please, remember that it is not allowed.\n" + " It is also possible you are using a rhist file create before commit 6b7d441330d. That commit changed how rhist file names\n" + " are formed. In particular, we no longer use INSTANT.${REST_OPTION}_x${REST_N}, but we use the avg type, and freq/freq_option\n" + " of the output stream (to avoid name clashes if 2 streams only differ for one of those). If you want to use your rhist file,\n" + " please rename it, so that the avg-type, freq, and freq_option reflect those of the output stream.\n"); } } From 632faeb0ff03a1f012b658de68d4b11f7c6ced45 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Mon, 30 Sep 2024 14:53:49 -0500 Subject: [PATCH 827/904] Use same balance_check_tolerance for C and P --- components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index a2ad41bf6658..1a9b59f776ea 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -281,7 +281,7 @@ subroutine ColCBalanceCheck(bounds, & end if ! check for significant errors - if (abs(col_errcb(c)) > 1e-8_r8) then + if (abs(col_errcb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if @@ -734,7 +734,7 @@ subroutine ColPBalanceCheck(bounds, & col_errpb(c) = (col_pinputs(c) - col_poutputs(c))*dt - & (col_endpb(c) - col_begpb(c)) - if (abs(col_errpb(c)) > 1e-8_r8) then + if (abs(col_errpb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if From e6c67b9fd399f9b30b93c164fbd99f9af5c70998 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Mon, 30 Sep 2024 16:48:12 -0400 Subject: [PATCH 828/904] Revert "new" craygnuamdgpu build to libfabric/1.15.2.0 to avoid performance bug. --- cime_config/machines/config_machines.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index e2d4a50cab55..88f217e4e97b 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1140,6 +1140,7 @@ PrgEnv-gnu cpe/24.07 + libfabric/1.15.2.0 craype-accel-amd-gfx90a rocm/6.2.0 libunwind @@ -1196,7 +1197,7 @@ - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.10/install/craygnuamdgppu/cpe-24.07/libfabric-1.15.2.0; else echo "$ADIOS2_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} From f8dfc6e3eaef99d5fefe0f1af234e4e71969887a Mon Sep 17 00:00:00 2001 From: Erin Thomas Date: Tue, 1 Oct 2024 09:22:15 -0500 Subject: [PATCH 829/904] add rtd to mesh file name --- components/ww3/bld/build-namelist | 2 +- components/ww3/cime_config/buildnml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/ww3/bld/build-namelist b/components/ww3/bld/build-namelist index 915e6b0bf2aa..092dab3ffba5 100755 --- a/components/ww3/bld/build-namelist +++ b/components/ww3/bld/build-namelist @@ -363,7 +363,7 @@ if ($NML_TYPE eq "ww3_grid") { add_default($nl, 'grid%dmin'); add_default($nl, 'unst%sf'); - add_default($nl, 'unst%filename', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/${WAV_GRID}.msh'"); + add_default($nl, 'unst%filename', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/${WAV_GRID}_rtd.msh'"); add_default($nl, 'unst%idf'); add_default($nl, 'unst%idla'); add_default($nl, 'unst%idfm'); diff --git a/components/ww3/cime_config/buildnml b/components/ww3/cime_config/buildnml index f550d0321888..3fe5d79d3f8e 100755 --- a/components/ww3/cime_config/buildnml +++ b/components/ww3/cime_config/buildnml @@ -64,7 +64,7 @@ def buildnml(case, caseroot, compname): with open(os.path.join(casebuild, "ww3.input_data_list"), "w") as input_list: - input_list.write("mesh = {}/wav/ww3/{}.msh\n".format(din_loc_root,wav_grid)) + input_list.write("mesh = {}/wav/ww3/{}_rtd.msh\n".format(din_loc_root,wav_grid)) input_list.write("stations = {}/wav/ww3/stations.txt\n".format(din_loc_root)) input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) From a01889ed514de3b97d25b8ee0c5a8391f1d8ef51 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 1 Oct 2024 08:44:27 -0600 Subject: [PATCH 830/904] EAMxx: fix compilation error --- components/eamxx/src/share/field/field_impl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 68f7fe6115a3..bfc0d44e99a2 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -654,9 +654,8 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) using RangePolicy = typename KokkosTypes::RangePolicy; auto policy = RangePolicy(0,x_l.size()); - auto ext = layout.extents(); + auto ext = x_l.extents(); - auto policy = RangePolicy(0,x_l.size()); switch (x_l.rank()) { case 0: { From 51b4bf2034ae3fea6d5da7ece3030bb313024b13 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 1 Oct 2024 09:15:46 -0600 Subject: [PATCH 831/904] EAMxx: fix handling of ranks in filename Do not expose option in YAML files anymore. Instead: - for standalone: always add ranks to filename - for CIME tests: never add ranks to filename --- components/eamxx/src/share/io/scream_io_utils.cpp | 4 ++++ .../eamxx/src/share/io/scream_output_manager.cpp | 13 ++++--------- .../output.yaml | 1 - .../homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml | 1 - .../physics_only/mam/optics_rrtmgp/output.yaml | 1 - .../physics_only/shoc_p3_nudging/CMakeLists.txt | 2 -- .../physics_only/shoc_p3_nudging/output.yaml | 1 - .../shoc_p3_nudging/output_remapped.yaml | 1 - .../tests/single-process/mam/optics/output.yaml | 1 - 9 files changed, 8 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 3d79184bac9d..2d5e0a840d09 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -2,6 +2,7 @@ #include "share/io/scream_scorpio_interface.hpp" #include "share/util/scream_utils.hpp" +#include "share/scream_config.hpp" #include #include @@ -35,6 +36,9 @@ std::string find_filename_in_rpointer ( " - filename prefix: " + filename_prefix + "\n"); pattern_str += e2str(avg_type) + "." + control.frequency_units + "_x" + std::to_string(control.frequency); } + if (is_scream_standalone()) { + pattern_str += ".np" + std::to_string(comm.size()); + } pattern_str += "." + run_t0.to_string() + ".nc"; std::regex pattern (pattern_str); diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 7797a0f76bd9..4f35827ba3f4 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -611,8 +611,10 @@ compute_filename (const IOFileSpecs& file_specs, filename += "." + e2str(m_avg_type); filename += "." + control.frequency_units+ "_x" + std::to_string(control.frequency); - // Optionally, add number of mpi ranks (useful mostly in unit tests, to run multiple MPI configs in parallel) - if (m_params.get("MPI Ranks in Filename")) { + // For standalone EAMxx, we may have 2+ versions of the same test running with two + // different choices of ranks. To avoid name clashing for the output files, + // add the comm size to the output file name. + if (is_scream_standalone()) { filename += ".np" + std::to_string(m_io_comm.size()); } @@ -667,7 +669,6 @@ set_params (const ekat::ParameterList& params, m_filename_prefix = m_params.get("filename_prefix"); // Hard code some parameters in case we access them later - m_params.set("MPI Ranks in Filename",false); m_params.set("Floating Point Precision","real"); } else { auto avg_type = m_params.get("Averaging Type"); @@ -703,12 +704,6 @@ set_params (const ekat::ParameterList& params, "Error! Invalid/unsupported value for 'Floating Point Precision'.\n" " - input value: " + prec + "\n" " - supported values: float, single, double, real\n"); - - // If not set, hard code to false for CIME cases, and true for standalone, - // since standalone may be running multiple versions of the same test at once - if (not m_params.isParameter("MPI Ranks in Filename")) { - m_params.set("MPI Ranks in Filename",is_scream_standalone()); - } } // Output control diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml index f2408ab189a5..5922c718e6e6 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml @@ -120,5 +120,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml index 559f5158bb60..be4bb16b95d6 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml @@ -117,5 +117,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml index 24e1dfbb3e7a..bdda48ebe459 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml @@ -26,5 +26,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt index 973e58b7da53..62c4b6d2266e 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt @@ -19,7 +19,6 @@ CreateUnitTest(create_vert_remap_and_weights "create_vert_remap_and_weights.cpp" # Run a test to setup nudging source data: set (NUM_STEPS 5) set (POSTFIX source_data) -set (ADD_RANKS false) set (ATM_TIME_STEP 300) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_source_data.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_source_data.yaml) @@ -36,7 +35,6 @@ CreateUnitTestFromExec (shoc_p3_source shoc_p3_nudging set (NUM_STEPS 5) set (ATM_TIME_STEP 300) set (POSTFIX nudged) -set (ADD_RANKS true) set (VERT_TYPE TIME_DEPENDENT_3D_PROFILE) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_nudging.yaml) diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml index e740c5cdf1f1..d9b573068ae7 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml @@ -3,7 +3,6 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged Averaging Type: Instant Max Snapshots Per File: 2 -MPI Ranks in Filename: ${ADD_RANKS} Field Names: - p_mid - T_mid diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml index c75a6a862f71..26f53fbe57f5 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml @@ -4,7 +4,6 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged_remapped Averaging Type: Instant Max Snapshots Per File: 2 vertical_remap_file: vertical_remap.nc -MPI Ranks in Filename: ${ADD_RANKS} Field Names: - T_mid - qv diff --git a/components/eamxx/tests/single-process/mam/optics/output.yaml b/components/eamxx/tests/single-process/mam/optics/output.yaml index 8ea3d237933d..d0c27e042843 100644 --- a/components/eamxx/tests/single-process/mam/optics/output.yaml +++ b/components/eamxx/tests/single-process/mam/optics/output.yaml @@ -13,5 +13,4 @@ Fields: output_control: Frequency: 2 frequency_units: nsteps - MPI Ranks in Filename: true ... From e482b69a80619032d0ed44e02224c323df3a3184 Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Tue, 1 Oct 2024 12:26:14 -0400 Subject: [PATCH 832/904] adjusted 1. to 1._r8 in VerticalProfileMod --- .../elm/src/biogeochem/VerticalProfileMod.F90 | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index d5bd1d1c83fb..3d315b95eb67 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -212,10 +212,10 @@ subroutine decomp_vertprofiles(bounds, & end do else ! if fully frozen, or no roots, put everything in the top layer - froot_prof(p,1) = 1./dzsoi_decomp(1) - croot_prof(p,1) = 1./dzsoi_decomp(1) - leaf_prof(p,1) = 1./dzsoi_decomp(1) - stem_prof(p,1) = 1./dzsoi_decomp(1) + froot_prof(p,1) = 1._r8/dzsoi_decomp(1) + croot_prof(p,1) = 1._r8/dzsoi_decomp(1) + leaf_prof(p,1) = 1._r8/dzsoi_decomp(1) + stem_prof(p,1) = 1._r8/dzsoi_decomp(1) endif end do @@ -257,9 +257,9 @@ subroutine decomp_vertprofiles(bounds, & pdep_prof(c,j) = surface_prof(j)/ surface_prof_tot end do else - nfixation_prof(c,1) = 1./dzsoi_decomp(1) - ndep_prof(c,1) = 1./dzsoi_decomp(1) - pdep_prof(c,1) = 1./dzsoi_decomp(1) + nfixation_prof(c,1) = 1._r8/dzsoi_decomp(1) + ndep_prof(c,1) = 1._r8/dzsoi_decomp(1) + pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif else if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then @@ -271,9 +271,9 @@ subroutine decomp_vertprofiles(bounds, & end if end do else - nfixation_prof(c,1) = 1./dzsoi_decomp(1) - ndep_prof(c,1) = 1./dzsoi_decomp(1) - pdep_prof(c,1) = 1./dzsoi_decomp(1) + nfixation_prof(c,1) = 1._r8/dzsoi_decomp(1) + ndep_prof(c,1) = 1._r8/dzsoi_decomp(1) + pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif end if end do From bbacf3a8f16f33091880bc1850b0e4a9cbecca35 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 1 Oct 2024 17:19:33 -0500 Subject: [PATCH 833/904] Update README and other files for v3.0.1 Update README and other files for v3.0.1 --- CITATION.cff | 2 +- LICENSE | 16 +++++++++++----- README.md | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 9542a7d01ef4..78099055fc35 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ authors: - given-names: E3SM family-names: Project version: 3.0.0 -doi: 10.11578/E3SM/dc.20240301.3 +doi: 10.11578/E3SM/dc.20240930.1 repository-code: 'https://github.com/E3SM-Project/E3SM' url: 'https://e3sm.org' license: BSD-3-Clause diff --git a/LICENSE b/LICENSE index d74a2aa127a0..e5e69b7c906f 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ Except for the separable pieces descibed below, E3SM is released under the following 3-Clause BSD Open Source license. ******************************************************************************* -Copyright ©2023, UChicago Argonne, LLC All Rights Reserved +Copyright ©2043, UChicago Argonne, LLC All Rights Reserved Software Name: Energy Exascale Earth System Model (E3SM) @@ -55,15 +55,21 @@ GPTL share/timing author non-commeric MCT externals/mct ANL BSD YAKL externals/YAKL author BSD cub externals/cub author, NVIDIA BSD -kokkos externals/kokkos SNL BSD +kokkos externals/ekat SNL BSD +haero externals/haero SNL/Battelle BSD +mam4xx externals/mam4xx SNL/Battelle BSD Ocean/Ice under components/ ----------- ----------------- CICE cice LANL BSD MPAS Framework mpas-framework LANL BSD -MPAS Ocean mpas-ocean LANL BSD -MPAS SeaIce mpas-seaice LANL BSD -MPAS-Albany LandIce mpas-albany-landice LANL, SNL BSD +FFTW mpas-ocean/src/FFTW author/MIT GPL +MARBL mpas-ocean/src/MARBL NCAR BSD +SHTNS mpas-ocean/src/SHTNS CeCILL GPL + +Land-ice under components/mpas-albany-landice +----------- ------------------------------------ +SeaLevelModel src/SeaLevelModel author MIT Land under components/elm/src ----------- ------------------------ diff --git a/README.md b/README.md index 192c288d78e1..1601f9cb9035 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ the most challenging and demanding climate-change research problems and Department of Energy mission needs while efficiently using DOE Leadership Computing Facilities. -DOI: [10.11578/E3SM/dc.20240301.3](http://dx.doi.org/10.11578/E3SM/dc.20240301.3) +DOI: [10.11578/E3SM/dc.20240930.1](http://dx.doi.org/10.11578/E3SM/dc.20240930.1) Please visit the [project website](https://e3sm.org) or our [Confluence site](https://acme-climate.atlassian.net/wiki/spaces/DOC/overview) for further details. For questions about the model, use [Github Discussions](https://github.com/E3SM-Project/E3SM/discussions). -See our Github-hosted documentation at [https://e3sm-project.github.io/E3SM/](https://e3sm-project.github.io/E3SM/). +See our Github-hosted documentation at [https://docs.e3sm.org/E3SM](https://docs.e3sm.org/E3SM/). Table of Contents -------------------------------------------------------------------------------- From d5d8f6a8f1e00e5848b12e28aff10457dbf48548 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 1 Oct 2024 17:55:02 -0500 Subject: [PATCH 834/904] Disable linting on README.md Disable linting on README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1601f9cb9035..13c927b8c7f1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + [![E3SM Logo](https://e3sm.org/wp-content/themes/e3sm/assets/images/e3sm-logo.png)](https://e3sm.org) Energy Exascale Earth System Model (E3SM) From 00377667bab08e27ff7fccec82d94984dfab04e9 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 1 Oct 2024 18:23:43 -0500 Subject: [PATCH 835/904] Mention more packages in LICENSE Mention more packages in LICENSE --- LICENSE | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index e5e69b7c906f..ba380e465313 100644 --- a/LICENSE +++ b/LICENSE @@ -66,6 +66,14 @@ MPAS Framework mpas-framework LANL BSD FFTW mpas-ocean/src/FFTW author/MIT GPL MARBL mpas-ocean/src/MARBL NCAR BSD SHTNS mpas-ocean/src/SHTNS CeCILL GPL +cvmix mpas-ocean/src/cvmix NCAR LGPL +gotm mpas-ocean/src/gotm authors GPL +ppr mpas-ocean/src/ppr author custom +Icepack mpas-seaice/src/icepack LANL BSD + +Waves under components/ww3 +----------- -------------------- +WW3 src/WW3 NWS/NOAA custom Land-ice under components/mpas-albany-landice ----------- ------------------------------------ @@ -97,7 +105,7 @@ HOMMEXX components/homme/src/share/cxx SNL BSD Actual copyright holder for above Institutions: NCAR = University Corporation for Atmospheric Research -LANL = Los Alamos National Security, LLC +LANL = Los Alamos National Security, LLC, Triad National Security, LLC SNL = National Technology & Engineering Solutions of Sandia, LLC LBNL = The Regents of the University of California, through Lawrence Berkeley National Laboratory From d866026295488f8f657e42b8d4c487568e590078 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 2 Oct 2024 15:11:02 -0500 Subject: [PATCH 836/904] Add sed commands to modify env_archive.xml to test the conservation am --- .../testmods_dirs/mpaso/conservation_check/shell_commands | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands new file mode 100644 index 000000000000..764340d699f9 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands @@ -0,0 +1,5 @@ +# include mpas-ocean outputs in testing +sed -i 's/\(compname="mpaso" exclude_testing="true"\)/compname="mpaso"/' env_archive.xml +sed -i '/\(compname="mpaso"\)/,// s/hist/hist.am.conservationCheck\\..*\\.nc$/' env_archive.xml +sed -i 's/casename.mpaso.hist.am.globalStats.1976-01-01.nc/casename.mpaso.hist.am.conservationCheck.1976-01-01.nc/' env_archive.xml +sed -i '/casename.mpaso.hist.am.highFrequencyOutput.1976-01-01_00.00.00.nc/d' env_archive.xml From 20aab95002238f420c3b98877b55ecae031a7c96 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 09:33:33 -0600 Subject: [PATCH 837/904] Pass data_type to subfield --- components/eamxx/src/share/field/field.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 320e10a60273..5f7fb62132f8 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -92,7 +92,7 @@ subfield (const std::string& sf_name, const ekat::units::Units& sf_units, "Error! Subview dimension index must be either 0 or 1.\n"); // Create identifier for subfield - FieldIdentifier sf_id(sf_name,lt.clone().strip_dim(idim),sf_units,id.get_grid_name()); + FieldIdentifier sf_id(sf_name,lt.clone().strip_dim(idim),sf_units,id.get_grid_name(),id.data_type()); // Create empty subfield, then set header and views // Note: we can access protected members, since it's the same type From 639f70f8fa4388b7afed7915a100d1fa7d78d56f Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 10:07:05 -0600 Subject: [PATCH 838/904] Create sync_subfield tests This test will fail with current master. Following changes allow it to pass --- .../eamxx/src/share/tests/field_tests.cpp | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 1cb69f474dee..094e4ef057cf 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -812,4 +812,79 @@ TEST_CASE ("update") { } } + +TEST_CASE ("sync_subfields") { + // This test is for previously incorrect behavior, where syncing a subfield + // to host/device would deep copy the entire data view (including all entries of + // the parent view). Here, if memory space is not shared between host and device, + // syncing a subfield to host/device will not sync the data of the other subfields. + + using namespace scream; + using namespace ekat::units; + using namespace ShortFieldTagsNames; + using FID = FieldIdentifier; + using FL = FieldLayout; + + constexpr int ncols = 10; + constexpr int ndims = 4; + constexpr int nlevs = 8; + + // Create field with (col, cmp, lev) + FID fid ("V",FL({COL,CMP,LEV},{ncols,ndims,nlevs}),Units::nondimensional(),"the_grid",DataType::IntType); + Field f (fid); + f.allocate_view(); + + // Store whether mem space for host and device are the same for testing subfield values + const bool shared_mem_space = + f.get_view().data() == f.get_view().data(); + + // Deep copy all values to ndims on device and host + f.deep_copy(ndims); + f.sync_to_host(); + + // Set subfield values to their index on device + for (int c=0; c(); + for (int idx=0; idx(c); + } + + // Sync only component 0 to device + f.get_component(0).sync_to_dev(); + + // For components 1,...,ndims-1, if device and host do not share a + // memory space, device values should be equal to ndims, else device + // values should be equal to component index + for (int c=1; c(); + Kokkos::parallel_for(Kokkos::MDRangePolicy>({0,0}, {ncols,nlevs}), + KOKKOS_LAMBDA (const int icol, const int ilev) { + if (shared_mem_space) EKAT_KERNEL_ASSERT(device_subview(icol, ilev) == c); + else EKAT_KERNEL_ASSERT(device_subview(icol, ilev) == ndims); + }); + } +} + } // anonymous namespace From 4cb9fb85241350a23f692e29c41cfa6ff8b59d3e Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 10:38:44 -0600 Subject: [PATCH 839/904] New sync_to_host impl Now, syncing to host for subfield only deep copies subfield values (not entire data array for parent) --- components/eamxx/src/share/field/field.cpp | 17 +++- components/eamxx/src/share/field/field.hpp | 3 + .../eamxx/src/share/field/field_impl.hpp | 96 +++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 5f7fb62132f8..5a489ec618ae 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -65,7 +65,22 @@ sync_to_host () const { EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); - Kokkos::deep_copy(m_data.h_view,m_data.d_view); + // Check for early return if Host and Device are the same memory space + if (m_data.h_view.data() == m_data.d_view.data()) return; + + switch (data_type()) { + case DataType::IntType: + sync_to_host_impl(); + break; + case DataType::FloatType: + sync_to_host_impl(); + break; + case DataType::DoubleType: + sync_to_host_impl(); + break; + default: + EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); + } } void Field:: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index e48a570d8131..be17cdddbfb7 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -282,6 +282,9 @@ class Field { // Cuda requires methods enclosing __device__ lambda's to be public protected: #endif + template + void sync_to_host_impl () const; + template void deep_copy_impl (const ST value); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 67753605864c..b6f4f14a0068 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -222,6 +222,102 @@ get_strided_view_type { return DstView(get_ND_view()); } +template +void Field::sync_to_host_impl () const { + const auto alloc_props = get_header().get_alloc_properties(); + switch (rank()) { + case 0: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 1: + if (alloc_props.contiguous()) { + Kokkos::deep_copy(get_view(), get_view()); + } else { + auto host_view = get_strided_view< ST*, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST**, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST***, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST****, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST*****, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST******, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i void Field:: deep_copy (const Field& src) { From 15883006dc879aec0ec03019bb6241dc6c2946ce Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 10:45:58 -0600 Subject: [PATCH 840/904] New sync_to_dev impl Now, syncing to device for subfield only deep copies subfield values (not entire data array for parent) --- components/eamxx/src/share/field/field.cpp | 18 +++- components/eamxx/src/share/field/field.hpp | 3 + .../eamxx/src/share/field/field_impl.hpp | 96 +++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 5a489ec618ae..81f994cc3a50 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -89,8 +89,22 @@ sync_to_dev () const { EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); - // Ensure host view was created (lazy construction) - Kokkos::deep_copy(m_data.d_view,m_data.h_view); + // Check for early return if Host and Device are the same memory space + if (m_data.h_view.data() == m_data.d_view.data()) return; + + switch (data_type()) { + case DataType::IntType: + sync_to_dev_impl(); + break; + case DataType::FloatType: + sync_to_dev_impl(); + break; + case DataType::DoubleType: + sync_to_dev_impl(); + break; + default: + EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_dev.\n"); + } } Field Field:: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index be17cdddbfb7..1cbf36236f36 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -285,6 +285,9 @@ class Field { template void sync_to_host_impl () const; + template + void sync_to_dev_impl () const; + template void deep_copy_impl (const ST value); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index b6f4f14a0068..8dbfc3e35a6d 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -318,6 +318,102 @@ void Field::sync_to_host_impl () const { } } +template +void Field::sync_to_dev_impl () const { + const auto alloc_props = get_header().get_alloc_properties(); + switch (rank()) { + case 0: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 1: + if (alloc_props.contiguous()) { + Kokkos::deep_copy(get_view(), get_view()); + } else { + auto dev_view = get_strided_view< ST*, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST**, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST***, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST****, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST*****, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST******, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i void Field:: deep_copy (const Field& src) { From cfa4b46c2048415890a660619a3337b97a4947b9 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 16:25:22 -0600 Subject: [PATCH 841/904] Allow sync_to_host for read-only fields Use mutable keyword for m_is_read_only to temporarily disable for sync_to_host when device memspace != host memspace --- components/eamxx/src/share/field/field.cpp | 7 +++++++ components/eamxx/src/share/field/field.hpp | 2 +- .../eamxx/src/share/grid/remap/coarsening_remapper.cpp | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 81f994cc3a50..40551eed039b 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -68,6 +68,10 @@ sync_to_host () const { // Check for early return if Host and Device are the same memory space if (m_data.h_view.data() == m_data.d_view.data()) return; + // We allow sync_to_host for constant fields. Temporarily disable read only flag. + const bool original_read_only = m_is_read_only; + m_is_read_only = false; + switch (data_type()) { case DataType::IntType: sync_to_host_impl(); @@ -81,6 +85,9 @@ sync_to_host () const { default: EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); } + + // Return field to read-only state + m_is_read_only = original_read_only; } void Field:: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 1cbf36236f36..e40e9649fede 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -344,7 +344,7 @@ class Field { dual_view_t m_data; // Whether this field is read-only - bool m_is_read_only = false; + mutable bool m_is_read_only = false; }; // We use this to find a Field in a std container. diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index f915cb14d292..7d6590c327a6 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -3,6 +3,7 @@ #include "share/grid/point_grid.hpp" #include "share/grid/grid_import_export.hpp" #include "share/io/scorpio_input.hpp" +#include "share/field/field.hpp" #include #include From 55f473dcdec35a66b9b7dfa80ca70bea00a22b47 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Wed, 2 Oct 2024 16:03:28 -0500 Subject: [PATCH 842/904] Fix typo in LICENSE Fix typo --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index ba380e465313..247287ab4c00 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ Except for the separable pieces descibed below, E3SM is released under the following 3-Clause BSD Open Source license. ******************************************************************************* -Copyright ©2043, UChicago Argonne, LLC All Rights Reserved +Copyright 2024, UChicago Argonne, LLC All Rights Reserved Software Name: Energy Exascale Earth System Model (E3SM) From 1230ce601b42e344bbf60a902341ea79e56bd39c Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Wed, 2 Oct 2024 16:07:16 -0600 Subject: [PATCH 843/904] Add comment on need for mutable m_is_read_only --- components/eamxx/src/share/field/field.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index e40e9649fede..833e49a611bb 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -343,7 +343,9 @@ class Field { // Actual data. dual_view_t m_data; - // Whether this field is read-only + // Whether this field is read-only. This is given + // mutable keyword since it needs to be turned off/on + // to allow sync_to_host for constant, read-only fields. mutable bool m_is_read_only = false; }; From c97aa4c0687d8d341add03840bc1db78096c2bcd Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Wed, 2 Oct 2024 16:21:03 -0600 Subject: [PATCH 844/904] Consolidate sync_to_host_impl and sync_to_device_impl into sync_views_impl --- components/eamxx/src/share/field/field.cpp | 12 +- components/eamxx/src/share/field/field.hpp | 7 +- .../eamxx/src/share/field/field_impl.hpp | 198 +++++------------- 3 files changed, 59 insertions(+), 158 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 40551eed039b..1b3de68ba08f 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -74,13 +74,13 @@ sync_to_host () const { switch (data_type()) { case DataType::IntType: - sync_to_host_impl(); + sync_views_impl(); break; case DataType::FloatType: - sync_to_host_impl(); + sync_views_impl(); break; case DataType::DoubleType: - sync_to_host_impl(); + sync_views_impl(); break; default: EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); @@ -101,13 +101,13 @@ sync_to_dev () const { switch (data_type()) { case DataType::IntType: - sync_to_dev_impl(); + sync_views_impl(); break; case DataType::FloatType: - sync_to_dev_impl(); + sync_views_impl(); break; case DataType::DoubleType: - sync_to_dev_impl(); + sync_views_impl(); break; default: EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_dev.\n"); diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 833e49a611bb..90e5ccd652c0 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -282,11 +282,8 @@ class Field { // Cuda requires methods enclosing __device__ lambda's to be public protected: #endif - template - void sync_to_host_impl () const; - - template - void sync_to_dev_impl () const; + template + void sync_views_impl () const; template void deep_copy_impl (const ST value); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 8dbfc3e35a6d..becb7b9f2f8b 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -222,94 +222,94 @@ get_strided_view_type { return DstView(get_ND_view()); } -template -void Field::sync_to_host_impl () const { +template +void Field::sync_views_impl () const { const auto alloc_props = get_header().get_alloc_properties(); switch (rank()) { case 0: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); break; case 1: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST*, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST**, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST***, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST****, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST*****, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST******, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i -void Field::sync_to_dev_impl () const { - const auto alloc_props = get_header().get_alloc_properties(); - switch (rank()) { - case 0: - Kokkos::deep_copy(get_view(), get_view()); - break; - case 1: - if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); - } else { - auto dev_view = get_strided_view< ST*, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST**, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST***, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST****, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST*****, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST******, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i void Field:: deep_copy (const Field& src) { From 756ea4accfdd9582d708ffec869104363642f850 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 2 Oct 2024 17:02:35 -0600 Subject: [PATCH 845/904] EAMxx: fix usage of layout extents in field methods --- .../eamxx/src/share/field/field_impl.hpp | 25 ++++++++++++++++--- .../eamxx/src/share/field/field_layout.cpp | 6 ++--- .../eamxx/src/share/field/field_layout.hpp | 2 ++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index bfc0d44e99a2..5eb810be9d84 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -306,10 +306,19 @@ deep_copy_impl (const Field& src) { auto src_alloc_props = src.get_header().get_alloc_properties(); auto tgt_alloc_props = get_header().get_alloc_properties(); - using RangePolicy = typename KokkosTypes::RangePolicy; + using device_t = typename Field::get_device; + using exec_space = typename device_t::execution_space; + using RangePolicy = Kokkos::RangePolicy; + auto policy = RangePolicy(0,layout.size()); - auto ext = layout.extents(); + using extents_type = typename ekat::KokkosTypes::template view_1d; + extents_type ext; + if constexpr (HD==Device) { + ext = layout.extents(); + } else { + ext = layout.extents_h(); + } switch (rank) { case 1: { @@ -651,10 +660,18 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) " - x layout: " + x_l.to_string() + "\n" " - y layout: " + y_l.to_string() + "\n"); - using RangePolicy = typename KokkosTypes::RangePolicy; + using device_t = typename Field::get_device; + using exec_space = typename device_t::execution_space; + using RangePolicy = Kokkos::RangePolicy; auto policy = RangePolicy(0,x_l.size()); - auto ext = x_l.extents(); + using extents_type = typename ekat::KokkosTypes::template view_1d; + extents_type ext; + if constexpr (HD==Device) { + ext = x_l.extents(); + } else { + ext = x_l.extents_h(); + } switch (x_l.rank()) { case 0: diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 2ce924cd0a74..7cd9e953f657 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -250,9 +250,9 @@ FieldLayout& FieldLayout::reset_dim (const FieldTag t, const int extent, const b void FieldLayout::set_extents () { m_extents = decltype(m_extents)("",m_rank); - auto extents_h = Kokkos::create_mirror_view(m_extents); - std::copy_n(m_dims.begin(),m_rank,extents_h.data()); - Kokkos::deep_copy(m_extents,extents_h); + m_extents_h = Kokkos::create_mirror_view(m_extents); + std::copy_n(m_dims.begin(),m_rank,m_extents_h.data()); + Kokkos::deep_copy(m_extents,m_extents_h); } void FieldLayout::compute_type () { diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index 018831f8818c..481ca0c922e8 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -97,6 +97,7 @@ class FieldLayout { int dim (const int idim) const; const std::vector& dims () const { return m_dims; } const extents_type& extents () const { return m_extents; } + const extents_type::HostMirror& extents_h () const { return m_extents_h; } long long size () const; @@ -156,6 +157,7 @@ class FieldLayout { std::vector m_names; std::vector m_dims; extents_type m_extents; + extents_type::HostMirror m_extents_h; LayoutType m_type; }; From c99106082c6b5efcfddb7cf8631bd854b1fd95fa Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 3 Oct 2024 08:49:30 -0600 Subject: [PATCH 846/904] EAMxx: fix nc file names in some standalone tests --- .../eamxx/src/physics/nudging/tests/nudging_tests.cpp | 2 +- components/eamxx/src/share/io/tests/io_utils.cpp | 8 +++++--- .../physics_only/shoc_p3_nudging/input_nudging.yaml | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index 51773cb249df..a5a158f44a17 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -51,7 +51,7 @@ TEST_CASE("nudging_tests") { const int nlevs_fine = 2*nlevs_data -1; // Files names - auto postfix = ".INSTANT.nsteps_x1." + get_t0().to_string() + ".nc"; + auto postfix = ".INSTANT.nsteps_x1.np*." + get_t0().to_string() + ".nc"; auto nudging_data = "nudging_data" + postfix; auto nudging_data_filled = "nudging_data_filled" + postfix; auto map_file = "map_ncol" + std::to_string(ngcols_data) diff --git a/components/eamxx/src/share/io/tests/io_utils.cpp b/components/eamxx/src/share/io/tests/io_utils.cpp index b8187f31c3f4..cf7b66ad8596 100644 --- a/components/eamxx/src/share/io/tests/io_utils.cpp +++ b/components/eamxx/src/share/io/tests/io_utils.cpp @@ -25,9 +25,10 @@ TEST_CASE ("find_filename_in_rpointer") { bar_c.frequency = 1; bar_c.frequency_units = "ndays"; bar2_c.frequency = 6; bar2_c.frequency_units = "nhours"; - std::string foo_fname = "foo.r.INSTANT.nsteps_x3." + t0.to_string() + ".nc"; - std::string bar_fname = "bar.rhist.AVERAGE.ndays_x1." + t0.to_string() + ".nc"; - std::string bar2_fname = "bar.rhist.AVERAGE.nhours_x6." + t0.to_string() + ".nc"; + std::string suffix = ".np" + std::to_string(comm.size()) + "." + t0.to_string() + ".nc"; + std::string foo_fname = "foo.r.INSTANT.nsteps_x3" + suffix; + std::string bar_fname = "bar.rhist.AVERAGE.ndays_x1" + suffix; + std::string bar2_fname = "bar.rhist.AVERAGE.nhours_x6" + suffix; rpointer << foo_fname<< "\n"; rpointer << bar_fname<< "\n"; @@ -44,6 +45,7 @@ TEST_CASE ("find_filename_in_rpointer") { REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0,INST,foo_c)); // foo is model restart REQUIRE_THROWS (find_filename_in_rpointer("foo",true,comm,t0,AVG)); // model restart MUST be INSTANT + auto test = find_filename_in_rpointer("bar",false,comm,t0,AVG,bar_c); REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar_c)==bar_fname); REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar2_c)==bar2_fname); REQUIRE (find_filename_in_rpointer("foo",true, comm,t0)==foo_fname); diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml index 2b33d15f9f75..6a4c0003376e 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml @@ -14,7 +14,7 @@ atmosphere_processes: p3: max_total_ni: 720.0e3 nudging: - nudging_filenames_patterns: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.${RUN_T0}.nc] + nudging_filenames_patterns: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.np1.${RUN_T0}.nc] nudging_fields: ["T_mid", "qv"] nudging_timescale: 1000 use_nudging_weights: true From 48930bfd6ecb5cb8f28631339555439a95fbbc7f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 3 Oct 2024 09:53:59 -0600 Subject: [PATCH 847/904] EAMxx: fix usage of regex when parsing rpointer file --- components/eamxx/src/share/io/scream_io_utils.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 2d5e0a840d09..9318728d657a 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -51,7 +51,7 @@ std::string find_filename_in_rpointer ( while (std::getline(rpointer_file,line)) { content += line + "\n"; - if (std::regex_match(line,pattern)) { + if (std::regex_search(line,pattern)) { filename = line; found = true; break; @@ -70,12 +70,14 @@ std::string find_filename_in_rpointer ( EKAT_ERROR_MSG ( "Error! Restart requested, but no model restart file found in 'rpointer.atm'.\n" " model restart filename prefix: " + filename_prefix + "\n" + " model restart filename pattern: " + pattern_str + "\n" " run t0 : " + run_t0.to_string() + "\n" " rpointer content:\n" + content + "\n\n"); } else { EKAT_ERROR_MSG ( "Error! Restart requested, but no history restart file found in 'rpointer.atm'.\n" " hist restart filename prefix: " + filename_prefix + "\n" + " hist restart filename pattern: " + pattern_str + "\n" " run t0 : " + run_t0.to_string() + "\n" " avg_type : " + e2str(avg_type) + "\n" " output freq : " + std::to_string(control.frequency) + "\n" From f9a57c730917c3cb253f10b6cdf603077b6d88df Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 10:23:48 -0600 Subject: [PATCH 848/904] Try async deep copy for sync to host --- .../eamxx/src/share/field/field_impl.hpp | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index becb7b9f2f8b..cafce7d4622b 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -224,14 +224,16 @@ get_strided_view_type { template void Field::sync_views_impl () const { + using ExecSpace = typename Field::get_device::execution_space; + const auto alloc_props = get_header().get_alloc_properties(); switch (rank()) { case 0: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); break; case 1: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST*, To >(); auto from_view = get_strided_view(); @@ -239,13 +241,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i); auto from_view_i = Kokkos::subview(from_view, i); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 2: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST**, To >(); auto from_view = get_strided_view(); @@ -253,13 +255,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 3: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST***, To >(); auto from_view = get_strided_view(); @@ -267,13 +269,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL(), Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL(), Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 4: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST****, To >(); auto from_view = get_strided_view(); @@ -281,13 +283,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 5: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST*****, To >(); auto from_view = get_strided_view(); @@ -295,13 +297,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 6: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST******, To >(); auto from_view = get_strided_view(); @@ -309,13 +311,17 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; default: EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); } + // Since deep_copy are all async wrt host, we need to fence before returning + if constexpr (To == Host) { + Kokkos::fence(); + } } template From 3b66686189b23afb56d4f203acdbccc450d995e2 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 3 Oct 2024 13:07:22 -0600 Subject: [PATCH 849/904] Remove most allocations from rrtmgp interface --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/scream_rrtmgp_interface.hpp | 178 ++++++++++++------ 2 files changed, 123 insertions(+), 57 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 8ff525eeed1d..8d88d427e78f 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 8ff525eeed1d87a2ca6f251c4d16b46222c5554d +Subproject commit 8d88d427e78fe7958564e587d9f776c0b36d3762 diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 216722b3766b..a420c0c1dd4f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -565,6 +565,44 @@ static void rrtmgp_sw( int ngpt = k_dist.get_ngpt(); int ngas = gas_concs.get_num_gases(); + // Allocate temporaries from pool + const int size1 = nday; + const int size2 = nday*nlay; // 4 + const int size3 = nday*(nlay+1); // 5 + const int size4 = ncol*nlay; + const int size5 = nbnd*nday; //2 + const int size6 = nday*ngpt; + const int size7 = nday*(nlay+1)*nbnd; // 3 + const int size8 = ncol*nlay*(k_dist.get_ngas()+1); + + RealT* data = pool_t::template alloc_raw(size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8), *dcurr = data; + + auto mu0_day = view_t (dcurr, nday); dcurr += size1; + + auto p_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto vmr_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_limited = view_t (dcurr, nday, nlay); dcurr += size2; + + auto p_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto t_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_up_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_dir_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + + auto vmr = view_t (dcurr, ncol, nlay); dcurr += size4; + + auto sfc_alb_dir_T = view_t (dcurr, nbnd, nday); dcurr += size5; + auto sfc_alb_dif_T = view_t (dcurr, nbnd, nday); dcurr += size5; + + auto toa_flux = view_t (dcurr, nday, ngpt); dcurr += size6; + + auto bnd_flux_up_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_dir_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + + auto col_gas = view_t(dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size8; + // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; auto &flux_dn = fluxes.flux_dn; @@ -623,20 +661,15 @@ static void rrtmgp_sw( } // Subset mu0 - auto mu0_day = view_t("mu0_day", nday); Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { mu0_day(iday) = mu0(dayIndices(iday)); }); // subset state variables - auto p_lay_day = view_t("p_lay_day", nday, nlay); - auto t_lay_day = view_t("t_lay_day", nday, nlay); Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); }); - auto p_lev_day = view_t("p_lev_day", nday, nlay+1); - auto t_lev_day = view_t("t_lev_day", nday, nlay+1); Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); @@ -647,8 +680,6 @@ static void rrtmgp_sw( gas_concs_t gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = view_t("vmr_day", nday, nlay); - auto vmr = view_t("vmr" , ncol, nlay); gas_concs.get_vmr(gas_names[igas], vmr); Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); @@ -680,20 +711,12 @@ static void rrtmgp_sw( // RRTMGP assumes surface albedos have a screwy dimension ordering // for some strange reason, so we need to transpose these; also do // daytime subsetting in the same kernel - view_t sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - view_t sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); }); // Temporaries we need for daytime-only fluxes - auto flux_up_day = view_t("flux_up_day", nday, nlay+1); - auto flux_dn_day = view_t("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = view_t("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); fluxes_t fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; @@ -713,18 +736,14 @@ static void rrtmgp_sw( } // Limit temperatures for gas optics look-up tables - auto t_lay_limited = view_t("t_lay_limited", nday, nlay); limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); // Do gas optics - view_t toa_flux("toa_flux", nday, ngpt); bool top_at_1 = false; Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { val |= p_lay(0, 0) < p_lay(0, nlay-1); }, Kokkos::LOr(top_at_1)); - oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); if (extra_clnsky_diag) { k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); @@ -803,6 +822,8 @@ static void rrtmgp_sw( clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); }); } + + pool_t::dealloc(data, dcurr - data); } /* @@ -819,6 +840,24 @@ static void rrtmgp_lw( { // Problem size int nbnd = k_dist.get_nband(); + int constexpr max_gauss_pts = 4; + + const int size1 = ncol; + const int size2 = nbnd*ncol; + const int size3 = max_gauss_pts*max_gauss_pts; + const int size4 = ncol*nlay; + const int size5 = ncol*(nlay+1); + const int size6 = ncol*nlay*(k_dist.get_ngas()+1); + + RealT* data = pool_t::template alloc_raw(size1 + size2 + size3*2 + size4 + size5 + size6), *dcurr = data; + + view_t t_sfc (dcurr, ncol); dcurr += size1; + view_t emis_sfc (dcurr, nbnd,ncol); dcurr += size2; + view_t gauss_Ds (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t gauss_wts (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t t_lay_limited(dcurr, ncol, nlay); dcurr += size4; + view_t t_lev_limited(dcurr, ncol, nlay+1); dcurr += size5; + view_t col_gas (dcurr, std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); dcurr += size6; // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; @@ -863,8 +902,6 @@ static void rrtmgp_lw( // Boundary conditions source_func_t lw_sources; lw_sources.alloc(ncol, nlay, k_dist); - view_t t_sfc ("t_sfc" ,ncol); - view_t emis_sfc("emis_sfc",nbnd,ncol); bool top_at_1 = false; Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { @@ -882,32 +919,31 @@ static void rrtmgp_lw( // Weights and angle secants for first order (k=1) Gaussian quadrature. // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - hview_t gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - hview_t gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - view_t gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - view_t gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + RealT gauss_Ds_host_raw[max_gauss_pts][max_gauss_pts] = { + {1.66, 1.18350343, 1.09719858, 1.06056257}, + {0., 2.81649655, 1.69338507, 1.38282560}, + {0., 0., 4.70941630, 2.40148179}, + {0., 0., 0., 7.15513024} + }; + hview_t gauss_Ds_host (&gauss_Ds_host_raw[0][0], max_gauss_pts, max_gauss_pts); + + RealT gauss_wts_host_raw[max_gauss_pts][max_gauss_pts] = { + {0.5, 0.3180413817, 0.2009319137, 0.1355069134}, + {0., 0.1819586183, 0.2292411064, 0.2034645680}, + {0., 0., 0.0698269799, 0.1298475476}, + {0., 0., 0., 0.0311809710} + }; + + hview_t gauss_wts_host(&gauss_wts_host_raw[0][0],max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); Kokkos::deep_copy(gauss_wts, gauss_wts_host); // Limit temperatures for gas optics look-up tables - auto t_lay_limited = view_t("t_lay_limited", ncol, nlay); - auto t_lev_limited = view_t("t_lev_limited", ncol, nlay+1); limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); // Do gas optics - oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); if (extra_clnsky_diag) { k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); @@ -941,23 +977,22 @@ static void rrtmgp_lw( // Compute clean-sky fluxes rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); } + + pool_t::dealloc(data, dcurr - data); } /* * Return a subcolumn mask consistent with a specified overlap assumption */ -static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds) +static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds, int3dk& subcolumn_mask) { - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - int3dk subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - // Subcolumn generators are a means for producing a variable x(i,j,k), where // // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) // // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = view_t("cldx", ncol, nlay, ngpt); + auto cldx = pool_t::template alloc(ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1017,6 +1052,9 @@ static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, subcolumn_mask(icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldx); + return subcolumn_mask; } @@ -1029,7 +1067,7 @@ static void compute_cloud_area( { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = view_t("subcol_mask", ncol, ngpt); + auto subcol_mask = pool_t::template alloc(ncol, ngpt); Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but // using play/pmid does not @@ -1046,6 +1084,8 @@ static void compute_cloud_area( cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; } }); + + pool_t::dealloc(subcol_mask); } /* @@ -1078,7 +1118,7 @@ static void compute_aerocom_cloudtop( Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = view_t("aerocom_clr", ncol); + auto aerocom_clr = pool_t::template alloc(ncol); Kokkos::deep_copy(aerocom_clr, 1.0); // Get gravity acceleration constant from constants @@ -1151,6 +1191,8 @@ static void compute_aerocom_cloudtop( // (their products) cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); + + pool_t::dealloc(aerocom_clr); } /* @@ -1253,14 +1295,17 @@ static optical_props2_t get_cloud_optics_sw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = view_t("rel_limited", ncol, nlay); - auto rei_limited = view_t("rei_limited", ncol, nlay); + auto rel_limited = pool_t::template alloc(ncol, nlay); + auto rei_limited = pool_t::template alloc(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + // Return optics return clouds; } @@ -1279,14 +1324,17 @@ static optical_props1_t get_cloud_optics_lw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = view_t("rel_limited", ncol, nlay); - auto rei_limited = view_t("rei_limited", ncol, nlay); + auto rel_limited = pool_t::alloc(ncol, nlay); + auto rei_limited = pool_t::alloc(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + // Return optics return clouds; } @@ -1298,6 +1346,10 @@ static optical_props2_t get_subsampled_clouds( optical_props2_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_2str(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::alloc(ncol, nlay, ngpt); + // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud @@ -1305,7 +1357,7 @@ static optical_props2_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + auto cldfrac_rad = pool_t::alloc(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1319,11 +1371,11 @@ static optical_props2_t get_subsampled_clouds( int overlap = 1; // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = view_t("seeds", ncol); + auto seeds = pool_t::alloc(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { @@ -1338,6 +1390,11 @@ static optical_props2_t get_subsampled_clouds( subsampled_optics.g (icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + return subsampled_optics; } @@ -1349,6 +1406,10 @@ static optical_props1_t get_subsampled_clouds( optical_props1_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_1scl(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::alloc(ncol, nlay, ngpt); + // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud @@ -1356,7 +1417,7 @@ static optical_props1_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + auto cldfrac_rad = pool_t::alloc(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1367,11 +1428,11 @@ static optical_props1_t get_subsampled_clouds( // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM; use different // seed values for longwave and shortwave - auto seeds = view_t("seeds", ncol); + auto seeds = pool_t::alloc(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { @@ -1382,6 +1443,11 @@ static optical_props1_t get_subsampled_clouds( subsampled_optics.tau(icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + return subsampled_optics; } From 16e80809262ebc2652ada3ed43737c174f9d90b3 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 3 Oct 2024 14:38:05 -0500 Subject: [PATCH 850/904] Change GMPAS-JRA1p5-DIB-PISMF-DSGR test from SMS to PET to check omp --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1cbf28b83974..f23cea24012f 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -275,7 +275,7 @@ "ERS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "PEM_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-TMIX.mpaso-jra_1958", - "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", + "PET_P480_Ld2.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", ) }, From 3cf1406f74f2875b316cf3d06bc2622b196800a9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 3 Oct 2024 15:11:25 -0600 Subject: [PATCH 851/904] Update rrtmgp to latest kokkos-coversion-branch --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 8d88d427e78f..e64b99cce24e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 8d88d427e78fe7958564e587d9f776c0b36d3762 +Subproject commit e64b99cce24eb31bb6f317bddb6f0ffbdfaf8bb7 From aba69664dad83c9624b28173daed6cb90d491ce6 Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Thu, 3 Oct 2024 17:51:59 -0400 Subject: [PATCH 852/904] Added debugging info --- components/elm/src/biogeochem/VerticalProfileMod.F90 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index 3d315b95eb67..a1b84fe945f5 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -336,7 +336,15 @@ subroutine decomp_vertprofiles(bounds, & end do if ( ( abs(froot_prof_sum - 1._r8) > delta ) .or. ( abs(croot_prof_sum - 1._r8) > delta ) .or. & ( abs(stem_prof_sum - 1._r8) > delta ) .or. ( abs(leaf_prof_sum - 1._r8) > delta ) ) then + c = veg_pp%column(p) write(iulog, *) 'profile sums: ', froot_prof_sum, croot_prof_sum, leaf_prof_sum, stem_prof_sum + write(iulog, *) 'c: ',c + write(iulog, *) 'altmax_lastyear_indx: ', altmax_lastyear_indx(c) + write(iulog, *) 'cinput_rootfr: ', col_cinput_rootfr(c,:) + write(iulog, *) 'dzsoi_decomp: ', dzsoi_decomp(:) + write(iulog, *) 'surface_prof: ', surface_prof(:) + write(iulog, *) 'p, itype(p), wtcol(p): ', p, veg_pp%itype(p), veg_pp%wtcol(p) + write(iulog, *) 'cinput_rootfr(p,:): ', cinput_rootfr(p,:) call endrun(msg=' ERROR: sum-1 > delta'//errMsg(__FILE__, __LINE__)) endif end do From 463ffbc702fefb9c42926315148859169fdc1d12 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Thu, 3 Oct 2024 17:01:07 -0700 Subject: [PATCH 853/904] A quick change to recreate an aborted run mid simulation --- components/eamxx/cime_config/namelist_defaults_scream.xml | 1 + components/eamxx/src/control/atmosphere_driver.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 57611fc3aa91..9a13029cec6f 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -604,6 +604,7 @@ be lost if SCREAM_HACK_XML is not enabled. + -9999 0 0, "Error! Input time step must be positive.\n"); From cc1ff16763306a34335a13299f78f632b2b37248 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 3 Oct 2024 20:36:35 -0500 Subject: [PATCH 854/904] print out moab counter only with MOABDEBUG --- driver-moab/main/cime_comp_mod.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/driver-moab/main/cime_comp_mod.F90 b/driver-moab/main/cime_comp_mod.F90 index 265f8e11338e..59fa2daf4080 100644 --- a/driver-moab/main/cime_comp_mod.F90 +++ b/driver-moab/main/cime_comp_mod.F90 @@ -2896,10 +2896,12 @@ subroutine cime_run() num_moab_exports = num_moab_exports + 1! this is moab clock used for debugging call seq_timemgr_clockAdvance( seq_SyncClock, force_stop, force_stop_ymd, force_stop_tod) call seq_timemgr_EClockGetData(EClock_d, stepno=cur_step_no) +#ifdef MOABDEBUG if (iamroot_CPLID) then write(logunit,*) ' num_moab_exports , cur_step_no ',num_moab_exports, cur_step_no call shr_sys_flush(logunit) endif +#endif call seq_timemgr_EClockGetData( EClock_d, curr_ymd=ymd, curr_tod=tod) call shr_cal_date2ymd(ymd,year,month,day) stop_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_stop) From 9daab1a446be1621bdedfe879f21e7f899a15ec3 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:33:39 -0600 Subject: [PATCH 855/904] Add function to return if host == device mem space --- components/eamxx/src/share/field/field.cpp | 4 ++-- components/eamxx/src/share/field/field.hpp | 7 +++++++ components/eamxx/src/share/tests/field_tests.cpp | 3 +-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 1b3de68ba08f..a002c3b28813 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -66,7 +66,7 @@ sync_to_host () const { "Error! Input field must be allocated in order to sync host and device views.\n"); // Check for early return if Host and Device are the same memory space - if (m_data.h_view.data() == m_data.d_view.data()) return; + if (host_and_device_share_memory_space()) return; // We allow sync_to_host for constant fields. Temporarily disable read only flag. const bool original_read_only = m_is_read_only; @@ -97,7 +97,7 @@ sync_to_dev () const { "Error! Input field must be allocated in order to sync host and device views.\n"); // Check for early return if Host and Device are the same memory space - if (m_data.h_view.data() == m_data.d_view.data()) return; + if (host_and_device_share_memory_space()) return; switch (data_type()) { case DataType::IntType: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 90e5ccd652c0..679f293e3e1c 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -278,6 +278,13 @@ class Field { // Allocate the actual view void allocate_view (); + inline bool host_and_device_share_memory_space() const { + EKAT_REQUIRE_MSG(is_allocated(), + "Error! Must allocate view before querying " + "host_and_device_share_memory_space().\n"); + return m_data.h_view.data() == m_data.d_view.data(); + } + #ifndef KOKKOS_ENABLE_CUDA // Cuda requires methods enclosing __device__ lambda's to be public protected: diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 094e4ef057cf..0dfe2522de39 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -835,8 +835,7 @@ TEST_CASE ("sync_subfields") { f.allocate_view(); // Store whether mem space for host and device are the same for testing subfield values - const bool shared_mem_space = - f.get_view().data() == f.get_view().data(); + const bool shared_mem_space = f.host_and_device_share_memory_space(); // Deep copy all values to ndims on device and host f.deep_copy(ndims); From c265fb58c10490766e26ec3100a0dbfb41a3319b Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:36:47 -0600 Subject: [PATCH 856/904] Create contiguous helper field --- components/eamxx/src/share/field/field.cpp | 16 ++++++++++++++ components/eamxx/src/share/field/field.hpp | 25 ++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index a002c3b28813..bf1ecbb5dfaa 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -137,6 +137,14 @@ subfield (const std::string& sf_name, const ekat::units::Units& sf_units, sf.m_data = m_data; sf.m_is_read_only = m_is_read_only; + if (not sf.m_header->get_alloc_properties().contiguous() and + not sf.host_and_device_share_memory_space()) { + // If subfield is not contiguous and Host and Device do not + // share a memory space, we must initialize the helper field + // for sync_to functions. + sf.initialize_contiguous_helper_field(); + } + return sf; } @@ -178,6 +186,14 @@ Field Field::subfield(const std::string& sf_name, index_end); sf.m_data = m_data; + if (not sf.m_header->get_alloc_properties().contiguous() and + not sf.host_and_device_share_memory_space()) { + // If subfield is not contiguous and Host and Device do not + // share a memory space, we must initialize the helper field + // for sync_to functions. + sf.initialize_contiguous_helper_field(); + } + return sf; } diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 679f293e3e1c..a3779f1ebf24 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -278,6 +278,27 @@ class Field { // Allocate the actual view void allocate_view (); + // Create contiguous helper field for running sync_to_host + // and sync_to_device with non-contiguous fields + void initialize_contiguous_helper_field () { + EKAT_REQUIRE_MSG(not m_header->get_alloc_properties().contiguous(), + "Error! We should not setup contiguous helper field " + "for an already contiguous field.\n"); + EKAT_REQUIRE_MSG(not host_and_device_share_memory_space(), + "Error! We should not setup contiguous helper field for a field " + "when host and device share a memory space.\n"); + + auto id = m_header->get_identifier(); + Field contig(id.alias(name()+std::string("_contiguous"))); + contig.allocate_view(); + + // Sanity check + EKAT_REQUIRE_MSG(contig.get_header().get_alloc_properties().contiguous(), + "Error! Contiguous helper field must be contiguous.\n"); + + m_contiguous_field = std::make_shared(contig); + } + inline bool host_and_device_share_memory_space() const { EKAT_REQUIRE_MSG(is_allocated(), "Error! Must allocate view before querying " @@ -347,6 +368,10 @@ class Field { // Actual data. dual_view_t m_data; + // Field needed for sync host/device in case of non-contiguous + // field when host and device do not share a memory space. + std::shared_ptr m_contiguous_field; + // Whether this field is read-only. This is given // mutable keyword since it needs to be turned off/on // to allow sync_to_host for constant, read-only fields. From e5e8a10fe0c24ba68dfb4d64cbde4f3574da00da Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:39:09 -0600 Subject: [PATCH 857/904] Make deep_copy const --- components/eamxx/src/share/field/field.hpp | 8 ++++---- components/eamxx/src/share/field/field_impl.hpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index a3779f1ebf24..c071a79c840c 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -198,11 +198,11 @@ class Field { // Set the field to a constant value (on host or device) template - void deep_copy (const T value); + void deep_copy (const T value) const; // Copy the data from one field to this field template - void deep_copy (const Field& src); + void deep_copy (const Field& src) const; // Updates this field y as y=alpha*x+beta*y // NOTE: ST=void is just so we can give a default to HD, @@ -314,10 +314,10 @@ class Field { void sync_views_impl () const; template - void deep_copy_impl (const ST value); + void deep_copy_impl (const ST value) const; template - void deep_copy_impl (const Field& src); + void deep_copy_impl (const Field& src) const; template void update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index cafce7d4622b..d5917b944177 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -326,7 +326,7 @@ void Field::sync_views_impl () const { template void Field:: -deep_copy (const Field& src) { +deep_copy (const Field& src) const { EKAT_REQUIRE_MSG (not m_is_read_only, "Error! Cannot call deep_copy on read-only fields.\n"); @@ -350,7 +350,7 @@ deep_copy (const Field& src) { template void Field:: -deep_copy (const ST value) { +deep_copy (const ST value) const { EKAT_REQUIRE_MSG (not m_is_read_only, "Error! Cannot call deep_copy on read-only fields.\n"); @@ -384,7 +384,7 @@ deep_copy (const ST value) { template void Field:: -deep_copy_impl (const Field& src) { +deep_copy_impl (const Field& src) const { const auto& layout = get_header().get_identifier().get_layout(); const auto& layout_src = src.get_header().get_identifier().get_layout(); @@ -528,7 +528,7 @@ deep_copy_impl (const Field& src) { } template -void Field::deep_copy_impl (const ST value) { +void Field::deep_copy_impl (const ST value) const { // Note: we can't just do a deep copy on get_view_impl(), since this // field might be a subfield of another. Instead, get the From a1ca668f646c05e01ca329f0f7213402ca067c02 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:39:45 -0600 Subject: [PATCH 858/904] Pass data_type to other subfield impl --- components/eamxx/src/share/field/field.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index bf1ecbb5dfaa..686e70bd5881 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -177,7 +177,7 @@ Field Field::subfield(const std::string& sf_name, auto sf_layout = lt.clone(); sf_layout.reset_dim(idim, index_end - index_beg); // Create identifier for subfield - FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name()); + FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name(), id.data_type()); // Create empty subfield, then set header and views // Note: we can access protected members, since it's the same type From 9333fa54fbe75decee8ae7a861a1022df6cd5865 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:56:30 -0600 Subject: [PATCH 859/904] Change sync_views_impl to use scratch contiguous field --- .../eamxx/src/share/field/field_impl.hpp | 139 ++++++------------ 1 file changed, 42 insertions(+), 97 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index d5917b944177..076ffc9d2da0 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -224,103 +224,48 @@ get_strided_view_type { template void Field::sync_views_impl () const { - using ExecSpace = typename Field::get_device::execution_space; - - const auto alloc_props = get_header().get_alloc_properties(); - switch (rank()) { - case 0: - Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); - break; - case 1: - if (alloc_props.contiguous()) { - Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); - } else { - auto to_view = get_strided_view< ST*, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST**, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST***, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST****, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST*****, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST******, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); + break; + case 1: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 2: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 3: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 4: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 5: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 6: + Kokkos::deep_copy(get_view(), get_view()); + break; + default: + EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); + } + } else { + // Store values in contiguous helper field + m_contiguous_field->deep_copy(*this); + + // Sync helper field + if constexpr (To==Host) { + m_contiguous_field->sync_to_host(); + } else { + m_contiguous_field->sync_to_dev(); + } + + // Copy values back to this field + deep_copy(*m_contiguous_field); } } From e52c5cf6bb0c4fee3d7fc73dae254b299f1a2244 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 3 Oct 2024 14:26:28 -0600 Subject: [PATCH 860/904] EAMxx: allow non-contiguous set of cpus in test-all-scream --- components/eamxx/scripts/test_all_scream.py | 28 ++++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index f5350954b1c3..61ce833e5640 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -398,7 +398,7 @@ def generate_cmake_config(self, test, for_ctest=False): return result ############################################################################### - def get_taskset_range(self, test, for_compile=True): + def get_taskset_resources(self, test, for_compile=True): ############################################################################### res_name = "compile_res_count" if for_compile else "testing_res_count" @@ -419,13 +419,11 @@ def get_taskset_range(self, test, for_compile=True): expect(offset < len(affinity_cp), f"Offset {offset} out of bounds (max={len(affinity_cp)}) for test {test}\naffinity_cp: {affinity_cp}") - start = affinity_cp[offset] - end = start - for i in range(1, getattr(test, res_name)): - expect(affinity_cp[offset+i] == start+i, f"Could not get contiguous range for test {test}") - end = affinity_cp[offset+i] + resources = [] + for i in range(0, getattr(test, res_name)): + resources.append(affinity_cp[offset+i]) - return start, end + return resources ############################################################################### def create_ctest_resource_file(self, test, build_dir): @@ -438,7 +436,7 @@ def create_ctest_resource_file(self, test, build_dir): # res group is where we usually bind an individual MPI rank. # The id of the res groups on is offset so that it is unique across all builds - start, end = self.get_taskset_range(test, for_compile=False) + resources = self.get_taskset_resources(test, for_compile=False) data = {} @@ -448,7 +446,7 @@ def create_ctest_resource_file(self, test, build_dir): # We add leading zeroes to ensure that ids will sort correctly # both alphabetically and numerically devices = [] - for res_id in range(start,end+1): + for res_id in resources: devices.append({"id":f"{res_id:05d}"}) # Add resource groups @@ -457,7 +455,7 @@ def create_ctest_resource_file(self, test, build_dir): with (build_dir/"ctest_resource_file.json").open("w", encoding="utf-8") as outfile: json.dump(data,outfile,indent=2) - return (end-start)+1 + return len(resources) ############################################################################### def generate_ctest_config(self, cmake_config, extra_configs, test): @@ -498,11 +496,11 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): # Ctest can only competently manage test pinning across a single instance of ctest. For # multiple concurrent instances of ctest, we have to help it. It's OK to use the compile_res_count - # taskset range even though the ctest script is also running the tests + # taskset resources even though the ctest script is also running the tests if self._parallel: - start, end = self.get_taskset_range(test) + resources = self.get_taskset_resources(test) result = result.replace("'", r"'\''") # handle nested quoting - result = f"taskset -c {start}-{end} sh -c '{result}'" + result = f"taskset -c {','.join([str(r) for r in resources])} sh -c '{result}'" return result @@ -539,8 +537,8 @@ def generate_baselines(self, test): cmd = f"make -j{test.compile_res_count}" if self._parallel: - start, end = self.get_taskset_range(test) - cmd = f"taskset -c {start}-{end} sh -c '{cmd}'" + resources = self.get_taskset_resources(test) + cmd = f"taskset -c {','.join([str(r) for r in resources])} sh -c '{cmd}'" stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True) From f1b3a4bb720ec7f8f579072b2f291e436da36999 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Fri, 4 Oct 2024 13:53:21 -0700 Subject: [PATCH 861/904] Change the name of fail option to be self-descriptive --- .../cime_config/namelist_defaults_scream.xml | 15 +++++++++++++-- .../eamxx/src/control/atmosphere_driver.cpp | 9 ++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 9a13029cec6f..7e92482e6767 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -48,7 +48,7 @@ be lost if SCREAM_HACK_XML is not enabled. ctl_nl - driver_options,iop_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters + driver_debug_options,driver_options,iop_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters @@ -602,9 +602,20 @@ be lost if SCREAM_HACK_XML is not enabled. + + + + + -9999 + + + - -9999 0 ("force_crash_nsteps", -9999)==m_current_ts.get_num_steps(); + if (force_fail_nstep) { + abort(); + } } // Make sure the end of the time step is after the current start_time EKAT_REQUIRE_MSG (dt>0, "Error! Input time step must be positive.\n"); From 61874d7d569ac84b694dc74913894e10e8a923d1 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 4 Oct 2024 12:41:14 -0600 Subject: [PATCH 862/904] Use asynchronous deep_copy in non-contig sync_views_impl --- components/eamxx/src/share/field/field.cpp | 8 +- components/eamxx/src/share/field/field.hpp | 7 +- .../eamxx/src/share/field/field_impl.hpp | 97 +++++++++++++++---- 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 686e70bd5881..00a2c9b593fc 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -60,7 +60,7 @@ Field::clone(const std::string& name) const { } void Field:: -sync_to_host () const { +sync_to_host (const bool fence) const { // Sanity check EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); @@ -86,12 +86,14 @@ sync_to_host () const { EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); } + if (fence) Kokkos::fence(); + // Return field to read-only state m_is_read_only = original_read_only; } void Field:: -sync_to_dev () const { +sync_to_dev (const bool fence) const { // Sanity check EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); @@ -112,6 +114,8 @@ sync_to_dev () const { default: EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_dev.\n"); } + + if (fence) Kokkos::fence(); } Field Field:: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index c071a79c840c..a338199f2ef0 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -193,8 +193,11 @@ class Field { // Note: this class takes no responsibility in keeping track of whether // a sync is required in either direction. Mainly because we expect // host views to be seldom used, and even less frequently modified. - void sync_to_host () const; - void sync_to_dev () const; + // The fence input controls whether a fence is done at the end of the sync. + // If multiple syncs are performed in a row on different data, the user may + // want to run them asynchronously and fence the final sync_to call. + void sync_to_host (const bool fence = true) const; + void sync_to_dev (const bool fence = true) const; // Set the field to a constant value (on host or device) template diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 076ffc9d2da0..ba484466747e 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -224,48 +224,103 @@ get_strided_view_type { template void Field::sync_views_impl () const { + // All deep_copy call we want to be async for host. + using DeviceExecSpace = typename Field::get_device::execution_space; + + // Rank 0 will always be contiguous. Copy and return early. + if (rank() == 0) { + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + return; + } const bool is_contiguous = get_header().get_alloc_properties().contiguous(); if (is_contiguous) { - // For contiguous fields, simply use Kokkos::deep_copy() + // For contiguous fields, simply use Kokkos::deep_copy(). switch (rank()) { - case 0: - Kokkos::deep_copy(get_view(), get_view()); - break; case 1: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 2: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 3: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 4: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 5: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 6: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; default: EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); } } else { - // Store values in contiguous helper field - m_contiguous_field->deep_copy(*this); - - // Sync helper field - if constexpr (To==Host) { - m_contiguous_field->sync_to_host(); - } else { - m_contiguous_field->sync_to_dev(); + auto sync_helper = [this] () { + if constexpr (To==Host) m_contiguous_field->sync_to_host(); + else m_contiguous_field->sync_to_dev(); + }; + switch (rank()) { + case 1: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 2: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 3: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 4: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 5: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 6: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + default: + EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); } - - // Copy values back to this field - deep_copy(*m_contiguous_field); } } From 1db9a5aae70cd5c89018eccf1a61c68bc5019723 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 4 Oct 2024 15:35:11 -0600 Subject: [PATCH 863/904] better comment --- components/eamxx/src/share/field/field_impl.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index ba484466747e..1c0d8257647f 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -224,7 +224,8 @@ get_strided_view_type { template void Field::sync_views_impl () const { - // All deep_copy call we want to be async for host. + // For all Kokkos::deep_copy() calls we will pass in an instance of the + // device execution space so that we are asynchronous w.r.t. host. using DeviceExecSpace = typename Field::get_device::execution_space; // Rank 0 will always be contiguous. Copy and return early. From 60c99c36012bbb5c8e3bc0ab2a2e7ac68ad73501 Mon Sep 17 00:00:00 2001 From: Darin Comeau Date: Sun, 6 Oct 2024 11:08:42 -0500 Subject: [PATCH 864/904] Tweaking Chrysalis PE layout --- cime_config/allactive/config_pesall.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 53e1dd2e3376..96ca63f5fbe3 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -848,17 +848,17 @@ -compset WCYCL*/CRYO* -res SOwISC12to30E3r3* on 52 nodes pure-MPI, ~8.5 sypd - 1350 - 128 - 128 - 1280 + 1408 + 384 + 384 + 1024 1920 1408 0 - 1280 - 1280 + 1024 + 1024 0 1408 0 From 2481ef6af48f0eecbd0c800e927166e092257d7a Mon Sep 17 00:00:00 2001 From: Michael Kelleher Date: Mon, 7 Oct 2024 10:01:24 -0500 Subject: [PATCH 865/904] Fix import of moved method --- components/mpas-ocean/cime_config/SystemTests/mvko.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/cime_config/SystemTests/mvko.py b/components/mpas-ocean/cime_config/SystemTests/mvko.py index 71477a4dfc12..90fa4b7a5539 100644 --- a/components/mpas-ocean/cime_config/SystemTests/mvko.py +++ b/components/mpas-ocean/cime_config/SystemTests/mvko.py @@ -18,6 +18,7 @@ import netCDF4 as nc import CIME.test_status +from CIME.status import append_testlog import CIME.utils from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup @@ -394,4 +395,4 @@ def _compare_baseline(self): f" {viewing}" ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) From f5b8ad73403b8390a9c587ea67c201a95925d97c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 7 Oct 2024 11:38:53 -0600 Subject: [PATCH 866/904] Adds NOOPT_FILES concept to mpas cmake system And uses this system to deoptimize icepack_shortwave_data.F90, a file that takes a very long time to compile and is just setting up consts. --- components/mpas-framework/src/build_core.cmake | 7 +++++++ components/mpas-seaice/src/seaice.cmake | 1 + 2 files changed, 8 insertions(+) diff --git a/components/mpas-framework/src/build_core.cmake b/components/mpas-framework/src/build_core.cmake index 23c1935a8abb..a2c361e069e5 100644 --- a/components/mpas-framework/src/build_core.cmake +++ b/components/mpas-framework/src/build_core.cmake @@ -62,6 +62,13 @@ function(build_core CORE) endforeach() endif() + # Disable optimizations on some files that would take too long to compile, expect these to all be fortran files + foreach (SOURCE_FILE IN LISTS NOOPT_FILES) + get_filename_component(SOURCE_EXT ${SOURCE_FILE} EXT) + string(REPLACE "${SOURCE_EXT}" ".f90" SOURCE_F90 ${SOURCE_FILE}) + e3sm_deoptimize_file(${CMAKE_BINARY_DIR}/${SOURCE_F90}) + endforeach() + genf90_targets("${RAW_SOURCES}" "${INCLUDES}" "${CPPDEFS}" "${NO_PREPROCESS}" "${INC_DIR}") target_sources(${COMPONENT} PRIVATE ${SOURCES}) diff --git a/components/mpas-seaice/src/seaice.cmake b/components/mpas-seaice/src/seaice.cmake index 8b72de8ec7d7..5d314fd7594d 100644 --- a/components/mpas-seaice/src/seaice.cmake +++ b/components/mpas-seaice/src/seaice.cmake @@ -144,6 +144,7 @@ set(SEAICE_MODEL_FORWARD ) list(APPEND RAW_SOURCES ${SEAICE_MODEL_FORWARD}) list(APPEND DISABLE_QSMP ${SEAICE_MODEL_FORWARD}) +list(APPEND NOOPT_FILES "core_seaice/icepack/columnphysics/icepack_shortwave_data.F90") # Generate core input handle_st_nl_gen("namelist.seaice" "streams.seaice stream_list.seaice. listed" ${CORE_INPUT_DIR} ${CORE_BLDDIR}) From f3ef2e69ab453fe034fbf9554a4f05a070965e33 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Mon, 7 Oct 2024 11:34:22 -0700 Subject: [PATCH 867/904] Simply call to new abort feature in the AD code --- components/eamxx/src/control/atmosphere_driver.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index faf2a725ef6c..a4bd4fabc528 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -1595,12 +1595,12 @@ void AtmosphereDriver::run (const int dt) { start_timer("EAMxx::run"); // DEBUG option: Check if user has set the run to fail at a specific timestep. - if (m_atm_params.isSublist("driver_debug_options")) { - bool force_fail_nstep = m_atm_params.sublist("driver_debug_options").get("force_crash_nsteps", -9999)==m_current_ts.get_num_steps(); - if (force_fail_nstep) { - abort(); - } + auto& debug = m_atm_params.sublist("driver_debug_options"); + auto fail_step = debug.get("force_crash_nsteps",-1); + if (fail_step==m_current_ts.get_num_steps()) { + std::abort(); } + // Make sure the end of the time step is after the current start_time EKAT_REQUIRE_MSG (dt>0, "Error! Input time step must be positive.\n"); From e1dff58884c2114e76c953b2460e03a2092447d1 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 7 Oct 2024 15:37:53 -0700 Subject: [PATCH 868/904] use FI_MR_CACHE_MONITOR=kdreg2 for all nersc machines --- cime_config/machines/config_machines.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 140b733f83db..76c206ed0b27 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -278,6 +278,7 @@ FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -444,6 +445,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -587,6 +589,7 @@ FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software + kdreg2 MPI_Bcast $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} @@ -756,6 +759,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -898,6 +902,7 @@ FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} From f37edf7fdc23b23b01dcf4f73d62f79a44a6d92f Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 7 Oct 2024 16:31:11 -0700 Subject: [PATCH 869/904] correct default fates parameter for API36.1 --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 7d2390d267b1..896fe5d73969 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -134,7 +134,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.36.1.0_12pft_c240926.nc +lnd/clm2/paramdata/fates_params_api.36.1.0_14pft_c241003.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc From 39ca74583f72e8fb9286954c10ded5c8ba994fd1 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 7 Oct 2024 16:36:39 -0700 Subject: [PATCH 870/904] update fates submodule to sci.1.78.3_api.36.1.0 --- components/elm/src/external_models/fates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/external_models/fates b/components/elm/src/external_models/fates index 1982b0032c3c..825579d0b406 160000 --- a/components/elm/src/external_models/fates +++ b/components/elm/src/external_models/fates @@ -1 +1 @@ -Subproject commit 1982b0032c3cab6278892eccb85f643114ffb1af +Subproject commit 825579d0b406fe99344591b5ed8356e5c7aeebec From ed346d4bb74a67593308a85e8a7dd8ab5ffec564 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 7 Oct 2024 16:39:45 -0700 Subject: [PATCH 871/904] update fates hydro test module to change sapwood allometry mode This update calls the fates parameter modification tooling to set the grass allometry smode to 1, which is the default for all other fates PFTs. This is a temporary workaround for NGEET/fates#1254. --- .../testmods_dirs/elm/fateshydro/shell_commands | 15 +++++++++++++++ .../testmods_dirs/elm/fateshydro/user_nl_elm | 1 + 2 files changed, 16 insertions(+) diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands index 84648f9222c6..e82b5ce85ff1 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands @@ -1,2 +1,17 @@ #!/bin/bash if [ `./xmlquery --value MACH` == bebop ]; then ./xmlchange --id LND_PIO_TYPENAME --val netcdf; fi +module load e4s +spack env activate gcc +spack load nco + +SRCDIR=`./xmlquery SRCROOT --value` +CASEDIR=`./xmlquery CASEROOT --value` +FATESDIR=$SRCDIR/src/fates +FATESPARAMFILE=$CASEDIR/fates_params_hydrograsstempfix.nc + +ncgen -o $FATESPARAMFILE $FATESDIR/parameter_files/fates_params_default.cdl + +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_allom_smode --val 1 --allpfts + +spack unload nco +module unload e4s diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm index c74b77d05dc6..ea831e5483b9 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm @@ -5,6 +5,7 @@ hist_nhtfrq = -24 hist_empty_htapes = .true. use_fates_planthydro = .true. fates_spitfire_mode = 1 +fates_paramfile = '$CASEROOT/fates_params_hydrograsstempfix.nc' hist_fincl1 = 'FATES_ERRH2O_SZPF', 'FATES_TRAN_SZPF', 'FATES_SAPFLOW_SZPF', 'FATES_ITERH1_SZPF','FATES_ABSROOT_H2O_SZPF', 'FATES_TRANSROOT_H2O_SZPF','FATES_STEM_H2O_SZPF','FATES_LEAF_H2O_SZPF', From 58125633d7253aca8e22fe83e56eaa6722f7eee1 Mon Sep 17 00:00:00 2001 From: Irena Vankova Date: Tue, 8 Oct 2024 08:31:51 -0600 Subject: [PATCH 872/904] replaceconfig_flux_attenuation_coefficient_runoff with config_flux_attenuation_coefficient_subglacial_runoff in subglacial kpp loop --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 1777e1f6d1e9..0334667f2f10 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3574,8 +3574,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & do k = maxLevelCell(iCell), minLevelCell(iCell), -1 zBot = zTop - layerThickness(k,iCell) if (k == minLevelCell(iCell)) then - transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) - transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) + transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_subglacial_runoff, -100.0_RKIND) ) + transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_subglacial_runoff, -100.0_RKIND) ) fracAbsorbedSubglacialRunoff = transmissionCoeffTop - transmissionCoeffBot end if zTop = zBot From 7ba6c85a8e907d56713aaa78a13b0966586cb9f4 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Wed, 9 Oct 2024 07:29:38 -0400 Subject: [PATCH 873/904] Address review comments --- components/elm/docs/user-guide/interpinic.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md index fcb897834f22..f868ee7e8d2a 100644 --- a/components/elm/docs/user-guide/interpinic.md +++ b/components/elm/docs/user-guide/interpinic.md @@ -47,6 +47,8 @@ cd components/elm/tools/interpinic/src export USER_LDFLAGS="-L$NETCDF_C_DIR/lib -lnetcdf -L$NETCDF_F_DIR/lib -lnetcdff -L$HDF5_DIR/lib -lhdf5" USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 + +cd ../ ``` ## 4. Run `interpinic` @@ -66,10 +68,10 @@ The `interpinic` can then be run via the following batch job (e.g., `remap.r025_ # Load relevant modules. cd -eval $(./cime/CIME/Tools/get_case_env/get_case_env) +eval $(./cime/CIME/Tools/get_case_env) # Change dir to `interpinic` -cd components/elm/tools/interpinic/src +cd components/elm/tools/interpinic/ srun -n 1 ./interpinic \ -i /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc \ From 468866d8b3c603ea239eb111fb8df6216faac348 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 9 Oct 2024 11:45:49 -0500 Subject: [PATCH 874/904] Make bld files consistent with Registry --- .../bld/namelist_files/namelist_definition_mpassi.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml index efbaef07cb9d..ba6ae4be4bb1 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml @@ -1077,7 +1077,7 @@ Default: Defined in namelist_defaults.xml -Use the humic matter tracer +Use the humic (refractory dissolved organic matter) tracer Valid values: true or false Default: Defined in namelist_defaults.xml @@ -1949,7 +1949,7 @@ Default: Defined in namelist_defaults.xml -Transport type of humics +Transport type of humics (refractory dissolved organic matter) Valid values: -1 = entirely in the mobile phase; 0 = retention dominated; 1 = release dominated; 0.5 = equal but rapid exchange; 2 = equal but slow exchange Default: Defined in namelist_defaults.xml From 6f3edeae3bd2a56ff3fb803bbba3d5986a8090d4 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 9 Oct 2024 17:09:34 -0400 Subject: [PATCH 875/904] keep github actions working for e3sm repo --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 04f7fcb4ffcf..6c9ee1ab114c 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -25,7 +25,7 @@ on: jobs: ci: - if: false + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index d39717589914..852bceec5359 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -13,7 +13,7 @@ on: jobs: scream-defaults: - if: false + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} From 64f041b9f8db1089902c956fe293d900d78ae159 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 10 Oct 2024 11:22:10 -0700 Subject: [PATCH 876/904] Make sure SCREAM_LIBS_ONLY is on We don't want testing stuff or small kernel stuff --- components/eam/src/physics/crm/pam/CMakeLists.txt | 3 ++- components/eamxx/CMakeLists.txt | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 4a8f88f669b0..ac629fa89fde 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -9,8 +9,9 @@ set(PAM_DRIVER_SRC params.F90) add_library(pam_driver - ${PAM_DRIVER_SRC}) + ${PAM_DRIVER_SRC}) +set(SCREAM_LIBS_ONLY TRUE) set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 07c3cda76728..9b6f54e5b056 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -200,6 +200,9 @@ option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them") set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") +message("JGF DEFAUL_SMALL_KERNELS is ${DEFAULT_SMALL_KERNELS}") +message("JGF SCREAM_SMALL_KERNELS is ${SCREAM_SMALL_KERNELS}") +message("JGF SCREAM_P3_SMALL_KERNELS is ${SCREAM_P3_SMALL_KERNELS}") set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") From 10099ac1b8d7a9a8b5ba3b49ac01e38eb30dba8d Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 10 Oct 2024 11:22:10 -0700 Subject: [PATCH 877/904] Make sure SCREAM_LIBS_ONLY is on We don't want testing stuff or small kernel stuff --- components/eam/src/physics/crm/pam/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 4a8f88f669b0..c55594523843 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -11,6 +11,7 @@ set(PAM_DRIVER_SRC add_library(pam_driver ${PAM_DRIVER_SRC}) +set(SCREAM_LIBS_ONLY TRUE) set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) From 2cdecfdd5cebf8a67fc687f743f49cac538de67d Mon Sep 17 00:00:00 2001 From: Peter Schwartz Date: Thu, 10 Oct 2024 15:44:43 -0400 Subject: [PATCH 878/904] Added more output and changed > 0._r8 checks to > tiny --- components/elm/src/biogeochem/VerticalProfileMod.F90 | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index a1b84fe945f5..1fa3db093b46 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -84,6 +84,7 @@ subroutine decomp_vertprofiles(bounds, & real(r8) :: nfixation_prof_sum real(r8) :: pdep_prof_sum real(r8) :: delta = 1.e-10_r8 + real(r8), parameter :: smallparameter = tiny(1._r8) character(len=32) :: subname = 'decomp_vertprofiles' !----------------------------------------------------------------------- @@ -191,7 +192,7 @@ subroutine decomp_vertprofiles(bounds, & surface_prof_tot = surface_prof_tot + surface_prof(j) * dzsoi_decomp(j) end if end do - if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > smallparameter) .and. (surface_prof_tot > smallparameter) ) then ! where there is not permafrost extending to the surface, integrate the profiles over the active layer ! this is equivalnet to integrating over all soil layers outside of permafrost regions do j = 1, min(max(altmax_lastyear_indx(c), 1), nlevdecomp) @@ -250,7 +251,7 @@ subroutine decomp_vertprofiles(bounds, & surface_prof_tot = surface_prof_tot + surface_prof(j) * dzsoi_decomp(j) end do if(col_pp%is_fates(c))then - if ( (altmax_lastyear_indx(c) > 0) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (surface_prof_tot > smallparameter) ) then do j = 1,min(alt_ind, nlevbed) nfixation_prof(c,j) = surface_prof(j)/ surface_prof_tot ndep_prof(c,j) = surface_prof(j)/ surface_prof_tot @@ -262,7 +263,7 @@ subroutine decomp_vertprofiles(bounds, & pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif else - if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > smallparameter) .and. (surface_prof_tot > smallparameter) ) then do j = 1, min(max(altmax_lastyear_indx(c), 1), nlevdecomp) nfixation_prof(c,j) = col_cinput_rootfr(c,j) / rootfr_tot if (j <= nlevbed) then @@ -345,6 +346,10 @@ subroutine decomp_vertprofiles(bounds, & write(iulog, *) 'surface_prof: ', surface_prof(:) write(iulog, *) 'p, itype(p), wtcol(p): ', p, veg_pp%itype(p), veg_pp%wtcol(p) write(iulog, *) 'cinput_rootfr(p,:): ', cinput_rootfr(p,:) + write(iulog,*) 'croot_prof(p,:): ',croot_prof(p,:) + write(iulog,*) 'froot_prof(p,:): ',froot_prof(p,:) + write(iulog,*) 'leaf_prof(p,:): ',leaf_prof(p,:) + write(iulog,*) 'stem_prof(p,:): ',stem_prof(p,:) call endrun(msg=' ERROR: sum-1 > delta'//errMsg(__FILE__, __LINE__)) endif end do From f5664a0d2aa06ca85138c3a78d5e3143c3d4d620 Mon Sep 17 00:00:00 2001 From: noel Date: Thu, 10 Oct 2024 15:32:34 -0700 Subject: [PATCH 879/904] Update pelayout on pm-cpu for ne4 scream tests to ensure 96 tasks. Rename a machinefile to reflect machine name --- cime_config/allactive/config_pesall.xml | 15 +++++++++++++++ .../machine-files/{gcp.cmake => gcp12.cmake} | 0 2 files changed, 15 insertions(+) rename components/eamxx/cmake/machine-files/{gcp.cmake => gcp12.cmake} (100%) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 96ca63f5fbe3..0d5f24565966 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -1797,6 +1797,21 @@ + + + allactive+pm-cpu: default, 1 node, 96 tasks, 1 thread + + 96 + 96 + 96 + 96 + 96 + 96 + 96 + 96 + + + diff --git a/components/eamxx/cmake/machine-files/gcp.cmake b/components/eamxx/cmake/machine-files/gcp12.cmake similarity index 100% rename from components/eamxx/cmake/machine-files/gcp.cmake rename to components/eamxx/cmake/machine-files/gcp12.cmake From c34cb650b81d231ccc8bc49e93c317be2b13c97e Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 10 Oct 2024 15:43:33 -0700 Subject: [PATCH 880/904] bug fix for pam_statistics.h --- .../eam/src/physics/crm/pam/pam_statistics.h | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_statistics.h b/components/eam/src/physics/crm/pam/pam_statistics.h index bb277600dfa1..8166b9912d26 100644 --- a/components/eam/src/physics/crm/pam/pam_statistics.h +++ b/components/eam/src/physics/crm/pam/pam_statistics.h @@ -458,29 +458,34 @@ inline void pam_statistics_compute_means( pam::PamCoupler &coupler ) { if (clear_rh_cnt(k,iens)>0) { clear_rh(k,iens) = clear_rh(k,iens) / clear_rh_cnt(k,iens); } - phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); - - phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); - - phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); - - phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + if (phys_tend_sgs_cnt(iens)>0) { + phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); + } + if (phys_tend_micro_cnt(iens)>0) { + phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); + } + if (phys_tend_dycor_cnt(iens)>0) { + phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); + } + if (phys_tend_sponge_cnt(iens)>0) { + phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + } }); //------------------------------------------------------------------------------------------------ } From 12c6dfd3f28ade7398e4c443ffa11d7cf2796f1a Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 11 Oct 2024 08:08:33 -0700 Subject: [PATCH 881/904] update PAM submodule --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 3ea20ad38f28..c3b6522c57b7 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 3ea20ad38f286730973429e0d491420a6f599f11 +Subproject commit c3b6522c57b754d073e1deaad5ce8125f7f88325 From 1b5fb62f8e37402a1a1f2cdf17405ce851112385 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 11 Oct 2024 11:51:37 -0400 Subject: [PATCH 882/904] enable shared builds for gh-standalone --- .github/workflows/eamxx-gh-ci-standalone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index 9fde89ed35bf..d276d600357e 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -44,7 +44,7 @@ jobs: run: | # TODO: get rid of this extra line if we can? git config --global safe.directory '*' - ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} + ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} -c BUILD_SHARED_LIBS=ON - name: Artifacts uses: actions/upload-artifact@v4 From b8a44d4038ff666e73b075f1d82dc6ce52c14de8 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Fri, 11 Oct 2024 09:30:25 -0700 Subject: [PATCH 883/904] Revert "update fates hydro test module to change sapwood allometry mode" This reverts commit ed346d4bb74a67593308a85e8a7dd8ab5ffec564. --- .../testmods_dirs/elm/fateshydro/shell_commands | 15 --------------- .../testmods_dirs/elm/fateshydro/user_nl_elm | 1 - 2 files changed, 16 deletions(-) diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands index e82b5ce85ff1..84648f9222c6 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/shell_commands @@ -1,17 +1,2 @@ #!/bin/bash if [ `./xmlquery --value MACH` == bebop ]; then ./xmlchange --id LND_PIO_TYPENAME --val netcdf; fi -module load e4s -spack env activate gcc -spack load nco - -SRCDIR=`./xmlquery SRCROOT --value` -CASEDIR=`./xmlquery CASEROOT --value` -FATESDIR=$SRCDIR/src/fates -FATESPARAMFILE=$CASEDIR/fates_params_hydrograsstempfix.nc - -ncgen -o $FATESPARAMFILE $FATESDIR/parameter_files/fates_params_default.cdl - -$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_allom_smode --val 1 --allpfts - -spack unload nco -module unload e4s diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm index ea831e5483b9..c74b77d05dc6 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm @@ -5,7 +5,6 @@ hist_nhtfrq = -24 hist_empty_htapes = .true. use_fates_planthydro = .true. fates_spitfire_mode = 1 -fates_paramfile = '$CASEROOT/fates_params_hydrograsstempfix.nc' hist_fincl1 = 'FATES_ERRH2O_SZPF', 'FATES_TRAN_SZPF', 'FATES_SAPFLOW_SZPF', 'FATES_ITERH1_SZPF','FATES_ABSROOT_H2O_SZPF', 'FATES_TRANSROOT_H2O_SZPF','FATES_STEM_H2O_SZPF','FATES_LEAF_H2O_SZPF', From 9059431e06373527939ac1dde50c4c458f87ff43 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Fri, 11 Oct 2024 12:31:57 -0500 Subject: [PATCH 884/904] Add an fsurdat file for r025 and year 2000 --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index cc1bd512e69a..bd058707f765 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -360,6 +360,8 @@ lnd/clm2/surfdata_map/surfdata_ne256np4_simyr2010_c220518.nc lnd/clm2/surfdata_map/surfdata_ne256np4_simyr2010_c220518.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr2000_c200624.nc + +lnd/clm2/surfdata_map/surfdata_r025_simyr1980_c240920.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr2000_c190730.nc From 9fe1cd9b7df6b65f3512c766b69ee79c158c99a7 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 11 Oct 2024 15:29:00 -0500 Subject: [PATCH 885/904] Upgrade NetCDF modules for Intel and GNU compilers on Chrysalis Upgraded NetCDF modules on Chrysalis to support CDF5 types in netcdf-fortran. netcdf-c: Updated from 4.4.1 to 4.7.1 (new netcdf-fortran 4.5.3 modules were built with netcdf-c 4.7.1) netcdf-fortran: Updated from 4.4.4 to 4.5.3 (4.4.5 adds nf_64bit_data; 4.5.0 introduces support for unsigned and 64-bit integer types) netcdf-cxx: Removed (not in use, and no modules available built with netcdf-c 4.7.1) Additionally, restored gcc/9.2.0 (required by the specific module intel-mpi/2019.9.304-jdih7h5) for compiler="gnu" and mpilib="impi" configurations. --- cime_config/machines/config_machines.xml | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 140b733f83db..8eb859b91905 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2644,17 +2644,15 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss openmpi/4.1.6-2mm63n2 hdf5/1.10.7-4cghwvq - netcdf-c/4.4.1-a4hji6e - netcdf-cxx/4.2-ldoxr43 - netcdf-fortran/4.4.4-husened + netcdf-c/4.7.4-4qjdadt + netcdf-fortran/4.5.3-qozrykr parallel-netcdf/1.11.0-icrpxty intel-mpi/2019.9.304-tkzvizk - hdf5/1.8.16-se4xyo7 - netcdf-c/4.4.1-qvxyzq2 - netcdf-cxx/4.2-binixgj - netcdf-fortran/4.4.4-rdxohvp + hdf5/1.10.7-wczt56s + netcdf-c/4.7.4-ba6agmb + netcdf-fortran/4.5.3-5lvy5p4 parallel-netcdf/1.11.0-b74wv4m @@ -2664,17 +2662,19 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss openmpi/4.1.6-ggebj5o hdf5/1.10.7-ol6xuae - netcdf-c/4.4.1-2njo6xx - netcdf-cxx/4.2-7pdzqua - netcdf-fortran/4.4.4-52c6oqi + netcdf-c/4.7.4-pfocec2 + netcdf-fortran/4.5.3-va3hoor parallel-netcdf/1.11.0-d7h4ysd + gcc/11.2.0-bgddrif + intel-oneapi-mkl/2022.1.0-w4kgsn4 + gcc/9.2.0-ugetvbp + intel-mkl/2020.4.304-n3b5fye intel-mpi/2019.9.304-jdih7h5 hdf5/1.8.16-dtbpce3 - netcdf-c/4.4.1-zcoa44z - netcdf-cxx/4.2-ayxg4c7 - netcdf-fortran/4.4.4-2lfr2lr + netcdf-c/4.7.4-seagl7g + netcdf-fortran/4.5.3-ova6t37 parallel-netcdf/1.11.0-ifdodru From 7182613fb62cf7a4014d2e9938a54350f0d0fae8 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 15 Oct 2024 12:18:59 -0400 Subject: [PATCH 886/904] upgrades and fixes to workflows --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 8 ++++++-- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 8 ++++++-- .github/workflows/e3sm-gh-md-linter.yml | 5 +++++ .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml | 5 +++++ .github/workflows/eamxx-gh-ci-standalone.yml | 1 + .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 6 +++++- 8 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 6c9ee1ab114c..5c6ff081f730 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -22,10 +22,14 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: ci: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -36,7 +40,7 @@ jobs: - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu - ERS_Ld5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.0 steps: - diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index 48c367c8f625..d5687a32b375 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -11,10 +11,14 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: ci-w: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -23,7 +27,7 @@ jobs: - SMS_D_Ld1_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - ERS_Ld3_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.0 steps: - diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 424a871637b6..46319b08658b 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -10,8 +10,13 @@ on: # for now let's not lint files in eamxx - '!components/eamxx/**/*.md' +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: linter: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ebd2ac9c1e97..dec9bc696bfb 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -15,7 +15,7 @@ concurrency: jobs: Build-and-Deploy-docs: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml index 8fe212886d9f..fcbf439f4097 100644 --- a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml +++ b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml @@ -11,8 +11,13 @@ on: - cron: '00 15 * * 2' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: mkatmsrffile-test: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index d276d600357e..17f79c09eeb4 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -18,6 +18,7 @@ on: jobs: ci: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index 2ce70d97843c..488c2b11a020 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -34,7 +34,7 @@ concurrency: jobs: eamxx-docs: - if: ${{ github.event.repository.name == 'scream' }} + if: ${{ github.repository == 'E3SM-Project/scream' }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 852bceec5359..38c528306c40 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -11,9 +11,13 @@ on: - cron: '00 00 * * *' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: scream-defaults: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} From c79b9e4a46910b4e1b7719a7253607ed59787016 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 15 Oct 2024 12:25:32 -0400 Subject: [PATCH 887/904] updates to miniforge settings --- .github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml index fcbf439f4097..cacb951b8a83 100644 --- a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml +++ b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml @@ -36,10 +36,7 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: activate-environment: "envmkatmsrffile" - miniforge-variant: Mambaforge miniforge-version: latest - use-mamba: true - mamba-version: "*" channel-priority: strict auto-update-conda: true python-version: 3.11 @@ -47,7 +44,7 @@ jobs: name: Install dependencies run: | echo $CONDA_PREFIX - mamba install -y nco xarray numba numpy netcdf4 + conda install -y nco xarray numba numpy netcdf4 -c conda-forge - name: Run tests working-directory: components/eam/tools/mkatmsrffile From 2dc3601547da390460ef0b408e79a45989dbc8bc Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 15 Oct 2024 12:26:53 -0400 Subject: [PATCH 888/904] add concurrency check for standalone --- .github/workflows/eamxx-gh-ci-standalone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index 17f79c09eeb4..19a2ec9cd8e9 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -15,6 +15,10 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: ci: From aef19a4edb4332465ccf531a044df991222783ad Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 15 Oct 2024 12:48:46 -0400 Subject: [PATCH 889/904] rename gh/ci-w to gh-w/ci --- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index d5687a32b375..f51aa88a34c4 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -1,4 +1,4 @@ -name: gh +name: gh-w on: pull_request: @@ -17,7 +17,7 @@ concurrency: jobs: - ci-w: + ci: if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: From 321b886377d9b4482cde7fbdff2262c911ad60d0 Mon Sep 17 00:00:00 2001 From: noel Date: Tue, 15 Oct 2024 15:00:38 -0700 Subject: [PATCH 890/904] For pm-cpu, update compiler versions for GCC, nvidia, AMD compilers. For those compilers, update other module versions to now be same as Intel uses. Various updates to muller-cpu/muller-gpu/alvarez. Changes to make machine entries of the NERSC machines more consistent. --- cime_config/machines/config_machines.xml | 117 +++++++++++++++-------- 1 file changed, 78 insertions(+), 39 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7af65ddcccbf..055909533c61 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -190,6 +190,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -218,8 +219,8 @@ PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.02.1.1 + gcc-native/12.3 + cray-libsci/23.12.5 @@ -229,35 +230,23 @@ PrgEnv-nvidia - nvidia/22.7 - cray-libsci/23.02.1.1 + nvidia/23.9 + cray-libsci/23.12.5 PrgEnv-aocc - aocc/4.0.0 - cray-libsci/23.02.1.1 + aocc/4.1.0 + cray-libsci/23.12.5 - + craype-accel-host craype/2.7.30 cray-mpich/8.1.28 cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - - - - craype-accel-host - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - - - cmake/3.24.3 evp-patch @@ -367,6 +356,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -377,13 +367,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit - cray-libsci climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -435,6 +426,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -590,12 +582,11 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast - $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} - $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} @@ -603,6 +594,8 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} @@ -618,6 +611,13 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 @@ -684,13 +684,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit - cray-libsci climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -742,6 +743,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -752,6 +754,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -762,6 +765,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} @@ -819,6 +825,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -846,36 +853,35 @@ - PrgEnv-gnu - gcc-native - cray-libsci + PrgEnv-gnu/8.5.0 + gcc-native/13.2 + cray-libsci/24.03.0 - PrgEnv-intel - intel + PrgEnv-intel/8.5.0 + intel/2024.1.0 PrgEnv-nvidia nvidia/24.5 - cray-libsci + cray-libsci/24.03.0 PrgEnv-aocc - aocc/4.0.1 - cray-libsci + aocc/4.1.0 + cray-libsci/24.03.0 craype-accel-host - cray-libsci - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 + craype/2.7.31.11 + cray-mpich/8.1.29 + cray-hdf5-parallel/1.12.2.11 + cray-netcdf-hdf5parallel/4.9.0.11 + cray-parallel-netcdf/1.12.3.11 cmake/3.24.3 @@ -883,6 +889,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -893,16 +900,48 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} + Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$BLAS_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$BLAS_ROOT"; fi} + $SHELL{if [ -z "$LAPACK_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$LAPACK_ROOT"; fi} + NVHPC + + + Intel10_64_dyn + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 + Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. .*spock.* From 3cf430e36de5b7abc533607ccc464b1709f4979a Mon Sep 17 00:00:00 2001 From: noel Date: Tue, 15 Oct 2024 19:00:06 -0700 Subject: [PATCH 891/904] remove special flags for gnu builds as kdreg2 seems to fix the issue --- .../machines/Depends.muller-cpu.gnu.cmake | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/cime_config/machines/Depends.muller-cpu.gnu.cmake b/cime_config/machines/Depends.muller-cpu.gnu.cmake index 5c7331f979a9..53f8b536651e 100644 --- a/cime_config/machines/Depends.muller-cpu.gnu.cmake +++ b/cime_config/machines/Depends.muller-cpu.gnu.cmake @@ -7,23 +7,3 @@ if (NOT DEBUG) e3sm_deoptimize_file("${ITEM}") endforeach() endif() - -# On pm-cpu (and muller-cpu), with gcc-native/12.3, we see hang with DEBUG runs of certain tests. -# https://github.com/E3SM-Project/E3SM/issues/6516 -# Currently, we have pm-cpu using gcc/12.2.0 which does not have this issue, but using muller-cpu to test 12.3 -# Turning off -O0 for these 2 files (by adding -O) at least avoids hang and will produce FPE in HOMME code -if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3) - if (DEBUG) - - set(ADJUST - eam/src/dynamics/se/inidat.F90 - eam/src/dynamics/se/dyn_comp.F90 - ) - - foreach(ITEM IN LISTS ADJUST) - e3sm_add_flags("${ITEM}" "-O") - #e3sm_add_flags("${ITEM}" "-DNDEBUG -O") - endforeach() - - endif() -endif() From 224a8397431176cefc20b5c8e9308b2efd9ea46b Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 16 Oct 2024 09:15:11 -0400 Subject: [PATCH 892/904] load Core/24.07 module and remove versions from git, submodule, and zlib modules --- cime_config/machines/config_machines.xml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7af65ddcccbf..49dcfdb2d25b 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1029,6 +1029,7 @@ /usr/share/lmod/lmod/libexec/lmod python + Core/24.07 PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 @@ -1041,6 +1042,7 @@ + Core/24.07 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 @@ -1049,6 +1051,7 @@ + Core/24.07 PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/12.2.0 @@ -1057,12 +1060,12 @@ rocm/5.4.0 - cray-python/3.9.13.1 + cray-python/3.11.5 cray-libsci - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - zlib/1.2.11 + cmake/3.27.9 + subversion + git + zlib cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 From 442e0686220a47a22e47a406714f4b9444fa5d8a Mon Sep 17 00:00:00 2001 From: Eva Sinha Date: Mon, 16 Oct 2023 12:23:36 -0500 Subject: [PATCH 893/904] Fix for bad leaf C:N ratios during transient land simulations --- components/elm/src/biogeochem/ComputeSeedMod.F90 | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/components/elm/src/biogeochem/ComputeSeedMod.F90 b/components/elm/src/biogeochem/ComputeSeedMod.F90 index 61b916d64f35..292aa036603f 100644 --- a/components/elm/src/biogeochem/ComputeSeedMod.F90 +++ b/components/elm/src/biogeochem/ComputeSeedMod.F90 @@ -183,16 +183,10 @@ subroutine LeafProportions(pft_type, ignore_current_state, & pstorage = 0._r8 pxfer = 0._r8 - if (tot_leaf == 0._r8 .or. ignore_current_state) then - if (veg_vp%evergreen(pft_type) == 1._r8) then - pleaf = 1._r8 - else - pstorage = 1._r8 - end if + if (veg_vp%evergreen(pft_type) == 1._r8) then + pleaf = 1._r8 else - pleaf = leaf /tot_leaf - pstorage = leaf_storage/tot_leaf - pxfer = leaf_xfer /tot_leaf + pstorage = 1._r8 end if end subroutine LeafProportions From ca312d9ff828136f7def5888798bfb39423c962c Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 21 Oct 2024 07:33:04 -0700 Subject: [PATCH 894/904] bump nvidia compiler version on pm-cpu --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 055909533c61..40b3e1b5cc3f 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -230,7 +230,7 @@ PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 cray-libsci/23.12.5 From 8d89f6e61c30aa2576d24cc51a5de0acff16dafe Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 21 Oct 2024 07:51:38 -0700 Subject: [PATCH 895/904] remove kdreg2 change as it's already on master --- cime_config/machines/config_machines.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 40b3e1b5cc3f..1b7520a69714 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -582,7 +582,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -754,7 +753,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -900,7 +898,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} From bdcc2f551cfae2fca53bd8aa4ec604601ddf1c68 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 21 Oct 2024 08:14:43 -0700 Subject: [PATCH 896/904] revert removing software as I hit merge conflict --- cime_config/machines/config_machines.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 1b7520a69714..593acc1b067c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -582,6 +582,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + software MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -898,6 +899,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + software MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} From ce98c5a075a3ebf4293fb923ce8ea3bc5962047b Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 11 Oct 2024 17:46:28 -0600 Subject: [PATCH 897/904] Update chicoma-cpu modules --- cime_config/machines/config_machines.xml | 50 ++++++++++++++---------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index acd10f74110b..b882536897ea 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4003,11 +4003,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4019,26 +4019,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 + PrgEnv-gnu/8.5.0 gcc/12.2.0 cray-libsci/23.05.1.4 @@ -4050,8 +4051,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -4060,15 +4061,22 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-libsci/23.05.1.4 - + + craype-accel-host + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + + + craype-accel-host craype/2.7.21 cray-mpich/8.1.26 - libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 @@ -4090,6 +4098,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + -1 @@ -4131,11 +4142,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4202,7 +4213,6 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 From 457c9439ad86c41a8ef80db04b086e3d8957983c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 14 Oct 2024 14:43:58 -0600 Subject: [PATCH 898/904] Add latest cmake --- cime_config/machines/config_machines.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index b882536897ea..3c20372aa53b 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4068,6 +4068,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 + cmake/3.27.7 @@ -4077,6 +4078,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + cmake/3.27.7 @@ -4213,6 +4215,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + cmake/3.27.7 From a44f71e8c87618bd43961529feb6378306ca5a26 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 17 Oct 2024 09:34:47 -0600 Subject: [PATCH 899/904] Updates to get E3SM working on chicoma-cpu --- cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake | 1 + cime_config/machines/config_machines.xml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake index a6148451eb76..a6c13942620f 100644 --- a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake @@ -5,6 +5,7 @@ endif() set(PIO_FILESYSTEM_HINTS "lustre") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3c20372aa53b..f893663f0ad8 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3983,7 +3983,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4068,7 +4068,6 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - cmake/3.27.7 @@ -4078,6 +4077,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + + + cmake/3.27.7 From 699dea0bf09e638ee9a897381e3528cee44821a0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 19 Oct 2024 15:07:53 -0600 Subject: [PATCH 900/904] Update nvidia modules --- cime_config/machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index f893663f0ad8..941bcef51707 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4045,8 +4045,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - PrgEnv-nvidia/8.4.0 - nvidia/22.7 + PrgEnv-nvidia/8.5.0 + nvidia/24.7 cray-libsci/23.05.1.4 From 6453565d37e8bc24930c753fa5b9ff4867b42f85 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 18:31:01 -0600 Subject: [PATCH 901/904] Add GNU_CRAY_LDFLAGS flag to MPAS standalone builds This should be set to: ``` export GNU_CRAY_LDFLAGS="-Wl,--enable-new-dtags" ``` on Chicoma-CPU with gcc. --- components/mpas-framework/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 1f9b00844e72..f92606987bbb 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -396,11 +396,11 @@ gnu-cray: "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ - "LDFLAGS_OPT = -O3 -m64" \ + "LDFLAGS_OPT = -O3 -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -g -m64" \ - "LDFLAGS_DEBUG = -g -m64" \ + "LDFLAGS_DEBUG = -g -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_OMP = -fopenmp" \ "CFLAGS_OMP = -fopenmp" \ "BUILD_TARGET = $(@)" \ From 750707a82541eb245db50da51f0a686afcd14536 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 21:43:12 -0600 Subject: [PATCH 902/904] Revert chicoma-gpu changes (for now) --- cime_config/machines/config_machines.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 941bcef51707..3964b1405a2a 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4146,11 +4146,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/lmod/init/perl + /usr/share/lmod/8.3.1/init/perl - /usr/share/lmod/lmod/init/python - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh + /usr/share/lmod/8.3.1/init/python + /usr/share/lmod/8.3.1/init/sh + /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4217,7 +4217,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.27.7 + cmake/3.25.1 From 0cb78eaff39e5290982f59e9eaf840f26de6a3c3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 21 Oct 2024 11:44:41 -0600 Subject: [PATCH 903/904] Update gcc and nvidia to newer versions This is to match proposed updates to Perlmutter-CPU https://github.com/E3SM-Project/E3SM/pull/6702 --- cime_config/machines/config_machines.xml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3964b1405a2a..b1df10a96e11 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4019,6 +4019,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 + gcc + gcc-native intel intel-oneapi nvidia @@ -4040,14 +4042,14 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.05.1.4 + gcc-native/12.3 + cray-libsci/23.12.5 PrgEnv-nvidia/8.5.0 nvidia/24.7 - cray-libsci/23.05.1.4 + cray-libsci/23.12.5 @@ -4061,25 +4063,13 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-libsci/23.05.1.4 - + craype-accel-host craype/2.7.30 cray-mpich/8.1.28 cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - - - - craype-accel-host - craype/2.7.21 - cray-mpich/8.1.26 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - - - cmake/3.27.7 @@ -4103,7 +4093,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_PARALLEL_NETCDF_PREFIX} - /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + /usr/lib64/gcc/x86_64-suse-linux/12:$ENV{LD_LIBRARY_PATH} -1 From 85cd5d3fa02b2e042cd571ed61dc20536612738d Mon Sep 17 00:00:00 2001 From: noel Date: Thu, 24 Oct 2024 14:49:00 -0700 Subject: [PATCH 904/904] Add overrun qos for pm-cpu/pm-gpu --- cime_config/machines/config_batch.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 923e497367f7..ee271d1f9dc0 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -448,6 +448,7 @@ regular preempt shared + overrun debug @@ -502,10 +503,11 @@ regular preempt shared + overrun debug - + --constraint=cpu