Main branch fixes for Rocoto and ecflow workflows

NOAA-EMC · Dec 12, 2024 · 54d925e · 54d925e
1 parent 5dcee15
commit 54d925e
Show file tree

Hide file tree

Showing 70 changed files with 1,366 additions and 1,545 deletions.
diff --git a/jobs/JRRFS_ANALYSIS_ENKF b/jobs/JRRFS_ANALYSIS_ENKF
@@ -61,17 +61,21 @@ specified cycle.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
 export CYCLE_TYPE=${CYCLE_TYPE:-prod}
 if [ "${CYCLE_TYPE}" = "spinup" ]; then
+  export jobid=${RUN}_enkfupdt_${OB_TYPE}_spinup_${envir}_${cyc}
   export COMOUT=${COMOUT:-$(compath.py -o ${NET}/${rrfs_ver}/${RUN}.${PDY}/${cyc}_spinup)}
 else
+  export jobid=${RUN}_enkfupdt_${OB_TYPE}_${envir}_${cyc}
   export COMOUT=${COMOUT:-$(compath.py -o ${NET}/${rrfs_ver}/${RUN}.${PDY}/${cyc})}
 fi
 
-export DATA=${DATAROOT}/${jobid}
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 rm -fr ${DATA}
 mkdir -p ${DATA}
 cd ${DATA}

diff --git a/jobs/JRRFS_ANALYSIS_GSI b/jobs/JRRFS_ANALYSIS_GSI
@@ -62,10 +62,11 @@ for the specified cycle.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
-export mem_num=m$(echo "${ENSMEM_INDX}")
+if [ ! "${MEM_TYPE}" = "MEAN" ] && [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+  export ENSMEM_INDX=${ENSMEM_INDX:-${MEMBER_NAME}}
+  export mem_num=m$(echo "${ENSMEM_INDX}")
+fi
 
 export CYCLE_TYPE=${CYCLE_TYPE:-prod}
 export observer_nwges_dir=""
@@ -80,15 +81,31 @@ if [[ ! -v OB_TYPE ]]; then
   OB_TYPE="conv"
 fi
 
-if [ "${MEM_TYPE}" = "MEAN" ]; then
-  observer_nwges_dir="${GESROOT}/${RUN}.${PDY}/${cyc}/ensmean/observer_${workname}"
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
 else
-# GSI_TYPE = OBSERVER is only for EnKF (ensemble forecasts do not have ANALYSIS tasks)
-  observer_nwges_dir="${GESROOT}/${RUN}.${PDY}/${cyc}/${mem_num}/observer_${workname}"
+  if [ "${GSI_TYPE}" = "OBSERVER" ]; then
+    if [ "${MEM_TYPE}" = "MEAN" ]; then
+      export jobid=${RUN}_observer_${workname}_ensmean_${envir}_${cyc}
+    else
+      export jobid=${RUN}_observer_${workname}_${mem_num}_${envir}_${cyc}
+    fi
+  else	# For control member, GSI_TYPE is always ANALYSIS
+    export jobid=${RUN}_analysis_${OB_TYPE}_${workname}_${envir}_${cyc}
+  fi
+fi
+
+if [ "${GSI_TYPE}" = "OBSERVER" ]; then
+  if [ "${MEM_TYPE}" = "MEAN" ]; then
+    observer_nwges_dir="${GESROOT}/${RUN}.${PDY}/${cyc}/ensmean/observer_${workname}"
+  else
+  # GSI_TYPE = OBSERVER is only for EnKF (ensemble forecasts do not have ANALYSIS tasks)
+    observer_nwges_dir="${GESROOT}/${RUN}.${PDY}/${cyc}/${mem_num}/observer_${workname}"
+  fi
+  mkdir -p ${observer_nwges_dir}
 fi
-mkdir -p ${observer_nwges_dir}
 
-export DATA=${DATAROOT}/${jobid}
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 rm -fr ${DATA}
 mkdir -p ${DATA}
 cd ${DATA}
@@ -117,7 +134,7 @@ export OB_TYPE=${OB_TYPE}
 #-----------------------------------------------------------------------
 
 if [ "${CYCLE_TYPE}" = "spinup" ]; then
-  if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+  if [ "${DO_ENSEMBLE}" = "TRUE" ] && [ ! "${MEM_TYPE}" = "MEAN" ]; then
     export COMOUT=${COMOUT:-$(compath.py -o ${NET}/${rrfs_ver}/${RUN}.${PDY}/${cyc}_spinup/${mem_num})}
   else
     export COMOUT=${COMOUT:-$(compath.py -o ${NET}/${rrfs_ver}/${RUN}.${PDY}/${cyc}_spinup)}

diff --git a/jobs/JRRFS_ANALYSIS_GSIDIAG b/jobs/JRRFS_ANALYSIS_GSIDIAG
@@ -61,12 +61,20 @@ the specified cycle.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
 export CYCLE_TYPE=${CYCLE_TYPE:-prod}
 
-export DATA="${DATAROOT}/${jobid}"
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  if [ "${CYCLE_TYPE}" = "spinup" ]; then
+    export jobid=${RUN}_analysis_conv_dbz_gsi_spinup_${envir}_${cyc}
+  else
+    export jobid=${RUN}_analysis_conv_dbz_gsi_${envir}_${cyc}
+  fi
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 if [ ! -d "${DATA}" ]; then
   echo "WARNING: ${DATA} does not exist" 
 fi

diff --git a/jobs/JRRFS_ANALYSIS_NONVARCLD b/jobs/JRRFS_ANALYSIS_NONVARCLD
@@ -61,12 +61,30 @@ analysis with RRFS for the specified cycle.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
-export mem_num=m$(echo "${ENSMEM_INDX}")
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  if [ "${CYCLE_TYPE}" = "spinup" ]; then
+    export workname="nonvar_cldanl_spinup"
+  else
+    export workname="nonvar_cldanl"
+  fi
+  if [ ${MEM_TYPE} = "MEAN" ]; then
+    export jobid=${RUN}_${workname}_ensmean_${envir}_${cyc}
+  else
+    if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+      export ENSMEM_INDX=${ENSMEM_INDX:-${MEMBER_NAME}}
+      export mem_num=m$(echo "${ENSMEM_INDX}")
+      export jobid=${RUN}_${workname}_${mem_num}_${envir}_${cyc}
+    else
+      export jobid=${RUN}_${workname}_${envir}_${cyc}
+    fi
+  fi
+fi
+export mem_num=${mem_num:-}
 
-export DATA=${DATAROOT}/${jobid}
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 rm -fr ${DATA}
 mkdir -p ${DATA}
 cd ${DATA}

diff --git a/jobs/JRRFS_BLEND_ICS b/jobs/JRRFS_BLEND_ICS
@@ -59,13 +59,18 @@ on the RRFS initial conditions.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
+export ENSMEM_INDX=${ENSMEM_INDX:-${MEMBER_NAME}}
 export mem_num=m$(echo "${ENSMEM_INDX}")
 export NWGES_DIR="${GESROOT}/${RUN}.${PDY}/${cyc}/${mem_num}/ics"
 
-export DATA=${DATAROOT}/${jobid}
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  export jobid=${RUN}_make_ics_${mem_num}_${envir}_${cyc}
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 mkdir -p $DATA
 
 mkdir -p ${DATA}/tmp_ICS

diff --git a/jobs/JRRFS_BUFRSND b/jobs/JRRFS_BUFRSND
@@ -62,13 +62,14 @@ on the output files corresponding to a specified forecast hour.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
 export fhr=01
 export tmmark=tm00
 
-export mem_num=m$(echo "${ENSMEM_INDX}")
+if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+  export ENSMEM_INDX=${ENSMEM_INDX:-${MEMBER_NAME}}
+  export mem_num=m$(printf "%03d" "$((10#${ENSMEM_INDX}))")
+fi
 
 CYCLE_TYPE=${CYCLE_TYPE:-prod}
 if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
@@ -82,7 +83,17 @@ fi
 mkdir -p "${NWGES_DIR}/RESTART"
 mkdir -p "${NWGES_DIR}/INPUT"
 
-export DATA=${DATAROOT}/${jobid}
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+    export jobid=${RUN}_bufrsnd_${mem_num}_${envir}_${cyc}
+  else
+    export jobid=${RUN}_bufrsnd_${envir}_${cyc}
+  fi
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 rm -rf ${DATA}
 mkdir -p ${DATA}
 cd ${DATA}

diff --git a/jobs/JRRFS_CALC_ENSMEAN b/jobs/JRRFS_CALC_ENSMEAN
@@ -59,14 +59,20 @@ RRFS for the specified cycle.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
-
-#export mem_num=m$(echo "${ENSMEM_INDX}")
 
 export CYCLE_TYPE=${CYCLE_TYPE:-prod}
 
-export DATA=${DATAROOT}/${jobid}
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  if [ "${CYCLE_TYPE}" = "spinup" ]; then
+    export jobid=${RUN}_calc_ensmean_spinup_${envir}_${cyc}
+  else
+    export jobid=${RUN}_calc_ensmean_${envir}_${cyc}
+  fi
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 rm -fr ${DATA}
 mkdir -p ${DATA}
 cd ${DATA}

diff --git a/jobs/JRRFS_FORECAST b/jobs/JRRFS_FORECAST
@@ -64,15 +64,42 @@ the specified cycle.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
-export mem_num=m$(echo "${ENSMEM_INDX}")
+if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+  export ENSMEM_INDX=${ENSMEM_INDX:-${MEMBER_NAME}}
+  # export mem_num=m$(echo "${ENSMEM_INDX}")
+  export mem_num=m$(printf "%03d" "$((10#${ENSMEM_INDX}))")
+fi
 
 export CYCLE_TYPE=${CYCLE_TYPE:-prod}
 export CYCLE_SUBTYPE=${CYCLE_SUBTYPE:-empty}
 
-export DATA=${DATAROOT}/${jobid}
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  if [ ${CYCLE_TYPE} == "spinup" ]; then
+    if [ "${DO_ENSEMBLE}" = "TRUE" ]; then	# EnKF has spinup forecasts
+      export jobid=${RUN}_forecast_spinup_${mem_num}_${envir}_${cyc}
+      if  [ ${CYCLE_SUBTYPE} == "ensinit" ]; then
+        export jobid=${RUN}_forecast_ensinit_${mem_num}_${envir}_${cyc}
+      fi
+    else
+      export jobid=${RUN}_forecast_spinup_${envir}_${cyc}
+    fi
+  else
+    if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+      export jobid=${RUN}_forecast_${mem_num}_${envir}_${cyc}
+    else
+      if [ ${WGF} == "firewx" ]; then
+        export jobid=firewx_forecast_${envir}_${cyc}
+      else
+        export jobid=${RUN}_forecast_${envir}_${cyc}
+      fi
+    fi
+  fi
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 mkdir -p ${DATA}
 
 mkdir -p ${DATA}/INPUT

diff --git a/jobs/JRRFS_MAKE_GRID b/jobs/JRRFS_MAKE_GRID
@@ -61,10 +61,14 @@ This is the J-job script for the task that generates grid files.
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
-export DATA=${DATAROOT}/${jobid}
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  export jobid=${RUN}_make_grid_${envir}_${cyc}
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 mkdir -p ${DATA}
 cd ${DATA}
 

diff --git a/jobs/JRRFS_MAKE_ICS b/jobs/JRRFS_MAKE_ICS
@@ -62,7 +62,7 @@ for the RRFS (in NetCDF format).
 # Set the parameters for the external model files.
 #-----------------------------------------------------------------------
 
-extrn_mdl_name="${EXTRN_MDL_NAME}"
+extrn_mdl_name="${EXTRN_MDL_NAME_ICS}"
 sysbasedir="${EXTRN_MDL_SYSBASEDIR_ICS}"
 gfs_file_fmt="${GFS_FILE_FMT_ICS}"
 time_offset_hrs="${EXTRN_MDL_ICS_OFFSET_HRS}"
@@ -118,7 +118,7 @@ case "${extrn_mdl_name}" in
     else
       sysdir="$sysbasedir/rrfs.${yyyymmdd}/${hh}"
     fi
-    fns_on_disk=( "rrfs.t${hh}z.natlev.f0${fcst_hh}.grib2" )
+    fns_on_disk=( "rrfs.t${hh}z.natlev.3km.f0${fcst_hh}.na.grib2" )
     ;;
 
   *)
@@ -134,17 +134,29 @@ extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}" )")"
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
-export mem_num=m$(echo "${ENSMEM_INDX}")
+if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+  export ENSMEM_INDX=${ENSMEM_INDX:-${MEMBER_NAME}}
+  export mem_num=m$(echo "${ENSMEM_INDX}")
+fi
+
 if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
   export NWGES_DIR="${GESROOT}/${RUN}.${PDY}/${cyc}/${mem_num}/ics"
 else
   export NWGES_DIR="${GESROOT}/${RUN}.${PDY}/${cyc}/ics"
 fi
 
-export DATA=${DATAROOT}/${jobid}
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+    export jobid=${RUN}_make_ics_${mem_num}_${envir}_${cyc}
+  else
+    export jobid=${RUN}_make_ics_${envir}_${cyc}
+  fi
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 mkdir -p $DATA
 
 mkdir -p ${DATA}/tmp_ICS

diff --git a/jobs/JRRFS_MAKE_LBCS b/jobs/JRRFS_MAKE_LBCS
@@ -61,7 +61,7 @@ hour zero).
 # Set the parameters for the external model files.
 #-----------------------------------------------------------------------
 
-export extrn_mdl_name="${EXTRN_MDL_NAME}"
+export extrn_mdl_name="${EXTRN_MDL_NAME_LBCS}"
 sysbasedir="${EXTRN_MDL_SYSBASEDIR_LBCS}"
 gfs_file_fmt="${GFS_FILE_FMT_LBCS}"
 extrn_mdl_memhead="${GEFS_INPUT_SUBDIR}"
@@ -149,9 +149,9 @@ case "${extrn_mdl_name}" in
     fi
     sysdir2=""
     fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) )
-    prefix="rrfs.t${hh}z.natlev.f"
+    prefix="rrfs.t${hh}z.natlev.3km.f"
     fns=( "${fcst_hhh[@]/#/$prefix}" )
-    suffix=".grib2"
+    suffix=".na.grib2"
     fns_on_disk=( "${fns[@]/%/$suffix}" )
     ;;
 
@@ -173,17 +173,26 @@ extrn_mdl_fns_on_disk_str2="( "$( printf "\"%s\" " "${fns_on_disk2[@]}" )")"
 
 export pid=${pid:-$$}
 export RUN=${RUN:-rrfs}
-#export jobid=${jobid:-${taskid}}
-export jobid=${taskid}
 
-export mem_num=m$(echo "${ENSMEM_INDX}")
 if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+  export ENSMEM_INDX=${ENSMEM_INDX:-${MEMBER_NAME}}
+  export mem_num=m$(echo "${ENSMEM_INDX}")
   export NWGES_DIR="${GESROOT}/${RUN}.${PDY}/${cyc}/${mem_num}/lbcs"
 else
   export NWGES_DIR="${GESROOT}/${RUN}.${PDY}/${cyc}/lbcs"
 fi
 
-export DATA=${DATAROOT}/${jobid}
+if [ ${WORKFLOW_MANAGER} == "rocoto" ] && [[ ! -z ${taskid} ]]; then
+  export jobid=${taskid}
+else
+  if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
+    export jobid=${RUN}_make_lbcs_${mem_num}_${envir}_${cyc}
+  else
+    export jobid=${RUN}_make_lbcs_${envir}_${cyc}
+  fi
+fi
+
+export DATA=${DATA:-${DATAROOT}/${jobid}}
 mkdir -p $DATA
 
 mkdir -p ${DATA}/tmp_LBCS_${bcgrp}