From e53c5e8e0abbc0edf95970a71df0e6e8a2be9f31 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Tue, 28 May 2024 17:16:23 -0400 Subject: [PATCH] Add atmensanlfv3inc job (#2592) This PR creates the atmensanlfv3inc job, the ensemble version of atmanlfv3inc, created in GW PR #2420. Its GDASApp companion PR is #[1104](https://github.com/NOAA-EMC/GDASApp/pull/1104), and its JCB-GDAS companion PR is #[3](https://github.com/NOAA-EMC/jcb-gdas/pull/3). --- env/AWSPW.env | 2 +- env/CONTAINER.env | 2 +- env/HERA.env | 20 +++- env/HERCULES.env | 18 ++- env/JET.env | 20 +++- env/ORION.env | 20 +++- env/S4.env | 20 +++- env/WCOSS2.env | 20 +++- jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT | 35 ++++++ ...YSIS_RUN => JGLOBAL_ATMENS_ANALYSIS_LETKF} | 4 +- jobs/rocoto/atmensanlfv3inc.sh | 24 ++++ .../{atmensanlrun.sh => atmensanlletkf.sh} | 4 +- parm/config/gfs/config.atmensanlfv3inc | 14 +++ parm/config/gfs/config.atmensanlletkf | 11 ++ parm/config/gfs/config.atmensanlrun | 11 -- parm/config/gfs/config.resources | 32 ++++-- .../exglobal_atm_analysis_fv3_increment.py | 4 +- .../exglobal_atmens_analysis_fv3_increment.py | 23 ++++ ...n.py => exglobal_atmens_analysis_letkf.py} | 6 +- sorc/gdas.cd | 2 +- sorc/gsi_utils.fd | 2 +- ush/python/pygfs/task/atmens_analysis.py | 105 +++++++----------- workflow/applications/applications.py | 2 +- workflow/applications/gfs_cycled.py | 4 +- workflow/rocoto/gfs_tasks.py | 36 +++++- workflow/rocoto/tasks.py | 2 +- 26 files changed, 304 insertions(+), 139 deletions(-) create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT rename jobs/{JGLOBAL_ATMENS_ANALYSIS_RUN => JGLOBAL_ATMENS_ANALYSIS_LETKF} (83%) create mode 100755 jobs/rocoto/atmensanlfv3inc.sh rename jobs/rocoto/{atmensanlrun.sh => atmensanlletkf.sh} (89%) create mode 100644 parm/config/gfs/config.atmensanlfv3inc create mode 100644 parm/config/gfs/config.atmensanlletkf delete mode 100644 parm/config/gfs/config.atmensanlrun create mode 100755 scripts/exglobal_atmens_analysis_fv3_increment.py rename scripts/{exglobal_atmens_analysis_run.py => exglobal_atmens_analysis_letkf.py} (86%) diff --git a/env/AWSPW.env b/env/AWSPW.env index 5cf819ba2b..7d81000f5c 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmanlfv3inc atmensanlrun aeroanlrun snowanl" + echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" diff --git a/env/CONTAINER.env b/env/CONTAINER.env index 700460b755..77768b485b 100755 --- a/env/CONTAINER.env +++ b/env/CONTAINER.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmanlfv3inc atmensanlrun aeroanlrun snowanl" + echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" diff --git a/env/HERA.env b/env/HERA.env index fbfdb68e92..ccaaea32e7 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -68,13 +68,21 @@ elif [[ "${step}" = "atmanlvar" ]]; then [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}" -elif [[ "${step}" = "atmensanlrun" ]]; then +elif [[ "${step}" = "atmensanlletkf" ]]; then - nth_max=$((npe_node_max / npe_node_atmensanlrun)) + nth_max=$((npe_node_max / npe_node_atmensanlletkf)) - export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} - export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}" + export NTHREADS_ATMENSANLLETKF=${nth_atmensanlletkf:-${nth_max}} + [[ ${NTHREADS_ATMENSANLLETKF} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLLETKF=${nth_max} + export APRUN_ATMENSANLLETKF="${launcher} -n ${npe_atmensanlletkf} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" + +elif [[ "${step}" = "atmensanlfv3inc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmensanlfv3inc)) + + export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max} + export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index 0b62120536..0824ba913a 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -73,13 +73,21 @@ case ${step} in [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" ;; - "atmensanlrun") + "atmensanlletkf") - nth_max=$((npe_node_max / npe_node_atmensanlrun)) + nth_max=$((npe_node_max / npe_node_atmensanlletkf)) - export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} - export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}" + export NTHREADS_ATMENSANLLETKF=${nth_atmensanlletkf:-${nth_max}} + [[ ${NTHREADS_ATMENSANLLETKF} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLLETKF=${nth_max} + export APRUN_ATMENSANLLETKF="${launcher} -n ${npe_atmensanlletkf} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" + ;; + "atmensanlfv3inc") + + nth_max=$((npe_node_max / npe_node_atmensanlfv3inc)) + + export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max} + export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" ;; "aeroanlrun") diff --git a/env/JET.env b/env/JET.env index 976e42a025..5bd88dc93a 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -56,13 +56,21 @@ elif [[ "${step}" = "atmanlvar" ]]; then [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}" -elif [[ "${step}" = "atmensanlrun" ]]; then +elif [[ "${step}" = "atmensanlletkf" ]]; then - nth_max=$((npe_node_max / npe_node_atmensanlrun)) + nth_max=$((npe_node_max / npe_node_atmensanlletkf)) - export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} - export APRUN_ATMENSANL="${launcher} ${npe_atmensanlrun}" + export NTHREADS_ATMENSANLLETKF=${nth_atmensanlletkf:-${nth_max}} + [[ ${NTHREADS_ATMENSANLLETKF} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLLETKF=${nth_max} + export APRUN_ATMENSANLLETKF="${launcher} ${npe_atmensanlletkf}" + +elif [[ "${step}" = "atmensanlfv3inc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmensanlfv3inc)) + + export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max} + export APRUN_ATMENSANLFV3INC="${launcher} ${npe_atmensanlfv3inc}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 1b66ca65c0..f701e55aa2 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -64,13 +64,21 @@ elif [[ "${step}" = "atmanlvar" ]]; then [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}" -elif [[ "${step}" = "atmensanlrun" ]]; then +elif [[ "${step}" = "atmensanlletkf" ]]; then - nth_max=$((npe_node_max / npe_node_atmensanlrun)) + nth_max=$((npe_node_max / npe_node_atmensanlletkf)) - export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} - export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}" + export NTHREADS_ATMENSANLLETKF=${nth_atmensanlletkf:-${nth_max}} + [[ ${NTHREADS_ATMENSANLLETKF} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLLETKF=${nth_max} + export APRUN_ATMENSANLLETKF="${launcher} -n ${npe_atmensanlletkf} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" + +elif [[ "${step}" = "atmensanlfv3inc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmensanlfv3inc)) + + export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max} + export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/env/S4.env b/env/S4.env index ce68fddb89..9ba3a61b01 100755 --- a/env/S4.env +++ b/env/S4.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -56,13 +56,21 @@ elif [[ "${step}" = "atmanlvar" ]]; then [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}" -elif [[ "${step}" = "atmensanlrun" ]]; then +elif [[ "${step}" = "atmensanlletkf" ]]; then - nth_max=$((npe_node_max / npe_node_atmensanlrun)) + nth_max=$((npe_node_max / npe_node_atmensanlletkf)) - export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} - export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun}" + export NTHREADS_ATMENSANLLETKF=${nth_atmensanlletkf:-${nth_max}} + [[ ${NTHREADS_ATMENSANLLETKF} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLLETKF=${nth_max} + export APRUN_ATMENSANLLETKF="${launcher} -n ${npe_atmensanlletkf}" + +elif [[ "${step}" = "atmensanlfv3inc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmensanlfv3inc)) + + export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max} + export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index ff0121e034..0876e4127d 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen esfc efcs epos" echo "postsnd awips gempak" @@ -50,13 +50,21 @@ elif [[ "${step}" = "atmanlvar" ]]; then [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}" -elif [[ "${step}" = "atmensanlrun" ]]; then +elif [[ "${step}" = "atmensanlletkf" ]]; then - nth_max=$((npe_node_max / npe_node_atmensanlrun)) + nth_max=$((npe_node_max / npe_node_atmensanlletkf)) - export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} - export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun}" + export NTHREADS_ATMENSANLLETKF=${nth_atmensanlletkf:-${nth_max}} + [[ ${NTHREADS_ATMENSANLLETKF} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLLETKF=${nth_max} + export APRUN_ATMENSANLLETKF="${launcher} -n ${npe_atmensanlletkf}" + +elif [[ "${step}" = "atmensanlfv3inc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmensanlfv3inc)) + + export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max} + export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT b/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT new file mode 100755 index 0000000000..7179ae0624 --- /dev/null +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlfv3inc" -c "base atmensanl atmensanlfv3inc" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASATMENSRUNSH:-${SCRgfs}/exglobal_atmens_analysis_fv3_increment.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN b/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF similarity index 83% rename from jobs/JGLOBAL_ATMENS_ANALYSIS_RUN rename to jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF index 65eeb5e0d8..060b7abd06 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlrun" -c "base atmensanl atmensanlrun" +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlletkf" -c "base atmensanl atmensanlletkf" ############################################## # Set variables used in the script @@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlrun" -c "base atmensanl atme ############################################################### # Run relevant script -EXSCRIPT=${GDASATMENSRUNSH:-${SCRgfs}/exglobal_atmens_analysis_run.py} +EXSCRIPT=${GDASATMENSRUNSH:-${SCRgfs}/exglobal_atmens_analysis_letkf.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/atmensanlfv3inc.sh b/jobs/rocoto/atmensanlfv3inc.sh new file mode 100755 index 0000000000..bb44ddc3a0 --- /dev/null +++ b/jobs/rocoto/atmensanlfv3inc.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="atmensanlfv3inc" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT" +status=$? +exit "${status}" diff --git a/jobs/rocoto/atmensanlrun.sh b/jobs/rocoto/atmensanlletkf.sh similarity index 89% rename from jobs/rocoto/atmensanlrun.sh rename to jobs/rocoto/atmensanlletkf.sh index d991e3eb82..b4a1a73a80 100755 --- a/jobs/rocoto/atmensanlrun.sh +++ b/jobs/rocoto/atmensanlletkf.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="atmensanlrun" +export job="atmensanlletkf" export jobid="${job}.$$" ############################################################### @@ -19,6 +19,6 @@ export PYTHONPATH ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN" +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF" status=$? exit "${status}" diff --git a/parm/config/gfs/config.atmensanlfv3inc b/parm/config/gfs/config.atmensanlfv3inc new file mode 100644 index 0000000000..2dc73f3f6e --- /dev/null +++ b/parm/config/gfs/config.atmensanlfv3inc @@ -0,0 +1,14 @@ +#! /usr/bin/env bash + +########## config.atmensanlfv3inc ########## +# Atm Var Analysis specific + +echo "BEGIN: config.atmensanlfv3inc" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlfv3inc + +export JCB_ALGO=fv3jedi_fv3inc_lgetkf +export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x + +echo "END: config.atmensanlfv3inc" diff --git a/parm/config/gfs/config.atmensanlletkf b/parm/config/gfs/config.atmensanlletkf new file mode 100644 index 0000000000..1fdc57ae62 --- /dev/null +++ b/parm/config/gfs/config.atmensanlletkf @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.atmensanlletkf ########## +# Atm Ens Analysis specific + +echo "BEGIN: config.atmensanlletkf" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlletkf + +echo "END: config.atmensanlletkf" diff --git a/parm/config/gfs/config.atmensanlrun b/parm/config/gfs/config.atmensanlrun deleted file mode 100644 index 01f211a17a..0000000000 --- a/parm/config/gfs/config.atmensanlrun +++ /dev/null @@ -1,11 +0,0 @@ -#! /usr/bin/env bash - -########## config.atmensanlrun ########## -# Atm Ens Analysis specific - -echo "BEGIN: config.atmensanlrun" - -# Get task specific resources -. "${EXPDIR}/config.resources" atmensanlrun - -echo "END: config.atmensanlrun" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index c583e0c04e..affb0da04f 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -11,7 +11,7 @@ if (( $# != 1 )); then echo "stage_ic aerosol_init" echo "prep prepsnowobs prepatmiodaobs" echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" - echo "atmensanlinit atmensanlrun atmensanlfinal" + echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl" echo "aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" @@ -957,17 +957,31 @@ case ${step} in export memory_atmensanlinit="3072M" ;; - "atmensanlrun") + "atmensanlletkf") export layout_x=${layout_x_atmensanl} export layout_y=${layout_y_atmensanl} - export wtime_atmensanlrun="00:30:00" - export npe_atmensanlrun=$(( layout_x * layout_y * 6 )) - export npe_atmensanlrun_gfs=$(( layout_x * layout_y * 6 )) - export nth_atmensanlrun=1 - export nth_atmensanlrun_gfs=${nth_atmensanlrun} - export npe_node_atmensanlrun=$(( npe_node_max / nth_atmensanlrun )) - export memory_atmensanlrun="96GB" + export wtime_atmensanlletkf="00:30:00" + export npe_atmensanlletkf=$(( layout_x * layout_y * 6 )) + export npe_atmensanlletkf_gfs=$(( layout_x * layout_y * 6 )) + export nth_atmensanlletkf=1 + export nth_atmensanlletkf_gfs=${nth_atmensanlletkf} + export npe_node_atmensanlletkf=$(( npe_node_max / nth_atmensanlletkf )) + export memory_atmensanlletkf="96GB" + export is_exclusive=True + ;; + + "atmensanlfv3inc") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + export wtime_atmensanlfv3inc="00:30:00" + export npe_atmensanlfv3inc=$(( layout_x * layout_y * 6 )) + export npe_atmensanlfv3inc_gfs=$(( layout_x * layout_y * 6 )) + export nth_atmensanlfv3inc=1 + export nth_atmensanlfv3inc_gfs=${nth_atmensanlfv3inc} + export npe_node_atmensanlfv3inc=$(( npe_node_max / nth_atmensanlfv3inc )) + export memory_atmensanlfv3inc="96GB" export is_exclusive=True ;; diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 57f2e7c9ee..66f6796343 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_fv3_increment.py # This script creates an AtmAnalysis object -# and runs the increment method -# which converts the JEDI increment into an FV3 increment +# and runs the init_fv3_increment and fv3_increment methods +# which convert the JEDI increment into an FV3 increment import os from wxflow import Logger, cast_strdict_as_dtypedict diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py new file mode 100755 index 0000000000..c50b00548f --- /dev/null +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# exglobal_atmens_analysis_fv3_increment.py +# This script creates an AtmEnsAnalysis object +# and runs the init_fv3_increment and fv3_increment methods +# which convert the JEDI increment into an FV3 increment +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.atmens_analysis import AtmEnsAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atmens analysis task + AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl.init_fv3_increment() + AtmEnsAnl.fv3_increment() diff --git a/scripts/exglobal_atmens_analysis_run.py b/scripts/exglobal_atmens_analysis_letkf.py similarity index 86% rename from scripts/exglobal_atmens_analysis_run.py rename to scripts/exglobal_atmens_analysis_letkf.py index b2eb9fb2e4..30394537cd 100755 --- a/scripts/exglobal_atmens_analysis_run.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# exglobal_atmens_analysis_run.py +# exglobal_atmens_analysis_letkf.py # This script creates an AtmEnsAnalysis object -# and runs the execute method +# and runs the letkf method # which executes the global atm local ensemble analysis import os @@ -19,4 +19,4 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.execute() + AtmEnsAnl.letkf() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 249e242e33..6742ec62a1 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 249e242e33a33feeb1c81bedd51198309f669de0 +Subproject commit 6742ec62a12d7d6f8129057bcf77cee0e2175022 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 0cdc3b4f7f..bb03e172e0 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 0cdc3b4f7ff8d4f0c54da3dab70ea2743bd68478 +Subproject commit bb03e172e0d0d9c56d6da7788ca033bfb5ef5119 diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 5aaacc42e8..37ac613736 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -132,7 +132,7 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def execute(self: Analysis) -> None: + def letkf(self: Analysis) -> None: """Execute a global atmens analysis This method will execute a global atmens analysis using JEDI. @@ -150,8 +150,9 @@ def execute(self: Analysis) -> None: """ chdir(self.task_config.DATA) - exec_cmd = Executable(self.task_config.APRUN_ATMENSANL) + exec_cmd = Executable(self.task_config.APRUN_ATMENSANLLETKF) exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('fv3jedi') exec_cmd.add_default_arg('localensembleda') @@ -167,6 +168,31 @@ def execute(self: Analysis) -> None: pass + @logit(logger) + def init_fv3_increment(self: Analysis) -> None: + # Setup JEDI YAML file + self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, + f"{self.task_config.JCB_ALGO}.yaml") + save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml) + + # Link JEDI executable to run directory + self.task_config.jedi_exe = self.link_jediexe() + + @logit(logger) + def fv3_increment(self: Analysis) -> None: + # Run executable + exec_cmd = Executable(self.task_config.APRUN_ATMENSANLFV3INC) + exec_cmd.add_default_arg(self.task_config.jedi_exe) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + @logit(logger) def finalize(self: Analysis) -> None: """Finalize a global atmens analysis @@ -218,42 +244,6 @@ def finalize(self: Analysis) -> None: } FileHandler(yaml_copy).sync() - # Create UFS model readable atm increment file from UFS-DA atm increment - logger.info("Create UFS model readable atm increment file from UFS-DA atm increment") - self.jedi2fv3inc() - - def clean(self): - super().clean() - - @logit(logger) - def jedi2fv3inc(self: Analysis) -> None: - """Generate UFS model readable analysis increment - - This method writes a UFS DA atm increment in UFS model readable format. - This includes: - - write UFS-DA atm increments using variable names expected by UFS model - - compute and write delp increment - - compute and write hydrostatic delz increment - - Please note that some of these steps are temporary and will be modified - once the modle is able to directly read atm increments. - - Parameters - ---------- - Analysis: parent class for GDAS task - - Returns - ---------- - None - """ - # Select the atm guess file based on the analysis and background resolutions - # Fields from the atm guess are used to compute the delp and delz increments - cdate = to_fv3time(self.task_config.current_cycle) - cdate_inc = cdate.replace('.', '_') - - # Reference the python script which does the actual work - incpy = os.path.join(self.task_config.HOMEgfs, 'ush/jediinc2fv3.py') - # create template dictionaries template_inc = self.task_config.COM_ATMOS_ANALYSIS_TMPL tmpl_inc_dict = { @@ -263,14 +253,10 @@ def jedi2fv3inc(self: Analysis) -> None: 'HH': self.task_config.current_cycle.strftime('%H') } - template_ges = self.task_config.COM_ATMOS_HISTORY_TMPL - tmpl_ges_dict = { - 'ROTDIR': self.task_config.ROTDIR, - 'RUN': self.task_config.RUN, - 'YMD': to_YMD(self.task_config.previous_cycle), - 'HH': self.task_config.previous_cycle.strftime('%H') - } - + # copy FV3 atm increment to comrot directory + logger.info("Copy UFS model readable atm increment file") + cdate = to_fv3time(self.task_config.current_cycle) + cdate_inc = cdate.replace('.', '_') # loop over ensemble members for imem in range(1, self.task_config.NMEM_ENS + 1): memchar = f"mem{imem:03d}" @@ -278,20 +264,15 @@ def jedi2fv3inc(self: Analysis) -> None: # create output path for member analysis increment tmpl_inc_dict['MEMDIR'] = memchar incdir = Template.substitute_structure(template_inc, TemplateConstants.DOLLAR_CURLY_BRACE, tmpl_inc_dict.get) + src = os.path.join(self.task_config.DATA, 'anl', memchar, f"atminc.{cdate_inc}z.nc4") + dest = os.path.join(incdir, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc") + + # copy increment + logger.debug(f"Copying {src} to {dest}") + inc_copy = { + 'copy': [[src, dest]] + } + FileHandler(inc_copy).sync() - # rewrite UFS-DA atmens increments - tmpl_ges_dict['MEMDIR'] = memchar - gesdir = Template.substitute_structure(template_ges, TemplateConstants.DOLLAR_CURLY_BRACE, tmpl_ges_dict.get) - atmges_fv3 = os.path.join(gesdir, f"{self.task_config.CDUMP}.t{self.task_config.previous_cycle.hour:02d}z.atmf006.nc") - atminc_jedi = os.path.join(self.task_config.DATA, 'anl', memchar, f'atminc.{cdate_inc}z.nc4') - atminc_fv3 = os.path.join(incdir, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc") - - # Execute incpy to create the UFS model atm increment file - # TODO: use MPMD or parallelize with mpi4py - # See https://github.com/NOAA-EMC/global-workflow/pull/1373#discussion_r1173060656 - cmd = Executable(incpy) - cmd.add_default_arg(atmges_fv3) - cmd.add_default_arg(atminc_jedi) - cmd.add_default_arg(atminc_fv3) - logger.debug(f"Executing {cmd}") - cmd(output='stdout', error='stderr') + def clean(self): + super().clean() diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index adfab16496..50a9a7cdd0 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -155,7 +155,7 @@ def _source_configs(self, conf: Configuration) -> Dict[str, Any]: files += ['config.fcst', 'config.efcs'] elif config in ['atmanlinit', 'atmanlvar', 'atmanlfv3inc']: files += ['config.atmanl', f'config.{config}'] - elif config in ['atmensanlinit', 'atmensanlrun']: + elif config in ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc']: files += ['config.atmensanl', f'config.{config}'] elif 'wave' in config: files += ['config.wave', f'config.{config}'] diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index ad13876528..f7f9b5b5e6 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -57,7 +57,7 @@ def _get_app_configs(self): if self.do_hybvar: if self.do_jediatmens: - configs += ['atmensanlinit', 'atmensanlrun', 'atmensanlfinal'] + configs += ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] @@ -161,7 +161,7 @@ def get_task_names(self): hybrid_after_eupd_tasks = [] if self.do_hybvar: if self.do_jediatmens: - hybrid_tasks += ['atmensanlinit', 'atmensanlrun', 'atmensanlfinal', 'echgres'] + hybrid_tasks += ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] else: hybrid_tasks += ['eobs', 'eupd', 'echgres'] hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index d61060a596..24da085bcc 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2397,7 +2397,7 @@ def atmensanlinit(self): return task - def atmensanlrun(self): + def atmensanlletkf(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlinit'} @@ -2406,14 +2406,40 @@ def atmensanlrun(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('atmensanlrun') - task_name = f'{self.cdump}atmensanlrun' + resources = self.get_resource('atmensanlletkf') + task_name = f'{self.cdump}atmensanlletkf' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlletkf.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + + def atmensanlfv3inc(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlletkf'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmensanlfv3inc') + task_name = f'{self.cdump}atmensanlfv3inc' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlfv3inc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2426,7 +2452,7 @@ def atmensanlrun(self): def atmensanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlfv3inc'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index a8b4eb9fac..eb13941002 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -18,7 +18,7 @@ class Tasks: 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', - 'atmensanlinit', 'atmensanlrun', 'atmensanlfinal', + 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'fcst',