Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[develop] Bring change of production/AQM.v7 back to develop #828

Merged
merged 20 commits into from
Jun 29, 2023
Merged

[develop] Bring change of production/AQM.v7 back to develop #828

Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
777c0a3
update j-job scripts
chan-hoo Jun 11, 2023
6d19532
update jjob scripts
chan-hoo Jun 11, 2023
1158568
update scripts
chan-hoo Jun 12, 2023
2acbe92
add missing change
chan-hoo Jun 12, 2023
0217016
Merge branch 'develop' of https://github.com/chan-hoo/ufs-srweather-a…
chan-hoo Jun 12, 2023
e500744
add missing change
chan-hoo Jun 12, 2023
cc2031f
fix typo
chan-hoo Jun 12, 2023
1da7eb6
change wtime for nexus_emission
chan-hoo Jun 12, 2023
57896b3
fix wrong change in nexus
chan-hoo Jun 13, 2023
8b4e998
fix typo
chan-hoo Jun 14, 2023
818b8f3
update aqm_utils to ensure source attribute of ic file
chan-hoo Jun 16, 2023
0c9b8a4
update scripts to meet reviewers comments
chan-hoo Jun 17, 2023
5f1be8e
add comment for i_pdym
chan-hoo Jun 21, 2023
193656c
Merge branch 'develop' of https://github.com/chan-hoo/ufs-srweather-a…
chan-hoo Jun 23, 2023
e0f79fa
add missing _default to nco
chan-hoo Jun 23, 2023
f8e479e
additional update from production
chan-hoo Jun 27, 2023
7e5c1c0
add missing _default
chan-hoo Jun 28, 2023
53a7248
add missing _default
chan-hoo Jun 28, 2023
0055e36
add slash_ensem_subdir to shared pid for nco
chan-hoo Jun 28, 2023
cec9168
fix typo
chan-hoo Jun 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ fix/
include/
lib/
share/
modulefiles/extrn_comp_build/
sorc/*/
tests/WE2E/WE2E_tests_*.yaml
tests/WE2E/*.txt
Expand Down
2 changes: 1 addition & 1 deletion Externals.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ protocol = git
repo_url = https://github.com/NOAA-EMC/AQM-utils
# Specify either a branch name or a hash but not both.
#branch = develop
hash = 0a86f73
hash = 17c7957
local_path = sorc/AQM-utils
required = True

Expand Down
8 changes: 6 additions & 2 deletions jobs/JREGIONAL_AQM_ICS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,12 @@ mkdir_vrfy -p "${INPUT_DATA}"
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_AQM_ICS}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#-----------------------------------------------------------------------
#
Expand Down
9 changes: 7 additions & 2 deletions jobs/JREGIONAL_AQM_LBCS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,13 @@ mkdir_vrfy -p "${INPUT_DATA}"
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_AQM_LBCS}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi

#
#-----------------------------------------------------------------------
#
Expand Down
9 changes: 8 additions & 1 deletion jobs/JREGIONAL_BIAS_CORRECTION_O3
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,14 @@ This is the J-job script for the task that runs BIAS-CORRECTION-O3.
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_BIAS_CORRECTION_O3}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi

mkdir_vrfy -p ${COMOUTwmo}

export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"

Expand Down
9 changes: 8 additions & 1 deletion jobs/JREGIONAL_BIAS_CORRECTION_PM25
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,14 @@ This is the J-job script for the task that runs BIAS-CORRECTION-PM25.
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_BIAS_CORRECTION_PM25}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi

mkdir_vrfy -p ${COMOUTwmo}

export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"

Expand Down
10 changes: 7 additions & 3 deletions jobs/JREGIONAL_FIRE_EMISSION
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,16 +90,20 @@ fi
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_FIRE_EMISSION}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#-----------------------------------------------------------------------
#
# Create the directory where the RAVE fire emission files should be stored
#
#-----------------------------------------------------------------------
#
export FIRE_EMISSION_STAGING_DIR="${COMINext}/FIRE_EMISSION"
export FIRE_EMISSION_STAGING_DIR="${FIRE_EMISSION_STAGING_DIR:-${COMIN}/FIRE_EMISSION}"
mkdir_vrfy -p "${FIRE_EMISSION_STAGING_DIR}"
#
#-----------------------------------------------------------------------
Expand Down
7 changes: 3 additions & 4 deletions jobs/JREGIONAL_GET_EXTRN_MDL_FILES
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
#
. $USHdir/source_util_funcs.sh
source_config_for_task "task_get_extrn_ics|task_get_extrn_lbcs" ${GLOBAL_VAR_DEFNS_FP}
. $USHdir/job_preamble.sh
. $USHdir/job_preamble.sh "TRUE"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It doesn't look like job_preamble.sh should accept an argument. If it does and I missed it, please add documentation about it in that script.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@christinaholtNOAA, it accepts an argument for the shared job id. If an argument exists, a shared pid is used.

if [ $# -ne 0 ]; then
    export pid=$share_pid
    export jobid=${job}.${pid}
fi

I'll add a comment to the script.

#
#-----------------------------------------------------------------------
#
Expand Down Expand Up @@ -221,8 +221,7 @@ esac
#-----------------------------------------------------------------------
#
if [ $RUN_ENVIR = "nco" ]; then
export EXTRN_MDL_STAGING_DIR="${COMINext}"
mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}"
export EXTRN_MDL_STAGING_DIR="${EXTRN_MDL_STAGING_DIR:-${DATA}}"
else
export EXTRN_MDL_STAGING_DIR="${COMIN}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}"
mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}"
Expand All @@ -245,7 +244,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
#
#-----------------------------------------------------------------------
#
job_postamble
job_postamble "FALSE"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar comment as above. I see that this function does accept TRUE as an action here, but this seems to be an undefined keyword that could be literally anything.

Copy link
Collaborator Author

@chan-hoo chan-hoo Jun 16, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I add a new function to job_postamble for the shared tasks. When an argument exists, the working directory will not be removed even when KEEPDATA=false. Only when an argument is true, all existing working directories will be removed from the tmp directory.

#
#-----------------------------------------------------------------------
#
Expand Down
8 changes: 6 additions & 2 deletions jobs/JREGIONAL_MAKE_ICS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,12 @@ mkdir_vrfy -p "${INPUT_DATA_NWGES}"
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ $RUN_ENVIR = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_MAKE_ICS}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#-----------------------------------------------------------------------
#
Expand Down
8 changes: 6 additions & 2 deletions jobs/JREGIONAL_MAKE_LBCS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,12 @@ mkdir_vrfy -p "${INPUT_DATA_NWGES}"
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_MAKE_LBCS}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#-----------------------------------------------------------------------
#
Expand Down
8 changes: 6 additions & 2 deletions jobs/JREGIONAL_NEXUS_EMISSION
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,12 @@ mkdir_vrfy -p "${INPUT_DATA}"
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_NEXUS_EMISSION_${nspt}}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#-----------------------------------------------------------------------
#
Expand Down
21 changes: 8 additions & 13 deletions jobs/JREGIONAL_NEXUS_GFS_SFC
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
#
. $USHdir/source_util_funcs.sh
source_config_for_task "cpl_aqm_parm|task_nexus_gfs_sfc" ${GLOBAL_VAR_DEFNS_FP}
. $USHdir/job_preamble.sh
. $USHdir/job_preamble.sh "TRUE"
#
#-----------------------------------------------------------------------
#
Expand Down Expand Up @@ -91,21 +91,16 @@ fi
#
#-----------------------------------------------------------------------
#
# Create the directory where the RAVE fire emission files should be stored
#
#-----------------------------------------------------------------------
#
export GFS_SFC_STAGING_DIR="${COMINext}/GFS_SFC"
mkdir_vrfy -p "${GFS_SFC_STAGING_DIR}"
#
#-----------------------------------------------------------------------
#
# Set the run directory
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_NEXUS_GFS_SFC}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#-----------------------------------------------------------------------
#
Expand All @@ -123,7 +118,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
#
#-----------------------------------------------------------------------
#
job_postamble
job_postamble "FALSE"
#
#-----------------------------------------------------------------------
#
Expand Down
8 changes: 6 additions & 2 deletions jobs/JREGIONAL_NEXUS_POST_SPLIT
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,12 @@ mkdir_vrfy -p "${INPUT_DATA}"
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_NEXUS_POST_SPLIT}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#-----------------------------------------------------------------------
#
Expand Down
8 changes: 6 additions & 2 deletions jobs/JREGIONAL_POINT_SOURCE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,12 @@ mkdir_vrfy -p "${INPUT_DATA}"
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p "${DATA}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_POINT_SOURCE}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#
#-----------------------------------------------------------------------
Expand Down
9 changes: 8 additions & 1 deletion jobs/JREGIONAL_POST_STAT_O3
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,14 @@ This is the J-job script for the task that runs POST-STAT-O3.
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_POST_STAT_O3}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi

mkdir_vrfy -p ${COMOUTwmo}

export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"
#
Expand Down
9 changes: 8 additions & 1 deletion jobs/JREGIONAL_POST_STAT_PM25
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,14 @@ This is the J-job script for the task that runs POST-UPP-STAT.
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_POST_STAT_PM25}"
if [ "${RUN_ENVIR}" = "community" ]; then
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi

mkdir_vrfy -p ${COMOUTwmo}

export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"
#
Expand Down
7 changes: 6 additions & 1 deletion jobs/JREGIONAL_PRE_POST_STAT
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,12 @@ This is the J-job script for the task that runs POST-UPP-STAT.
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_PRE_POST_STAT}"
check_for_preexist_dir_file "$DATA" "delete"
mkdir_vrfy -p $DATA
cd_vrfy $DATA
fi
#
#-----------------------------------------------------------------------
#
Expand Down
4 changes: 2 additions & 2 deletions jobs/JREGIONAL_RUN_FCST
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ the specified cycle.
if [ $RUN_ENVIR = "nco" ]; then
export INPUT_DATA="${COMIN}"
else
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT"
fi
#
Expand All @@ -75,7 +76,6 @@ fi
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
mkdir_vrfy -p ${DATA}/INPUT
mkdir_vrfy -p ${DATA}/RESTART
#
Expand All @@ -96,7 +96,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
#
#-----------------------------------------------------------------------
#
job_postamble
job_postamble "FALSE"
#
#-----------------------------------------------------------------------
#
Expand Down
28 changes: 14 additions & 14 deletions jobs/JREGIONAL_RUN_POST
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,6 @@ This is the J-job script for the task that runs the post-processor (UPP)
on the output files corresponding to a specified forecast hour.
========================================================================"
#
#-----------------------------------------------------------------------
#
# Set the run directory.
#
#-----------------------------------------------------------------------
#
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
#
# If SUB_HOURLY_POST is not set to "TRUE", ensure that the forecast
# minutes (fmn) are set to "00". This is necessary in order to pass
# "fmn" into the post ex-script for the calculation of post_time.
Expand All @@ -90,10 +82,11 @@ fi
#
#-----------------------------------------------------------------------
#
if [ "${RUN_ENVIR}" != "nco" ]; then
if [ "${RUN_ENVIR}" = "community" ]; then
DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
export COMOUT="${DATA}/postprd"
mkdir_vrfy -p "${COMOUT}"
fi
mkdir_vrfy -p "${COMOUT}"

if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then
export DATA_FHR="${DATA:-$COMOUT}/$fhr$fmn"
Expand Down Expand Up @@ -143,9 +136,11 @@ if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then
CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} ))
FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]}

fcst_len_hrs=$( printf "%03d" "${FCST_LEN_HRS}" )
if [ "${fhr}" = "${fcst_len_hrs}" ]; then
touch "${COMIN}/post_${PDY}${cyc}_task_complete.txt"
if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then
fcst_len_hrs=$( printf "%03d" "${FCST_LEN_HRS}" )
if [ "${fhr}" = "${fcst_len_hrs}" ]; then
touch "${COMIN}/post_${PDY}${cyc}_task_complete.txt"
fi
fi
fi
#
Expand All @@ -155,7 +150,12 @@ fi
#
#-----------------------------------------------------------------------
#
job_postamble
fcst_len_hrs=$( printf "%03d" "${FCST_LEN_HRS}" )
if [ "${fhr}" = "${fcst_len_hrs}" ]; then
job_postamble "TRUE"
else
job_postamble
fi
#
#-----------------------------------------------------------------------
#
Expand Down
2 changes: 2 additions & 0 deletions modulefiles/tasks/wcoss2/python_srw.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
load(pathJoin("PrgEnv-intel", os.getenv("PrgEnv_intel_ver")))
load(pathJoin("craype", os.getenv("craype_ver")))
load(pathJoin("intel", os.getenv("intel_ver")))
load(pathJoin("python", os.getenv("python_ver")))
load(pathJoin("prod_util", os.getenv("prod_util_ver")))
2 changes: 1 addition & 1 deletion parm/aqm.rc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ bio_format: netcdf
bio_file: {{ aqm_rc_bio_file_fp }}
bio_frequency: static
bio_period: summer
bio_speciation_file: {{ aqm_bio_dir }}/gspro_biogenics_1mar2017.txt
bio_speciation_file: {{ dcominbio }}/gspro_biogenics_1mar2017.txt
bio_speciation_profile: B10C6
bio_species::
AVG_NOAG_GROW 1.00000 AVG_NOAG_GROW gmN/hr
Expand Down
Loading