From e3bb9ea6d742fa0ef1167af76d1ad51d3f7e280e Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 5 Jun 2023 10:00:16 -0600 Subject: [PATCH 01/16] [develop] Split EnsembleStat vx tasks into GenEnsProd and EnsembleStat (#809) The latest versions of MET/METplus have introduced a new tool named GenEnsProd that performs many of the ensemble-related calculations that EnsembleStat used to do. Although EnsembleStat can still perform these calculations, that will change in future versions of MET/METplus. Thus, this PR creates new vx tasks that call GenEnsProd to perform the necessary calculations, removing them from the EnsembleStat tasks. --- jobs/JREGIONAL_CHECK_POST_OUTPUT | 99 +++++++ parm/metplus/EnsembleStat_APCP01h.conf | 85 +++--- parm/metplus/EnsembleStat_APCPgt01h.conf | 93 +++---- parm/metplus/EnsembleStat_REFC.conf | 83 +++--- parm/metplus/EnsembleStat_RETOP.conf | 85 +++--- parm/metplus/EnsembleStat_SFC.conf | 103 +++---- parm/metplus/EnsembleStat_UPA.conf | 124 +++------ parm/metplus/GenEnsProd_APCP01h.conf | 191 +++++++++++++ parm/metplus/GenEnsProd_APCPgt01h.conf | 204 ++++++++++++++ parm/metplus/GenEnsProd_REFC.conf | 191 +++++++++++++ parm/metplus/GenEnsProd_RETOP.conf | 193 ++++++++++++++ parm/metplus/GenEnsProd_SFC.conf | 216 +++++++++++++++ parm/metplus/GenEnsProd_UPA.conf | 236 ++++++++++++++++ parm/metplus/GridStat_APCP01h.conf | 29 +- parm/metplus/GridStat_APCPgt01h.conf | 27 +- parm/metplus/GridStat_REFC.conf | 42 +-- parm/metplus/GridStat_RETOP.conf | 42 +-- parm/metplus/GridStat_ensmean_APCP01h.conf | 43 +-- parm/metplus/GridStat_ensmean_APCPgt01h.conf | 33 ++- parm/metplus/GridStat_ensmean_REFC.conf | 42 +-- parm/metplus/GridStat_ensmean_RETOP.conf | 38 +-- parm/metplus/GridStat_ensprob_APCP01h.conf | 24 +- parm/metplus/GridStat_ensprob_APCP03h.conf | 31 ++- parm/metplus/GridStat_ensprob_APCP06h.conf | 31 ++- parm/metplus/GridStat_ensprob_APCP24h.conf | 31 ++- parm/metplus/GridStat_ensprob_REFC.conf | 38 +-- parm/metplus/GridStat_ensprob_RETOP.conf | 40 ++- parm/metplus/Pb2nc_obs.conf | 55 ++-- parm/metplus/PcpCombine_fcst.conf | 38 ++- parm/metplus/PcpCombine_obs.conf | 38 ++- parm/metplus/PointStat_SFC.conf | 29 +- parm/metplus/PointStat_UPA.conf | 29 +- parm/metplus/PointStat_ensmean_SFC.conf | 24 +- parm/metplus/PointStat_ensmean_UPA.conf | 24 +- parm/metplus/PointStat_ensprob_SFC.conf | 24 +- parm/metplus/PointStat_ensprob_UPA.conf | 24 +- parm/wflow/coldstart.yaml | 2 +- parm/wflow/post.yaml | 4 +- parm/wflow/verify.yaml | 252 ------------------ parm/wflow/verify_det.yaml | 116 ++++++++ parm/wflow/verify_ens.yaml | 189 +++++++++++++ parm/wflow/verify_ensgrid.yaml | 160 ----------- parm/wflow/verify_pre.yaml | 222 +++++++++++++++ scripts/exregional_check_post_output.sh | 120 +++++++++ scripts/exregional_get_obs_ccpa.sh | 20 +- scripts/exregional_get_obs_mrms.sh | 4 +- scripts/exregional_get_obs_ndas.sh | 16 +- ...onal_run_met_genensprod_or_ensemblestat.sh | 18 +- ...gional_run_met_gridstat_or_pointstat_vx.sh | 46 +++- ...un_met_gridstat_or_pointstat_vx_ensmean.sh | 18 +- ...un_met_gridstat_or_pointstat_vx_ensprob.sh | 18 +- scripts/exregional_run_met_pb2nc_obs.sh | 14 +- scripts/exregional_run_met_pcpcombine.sh | 51 ++-- tests/WE2E/machine_suites/comprehensive | 2 + ...S_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml | 2 +- ...S_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml | 6 +- ...fig.MET_ensemble_verification_only_vx.yaml | 37 +-- ...nsemble_verification_only_vx_time_lag.yaml | 43 +++ .../config.MET_verification_only_vx.yaml | 17 +- ush/bash_utils/eval_METplus_timestr_tmpl.sh | 6 +- ush/config_defaults.yaml | 37 ++- ush/get_met_metplus_tool_name.sh | 8 +- ush/set_vx_fhr_list.sh | 26 +- ush/setup.py | 116 ++++++-- ush/valid_param_vals.yaml | 2 + 65 files changed, 2984 insertions(+), 1267 deletions(-) create mode 100755 jobs/JREGIONAL_CHECK_POST_OUTPUT create mode 100644 parm/metplus/GenEnsProd_APCP01h.conf create mode 100644 parm/metplus/GenEnsProd_APCPgt01h.conf create mode 100644 parm/metplus/GenEnsProd_REFC.conf create mode 100644 parm/metplus/GenEnsProd_RETOP.conf create mode 100644 parm/metplus/GenEnsProd_SFC.conf create mode 100644 parm/metplus/GenEnsProd_UPA.conf delete mode 100644 parm/wflow/verify.yaml create mode 100644 parm/wflow/verify_det.yaml create mode 100644 parm/wflow/verify_ens.yaml delete mode 100644 parm/wflow/verify_ensgrid.yaml create mode 100644 parm/wflow/verify_pre.yaml create mode 100755 scripts/exregional_check_post_output.sh create mode 100644 tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml diff --git a/jobs/JREGIONAL_CHECK_POST_OUTPUT b/jobs/JREGIONAL_CHECK_POST_OUTPUT new file mode 100755 index 0000000000..b9c1868282 --- /dev/null +++ b/jobs/JREGIONAL_CHECK_POST_OUTPUT @@ -0,0 +1,99 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. $USHdir/source_util_funcs.sh +source_config_for_task "" ${GLOBAL_VAR_DEFNS_FP} +. $USHdir/job_preamble.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that checks that all the post- +processed output files in fact exist and are at least a certain age. +These files may have been generated by UPP as part of the current SRW +App workflow, or they may be user-staged. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSdir/exregional_check_post_output.sh || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Create a flag file to make rocoto aware that the make_grid task has +# successfully completed (so that other tasks that depend on it can be +# launched). +# +#----------------------------------------------------------------------- +# +ensmem_name="mem${ENSMEM_INDX}" +cycle_dir="$EXPTDIR/$CDATE" +mkdir_vrfy -p "${cycle_dir}" +touch "${cycle_dir}/post_files_exist_${ensmem_name}.txt" +# +#----------------------------------------------------------------------- +# +# Run job postamble. +# +#----------------------------------------------------------------------- +# +job_postamble +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf index 4f78827e08..aa85f5dc51 100644 --- a/parm/metplus/EnsembleStat_APCP01h.conf +++ b/parm/metplus/EnsembleStat_APCP01h.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -54,19 +54,23 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - -# Name to identify observation data in output +# +# Name to identify observation data in output. +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -89,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. @@ -144,7 +146,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = @@ -164,36 +166,22 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -# ENS_VARn_THRESH must be included here in order for the netCDF file -# generated by EnsembleStat to include the ensemble probability (aka -# ensembe frequency) fields corresponding to the specified thresholds. -# These must be included because the corresonding GridStat task for -# ensemble probabilities expects to read them in as input. -# -ENS_VAR1_NAME = {{fieldname_in_fcst_input}} -ENS_VAR1_LEVELS = A{ACCUM_HH} -ENS_VAR1_THRESH = {{field_thresholds}} -# -# Get the variable ACCUM_HH from the environment. +# Precipitation accumulation interval in 2-digit hours. # ACCUM_HH = {{accum_hh}} # @@ -216,7 +204,6 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -242,31 +229,37 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -278,7 +271,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. # -ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_APCPgt01h.conf b/parm/metplus/EnsembleStat_APCPgt01h.conf index e8b64b6763..e748642b9b 100644 --- a/parm/metplus/EnsembleStat_APCPgt01h.conf +++ b/parm/metplus/EnsembleStat_APCPgt01h.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -54,24 +54,29 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} -FCST_NATIVE_DATA_TYPE = GRIB - ENSEMBLE_STAT_DESC = NA - -# Name to identify observation data in output +# +# Name to identify observation data in output. +# OBTYPE = {{obtype}} -OBS_NATIVE_DATA_TYPE = GRIB - -# Forecast data description variables +# +# Forecast data description variables. +# FCST_IS_PROB = false -#ENSEMBLE_STAT_DESC = +FCST_NATIVE_DATA_TYPE = GRIB +OBS_NATIVE_DATA_TYPE = GRIB # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. @@ -95,8 +100,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. @@ -150,7 +153,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = @@ -170,36 +173,22 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -# ENS_VARn_THRESH must be included here in order for the netCDF file -# generated by EnsembleStat to include the ensemble probability (aka -# ensembe frequency) fields corresponding to the specified thresholds. -# These must be included because the corresonding GridStat task for -# ensemble probabilities expects to read them in as input. -# -ENS_VAR1_NAME = {{fieldname_in_met_output}} -ENS_VAR1_LEVELS = A{ACCUM_HH} -ENS_VAR1_THRESH = {{field_thresholds}} -# -# Get the variable ACCUM_HH from the environment. +# Precipitation accumulation interval in 2-digit hours. # ACCUM_HH = {{accum_hh}} # @@ -222,7 +211,6 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -248,31 +236,38 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -284,7 +279,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. # -ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf index 5379540b76..d6fe4874c8 100644 --- a/parm/metplus/EnsembleStat_REFC.conf +++ b/parm/metplus/EnsembleStat_REFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -54,19 +54,23 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - -# Name to identify observation data in output +# +# Name to identify observation data in output. +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -89,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. @@ -146,7 +148,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = FULL @@ -166,35 +168,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -# ENS_VARn_THRESH must be included here in order for the netCDF file -# generated by EnsembleStat to include the ensemble probability (aka -# ensembe frequency) fields corresponding to the specified thresholds. -# These must be included because the corresonding GridStat task for -# ensemble probabilities expects to read them in as input. -# -ENS_VAR1_NAME = {{fieldname_in_fcst_input}} -ENS_VAR1_LEVELS = L0 -ENS_VAR1_THRESH = {{field_thresholds}} -# # Forecast and observation variables and levels as specified in the fcst # field dictionary of the MET configuration file. Specify as FCST_VARn_NAME, # FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION and OBS_VARn_NAME, @@ -214,7 +202,6 @@ OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -240,31 +227,37 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -276,7 +269,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. # -ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf index 46286e1632..55ac5fc8b1 100644 --- a/parm/metplus/EnsembleStat_RETOP.conf +++ b/parm/metplus/EnsembleStat_RETOP.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -54,19 +54,23 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - -# Name to identify observation data in output +# +# Name to identify observation data in output. +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -89,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. @@ -146,7 +148,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = FULL @@ -166,37 +168,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -# ENS_VARn_THRESH must be included here in order for the netCDF file -# generated by EnsembleStat to include the ensemble probability (aka -# ensembe frequency) fields corresponding to the specified thresholds. -# These must be included because the corresonding GridStat task for -# ensemble probabilities expects to read them in as input. -# -ENS_VAR1_NAME = {{fieldname_in_fcst_input}} -ENS_VAR1_LEVELS = L0 -ENS_VAR1_THRESH = {{field_thresholds}} -# convert from meters to kilofeet: M_to_KFT(x) = x * 3.28084 * 0.001; -ENS_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; -# # Forecast and observation variables and levels as specified in the fcst # field dictionary of the MET configuration file. Specify as FCST_VARn_NAME, # FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION and OBS_VARn_NAME, @@ -218,7 +204,6 @@ OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -244,31 +229,37 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -280,7 +271,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. # -ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_SFC.conf b/parm/metplus/EnsembleStat_SFC.conf index 1cd1495ca6..14c5c7b85a 100644 --- a/parm/metplus/EnsembleStat_SFC.conf +++ b/parm/metplus/EnsembleStat_SFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -54,19 +54,23 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - -# Name to identify observation data in output +# +# Name to identify observation data in output. +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -89,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA @@ -147,7 +149,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 #ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 #ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -#ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = @@ -167,53 +169,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -ENS_VAR1_NAME = TMP -ENS_VAR1_LEVELS = Z02 -ENS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 - -ENS_VAR2_NAME = DPT -ENS_VAR2_LEVELS = Z2 -ENS_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 - -ENS_VAR3_NAME = WIND -ENS_VAR3_LEVELS = Z10 -ENS_VAR3_THRESH = >=5, >=10, >=15 -ENS_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind - -ENS_VAR4_NAME = TCDC -ENS_VAR4_LEVELS = L0 -ENS_VAR4_THRESH = <25, >75 -ENS_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index=[27]; GRIB2_ipdtmpl_val=[255]; interp = { type = [ { method = NEAREST; width = 1; } ]; } - -ENS_VAR5_NAME = VIS -ENS_VAR5_LEVELS = L0 -ENS_VAR5_THRESH = <1609, <8045, >=8045 -ENS_VAR5_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } - -ENS_VAR6_NAME = HGT -ENS_VAR6_LEVELS = L0 -ENS_VAR6_THRESH = <152, <1520, >=914 -ENS_VAR6_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; -# # Forecast and observation variables and levels as specified in the fcst # field dictionary of the MET configuration file. Specify as FCST_VARn_NAME, # FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION and OBS_VARn_NAME, @@ -281,7 +251,6 @@ OBS_VAR6_THRESH = <152, <305, >=914 # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -307,31 +276,37 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -343,7 +318,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. # -ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_UPA.conf b/parm/metplus/EnsembleStat_UPA.conf index 9a60c5ae0f..4bc2bad3d0 100644 --- a/parm/metplus/EnsembleStat_UPA.conf +++ b/parm/metplus/EnsembleStat_UPA.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -54,19 +54,23 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - -# Name to identify observation data in output +# +# Name to identify observation data in output. +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -89,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA @@ -147,7 +149,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 #ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 #ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -#ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = @@ -167,74 +169,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -ENS_VAR1_NAME = TMP -ENS_VAR1_LEVELS = P850 -ENS_VAR1_THRESH = >=288, >=293, >=298 - -ENS_VAR2_NAME = TMP -ENS_VAR2_LEVELS = P700 -ENS_VAR2_THRESH = >=273, >=278, >=283 - -ENS_VAR3_NAME = TMP -ENS_VAR3_LEVELS = P500 -ENS_VAR3_THRESH = >=258, >=263, >=268 - -ENS_VAR4_NAME = DPT -ENS_VAR4_LEVELS = P850 -ENS_VAR4_THRESH = >=273, >=278, >=283 - -ENS_VAR5_NAME = DPT -ENS_VAR5_LEVELS = P700 -ENS_VAR5_THRESH = >=263, >=268, >=273 - -ENS_VAR6_NAME = WIND -ENS_VAR6_LEVELS = P850 -ENS_VAR6_THRESH = >=5, >=10, >=15 - -ENS_VAR7_NAME = WIND -ENS_VAR7_LEVELS = P700 -ENS_VAR7_THRESH = >=10, >=15, >=20 - -ENS_VAR8_NAME = WIND -ENS_VAR8_LEVELS = P500 -ENS_VAR8_THRESH = >=15, >=21, >=26 - -ENS_VAR9_NAME = WIND -ENS_VAR9_LEVELS = P250 -ENS_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 - -ENS_VAR10_NAME = HGT -ENS_VAR10_LEVELS = P500 -ENS_VAR10_THRESH = >=5400, >=5600, >=5880 - -ENS_VAR11_NAME = CAPE -ENS_VAR11_LEVELS = L0 -ENS_VAR11_OPTIONS = cnt_thresh = [ >0 ]; -ENS_VAR11_THRESH = <=1000, >1000&&<2500, >2500&&<4000, >2500 - -ENS_VAR12_NAME = HPBL -ENS_VAR12_LEVELS = Z0 -ENS_VAR12_THRESH = <500, <1500, >1500 -# # Forecast and observation variables and levels as specified in the fcst # field dictionary of the MET configuration file. Specify as FCST_VARn_NAME, # FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION and OBS_VARn_NAME, @@ -344,7 +293,6 @@ OBS_VAR12_THRESH = <500, <1500, >1500 # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -370,31 +318,37 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -406,7 +360,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. # -ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/GenEnsProd_APCP01h.conf b/parm/metplus/GenEnsProd_APCP01h.conf new file mode 100644 index 0000000000..d60c8d3ce0 --- /dev/null +++ b/parm/metplus/GenEnsProd_APCP01h.conf @@ -0,0 +1,191 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = {{fieldname_in_fcst_input}} +ENS_VAR1_LEVELS = A{{accum_hh}} +ENS_VAR1_THRESH = {{field_thresholds}} + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_APCPgt01h.conf b/parm/metplus/GenEnsProd_APCPgt01h.conf new file mode 100644 index 0000000000..08886087d9 --- /dev/null +++ b/parm/metplus/GenEnsProd_APCPgt01h.conf @@ -0,0 +1,204 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# Forecast data description variables +FCST_IS_PROB = false + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +# Note: +# For APCP variables with accumulation periods greater than 1 hour (e.g. +# 03h, 06h, 24h), ENS_VAR1_NAME in the corresponding METplus configuration +# file specifies the name of the field in the NetCDF file(s) generated by +# MET's pcp_combine tool. This tool reads in the grib2 file(s) containing +# 1 hour accumulation data and outputs NetCDF file(s) with the appropriate +# > 1 hour accumulation. These output NetCDF files name their arrays as +# specified by the environment variable fieldname_in_met_output. Thus, +# that is the value we set ENS_VAR1_NAME to here, not to the name of the +# variable in the grib2 file. +ENS_VAR1_NAME = {{fieldname_in_met_output}} +ENS_VAR1_LEVELS = A{{accum_hh}} +ENS_VAR1_THRESH = {{field_thresholds}} + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_REFC.conf b/parm/metplus/GenEnsProd_REFC.conf new file mode 100644 index 0000000000..8c02f71d64 --- /dev/null +++ b/parm/metplus/GenEnsProd_REFC.conf @@ -0,0 +1,191 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = {{fieldname_in_fcst_input}} +ENS_VAR1_LEVELS = L0 +ENS_VAR1_THRESH = {{field_thresholds}} + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_RETOP.conf b/parm/metplus/GenEnsProd_RETOP.conf new file mode 100644 index 0000000000..111b3bfccc --- /dev/null +++ b/parm/metplus/GenEnsProd_RETOP.conf @@ -0,0 +1,193 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = {{fieldname_in_fcst_input}} +ENS_VAR1_LEVELS = L0 +ENS_VAR1_THRESH = {{field_thresholds}} +# convert from meters to kilofeet: M_to_KFT(x) = x * 3.28084 * 0.001; +ENS_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_SFC.conf b/parm/metplus/GenEnsProd_SFC.conf new file mode 100644 index 0000000000..47b49b9adb --- /dev/null +++ b/parm/metplus/GenEnsProd_SFC.conf @@ -0,0 +1,216 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = TMP +ENS_VAR1_LEVELS = Z02 +ENS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 + +ENS_VAR2_NAME = DPT +ENS_VAR2_LEVELS = Z2 +ENS_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 + +ENS_VAR3_NAME = WIND +ENS_VAR3_LEVELS = Z10 +ENS_VAR3_THRESH = >=5, >=10, >=15 +ENS_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind + +ENS_VAR4_NAME = TCDC +ENS_VAR4_LEVELS = L0 +ENS_VAR4_THRESH = <25, >75 +ENS_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index=[27]; GRIB2_ipdtmpl_val=[255]; interp = { type = [ { method = NEAREST; width = 1; } ]; } + +ENS_VAR5_NAME = VIS +ENS_VAR5_LEVELS = L0 +ENS_VAR5_THRESH = <1609, <8045, >=8045 +ENS_VAR5_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +ENS_VAR6_NAME = HGT +ENS_VAR6_LEVELS = L0 +ENS_VAR6_THRESH = <152, <1520, >=914 +ENS_VAR6_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; + + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_UPA.conf b/parm/metplus/GenEnsProd_UPA.conf new file mode 100644 index 0000000000..ee1ceb42ca --- /dev/null +++ b/parm/metplus/GenEnsProd_UPA.conf @@ -0,0 +1,236 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = TMP +ENS_VAR1_LEVELS = P850 +ENS_VAR1_THRESH = >=288, >=293, >=298 + +ENS_VAR2_NAME = TMP +ENS_VAR2_LEVELS = P700 +ENS_VAR2_THRESH = >=273, >=278, >=283 + +ENS_VAR3_NAME = TMP +ENS_VAR3_LEVELS = P500 +ENS_VAR3_THRESH = >=258, >=263, >=268 + +ENS_VAR4_NAME = DPT +ENS_VAR4_LEVELS = P850 +ENS_VAR4_THRESH = >=273, >=278, >=283 + +ENS_VAR5_NAME = DPT +ENS_VAR5_LEVELS = P700 +ENS_VAR5_THRESH = >=263, >=268, >=273 + +ENS_VAR6_NAME = WIND +ENS_VAR6_LEVELS = P850 +ENS_VAR6_THRESH = >=5, >=10, >=15 + +ENS_VAR7_NAME = WIND +ENS_VAR7_LEVELS = P700 +ENS_VAR7_THRESH = >=10, >=15, >=20 + +ENS_VAR8_NAME = WIND +ENS_VAR8_LEVELS = P500 +ENS_VAR8_THRESH = >=15, >=21, >=26 + +ENS_VAR9_NAME = WIND +ENS_VAR9_LEVELS = P250 +ENS_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 + +ENS_VAR10_NAME = HGT +ENS_VAR10_LEVELS = P500 +ENS_VAR10_THRESH = >=5400, >=5600, >=5880 + +ENS_VAR11_NAME = CAPE +ENS_VAR11_LEVELS = L0 +ENS_VAR11_OPTIONS = cnt_thresh = [ >0 ]; +ENS_VAR11_THRESH = <=1000, >1000&&<2500, >2500&&<4000, >2500 + +ENS_VAR12_NAME = HPBL +ENS_VAR12_LEVELS = Z0 +ENS_VAR12_THRESH = <500, <1500, >1500 + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GridStat_APCP01h.conf b/parm/metplus/GridStat_APCP01h.conf index d8a686ab00..0ba9399203 100644 --- a/parm/metplus/GridStat_APCP01h.conf +++ b/parm/metplus/GridStat_APCP01h.conf @@ -1,10 +1,8 @@ # GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -59,6 +57,11 @@ METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to GridStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped # grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary @@ -85,7 +88,7 @@ GRID_STAT_REGRID_SHAPE = SQUARE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} FCST_NATIVE_DATA_TYPE = GRIB # @@ -284,19 +287,23 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR. +# Template for observation input to GridStat relative to +# OBS_GRID_STAT_INPUT_DIR. # -FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# Template for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR. +# Template for forecast input to GridStat relative to +# FCST_GRID_STAT_INPUT_DIR. # -OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # diff --git a/parm/metplus/GridStat_APCPgt01h.conf b/parm/metplus/GridStat_APCPgt01h.conf index 6d8968796d..d56bd21e23 100644 --- a/parm/metplus/GridStat_APCPgt01h.conf +++ b/parm/metplus/GridStat_APCPgt01h.conf @@ -1,10 +1,8 @@ # GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -85,7 +88,7 @@ GRID_STAT_REGRID_SHAPE = SQUARE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} FCST_NATIVE_DATA_TYPE = GRIB # @@ -143,7 +146,7 @@ FCST_VAR1_LEVELS = A{ACCUM_HH} FCST_VAR1_OPTIONS = set_attr_lead = "{lead?fmt=%H%M%S}"; # # List of thresholds to evaluate for each name/level combination for -# forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # @@ -271,12 +274,10 @@ GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE # # Directory containing observation input to GridStat. # -#OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/PcpCombine_obs OBS_GRID_STAT_INPUT_DIR = {{obs_input_dir}} # # Directory containing forecast input to GridStat. # -#FCST_GRID_STAT_INPUT_DIR = {OUTPUT_BASE}/metprd/PcpCombine_fcst FCST_GRID_STAT_INPUT_DIR = {{fcst_input_dir}} # # Directory containing climatology mean input to GridStat. Not used in @@ -295,14 +296,11 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # to its default value. But it is not explicitly used elsewhere in this # configuration file. # -#OUTPUT_BASE = {ENV[OUTPUT_BASE]} -#GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # -#STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. @@ -311,18 +309,15 @@ STAGING_DIR = {{staging_dir}} # Template for observation input to GridStat relative to # OBS_GRID_STAT_INPUT_DIR. # -#OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2_a{level?fmt=%HH}h.nc OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # # Template for forecast input to GridStat relative to # FCST_GRID_STAT_INPUT_DIR. # -#FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h.nc FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # -#GRID_STAT_OUTPUT_TEMPLATE = metprd/GridStat GRID_STAT_OUTPUT_TEMPLATE = # # Template for climatology input to GridStat relative to diff --git a/parm/metplus/GridStat_REFC.conf b/parm/metplus/GridStat_REFC.conf index 9fd7e2f86c..ce77614033 100644 --- a/parm/metplus/GridStat_REFC.conf +++ b/parm/metplus/GridStat_REFC.conf @@ -1,10 +1,8 @@ # GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -84,7 +87,7 @@ GRID_STAT_GRID_WEIGHT_FLAG = NONE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} FCST_NATIVE_DATA_TYPE = GRIB # @@ -156,6 +159,7 @@ OBS_VAR1_OPTIONS = censor_thresh = [eq-999, <-20]; censor_val = [-9999, -20]; cn # Time relative to valid time (in seconds) to allow files to be considered # valid. Set both BEGIN and END to 0 to require the exact time in the # filename. Not used in this example. +# #FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 #FCST_GRID_STAT_FILE_WINDOW_END = 0 OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 @@ -188,12 +192,14 @@ FCST_IS_PROB = False FCST_GRID_STAT_PROB_THRESH = ==0.1 GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - -# Set to true if observation data is probabilistic. -# Only used if configuring forecast data as the 'OBS' input. +# +# Set to true if observation data is probabilistic. Only used if +# configuring forecast data as the 'OBS' input. +# OBS_IS_PROB = false - +# # Only used if OBS_IS_PROB is true - sets probabilistic threshold. +# OBS_GRID_STAT_PROB_THRESH = ==0.1 # Climatology data @@ -291,19 +297,23 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR. +# Template for observation input to GridStat relative to +# OBS_GRID_STAT_INPUT_DIR. # -FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# Template for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR. +# Template for forecast input to GridStat relative to +# FCST_GRID_STAT_INPUT_DIR. # -OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # diff --git a/parm/metplus/GridStat_RETOP.conf b/parm/metplus/GridStat_RETOP.conf index fde17ded5d..67ed1e8b2b 100644 --- a/parm/metplus/GridStat_RETOP.conf +++ b/parm/metplus/GridStat_RETOP.conf @@ -1,10 +1,8 @@ # GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -84,7 +87,7 @@ GRID_STAT_GRID_WEIGHT_FLAG = NONE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} FCST_NATIVE_DATA_TYPE = GRIB # @@ -157,6 +160,7 @@ OBS_VAR1_OPTIONS = convert(x) = x * 3280.84 * 0.001; censor_thresh = [<=-9.84252 # Time relative to valid time (in seconds) to allow files to be considered # valid. Set both BEGIN and END to 0 to require the exact time in the # filename. Not used in this example. +# #FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 #FCST_GRID_STAT_FILE_WINDOW_END = 0 OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 @@ -189,12 +193,14 @@ FCST_IS_PROB = False FCST_GRID_STAT_PROB_THRESH = ==0.1 GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - -# Set to true if observation data is probabilistic. -# Only used if configuring forecast data as the 'OBS' input. +# +# Set to true if observation data is probabilistic. Only used if +# configuring forecast data as the 'OBS' input. +# OBS_IS_PROB = false - +# # Only used if OBS_IS_PROB is true - sets probabilistic threshold. +# OBS_GRID_STAT_PROB_THRESH = ==0.1 # Climatology data @@ -292,19 +298,23 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR. +# Template for observation input to GridStat relative to +# OBS_GRID_STAT_INPUT_DIR. # -FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# Template for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR. +# Template for forecast input to GridStat relative to +# FCST_GRID_STAT_INPUT_DIR. # -OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # diff --git a/parm/metplus/GridStat_ensmean_APCP01h.conf b/parm/metplus/GridStat_ensmean_APCP01h.conf index 5e215f5395..a8408f90d9 100644 --- a/parm/metplus/GridStat_ensmean_APCP01h.conf +++ b/parm/metplus/GridStat_ensmean_APCP01h.conf @@ -1,10 +1,8 @@ # Ensemble mean GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -93,8 +96,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -104,7 +108,9 @@ GRID_STAT_DESC = NA GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the grib2 file(s) (in this case @@ -135,11 +141,12 @@ ACCUM_HH = {{accum_hh}} FCST_VAR1_LEVELS = A{ACCUM_HH} # # List of thresholds to evaluate for each name/level combination for -# observation and forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -290,22 +297,24 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for forecast input to GridStat relative to -# FCST_GRID_STAT_INPUT_DIR. -# -FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} -# # Template for observation input to GridStat relative to # OBS_GRID_STAT_INPUT_DIR. # OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # +# Template for forecast input to GridStat relative to +# FCST_GRID_STAT_INPUT_DIR. +# +FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # GRID_STAT_OUTPUT_TEMPLATE = diff --git a/parm/metplus/GridStat_ensmean_APCPgt01h.conf b/parm/metplus/GridStat_ensmean_APCPgt01h.conf index 09dd54d68c..8eb87fafca 100644 --- a/parm/metplus/GridStat_ensmean_APCPgt01h.conf +++ b/parm/metplus/GridStat_ensmean_APCPgt01h.conf @@ -1,10 +1,8 @@ # Ensemble mean GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -93,8 +96,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -104,7 +108,9 @@ GRID_STAT_DESC = NA GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the NetCDF file(s) generated by @@ -131,11 +137,12 @@ ACCUM_HH = {{accum_hh}} FCST_VAR1_LEVELS = A{ACCUM_HH} # # List of thresholds to evaluate for each name/level combination for -# observation and forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the NetCDF file created by MET's @@ -286,7 +293,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensmean_REFC.conf b/parm/metplus/GridStat_ensmean_REFC.conf index 17a1996211..bfd4c65990 100644 --- a/parm/metplus/GridStat_ensmean_REFC.conf +++ b/parm/metplus/GridStat_ensmean_REFC.conf @@ -1,10 +1,8 @@ # Ensemble mean GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -90,8 +93,9 @@ MODEL = {{vx_fcst_model_name}}_ensmean # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -101,16 +105,18 @@ GRID_STAT_DESC = NA GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; # -# Name of forecast variable 1. +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the grib2 file(s) (in this case # of forecasts) and outputs NetCDF file(s) in which the array names -# consist of the value of FIELDNAME_IN_MET_OUTPUT plus a suffix that +# consist of the value of fieldname_in_met_output plus a suffix that # specifies additional properties of the data in the array such as the # level, the type of statistic, etc. In this case, this suffix is # "_L0_ENS_MEAN". Thus, below, FCST_VAR1_NAME must be set to the value -# of FIELDNAME_IN_MET_OUTPUT with "_L0_ENS_MEAN" appended to it. +# of fieldname_in_met_output with "_L0_ENS_MEAN" appended to it. # FCST_VAR1_NAME = {{fieldname_in_met_output}}_L0_ENS_MEAN # @@ -121,11 +127,12 @@ FCST_VAR1_LEVELS = L0 #FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; # # List of thresholds to evaluate for each name/level combination for -# observation and forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -142,7 +149,6 @@ OBS_VAR1_NAME = {{fieldname_in_obs_input}} # same length as FCST_VAR1_LEVELS. # OBS_VAR1_LEVELS = Z500 - OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; # # Time relative to valid time (in seconds) to allow files to be considered @@ -179,6 +185,8 @@ FCST_IS_PROB = False # Only used if FCST_IS_PROB is true - sets probabilistic threshold # FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} # # Set to true if observation data is probabilistic. Only used if # configuring forecast data as the 'OBS' input. @@ -189,8 +197,6 @@ OBS_IS_PROB = false # OBS_GRID_STAT_PROB_THRESH = ==0.1 -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - # Climatology data #GRID_STAT_CLIMO_MEAN_FILE_NAME = #GRID_STAT_CLIMO_MEAN_FIELD = @@ -286,7 +292,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensmean_RETOP.conf b/parm/metplus/GridStat_ensmean_RETOP.conf index 36c6282671..04e337e0d4 100644 --- a/parm/metplus/GridStat_ensmean_RETOP.conf +++ b/parm/metplus/GridStat_ensmean_RETOP.conf @@ -1,10 +1,8 @@ # Ensemble mean GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -90,8 +93,9 @@ MODEL = {{vx_fcst_model_name}}_ensmean # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -101,7 +105,8 @@ GRID_STAT_DESC = NA GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; # -# Name of forecast variable 1. +# String to search for in the foreast input files for forecast variable +# 1. # # Note: # This is the name of the field in the NetCDF file(s) created by MET's @@ -118,15 +123,15 @@ FCST_VAR1_NAME = {{fieldname_in_met_output}}_L0_ENS_MEAN # List of levels to evaluate for forecast variable 1. # FCST_VAR1_LEVELS = L0 - FCST_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; # # List of thresholds to evaluate for each name/level combination for -# observation and forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -143,7 +148,6 @@ OBS_VAR1_NAME = {{fieldname_in_obs_input}} # same length as FCST_VAR1_LEVELS. # OBS_VAR1_LEVELS = Z500 - OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = x * 3280.84 * 0.001; # # Time relative to valid time (in seconds) to allow files to be considered @@ -180,6 +184,8 @@ FCST_IS_PROB = False # Only used if FCST_IS_PROB is true - sets probabilistic threshold # FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} # # Set to true if observation data is probabilistic. Only used if # configuring forecast data as the 'OBS' input. @@ -190,8 +196,6 @@ OBS_IS_PROB = false # OBS_GRID_STAT_PROB_THRESH = ==0.1 -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - # Climatology data #GRID_STAT_CLIMO_MEAN_FILE_NAME = #GRID_STAT_CLIMO_MEAN_FIELD = @@ -287,7 +291,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_APCP01h.conf b/parm/metplus/GridStat_ensprob_APCP01h.conf index 728a80275c..289c407d31 100644 --- a/parm/metplus/GridStat_ensprob_APCP01h.conf +++ b/parm/metplus/GridStat_ensprob_APCP01h.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -93,8 +96,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -319,7 +323,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_APCP03h.conf b/parm/metplus/GridStat_ensprob_APCP03h.conf index 3b3f9a3e20..145f3e5fa1 100644 --- a/parm/metplus/GridStat_ensprob_APCP03h.conf +++ b/parm/metplus/GridStat_ensprob_APCP03h.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -93,8 +96,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -110,7 +114,9 @@ FIELDNAME_IN_MET_OUTPUT = {{fieldname_in_met_output}} # Process as probability # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the NetCDF file(s) generated by @@ -134,7 +140,8 @@ FCST_VAR1_NAME = {FIELDNAME_IN_MET_OUTPUT}_A3_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A03 FCST_VAR1_THRESH = ==0.1 # -# Name of observation variable 1 +# List of levels to evaluate for observation variable 1. Must be the +# same length as FCST_VAR1_LEVELS. # # Note: # This is the name of the field in the NetCDF file created by MET's @@ -368,7 +375,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_APCP06h.conf b/parm/metplus/GridStat_ensprob_APCP06h.conf index d9856518c6..f3e2243c15 100644 --- a/parm/metplus/GridStat_ensprob_APCP06h.conf +++ b/parm/metplus/GridStat_ensprob_APCP06h.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -93,8 +96,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -110,7 +114,9 @@ FIELDNAME_IN_MET_OUTPUT = {{fieldname_in_met_output}} # Process as probability # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the NetCDF file(s) generated by @@ -134,7 +140,8 @@ FCST_VAR1_NAME = {FIELDNAME_IN_MET_OUTPUT}_A6_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A06 FCST_VAR1_THRESH = ==0.1 # -# Name of observation variable 1 +# List of levels to evaluate for observation variable 1. Must be the +# same length as FCST_VAR1_LEVELS. # # Note: # This is the name of the field in the NetCDF file created by MET's @@ -368,7 +375,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_APCP24h.conf b/parm/metplus/GridStat_ensprob_APCP24h.conf index 4c82f06829..089b8d3821 100644 --- a/parm/metplus/GridStat_ensprob_APCP24h.conf +++ b/parm/metplus/GridStat_ensprob_APCP24h.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -93,8 +96,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -110,7 +114,9 @@ FIELDNAME_IN_MET_OUTPUT = {{fieldname_in_met_output}} # Process as probability # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the NetCDF file(s) generated by @@ -134,7 +140,8 @@ FCST_VAR1_NAME = {FIELDNAME_IN_MET_OUTPUT}_A24_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A24 FCST_VAR1_THRESH = ==0.1 # -# Name of observation variable 1 +# List of levels to evaluate for observation variable 1. Must be the +# same length as FCST_VAR1_LEVELS. # # Note: # This is the name of the field in the NetCDF file created by MET's @@ -368,7 +375,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_REFC.conf b/parm/metplus/GridStat_ensprob_REFC.conf index 2348b46832..671d351dfb 100644 --- a/parm/metplus/GridStat_ensprob_REFC.conf +++ b/parm/metplus/GridStat_ensprob_REFC.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -92,8 +95,9 @@ MODEL = {{vx_fcst_model_name}}_ensprob # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -107,7 +111,9 @@ GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = # Process as probability # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the grib2 file(s) (in this case @@ -122,10 +128,7 @@ GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = # #FCST_VAR1_NAME = REFC_L0_ENS_FREQ_ge20 FCST_VAR1_NAME = {{fieldname_in_met_output}}_L0_ENS_FREQ_ge20 - -# List of levels to evaluate for forecast variable 1 FCST_VAR1_LEVELS = L0 - #FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; # # List of thresholds to evaluate for each name/level combination for @@ -135,7 +138,8 @@ FCST_VAR1_THRESH = ==0.1 #FCST_GRID_STAT_FILE_TYPE = # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -274,6 +278,8 @@ FCST_PROB_IN_GRIB_PDS = False # Only used if FCST_IS_PROB is true - sets probabilistic threshold # FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} # # Set to true if observation data is probabilistic. Only used if # configuring forecast data as the 'OBS' input. @@ -284,8 +290,6 @@ OBS_IS_PROB = false # OBS_GRID_STAT_PROB_THRESH = ==0.1 -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - # Climatology data #GRID_STAT_CLIMO_MEAN_FILE_NAME = #GRID_STAT_CLIMO_MEAN_FIELD = @@ -379,7 +383,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_RETOP.conf b/parm/metplus/GridStat_ensprob_RETOP.conf index 071d47fdc7..581dd4164e 100644 --- a/parm/metplus/GridStat_ensprob_RETOP.conf +++ b/parm/metplus/GridStat_ensprob_RETOP.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -34,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -43,11 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,7 +55,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -92,8 +95,9 @@ MODEL = {{vx_fcst_model_name}}_ensprob # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -107,7 +111,8 @@ GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = # Process as probability # -# Name of forecast variable 1. +# String to search for in the foreast input files for forecast variable +# 1. # # Note: # This is the name of the field in the NetCDF file(s) created by MET's @@ -123,12 +128,17 @@ GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = # #FCST_VAR1_NAME = RETOP_L0_ENS_FREQ_ge20 FCST_VAR1_NAME = {{fieldname_in_met_output}}_L0_ENS_FREQ_ge20 -FCST_VAR1_LEVELS = L0 +FCST_VAR1_LEVELS = L0 +# +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1. +# FCST_VAR1_THRESH = ==0.1 #FCST_GRID_STAT_FILE_TYPE = # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -267,6 +277,8 @@ FCST_PROB_IN_GRIB_PDS = False # Only used if FCST_IS_PROB is true - sets probabilistic threshold # FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} # # Set to true if observation data is probabilistic. Only used if # configuring forecast data as the 'OBS' input. @@ -277,8 +289,6 @@ OBS_IS_PROB = false # OBS_GRID_STAT_PROB_THRESH = ==0.1 -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - # Climatology data #GRID_STAT_CLIMO_MEAN_FILE_NAME = #GRID_STAT_CLIMO_MEAN_FIELD = @@ -372,7 +382,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/Pb2nc_obs.conf b/parm/metplus/Pb2nc_obs.conf index 263ad864ed..ae2b5e19f0 100644 --- a/parm/metplus/Pb2nc_obs.conf +++ b/parm/metplus/Pb2nc_obs.conf @@ -1,6 +1,8 @@ +# Pb2nc METplus Configuration + [config] -# List of applications to run. +# List of applications (tools) to run. PROCESS_LIST = PB2NC # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -14,7 +16,7 @@ LOOP_BY = INIT # %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. # see www.strftime.org for more information # %Y%m%d%H expands to YYYYMMDDHH -INIT_TIME_FMT = %Y%m%d%H +INIT_TIME_FMT = %Y%m%d%H # Start time for METplus run - must match INIT_TIME_FMT INIT_BEG = {{cdate}} @@ -22,8 +24,8 @@ INIT_BEG = {{cdate}} # End time for METplus run - must match INIT_TIME_FMT INIT_END = {{cdate}} -# Increment between METplus runs (in seconds if no units are specified) -# Must be >= 60 seconds +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. INIT_INCREMENT = 3600 # List of forecast leads to process for each run time (init or valid) @@ -39,13 +41,31 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run -LOOP_ORDER = times +LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -# Location of MET config file to pass to PB2NC +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {PB2NC_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Location of MET configuration file to pass to Pb2nc. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -84,41 +104,30 @@ PB2NC_TIME_SUMMARY_BEG = 000000 PB2NC_TIME_SUMMARY_END = 235959 PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 -# -# Specify the name of the METplus log file. -# -LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} -# -# Specify the location and name of the final METplus configuration file. -# -METPLUS_CONF = {PB2NC_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} - -OBTYPE = NDAS # End of [config] section and start of [dir] section [dir] # -# Directory containing (obs) input to PB2NC. +# Directory containing (observation) input to Pb2nc. # PB2NC_INPUT_DIR = {{obs_input_dir}} # -# Directory in which to write output from PB2NC. -# -PB2NC_OUTPUT_DIR = {{output_dir}} +# Directory in which to write output from Pb2nc. # # OUTPUT_BASE apparently has to be set to something; it cannot be left # to its default value. But it is not explicitly used elsewhere in this # configuration file. # OUTPUT_BASE = {{output_base}} +PB2NC_OUTPUT_DIR = {{output_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for prepbufr input to PB2NC relative to PB2NC_INPUT_DIR. +# Template for prepbufr input to Pb2nc relative to PB2NC_INPUT_DIR. # PB2NC_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# Template for output from PB2NC. +# Template for output from Pb2nc relative to PB2NC_OUTPUT_DIR. # PB2NC_OUTPUT_TEMPLATE = {{output_fn_template}} diff --git a/parm/metplus/PcpCombine_fcst.conf b/parm/metplus/PcpCombine_fcst.conf index 1ab2b22425..65204c7caa 100644 --- a/parm/metplus/PcpCombine_fcst.conf +++ b/parm/metplus/PcpCombine_fcst.conf @@ -1,10 +1,8 @@ # PcpCombine METplus Configuration for Forecasts -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only PcpCombine for this case +# List of applications (tools) to run. PROCESS_LIST = PcpCombine # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -30,9 +28,9 @@ INIT_END = {{cdate}} # Must be >= 60 seconds. INIT_INCREMENT = 3600 -# List of forecast leads to process for each run time (init or valid). -# In hours if units are not specified. -# If unset, defaults to 0 (don't loop through forecast leads). +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} # Order of loops to process data - Options are times, processes @@ -45,6 +43,14 @@ LEAD_SEQ = {{fhr_list}} # wrappers have been run LOOP_ORDER = times # +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {FCST_PCP_COMBINE_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# # Run PcpCombine on forecast data but not observation (observation input # files are not provided). # @@ -83,14 +89,6 @@ FCST_PCP_COMBINE_MAX_FORECAST = 2d # Keep initialization time constant. FCST_PCP_COMBINE_CONSTANT_INIT = True -# -# Specify the name of the METplus log file. -# -LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} -# -# Specify the location and name of the final METplus configuration file. -# -METPLUS_CONF = {FCST_PCP_COMBINE_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB FCST_NATIVE_DATA_TYPE = GRIB @@ -104,13 +102,12 @@ FCST_PCP_COMBINE_INPUT_DIR = {{fcst_input_dir}} # # Directory in which to write output from PcpCombine. # -FCST_PCP_COMBINE_OUTPUT_DIR = {{output_dir}} -# # OUTPUT_BASE apparently has to be set to something; it cannot be left # to its default value. But it is not explicitly used elsewhere in this # configuration file. # OUTPUT_BASE = {{output_base}} +FCST_PCP_COMBINE_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # @@ -119,13 +116,12 @@ STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# The template FCST_PCP_COMBINE_INPUT_TEMPLATE is used to search for -# forecast input files. It is relative to FCST_PCP_COMBINE_INPUT_DIR. +# Template for forecast input to PcpCombine relative to +# FCST_PCP_COMBINE_INPUT_DIR. # FCST_PCP_COMBINE_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# FCST_PCP_COMBINE_OUTPUT_TEMPLATE is a path/file template that specifies -# the name of output file, including optional subdirectories. It is -# relative to FCST_PCP_COMBINE_OUTPUT_DIR. +# Template for output from PCPCOMBINE relative to +# FCST_PCP_COMBINE_OUTPUT_DIR. # FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {{output_fn_template}} diff --git a/parm/metplus/PcpCombine_obs.conf b/parm/metplus/PcpCombine_obs.conf index 8b9202d081..bc2c4180eb 100644 --- a/parm/metplus/PcpCombine_obs.conf +++ b/parm/metplus/PcpCombine_obs.conf @@ -1,10 +1,8 @@ # PcpCombine METplus Configuration for Observations -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only PcpCombine for this case +# List of applications (tools) to run. PROCESS_LIST = PcpCombine # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -30,9 +28,9 @@ INIT_END = {{cdate}} # Must be >= 60 seconds. INIT_INCREMENT = 3600 -# List of forecast leads to process for each run time (init or valid). -# In hours if units are not specified. -# If unset, defaults to 0 (don't loop through forecast leads). +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} # Order of loops to process data - Options are times, processes @@ -45,6 +43,14 @@ LEAD_SEQ = {{fhr_list}} # wrappers have been run LOOP_ORDER = times # +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {OBS_PCP_COMBINE_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# # Run PcpCombine on observation data but not forecast (forecast input # files are not provided). # @@ -90,14 +96,6 @@ OBS_PCP_COMBINE_OUTPUT_ACCUM = {{accum_hh}} # PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True # -# Specify the name of the METplus log file. -# -LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} -# -# Specify the location and name of the final METplus configuration file. -# -METPLUS_CONF = {OBS_PCP_COMBINE_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} -# # Name to identify observation data in output. # OBTYPE = CCPA @@ -112,13 +110,12 @@ OBS_PCP_COMBINE_INPUT_DIR = {{obs_input_dir}} # # Directory in which to write output from PcpCombine. # -OBS_PCP_COMBINE_OUTPUT_DIR = {{output_dir}} -# # OUTPUT_BASE apparently has to be set to something; it cannot be left # to its default value. But it is not explicitly used elsewhere in this # configuration file. # OUTPUT_BASE = {{output_base}} +OBS_PCP_COMBINE_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # @@ -127,13 +124,12 @@ STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# The template OBS_PCP_COMBINE_INPUT_TEMPLATE is used to search for -# observation input files. It is relative to OBS_PCP_COMBINE_INPUT_DIR. +# Template for observation input to PcpCombine relative to +# OBS_PCP_COMBINE_INPUT_DIR. # OBS_PCP_COMBINE_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# OBS_PCP_COMBINE_OUTPUT_TEMPLATE is a path/file template that specifies -# the name of output file, including optional subdirectories. It is -# relative to OBS_PCP_COMBINE_OUTPUT_DIR. +# Template for output from PcpCombine relative to +# OBS_PCP_COMBINE_OUTPUT_DIR. # OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {{output_fn_template}} diff --git a/parm/metplus/PointStat_SFC.conf b/parm/metplus/PointStat_SFC.conf index 2d5fe0a2e4..62f59328fe 100644 --- a/parm/metplus/PointStat_SFC.conf +++ b/parm/metplus/PointStat_SFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -24,7 +24,7 @@ INIT_BEG = {{cdate}} # End time for METplus run - must match INIT_TIME_FMT INIT_END = {{cdate}} -# Increment between METplus runs (in seconds if no units are specified) +# Increment between METplus runs (in seconds if no units are specified). # Must be >= 60 seconds. INIT_INCREMENT = 3600 @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,12 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -55,12 +54,14 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # Specify the location and name of the final METplus conf file. # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} - -# Location of MET config file to pass to GridStat +# +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE # if set. +# POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_wrapped POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA @@ -122,7 +123,11 @@ POINT_STAT_OFFSETS = 0 # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} # # Value to enter under the DESC column in the output stat file. Here, # we store the value of the original lead in this column, i.e. the lead @@ -134,10 +139,6 @@ MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} # Uncomment this line only after upgrading to METplus 5.x. #POINT_STAT_DESC = {lead?fmt=%H%M%S?shift={{time_lag}}} POINT_STAT_DESC = NA -# -# Name to identify observation data in output. -# -OBTYPE = {{obtype}} # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -145,7 +146,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -169,7 +170,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields diff --git a/parm/metplus/PointStat_UPA.conf b/parm/metplus/PointStat_UPA.conf index 1d20b16fd8..385b5387a2 100644 --- a/parm/metplus/PointStat_UPA.conf +++ b/parm/metplus/PointStat_UPA.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -24,7 +24,7 @@ INIT_BEG = {{cdate}} # End time for METplus run - must match INIT_TIME_FMT INIT_END = {{cdate}} -# Increment between METplus runs (in seconds if no units are specified) +# Increment between METplus runs (in seconds if no units are specified). # Must be >= 60 seconds. INIT_INCREMENT = 3600 @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,12 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -55,12 +54,14 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # Specify the location and name of the final METplus conf file. # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} - -# Location of MET config file to pass to GridStat +# +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE # if set. +# POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_wrapped POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA @@ -122,7 +123,11 @@ POINT_STAT_OFFSETS = 0 # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} # # Value to enter under the DESC column in the output stat file. Here, # we store the value of the original lead in this column, i.e. the lead @@ -134,10 +139,6 @@ MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} # Uncomment this line only after upgrading to METplus 5.x. #POINT_STAT_DESC = {lead?fmt=%H%M%S?shift={{time_lag}}} POINT_STAT_DESC = NA -# -# Name to identify observation data in output. -# -OBTYPE = {{obtype}} # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -145,7 +146,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -169,7 +170,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields diff --git a/parm/metplus/PointStat_ensmean_SFC.conf b/parm/metplus/PointStat_ensmean_SFC.conf index 9771081456..0d5ee3b696 100644 --- a/parm/metplus/PointStat_ensmean_SFC.conf +++ b/parm/metplus/PointStat_ensmean_SFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,12 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -56,7 +55,8 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE @@ -124,12 +124,14 @@ POINT_STAT_OFFSETS = 0 # mean. This makes it easier to identify each curve. # MODEL = {{vx_fcst_model_name}}_ensmean - -POINT_STAT_DESC = NA # # Name to identify observation data in output. # OBTYPE = {{obtype}} +# +# Value to enter under the DESC column in the output stat file. +# +POINT_STAT_DESC = NA # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -137,7 +139,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -161,7 +163,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields @@ -228,7 +230,9 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} POINT_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/PointStat_ensmean_UPA.conf b/parm/metplus/PointStat_ensmean_UPA.conf index 3fb3f1b790..1846e0cf6e 100644 --- a/parm/metplus/PointStat_ensmean_UPA.conf +++ b/parm/metplus/PointStat_ensmean_UPA.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,12 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -56,7 +55,8 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE @@ -124,12 +124,14 @@ POINT_STAT_OFFSETS = 0 # mean. This makes it easier to identify each curve. # MODEL = {{vx_fcst_model_name}}_ensmean - -POINT_STAT_DESC = NA # # Name to identify observation data in output. # OBTYPE = {{obtype}} +# +# Value to enter under the DESC column in the output stat file. +# +POINT_STAT_DESC = NA # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -137,7 +139,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -161,7 +163,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields @@ -303,7 +305,9 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} POINT_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/PointStat_ensprob_SFC.conf b/parm/metplus/PointStat_ensprob_SFC.conf index 10937c2ec7..84b3ac9445 100644 --- a/parm/metplus/PointStat_ensprob_SFC.conf +++ b/parm/metplus/PointStat_ensprob_SFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,12 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -56,7 +55,8 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE @@ -124,12 +124,14 @@ POINT_STAT_OFFSETS = 0 # This makes it easier to identify each curve. # MODEL = {{vx_fcst_model_name}}_ensprob - -POINT_STAT_DESC = NA # # Name to identify observation data in output. # OBTYPE = {{obtype}} +# +# Value to enter under the DESC column in the output stat file. +# +POINT_STAT_DESC = NA # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -137,7 +139,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -161,7 +163,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields @@ -406,7 +408,9 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} POINT_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/PointStat_ensprob_UPA.conf b/parm/metplus/PointStat_ensprob_UPA.conf index ea6e380422..6b4628112f 100644 --- a/parm/metplus/PointStat_ensprob_UPA.conf +++ b/parm/metplus/PointStat_ensprob_UPA.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,12 +41,11 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -56,7 +55,8 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE @@ -124,12 +124,14 @@ POINT_STAT_OFFSETS = 0 # This makes it easier to identify each curve. # MODEL = {{vx_fcst_model_name}}_ensprob - -POINT_STAT_DESC = NA # # Name to identify observation data in output. # OBTYPE = {{obtype}} +# +# Value to enter under the DESC column in the output stat file. +# +POINT_STAT_DESC = NA # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -137,7 +139,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -161,7 +163,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields @@ -528,7 +530,9 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} POINT_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index 98aa2e98fd..c64b62bbe3 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -75,7 +75,7 @@ task_get_extrn_lbcs: metatask_run_ensemble: var: - mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' task_make_ics_mem#mem#: <<: *default_task command: '&LOAD_MODULES_RUN_TASK_FP; "make_ics" "&JOBSdir;/JREGIONAL_MAKE_ICS"' diff --git a/parm/wflow/post.yaml b/parm/wflow/post.yaml index 782b21b020..ae02b4025b 100644 --- a/parm/wflow/post.yaml +++ b/parm/wflow/post.yaml @@ -25,8 +25,8 @@ default_task_post: &default_task metatask_run_ens_post: var: - mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - metatask_run_post: + mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + metatask_run_post_mem#mem#_all_fhrs: var: fhr: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{{ " %03d" % h }}{% endfor %}' cycledef: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{% if h <= workflow.FCST_LEN_CYCL|min %}forecast {% else %}long_forecast {% endif %}{% endfor %}' diff --git a/parm/wflow/verify.yaml b/parm/wflow/verify.yaml deleted file mode 100644 index b905dc2b68..0000000000 --- a/parm/wflow/verify.yaml +++ /dev/null @@ -1,252 +0,0 @@ -default_task_verify: &default_task - account: '&ACCOUNT;' - attrs: - cycledefs: forecast - maxtries: '1' - envars: &default_vars - GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' - USHdir: '&USHdir;' - PDY: !cycstr '@Y@m@d' - cyc: !cycstr "@H" - subcyc: !cycstr "@M" - LOGDIR: !cycstr "&LOGDIR;" - nprocs: '{{ parent.nnodes * parent.ppn }}' - FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' - join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - memory: '{% if user.MACHINE not in ["NOAACLOUD"] %}{{ "2G" }}{% endif %}' - native: '{{ platform.SCHED_NATIVE_CMD }}' - nnodes: 1 - nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: '&NCORES_PER_NODE;' - partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' - ppn: 1 - queue: '&QUEUE_DEFAULT;' - walltime: 00:30:00 - -task_get_obs_ccpa: - <<: *default_task - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_CCPA"' - envars: - <<: *default_vars - ACCUM_HH: '01' - OBS_DIR: '&CCPA_OBS_DIR;' - partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' - queue: "&QUEUE_HPSS;" - walltime: 00:45:00 - -task_get_obs_mrms: - <<: *default_task - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_MRMS"' - envars: - <<: *default_vars - OBS_DIR: '&MRMS_OBS_DIR;' - VAR: 'REFC RETOP' - partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' - queue: "&QUEUE_HPSS;" - walltime: 00:45:00 - -task_get_obs_ndas: - <<: *default_task - envars: - <<: *default_vars - OBS_DIR: '&NDAS_OBS_DIR;' - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_NDAS"' - queue: "&QUEUE_HPSS;" - partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' - walltime: 02:00:00 - -task_run_MET_Pb2nc_obs: - <<: *default_task - attrs: - cycledefs: forecast - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PB2NC_OBS"' - envars: - <<: *default_vars - VAR: SFC - ACCUM_HH: '01' - obs_or_fcst: obs - OBTYPE: NDAS - OBS_DIR: '&NDAS_OBS_DIR;' - MET_TOOL: 'PB2NC' - dependency: - and: - datadep: - text: "&NDAS_OBS_DIR;" - or: - not: - taskvalid: - attrs: - task: get_obs_ndas - taskdep: - attrs: - task: get_obs_ndas - -metatask_vx_ens_member: - var: - mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - metatask_GridStat_MRMS_mem#mem#: - var: - VAR: REFC RETOP - task_run_MET_GridStat_vx_#VAR#_mem#mem#: - <<: *default_task - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' - envars: - <<: *default_vars - OBS_DIR: '&MRMS_OBS_DIR;' - VAR: '#VAR#' - ACCUM_HH: '01' - MET_TOOL: 'GRIDSTAT' - OBTYPE: 'MRMS' - ENSMEM_INDX: "#mem#" - SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' - walltime: 02:00:00 - dependency: - and: - or_get_obs: # Ensure get_obs task is complete if it's turned on - not: - taskvalid: - attrs: - task: get_obs_mrms - and: - taskvalid: - attrs: - task: get_obs_mrms - taskdep: - attrs: - task: get_obs_mrms - or_do_post: &post_files_exist - and_run_post: # If post was meant to run, wait on the whole post metatask - taskvalid: - attrs: - task: run_post_mem#mem#_f000 - metataskdep: - attrs: - metatask: run_ens_post - and_inline_post: # If inline post ran, wait on the forecast task to complete - not: - taskvalid: - attrs: - task: run_post_mem#mem#_f000 - taskdep: - attrs: - task: run_fcst_mem#mem# - - metatask_PointStat_mem#mem#: - var: - VAR: SFC UPA - task_run_MET_PointStat_vx_#VAR#_mem#mem#: - <<: *default_task - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' - envars: - <<: *default_vars - FHR: '#FCST_HR_LIST#' - OBS_DIR: '&NDAS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'POINTSTAT' - OBTYPE: 'NDAS' - ACCUM_HH: '01' - ENSMEM_INDX: "#mem#" - SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' - walltime: 01:00:00 - dependency: - and: - taskdep_pb2nc: - attrs: - task: run_MET_Pb2nc_obs - or_do_post: - <<: *post_files_exist - or_get_obs: # Get obs complete if turned on - not: - taskvalid: - attrs: - task: get_obs_ndas - and: - taskvalid: - attrs: - task: get_obs_ndas - taskdep: - attrs: - task: get_obs_ndas - -metatask_GridStat_APCP_acc: - var: - ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - FCST_HR_LIST: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{% for h in range(ah, workflow.FCST_LEN_HRS+1, ah) %}{{ "%02d," % h }}{% endfor %}{% endif %} {% endfor %}' - - - task_run_MET_PcpCombine_obs_APCP#ACCUM_HR#h: - <<: *default_task - attrs: - cycledefs: forecast - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' - envars: - <<: *default_vars - VAR: APCP - ACCUM_HH: '#ACCUM_HR#' - obs_or_fcst: obs - OBTYPE: CCPA - OBS_DIR: '&CCPA_OBS_DIR;' - MET_TOOL: 'PCPCOMBINE' - dependency: - or: - not: - taskvalid: - attrs: - task: get_obs_ccpa - and: - taskdep: - attrs: - task: get_obs_ccpa - datadep: - text: "&CCPA_OBS_DIR;" - - metatask_vx_ens_member_acc: - var: - mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: - <<: *default_task - attrs: - cycledefs: forecast - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' - envars: - <<: *default_vars - VAR: APCP - ACCUM_HH: '#ACCUM_HR#' - obs_or_fcst: fcst - OBTYPE: CCPA - OBS_DIR: '&CCPA_OBS_DIR;' - MET_TOOL: 'PCPCOMBINE' - ENSMEM_INDX: "#mem#" - SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' - dependency: - or_do_post: - <<: *post_files_exist - - task_run_MET_GridStat_vx_APCP#ACCUM_HR#h_mem#mem#: - <<: *default_task - attrs: - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' - envars: - <<: *default_vars - FHR: '#FCST_HR_LIST#' - OBS_DIR: '&CCPA_OBS_DIR;' - VAR: APCP - ACCUM_HH: '#ACCUM_HR#' - MET_TOOL: 'GRIDSTAT' - OBTYPE: 'CCPA' - ENSMEM_INDX: "#mem#" - SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' - walltime: 02:00:00 - dependency: - and: - taskdep_obs: - attrs: - task: run_MET_PcpCombine_obs_APCP#ACCUM_HR#h - taskdep_fcst: - attrs: - task: run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem# diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml new file mode 100644 index 0000000000..804cc06107 --- /dev/null +++ b/parm/wflow/verify_det.yaml @@ -0,0 +1,116 @@ +default_task_verify_det: &default_task_verify_det + account: '&ACCOUNT;' + attrs: + cycledefs: forecast + maxtries: '1' + envars: &default_vars + GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' + USHdir: '&USHdir;' + PDY: !cycstr '@Y@m@d' + cyc: !cycstr "@H" + subcyc: !cycstr "@M" + LOGDIR: !cycstr "&LOGDIR;" + nprocs: '{{ parent.nnodes * parent.ppn }}' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: '{% if user.MACHINE not in ["NOAACLOUD"] %}{{ "2G" }}{% endif %}' + native: '{{ platform.SCHED_NATIVE_CMD }}' + nnodes: 1 + nodes: '{{ nnodes }}:ppn={{ ppn }}' + nodesize: '&NCORES_PER_NODE;' + partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' + ppn: 1 + queue: '&QUEUE_DEFAULT;' + walltime: 00:30:00 + +metatask_GridStat_CCPA_all_accums_all_mems: + var: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_mem#mem#: + <<: *default_task_verify_det + attrs: + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' + envars: + <<: *default_vars + OBS_DIR: '&CCPA_OBS_DIR;' + VAR: APCP + ACCUM_HH: '#ACCUM_HH#' + MET_TOOL: 'GRIDSTAT' + OBTYPE: 'CCPA' + ENSMEM_INDX: "#mem#" + SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' + walltime: 02:00:00 + dependency: + and: + taskdep_pcpcombine_obs: + attrs: + task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h + taskdep_pcpcombine_fcst: + attrs: + task: run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem# + +metatask_GridStat_MRMS_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + metatask_GridStat_MRMS_mem#mem#: + var: + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' + task_run_MET_GridStat_vx_#VAR#_mem#mem#: + <<: *default_task_verify_det + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' + envars: + <<: *default_vars + OBS_DIR: '&MRMS_OBS_DIR;' + VAR: '#VAR#' + ACCUM_HH: '01' + MET_TOOL: 'GRIDSTAT' + OBTYPE: 'MRMS' + ENSMEM_INDX: "#mem#" + SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' + walltime: 02:00:00 + dependency: + and: + or_get_obs: # Ensure get_obs task is complete if it's turned on + not: + taskvalid: + attrs: + task: get_obs_mrms + taskdep: + attrs: + task: get_obs_mrms + datadep_post_files_exist: + attrs: + age: 00:00:00:30 + text: !cycstr '{{ workflow.EXPTDIR }}/@Y@m@d@H/post_files_exist_mem#mem#.txt' + +metatask_PointStat_NDAS_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + metatask_PointStat_NDAS_mem#mem#: + var: + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' + task_run_MET_PointStat_vx_#VAR#_mem#mem#: + <<: *default_task_verify_det + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' + envars: + <<: *default_vars + OBS_DIR: '&NDAS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'POINTSTAT' + OBTYPE: 'NDAS' + ACCUM_HH: '01' + ENSMEM_INDX: "#mem#" + SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' + walltime: 01:00:00 + dependency: + and: + taskdep_pb2nc: + attrs: + task: run_MET_Pb2nc_obs + datadep_post_files_exist: + attrs: + age: 00:00:00:30 + text: !cycstr '{{ workflow.EXPTDIR }}/@Y@m@d@H/post_files_exist_mem#mem#.txt' diff --git a/parm/wflow/verify_ens.yaml b/parm/wflow/verify_ens.yaml new file mode 100644 index 0000000000..3dbf924ce8 --- /dev/null +++ b/parm/wflow/verify_ens.yaml @@ -0,0 +1,189 @@ +default_task_verify_ens: &default_task_verify_ens + account: '&ACCOUNT;' + attrs: + cycledefs: forecast + maxtries: '2' + envars: &default_vars + GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' + USHdir: '&USHdir;' + PDY: !cycstr '@Y@m@d' + cyc: !cycstr "@H" + subcyc: !cycstr "@M" + LOGDIR: !cycstr "&LOGDIR;" + nprocs: '{{ parent.nnodes * parent.ppn }}' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: '{% if user.MACHINE not in ["WCOSS2", "NOAACLOUD"] %}{{ "2G" }}{% endif %}' + native: '{{ platform.SCHED_NATIVE_CMD }}' + nnodes: 1 + nodes: '{{ nnodes }}:ppn={{ ppn }}' + nodesize: '&NCORES_PER_NODE;' + partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' + ppn: 1 + queue: '&QUEUE_DEFAULT;' + walltime: 01:00:00 + +metatask_GenEnsProd_EnsembleStat_CCPA: + var: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + task_run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h: &task_GenEnsProd_CCPA + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' + envars: &envars_GenEnsProd_CCPA + <<: *default_vars + ACCUM_HH: '#ACCUM_HH#' + OBS_DIR: '&CCPA_OBS_DIR;' + VAR: APCP + MET_TOOL: 'GENENSPROD' + OBTYPE: 'CCPA' + dependency: + and: + # The PcpCombine task for obs must be complete because this GenEnsProd + # task checks to see the forecast hours for which obs are available before + # processing the forecast for those hours. + taskdep_pcpcombine_obs: + attrs: + task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h + metataskdep_pcpcombine_fcst: + attrs: + metatask: PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems + task_run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h: + <<: *task_GenEnsProd_CCPA + envars: + <<: *envars_GenEnsProd_CCPA + MET_TOOL: 'ENSEMBLESTAT' + dependency: + taskdep_genensprod: + attrs: + task: run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h + +metatask_GenEnsProd_EnsembleStat_MRMS: + var: + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' + task_run_MET_GenEnsProd_vx_#VAR#: &task_GenEnsProd_MRMS + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' + envars: &envars_GenEnsProd_MRMS + <<: *default_vars + ACCUM_HH: '01' + OBS_DIR: '&MRMS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'GENENSPROD' + OBTYPE: 'MRMS' + dependency: + and: + or_get_obs: # Ensure get_obs task is complete if it's turned on + not: + taskvalid: + attrs: + task: get_obs_mrms + taskdep: + attrs: + task: get_obs_mrms + metataskdep_post_files_exist: &post_files_exist + attrs: + metatask: check_post_output_all_mems + + task_run_MET_EnsembleStat_vx_#VAR#: + <<: *task_GenEnsProd_MRMS + envars: + <<: *envars_GenEnsProd_MRMS + MET_TOOL: 'ENSEMBLESTAT' + dependency: + taskdep: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# + +metatask_GenEnsProd_EnsembleStat_NDAS: + var: + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' + task_run_MET_GenEnsProd_vx_#VAR#: &task_GenEnsProd_NDAS + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' + envars: &envars_GenEnsProd_NDAS + <<: *default_vars + OBS_DIR: '&NDAS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'GENENSPROD' + OBTYPE: 'NDAS' + ACCUM_HH: '01' + dependency: + and: + # The Pb2nc task (which is run only for obs) must be complete because + # this GenEnsProd task checks to see the forecast hours for which obs + # are available before processing the forecast for those hours. + taskdep_pb2nc: + attrs: + task: run_MET_Pb2nc_obs + metataskdep_post_files_exist: + <<: *post_files_exist + task_run_MET_EnsembleStat_vx_#VAR#: + <<: *task_GenEnsProd_NDAS + envars: + <<: *envars_GenEnsProd_NDAS + MET_TOOL: 'ENSEMBLESTAT' + dependency: + taskdep_genensprod: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# + +metatask_GridStat_CCPA_ensmeanprob_all_accums: + var: + stat: MEAN PROB + statlc: mean prob + metatask_GridStat_CCPA_ens#statlc#_all_accums: + var: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + task_run_MET_GridStat_vx_ens#statlc#_APCP#ACCUM_HH#h: + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' + envars: + <<: *default_vars + OBS_DIR: '&CCPA_OBS_DIR;' + VAR: APCP + MET_TOOL: 'GRIDSTAT' + OBTYPE: 'CCPA' + ACCUM_HH: '#ACCUM_HH#' + dependency: + taskdep: + attrs: + task: run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h + +metatask_GridStat_MRMS_ensprob: + var: + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' + task_run_MET_GridStat_vx_ensprob_#VAR#: + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSPROB"' + envars: + <<: *default_vars + ACCUM_HH: '01' + OBS_DIR: '&MRMS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'GRIDSTAT' + OBTYPE: 'MRMS' + dependency: + taskdep: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# + +metatask_PointStat_NDAS_ensmeanprob: + var: + stat: MEAN PROB + statlc: mean prob + metatask_PointStat_NDAS_ens#statlc#: + var: + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' + task_run_MET_PointStat_vx_ens#statlc#_#VAR#: + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' + envars: + <<: *default_vars + OBS_DIR: '&NDAS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'POINTSTAT' + OBTYPE: 'NDAS' + ACCUM_HH: '01' + dependency: + taskdep: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml deleted file mode 100644 index 4f8dd65ddf..0000000000 --- a/parm/wflow/verify_ensgrid.yaml +++ /dev/null @@ -1,160 +0,0 @@ -default_task_verify_ens: &default_task_verify_ens - account: '&ACCOUNT;' - attrs: - cycledefs: forecast - maxtries: '2' - envars: &default_vars - GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' - USHdir: '&USHdir;' - PDY: !cycstr '@Y@m@d' - cyc: !cycstr "@H" - subcyc: !cycstr "@M" - LOGDIR: !cycstr "&LOGDIR;" - FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' - nprocs: '{{ parent.nnodes * parent.ppn }}' - join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - memory: '{% if user.MACHINE not in ["WCOSS2", "NOAACLOUD"] %}{{ "2G" }}{% endif %}' - native: '{{ platform.SCHED_NATIVE_CMD }}' - nnodes: 1 - nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: '&NCORES_PER_NODE;' - partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' - ppn: 1 - queue: '&QUEUE_DEFAULT;' - walltime: 01:00:00 - -metatask_EnsembleStat: - var: - VAR: REFC RETOP - task_run_MET_EnsembleStat_vx_#VAR#: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' - envars: - <<: *default_vars - ACCUM_HH: '01' - OBS_DIR: '&MRMS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'ENSEMBLESTAT' - OBTYPE: 'MRMS' - dependency: - metataskdep: - attrs: - metatask: run_ens_post - -metatask_GridStat_ensprob: - var: - VAR: REFC RETOP - task_run_MET_GridStat_vx_ensprob_#VAR#: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSPROB"' - envars: - <<: *default_vars - ACCUM_HH: '01' - OBS_DIR: '&MRMS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'GRIDSTAT' - OBTYPE: 'MRMS' - dependency: - taskdep: - attrs: - task: run_MET_EnsembleStat_vx_#VAR# - -metatask_EnsembleStat_acc: - var: - ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - FCST_HR_LIST: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{% for h in range(ah, workflow.FCST_LEN_HRS+1, ah) %}{{ "%02d," % h }}{% endfor %}{% endif %} {% endfor %}' - task_run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' - envars: - <<: *default_vars - ACCUM_HH: '#ACCUM_HR#' - FHR: '#FCST_HR_LIST#' - OBS_DIR: '&CCPA_OBS_DIR;' - VAR: APCP - MET_TOOL: 'ENSEMBLESTAT' - OBTYPE: 'CCPA' - dependency: - and: - or_do_post: &post_files_exist - and_run_post: # If post was meant to run, wait on the whole post metatask - taskvalid: - attrs: - task: run_post_mem001_f000 - metataskdep: - attrs: - metatask: run_ens_post - and_inline_post: # If inline post ran, wait on the forecast ensemble to complete - not: - taskvalid: - attrs: - task: run_post_mem001_f000 - metataskdep: - attrs: - metatask: run_ensemble - metataskdep: - attrs: - metatask: GridStat_APCP_acc - -metatask_ensemble_vx_stats: - var: - stat: MEAN PROB - statlc: mean prob - metatask_GridStat_ens_acc: - var: - ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - FCST_HR_LIST: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{% for h in range(ah, workflow.FCST_LEN_HRS+1, ah) %}{{ "%02d," % h }}{% endfor %}{% endif %} {% endfor %}' - task_run_MET_GridStat_vx_ens#statlc#_APCP#ACCUM_HR#h: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' - envars: - <<: *default_vars - ACCUM_HH: '#ACCUM_HR#' - FHR: '#FCST_HR_LIST#' - OBS_DIR: '&CCPA_OBS_DIR;' - VAR: APCP - MET_TOOL: 'GRIDSTAT' - OBTYPE: 'CCPA' - dependency: - taskdep: - attrs: - task: run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h - metatask_PointStat_ens: - var: - VAR: SFC UPA - task_run_MET_PointStat_vx_ens#statlc#_#VAR#: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' - envars: - <<: *default_vars - OBS_DIR: '&NDAS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'POINTSTAT' - OBTYPE: 'NDAS' - ACCUM_HH: '01' - dependency: - taskdep: - attrs: - task: run_MET_EnsembleStat_vx_#VAR# - -metatask_EnsembleStat_point: - var: - VAR: SFC UPA - task_run_MET_EnsembleStat_vx_#VAR#: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' - envars: - <<: *default_vars - OBS_DIR: '&NDAS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'ENSEMBLESTAT' - OBTYPE: 'NDAS' - ACCUM_HH: '01' - dependency: - and: - metataskdep: - attrs: - metatask: run_ens_post - taskdep: - attrs: - task: run_MET_Pb2nc_obs diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml new file mode 100644 index 0000000000..2eccbf8027 --- /dev/null +++ b/parm/wflow/verify_pre.yaml @@ -0,0 +1,222 @@ +default_task_verify_pre: &default_task_verify_pre + account: '&ACCOUNT;' + attrs: + cycledefs: forecast + maxtries: '1' + envars: &default_vars + GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' + USHdir: '&USHdir;' + PDY: !cycstr '@Y@m@d' + cyc: !cycstr "@H" + subcyc: !cycstr "@M" + LOGDIR: !cycstr "&LOGDIR;" + nprocs: '{{ parent.nnodes * parent.ppn }}' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: '{% if user.MACHINE not in ["NOAACLOUD"] %}{{ "2G" }}{% endif %}' + native: '{{ platform.SCHED_NATIVE_CMD }}' + nnodes: 1 + nodes: '{{ nnodes }}:ppn={{ ppn }}' + nodesize: '&NCORES_PER_NODE;' + partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' + ppn: 1 + queue: '&QUEUE_DEFAULT;' + walltime: 00:30:00 + +task_get_obs_ccpa: + <<: *default_task_verify_pre + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_CCPA"' + envars: + <<: *default_vars + ACCUM_HH: '01' + OBS_DIR: '&CCPA_OBS_DIR;' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' + queue: "&QUEUE_HPSS;" + walltime: 00:45:00 + +task_get_obs_mrms: + <<: *default_task_verify_pre + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_MRMS"' + envars: + <<: *default_vars + OBS_DIR: '&MRMS_OBS_DIR;' + VAR: 'REFC RETOP' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' + queue: "&QUEUE_HPSS;" + walltime: 00:45:00 + +task_get_obs_ndas: + <<: *default_task_verify_pre + envars: + <<: *default_vars + OBS_DIR: '&NDAS_OBS_DIR;' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_NDAS"' + queue: "&QUEUE_HPSS;" + partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' + walltime: 02:00:00 + +task_run_MET_Pb2nc_obs: + <<: *default_task_verify_pre + attrs: + cycledefs: forecast + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PB2NC_OBS"' + envars: + <<: *default_vars + VAR: SFC + ACCUM_HH: '01' + obs_or_fcst: obs + OBTYPE: NDAS + OBS_DIR: '&NDAS_OBS_DIR;' + MET_TOOL: 'PB2NC' + dependency: + and: + datadep: + text: "&NDAS_OBS_DIR;" + or: + not: + taskvalid: + attrs: + task: get_obs_ndas + taskdep: + attrs: + task: get_obs_ndas + +metatask_PcpCombine_obs: + var: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: + <<: *default_task_verify_pre + attrs: + cycledefs: forecast + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' + envars: + <<: *default_vars + VAR: APCP + ACCUM_HH: '#ACCUM_HH#' + obs_or_fcst: obs + OBTYPE: CCPA + OBS_DIR: '&CCPA_OBS_DIR;' + MET_TOOL: 'PCPCOMBINE' + ENSMEM_INDX: '' + dependency: + and: + datadep: + text: "&CCPA_OBS_DIR;" + or: + not: + taskvalid: + attrs: + task: get_obs_ccpa + taskdep: + attrs: + task: get_obs_ccpa + +metatask_check_post_output_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + task_check_post_output_mem#mem#: + <<: *default_task_verify_pre + attrs: + cycledefs: forecast + maxtries: '1' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_CHECK_POST_OUTPUT"' + envars: + <<: *default_vars + VAR: APCP + ACCUM_HH: '01' + ENSMEM_INDX: '#mem#' + dependency: + # This "or" checks that the necessary stand-alone post tasks or forecast + # tasks with inline post have run to generate the post files that are + # needed as forecast input to various verification tasks. + or_post_files_exist: &post_files_exist + # This "and" is to check whether post is being run stand-alone (i.e. outside + # the weather model in tasks separate from the forecast task(s)), and if + # so, to ensure that the stand-alone post metatask for the current member + # has completed for all forecast hours. + and_standalone_post: + # This "or" is to check whether a stand-alane post task exists [either for + # mem000 (if running a deterministic forecast) or for mem001 (if running + # an ensemble forecast)]. Checking for forecast hour 000 is sufficient. + # If so, it implies post is being run stand-alone. + or_post_task_exists: &post_task_exists + taskvalid_mem000: + attrs: + task: run_post_mem000_f000 + taskvalid_mem001: + attrs: + task: run_post_mem001_f000 + # We would like a dependency here on the run_post_#mem#_all_fhrs metatask, + # but that is apparently not possible in rocoto with a metataskvalid/ + # metataskdep combination (currently, metataskvalid is not a defined + # dependency type in ROCOTO); that approach only works for tasks. Thus, + # here we use jinja directives to explicitly list all the tasks in the + # run_post_#mem#_all_fhrs metatask. + #or_post_metatask: + # not: + # metataskvalid: + # attrs: + # metatask: run_post_mem#mem#_all_fhrs + # metataskdep: + # attrs: + # metatask: run_post_mem#mem#_all_fhrs + taskdep: + attrs: + task: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{% if h > 0 %}{{" \n"}}{% endif %}{%- endfor -%}' + # This "and" is to check whether post is being run inline (i.e. as part of + # the weather model), and if so, to ensure that the forecast task for the + # current member has completed. + and_inline_post: + # This "not" is to check whether a stand-alone post task DOESN'T exist + # [either for mem000 (if running a deterministic forecast) or for mem001 + # (if running an ensemble forecast)]. If so, it implies that either + # post is being run inline (as part of the weather model) or is not being + # run at all. + not: + or_post_task_exists: + <<: *post_task_exists + # This "or" is to verify that either the forecast task for the current + # member exists and has completed, or that it doesn't exist. + or_run_fcst: + and: + taskvalid: &fcst_task_exists + attrs: + task: run_fcst_mem#mem# + taskdep: + attrs: + task: run_fcst_mem#mem# + not: + taskvalid: + <<: *fcst_task_exists + +metatask_PcpCombine_fcst_all_accums_all_mems: + var: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: + <<: *default_task_verify_pre + attrs: + cycledefs: forecast + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' + envars: + <<: *default_vars + VAR: APCP + ACCUM_HH: '#ACCUM_HH#' + obs_or_fcst: fcst + OBTYPE: CCPA + OBS_DIR: '&CCPA_OBS_DIR;' + MET_TOOL: 'PCPCOMBINE' + ENSMEM_INDX: "#mem#" + dependency: + datadep: + attrs: + age: 00:00:00:30 + text: !cycstr '{{ workflow.EXPTDIR }}/@Y@m@d@H/post_files_exist_mem#mem#.txt' + walltime: 00:05:00 diff --git a/scripts/exregional_check_post_output.sh b/scripts/exregional_check_post_output.sh new file mode 100755 index 0000000000..d088e218cd --- /dev/null +++ b/scripts/exregional_check_post_output.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. $USHdir/source_util_funcs.sh +source_config_for_task "task_run_met_pcpcombine|task_run_post" ${GLOBAL_VAR_DEFNS_FP} +# +#----------------------------------------------------------------------- +# +# Source files defining auxiliary functions for verification. +# +#----------------------------------------------------------------------- +# +. $USHdir/set_vx_fhr_list.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that checks that all the post-processed +output files in fact exist and are at least a certain age. These files +may have been generated by UPP as part of the current SRW App workflow, +or they may be user-staged. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Get the time lag for the current ensemble member. +# +#----------------------------------------------------------------------- +# +i="0" +if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + i=$( bc -l <<< "${ENSMEM_INDX}-1" ) +fi +time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) +# +#----------------------------------------------------------------------- +# +# Get the list of forecast hours for which there is a post-processed +# output file. Note that: +# +# 1) CDATE (in YYYYMMDDHH format) is already available via the call to +# the job_preamble.sh script in the j-job of this ex-script. +# 2) VAR is set to "APCP" and ACCUM_HH is set to "01" because we assume +# the output files are hourly, so these settings will result in the +# function set_vx_fhr_list checking for existence of hourly post output +# files. +# +#----------------------------------------------------------------------- +# +ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "${ENSMEM_INDX}") +ensmem_name="mem${ensmem_indx}" +FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) +set_vx_fhr_list \ + cdate="${CDATE}" \ + fcst_len_hrs="${FCST_LEN_HRS}" \ + field="$VAR" \ + accum_hh="${ACCUM_HH}" \ + base_dir="${VX_FCST_INPUT_BASEDIR}" \ + fn_template="${FCST_INPUT_FN_TEMPLATE}" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_FCST_FILES_MAX}" \ + outvarname_fhr_list="FHR_LIST" +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Done checking for existence of post-processed files for ensemble member ${ENSMEM_INDX}. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_get_obs_ccpa.sh b/scripts/exregional_get_obs_ccpa.sh index d3f53c8d5a..9b54e49d56 100755 --- a/scripts/exregional_get_obs_ccpa.sh +++ b/scripts/exregional_get_obs_ccpa.sh @@ -162,15 +162,15 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do if [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then cd_vrfy $ccpa_raw/${vyyyymmdd_p1} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` else cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` fi # One hour CCPA files have incorrect metadeta in the files under the "00" directory from 20180718 to 20210504. @@ -200,15 +200,15 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do if [[ ${vhh_noZero} -ne 21 ]]; then cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` elif [[ ${vhh_noZero} -eq 21 ]]; then cd_vrfy $ccpa_raw/${vyyyymmdd_p1} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` fi if [[ ${vhh_noZero} -eq 0 ]]; then @@ -226,9 +226,9 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do elif [[ ${accum} == "06" ]]; then cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` if [[ ${vhh_noZero} -eq 0 ]]; then cp_vrfy $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} diff --git a/scripts/exregional_get_obs_mrms.sh b/scripts/exregional_get_obs_mrms.sh index 2f2d7afb4f..2d4ecac82f 100755 --- a/scripts/exregional_get_obs_mrms.sh +++ b/scripts/exregional_get_obs_mrms.sh @@ -128,8 +128,8 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do echo "TAR FILE:${TarFile}" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz\" | awk '{print $7}'\`" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz" | awk '{print $7}'` + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz\" | awk '{print \$7}'\`" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz" | awk '{print \$7}'` Status=$? if [[ ${Status} != 0 ]]; then diff --git a/scripts/exregional_get_obs_ndas.sh b/scripts/exregional_get_obs_ndas.sh index 401013a9ce..4249faafca 100755 --- a/scripts/exregional_get_obs_ndas.sh +++ b/scripts/exregional_get_obs_ndas.sh @@ -91,24 +91,24 @@ echo "vhh_noZero=$vhh_noZero" # Name of NDAS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until July 2020. if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -le 20190820 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com2_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` elif [[ ${vyyyymmdd} -ge 20190821 && ${vyyyymmdd} -le 20200226 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/gpfs_dell1_nco_ops_com_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` elif [[ ${vyyyymmdd} -gt 20200226 && ${vyyyymmdd} -le 20220627 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` else TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_obsproc_v1.0_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` fi if [[ ! -d "$ndas_proc" ]]; then diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 9f5ddef0b7..49e2dfdd91 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -205,29 +205,27 @@ fi # tasks). This will be exported to the environment and read by the # METplus configuration files. # -NDIGITS_ENSMEM_NAMES=3 FCST_INPUT_FN_TEMPLATE="" for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do - mem_indx=$(($i+1)) - mem_indx_fmt=$(printf "%0${NDIGITS_ENSMEM_NAMES}d" "${mem_indx}") - time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) + ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "$((i+1))") + ensmem_name="mem${ensmem_indx}" if [ "${RUN_ENVIR}" = "nco" ]; then cdate_ensmem_subdir_or_null="" - DOT_ENSMEM_OR_NULL=".mem${mem_indx_fmt}" else - cdate_ensmem_subdir_or_null="${CDATE}/mem${mem_indx_fmt}" - DOT_ENSMEM_OR_NULL="" + cdate_ensmem_subdir_or_null="${CDATE}/${ensmem_name}" fi + time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) + if [ "${field_is_APCPgt01h}" = "TRUE" ]; then template="${cdate_ensmem_subdir_or_null:+${cdate_ensmem_subdir_or_null}/}metprd/PcpCombine_fcst/${FCST_FN_METPROC_TEMPLATE}" else template="${FCST_SUBDIR_TEMPLATE}/${FCST_FN_TEMPLATE}" fi - SLASH_ENSMEM_SUBDIR_OR_NULL="/mem${mem_indx_fmt}" + slash_ensmem_subdir_or_null="/${ensmem_name}" if [ -z "${FCST_INPUT_FN_TEMPLATE}" ]; then FCST_INPUT_FN_TEMPLATE=" $(eval echo ${template})" else @@ -255,7 +253,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- @@ -368,6 +367,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_indx': '${ENSMEM_INDX:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 62e5961f4e..a6a30671b6 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -117,10 +117,11 @@ set_vx_params \ # #----------------------------------------------------------------------- # -time_lag=$(( (${MEM_INDX_OR_NULL:+${ENS_TIME_LAG_HRS[${MEM_INDX_OR_NULL}-1]}}+0) )) -# Convert to seconds. We do this as a separate step using bc because -# bash's $((...)) arithmetic operator can't handle floats well. -time_lag=$( bc -l <<< "${time_lag}*${SECS_PER_HOUR}" ) +i="0" +if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + i=$( bc -l <<< "${ENSMEM_INDX}-1" ) +fi +time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) # #----------------------------------------------------------------------- # @@ -172,15 +173,30 @@ fi # vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) +ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "${ENSMEM_INDX}") +ensmem_name="mem${ensmem_indx}" if [ "${RUN_ENVIR}" = "nco" ]; then - if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - ENSMEM=$( echo ${SLASH_ENSMEM_SUBDIR_OR_NULL} | cut -d"/" -f2 ) - DOT_ENSMEM_OR_NULL=".$ENSMEM" + slash_cdate_or_null="" + slash_ensmem_subdir_or_null="" +else + slash_cdate_or_null="/${CDATE}" +# +# Since other aspects of a deterministic run use the "mem000" string (e.g. +# in rocoto workflow task names, in log file names), it seems reasonable +# that a deterministic run create a "mem000" subdirectory under the $CDATE +# directory. But since that is currently not the case in in the run_fcst +# task, we need the following if-statement. If and when such a modification +# is made for the run_fcst task, we would remove this if-statement and +# simply set +# slash_ensmem_subdir_or_null="/${ensmem_name}" +# or, better, just remove this variale and code "/${ensmem_name}" where +# slash_ensmem_subdir_or_null currently appears below. +# + if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + slash_ensmem_subdir_or_null="/${ensmem_name}" else - DOT_ENSMEM_OR_NULL="" + slash_ensmem_subdir_or_null="" fi -else - DOT_ENSMEM_OR_NULL="" fi if [ "${grid_or_point}" = "grid" ]; then @@ -189,7 +205,7 @@ if [ "${grid_or_point}" = "grid" ]; then if [ "${field_is_APCPgt01h}" = "TRUE" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/PcpCombine_obs" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_CCPA_APCPgt01h_FN_TEMPLATE} ) - FCST_INPUT_DIR="${vx_output_basedir}/${CDATE}/mem${ENSMEM_INDX}/metprd/PcpCombine_fcst" + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/${slash_ensmem_subdir_or_null}/metprd/PcpCombine_fcst" FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_FN_METPROC_TEMPLATE} ) else OBS_INPUT_DIR="${OBS_DIR}" @@ -218,7 +234,7 @@ elif [ "${grid_or_point}" = "point" ]; then fi -OUTPUT_BASE="${vx_output_basedir}/${CDATE}/mem${ENSMEM_INDX}" +OUTPUT_BASE="${vx_output_basedir}${slash_cdate_or_null}/${slash_ensmem_subdir_or_null}" OUTPUT_DIR="${OUTPUT_BASE}/metprd/${metplus_tool_name}" STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" # @@ -235,7 +251,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- @@ -300,7 +317,7 @@ else metplus_config_tmpl_fn="${FIELDNAME_IN_MET_FILEDIR_NAMES}" fi metplus_config_tmpl_fn="${metplus_tool_name}_${metplus_config_tmpl_fn}" -metplus_config_fn="${metplus_tool_name}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_mem${ENSMEM_INDX}" +metplus_config_fn="${metplus_tool_name}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${ensmem_name}" metplus_log_fn="${metplus_config_fn}" # # Add prefixes and suffixes (extensions) to the base file names. @@ -348,6 +365,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 15c2e34a56..c8cb4d9cac 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -177,21 +177,15 @@ if [ "${grid_or_point}" = "grid" ]; then esac OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) fi -# Keep for when splitting GenEnsProd from EnsembleStat. -# FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/gen_ens_prod" -# FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/EnsembleStat" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'ensemble_stat_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc' ) + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) elif [ "${grid_or_point}" = "point" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_NDAS_SFCorUPA_FN_METPROC_TEMPLATE} ) -# Keep for when splitting GenEnsProd from EnsembleStat. -# FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/gen_ens_prod" -# FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/EnsembleStat" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'ensemble_stat_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc' ) + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) fi @@ -212,7 +206,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- @@ -334,6 +329,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index fb33760892..473a17dcf5 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -136,21 +136,15 @@ if [ "${grid_or_point}" = "grid" ]; then esac OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) fi -# Keep for when splitting GenEnsProd from EnsembleStat. -# FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/gen_ens_prod" -# FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/EnsembleStat" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'ensemble_stat_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc' ) + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) elif [ "${grid_or_point}" = "point" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_NDAS_SFCorUPA_FN_METPROC_TEMPLATE} ) -# Keep for when splitting GenEnsProd from EnsembleStat. -# FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/gen_ens_prod" -# FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/EnsembleStat" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'ensemble_stat_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc' ) + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) fi @@ -171,7 +165,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- @@ -296,6 +291,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 528e7743dc..95feb701f4 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -108,16 +108,6 @@ set_vx_params \ #----------------------------------------------------------------------- # vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) -if [ "${RUN_ENVIR}" = "nco" ]; then - if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - ENSMEM=$( echo ${SLASH_ENSMEM_SUBDIR_OR_NULL} | cut -d"/" -f2 ) - DOT_ENSMEM_OR_NULL=".$ENSMEM" - else - DOT_ENSMEM_OR_NULL="" - fi -else - DOT_ENSMEM_OR_NULL="" -fi OBS_INPUT_DIR="${OBS_DIR}" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_NDAS_SFCorUPA_FN_TEMPLATE} ) @@ -140,7 +130,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- @@ -265,6 +256,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index 0ff30d270a..a40bece103 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -121,10 +121,11 @@ set_vx_params \ # time_lag="0" if [ "${obs_or_fcst}" = "fcst" ]; then - time_lag=$(( (${MEM_INDX_OR_NULL:+${ENS_TIME_LAG_HRS[${MEM_INDX_OR_NULL}-1]}}+0) )) -# Convert to seconds. We do this as a separate step using bc because -# bash's $((...)) arithmetic operator can't handle floats well. - time_lag=$( bc -l <<< "${time_lag}*${SECS_PER_HOUR}" ) + i="0" + if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + i=$( bc -l <<< "${ENSMEM_INDX}-1" ) + fi + time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) fi # #----------------------------------------------------------------------- @@ -136,15 +137,30 @@ fi # vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) +ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "${ENSMEM_INDX}") +ensmem_name="mem${ensmem_indx}" if [ "${RUN_ENVIR}" = "nco" ]; then - if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - ENSMEM=$( echo ${SLASH_ENSMEM_SUBDIR_OR_NULL} | cut -d"/" -f2 ) - DOT_ENSMEM_OR_NULL=".$ENSMEM" + slash_cdate_or_null="" + slash_ensmem_subdir_or_null="" +else + slash_cdate_or_null="/${CDATE}" +# +# Since other aspects of a deterministic run use the "mem000" string (e.g. +# in rocoto workflow task names, in log file names), it seems reasonable +# that a deterministic run create a "mem000" subdirectory under the $CDATE +# directory. But since that is currently not the case in in the run_fcst +# task, we need the following if-statement. If and when such a modification +# is made for the run_fcst task, we would remove this if-statement and +# simply set +# slash_ensmem_subdir_or_null="/${ensmem_name}" +# or, better, just remove this variale and code "/${ensmem_name}" where +# slash_ensmem_subdir_or_null currently appears below. +# + if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + slash_ensmem_subdir_or_null="/${ensmem_name}" else - DOT_ENSMEM_OR_NULL="" + slash_ensmem_subdir_or_null="" fi -else - DOT_ENSMEM_OR_NULL="" fi OBS_INPUT_DIR="" @@ -167,7 +183,7 @@ elif [ "${obs_or_fcst}" = "fcst" ]; then FCST_INPUT_DIR="${vx_fcst_input_basedir}" FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) - OUTPUT_BASE="${vx_output_basedir}/${CDATE}/mem${ENSMEM_INDX}" + OUTPUT_BASE="${vx_output_basedir}${slash_cdate_or_null}/${slash_ensmem_subdir_or_null}" OUTPUT_DIR="${OUTPUT_BASE}/metprd/${metplus_tool_name}_fcst" OUTPUT_FN_TEMPLATE=$( eval echo ${FCST_FN_METPROC_TEMPLATE} ) STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" @@ -183,9 +199,11 @@ fi if [ "${obs_or_fcst}" = "obs" ]; then base_dir="${OBS_INPUT_DIR}" fn_template="${OBS_INPUT_FN_TEMPLATE}" + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" elif [ "${obs_or_fcst}" = "fcst" ]; then base_dir="${FCST_INPUT_DIR}" fn_template="${FCST_INPUT_FN_TEMPLATE}" + num_missing_files_max="${NUM_MISSING_FCST_FILES_MAX}" fi set_vx_fhr_list \ @@ -195,7 +213,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${base_dir}" \ fn_template="${fn_template}" \ - check_hourly_files="TRUE" \ + check_accum_contrib_files="TRUE" \ + num_missing_files_max="${num_missing_files_max}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- @@ -252,15 +271,10 @@ fi # #----------------------------------------------------------------------- # -uscore_ensmem_name_or_null="" -if [ "${obs_or_fcst}" = "fcst" ]; then - uscore_ensmem_name_or_null="_mem${ENSMEM_INDX}" -fi -# # First, set the base file names. # metplus_config_tmpl_fn="${metplus_tool_name}_${obs_or_fcst}" -metplus_config_fn="${metplus_config_tmpl_fn}_${FIELDNAME_IN_MET_FILEDIR_NAMES}${uscore_ensmem_name_or_null}" +metplus_config_fn="${metplus_config_tmpl_fn}_${FIELDNAME_IN_MET_FILEDIR_NAMES}${ENSMEM_INDX:+_${ensmem_name}}" metplus_log_fn="${metplus_config_fn}_$CDATE" # # If operating on observation files, append the cycle date to the name @@ -318,6 +332,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/tests/WE2E/machine_suites/comprehensive b/tests/WE2E/machine_suites/comprehensive index 07ea344271..ce5baaff91 100644 --- a/tests/WE2E/machine_suites/comprehensive +++ b/tests/WE2E/machine_suites/comprehensive @@ -55,6 +55,8 @@ grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot GST_release_public_v1 MET_verification_only_vx +MET_ensemble_verification_only_vx +MET_ensemble_verification_only_vx_time_lag nco nco_ensemble nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml index 2d2ee706a1..41da098f00 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml @@ -17,7 +17,7 @@ workflow: PREEXISTING_DIR_METHOD: rename rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml"]|include }}' metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml index 9ec3925c59..25099ce9b8 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml @@ -16,10 +16,13 @@ workflow: PREEXISTING_DIR_METHOD: rename rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify.yaml", "parm/wflow/verify_ensgrid.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 + task_get_obs_ccpa: + task_get_obs_mrms: + task_get_obs_ndas: task_get_extrn_ics: EXTRN_MDL_NAME_ICS: NAM USE_USER_STAGED_EXTRN_FILES: true @@ -32,4 +35,3 @@ global: NUM_ENS_MEMBERS: 2 verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km - diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index 0c545761ab..ff7b05a906 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -17,45 +17,10 @@ workflow: rocoto: tasks: - taskgroups: '{{ ["parm/wflow/verify.yaml", "parm/wflow/verify_ensgrid.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: - metatask_vx_ens_member: - metatask_GridStat_MRMS_mem#mem#: - task_run_MET_GridStat_vx_#VAR#_mem#mem#: - dependency: - metatask_PointStat_mem#mem#: - task_run_MET_PointStat_vx_#VAR#_mem#mem#: - dependency: - and: - or_do_post: - or_get_obs: - - metatask_GridStat_APCP_acc: - metatask_vx_ens_member_acc: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: - dependency: - metatask_EnsembleStat_acc: - task_run_MET_GridStat_vx_ensprob_#VAR#: - dependency: - task_run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h: - dependency: - and: - metataskdep: - attrs: - metatask: GridStat_APCP_acc - metatask_EnsembleStat: - task_run_MET_EnsembleStat_vx_#VAR#: - dependency: - metatask_EnsembleStat_point: - task_run_MET_EnsembleStat_vx_#VAR#: - dependency: - and: - metataskdep: - taskdep: - attrs: - task: run_MET_Pb2nc_obs global: DO_ENSEMBLE: true diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml new file mode 100644 index 0000000000..d4a95812bb --- /dev/null +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -0,0 +1,43 @@ +metadata: + description: |- + This test is to ensure that the workflow running in community mode can + successfully run both the MET deterministic and ensemble verification + tasks using staged observation and forecast files (i.e. with other tasks + such as pre-processing, forecast, and post-processing deactivated) AND + assuming one of the ensemble members is time-lagged. + +user: + RUN_ENVIR: community + +workflow: + PREDEF_GRID_NAME: RRFS_CONUS_3km + DATE_FIRST_CYCL: '2021050500' + DATE_LAST_CYCL: '2021050500' + FCST_LEN_HRS: 6 + PREEXISTING_DIR_METHOD: rename + +rocoto: + tasks: + taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' + task_get_obs_ccpa: + task_get_obs_mrms: + task_get_obs_ndas: + +nco: + NET: 'RRFSE_CONUS' + +global: + DO_ENSEMBLE: true + NUM_ENS_MEMBERS: 2 + ENS_TIME_LAG_HRS: '[ 0, 12 ]' + +platform: + CCPA_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/ccpa/proc' + MRMS_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/mrms/proc' + NDAS_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/ndas/proc' + +verification: + VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km + VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' + VX_NDIGITS_ENSMEM_NAMES: 1 + FCST_FN_TEMPLATE: '${NET}.t{init?fmt=%H?shift=-${time_lag}}z{% if user.RUN_ENVIR == "nco" and global.DO_ENSEMBLE %}.${ensmem_name}{% endif %}.bgdawpf{lead?fmt=%HHH?shift=${time_lag}}.tm00.grib2' diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index 57d1ca3d53..1fc04a622a 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -21,26 +21,11 @@ workflow: # It also assumes staged observations, so turn off those tasks. rocoto: tasks: - taskgroups: '{{ ["parm/wflow/verify.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml"]|include }}' task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: - metatask_vx_ens_member: - metatask_GridStat_MRMS_mem#mem#: - task_run_MET_GridStat_vx_#VAR#_mem#mem#: - dependency: - metatask_PointStat_mem#mem#: - task_run_MET_PointStat_vx_#VAR#_mem#mem#: - dependency: - and: - or_do_post: - or_get_obs: - metatask_GridStat_APCP_acc: - metatask_vx_ens_member_acc: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: - dependency: verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' - diff --git a/ush/bash_utils/eval_METplus_timestr_tmpl.sh b/ush/bash_utils/eval_METplus_timestr_tmpl.sh index 0f5db76153..245369509b 100644 --- a/ush/bash_utils/eval_METplus_timestr_tmpl.sh +++ b/ush/bash_utils/eval_METplus_timestr_tmpl.sh @@ -185,8 +185,10 @@ METplus time string template passed to this function is: METplus_timestr_tmpl = \"${METplus_timestr_tmpl}\"" ;; esac - - time_shift_str=$(( ${METplus_time_shift} + 0 ))" seconds" +# +# Calculate the time shift as an integer in units of seconds. +# + time_shift_str=$(( $(printf "%.0f" "${METplus_time_shift}") + 0 ))" seconds" # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 6df35759e3..0087e85c21 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2295,7 +2295,9 @@ global: # experiment directory # # ENS_TIME_LAG_HRS: - # Time lag (in hours) to use for each ensemble member. + # Time lag (in hours) to use for each ensemble member. For a deterministic + # forecast, this is a one-element array. Default values of array elements + # are zero. # #----------------------------------------------------------------------- # @@ -2303,7 +2305,7 @@ global: NUM_ENS_MEMBERS: 0 ENSMEM_NAMES: '{% for m in range(NUM_ENS_MEMBERS) %}{{ "mem%03d, " % m }}{% endfor %}' FV3_NML_ENSMEM_FPS: '{% for mem in ENSMEM_NAMES %}{{ [EXPTDIR, "%s_%s" % FV3_NML_FN, mem]|path_join }}{% endfor %}' - ENS_TIME_LAG_HRS: '[ {% for m in range(NUM_ENS_MEMBERS) %} 0, {% endfor %} ]' + ENS_TIME_LAG_HRS: '[ {% for m in range([1,NUM_ENS_MEMBERS]|max) %} 0, {% endfor %} ]' # #----------------------------------------------------------------------- # @@ -2430,7 +2432,18 @@ verification: # This is used in forming the names of the verification output files as # well as in the contents of those files. # + # VX_FIELDS: + # The fields or groups of fields on which to run verification. + # + # VX_APCP_ACCUMS_HRS: + # The 2-digit accumulation periods (in units of hours) to consider for + # APCP (accumulated precipitation). If VX_FIELDS contains "APCP", then + # VX_APCP_ACCUMS_HRS must contain at least one element. If not, + # VX_APCP_ACCUMS_HRS will be ignored. + # VX_FCST_MODEL_NAME: '{{ nco.NET }}.{{ task_run_post.POST_OUTPUT_DOMAIN_NAME }}' + VX_FIELDS: [ "APCP", "REFC", "RETOP", "SFC", "UPA" ] + VX_APCP_ACCUMS_HRS: [ 1, 3, 6, 24 ] # # VX_FCST_INPUT_BASEDIR: # Template for top-level directory containing forecast (but not obs) @@ -2443,11 +2456,18 @@ verification: VX_FCST_INPUT_BASEDIR: '{% if user.RUN_ENVIR == "nco" %}$COMOUT/../..{% else %}{{ workflow.EXPTDIR }}{% endif %}' VX_OUTPUT_BASEDIR: '{% if user.RUN_ENVIR == "nco" %}$COMOUT/metout{% else %}{{ workflow.EXPTDIR }}{% endif %}' # + # Number of digits in the ensember member names. This is a configurable + # variable to allow users to change its value (e.g. to go from "mem004" + # to "mem02") when using staged forecast files that do not use the same + # number of digits as the SRW App. + # + VX_NDIGITS_ENSMEM_NAMES: 3 + # # File name and path templates used in the verification tasks. # - FCST_SUBDIR_TEMPLATE: '{% if user.RUN_ENVIR == "nco" %}${NET}.{init?fmt=%Y%m%d?shift=-${time_lag}}/{init?fmt=%H?shift=-${time_lag}}{% else %}{init?fmt=%Y%m%d%H?shift=-${time_lag}}${SLASH_ENSMEM_SUBDIR_OR_NULL}/postprd{% endif %}' - FCST_FN_TEMPLATE: '${NET}.t{init?fmt=%H?shift=-${time_lag}}z{% if user.RUN_ENVIR == "nco" %}${DOT_ENSMEM_OR_NULL}{% endif %}.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.${POST_OUTPUT_DOMAIN_NAME}.grib2' - FCST_FN_METPROC_TEMPLATE: '${NET}.t{init?fmt=%H}z{% if user.RUN_ENVIR == "nco" %}${DOT_ENSMEM_OR_NULL}{% endif %}.prslev.f{lead?fmt=%HHH}.${POST_OUTPUT_DOMAIN_NAME}_a${ACCUM_HH}h.nc' + FCST_SUBDIR_TEMPLATE: '{% if user.RUN_ENVIR == "nco" %}${NET}.{init?fmt=%Y%m%d?shift=-${time_lag}}/{init?fmt=%H?shift=-${time_lag}}{% else %}{init?fmt=%Y%m%d%H?shift=-${time_lag}}{% if global.DO_ENSEMBLE %}/${ensmem_name}{% endif %}/postprd{% endif %}' + FCST_FN_TEMPLATE: '${NET}.t{init?fmt=%H?shift=-${time_lag}}z{% if user.RUN_ENVIR == "nco" and global.DO_ENSEMBLE %}.${ensmem_name}{% endif %}.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.${POST_OUTPUT_DOMAIN_NAME}.grib2' + FCST_FN_METPROC_TEMPLATE: '${NET}.t{init?fmt=%H}z{% if user.RUN_ENVIR == "nco" and global.DO_ENSEMBLE %}.${ensmem_name}{% endif %}.prslev.f{lead?fmt=%HHH}.${POST_OUTPUT_DOMAIN_NAME}_a${ACCUM_HH}h.nc' # # For verification tasks that need observational data, this specifies # the maximum number of observation files that may be missing. If more @@ -2461,6 +2481,13 @@ verification: # number missing divided by the number that are expected to exist). # NUM_MISSING_OBS_FILES_MAX: 2 + # + # For verification tasks that need forecast data, this specifies the + # maximum number of post-processed forecast files that may be missing. + # If more than this number are missing, the verification task will not + # be run. + # + NUM_MISSING_FCST_FILES_MAX: 0 #---------------------------- # CPL_AQM config parameters diff --git a/ush/get_met_metplus_tool_name.sh b/ush/get_met_metplus_tool_name.sh index 9ecadfbcb8..3802ded893 100644 --- a/ush/get_met_metplus_tool_name.sh +++ b/ush/get_met_metplus_tool_name.sh @@ -65,8 +65,8 @@ function get_met_metplus_tool_name() { # generic_tool_name=${generic_tool_name,,} valid_vals_generic_tool_name=( \ - "PB2NC" "PCPCOMBINE" "GRIDSTAT" "POINTSTAT" "ENSEMBLESTAT" \ - "pb2nc" "pcpcombine" "gridstat" "pointstat" "ensemblestat" \ + "PB2NC" "PCPCOMBINE" "GRIDSTAT" "POINTSTAT" "GENENSPROD" "ENSEMBLESTAT" \ + "pb2nc" "pcpcombine" "gridstat" "pointstat" "genensprod" "ensemblestat" \ ) check_var_valid_value "generic_tool_name" "valid_vals_generic_tool_name" @@ -87,6 +87,10 @@ function get_met_metplus_tool_name() { _met_tool_name_="point_stat" _metplus_tool_name_="PointStat" ;; + "genensprod") + _met_tool_name_="gen_ens_prod" + _metplus_tool_name_="GenEnsProd" + ;; "ensemblestat") _met_tool_name_="ensemble_stat" _metplus_tool_name_="EnsembleStat" diff --git a/ush/set_vx_fhr_list.sh b/ush/set_vx_fhr_list.sh index d3403d09f2..490c7f5374 100644 --- a/ush/set_vx_fhr_list.sh +++ b/ush/set_vx_fhr_list.sh @@ -54,7 +54,8 @@ function set_vx_fhr_list() { "accum_hh" \ "base_dir" \ "fn_template" \ - "check_hourly_files" \ + "check_accum_contrib_files" \ + "num_missing_files_max" \ "outvarname_fhr_list" \ ) process_args valid_args "$@" @@ -105,19 +106,19 @@ function set_vx_fhr_list() { fhr_int="${accum_hh}" ;; "REFC") - fhr_min="01" + fhr_min="00" fhr_int="01" ;; "RETOP") - fhr_min="01" + fhr_min="00" fhr_int="01" ;; "SFC") - fhr_min="01" + fhr_min="00" fhr_int="01" ;; "UPA") - fhr_min="06" + fhr_min="00" fhr_int="06" ;; *) @@ -153,7 +154,7 @@ is: fhr_orig="${fhr_array[$i]}" - if [ "${check_hourly_files}" = "TRUE" ]; then + if [ "${check_accum_contrib_files}" = "TRUE" ]; then fhr=$(( ${fhr_orig} - ${accum_hh} + 1 )) num_back_hrs=${accum_hh} else @@ -165,6 +166,9 @@ is: for (( j=0; j<${num_back_hrs}; j++ )); do # # Use the provided template to set the name of/relative path to the file +# Note that the while-loop below is over all METplus time string templates +# of the form {...} in the template fn_template; it continues until all +# such templates have been evaluated to actual time strings. # fn="${fn_template}" regex_search_tmpl="(.*)(\{.*\})(.*)" @@ -247,18 +251,18 @@ Final (i.e. after filtering for missing files) set of foreast hours is # #----------------------------------------------------------------------- # -# If the number of missing files is greater than the user-specified -# variable NUM_MISSING_OBS_FILES_MAX, print out an error message and +# If the number of missing files is greater than the maximum allowed +# (specified by num_missing_files_max), print out an error message and # exit. # #----------------------------------------------------------------------- # - if [ "${num_missing_files}" -gt "${NUM_MISSING_OBS_FILES_MAX}" ]; then + if [ "${num_missing_files}" -gt "${num_missing_files_max}" ]; then print_err_msg_exit "\ The number of missing files (num_missing_files) is greater than the -maximum allowed number (NUM_MISSING_OBS_FILES_MAX): +maximum allowed number (num_missing_files_max): num_missing_files = ${num_missing_files} - NUM_MISSING_OBS_FILES_MAX = ${NUM_MISSING_OBS_FILES_MAX}" + num_missing_files_max = ${num_missing_files_max}" fi # #----------------------------------------------------------------------- diff --git a/ush/setup.py b/ush/setup.py index 002ccbf2df..6d387c0374 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -540,6 +540,58 @@ def remove_tag(tasks, tag): post_meta = rocoto_tasks.get("metatask_run_ens_post", {}) post_meta.pop("metatask_run_sub_hourly_post", None) post_meta.pop("metatask_sub_hourly_last_hour_post", None) + # + # ----------------------------------------------------------------------- + # + # Remove all verification [meta]tasks for which no fields are specified. + # + # ----------------------------------------------------------------------- + # + vx_fields_all = {} + vx_metatasks_all = {} + + vx_fields_all["CCPA"] = ["APCP"] + vx_metatasks_all["CCPA"] = ["metatask_PcpCombine_obs", + "metatask_PcpCombine_fcst_all_accums_all_mems", + "metatask_GridStat_CCPA_all_accums_all_mems", + "metatask_GenEnsProd_EnsembleStat_CCPA", + "metatask_GridStat_CCPA_ensmeanprob_all_accums"] + + vx_fields_all["MRMS"] = ["REFC", "RETOP"] + vx_metatasks_all["MRMS"] = ["metatask_GridStat_MRMS_all_mems", + "metatask_GenEnsProd_EnsembleStat_MRMS", + "metatask_GridStat_MRMS_ensprob"] + + vx_fields_all["NDAS"] = ["SFC", "UPA"] + vx_metatasks_all["NDAS"] = ["task_run_MET_Pb2nc_obs", + "metatask_PointStat_NDAS_all_mems", + "metatask_GenEnsProd_EnsembleStat_NDAS", + "metatask_PointStat_NDAS_ensmeanprob"] + + # Get the vx fields specified in the experiment configuration. + vx_fields_config = expt_config["verification"]["VX_FIELDS"] + + # If there are no vx fields specified, remove those tasks that are necessary + # for all observation types. + if not vx_fields_config: + metatask = "metatask_check_post_output_all_mems" + rocoto_config['tasks'].pop(metatask) + + # If for a given obstype no fields are specified, remove all vx metatasks + # for that obstype. + for obstype in vx_fields_all: + vx_fields_obstype = [field for field in vx_fields_config if field in vx_fields_all[obstype]] + if not vx_fields_obstype: + for metatask in vx_metatasks_all[obstype]: + if metatask in rocoto_config['tasks']: + logging.info(dedent( + f""" + Removing verification [meta]task + "{metatask}" + from workflow since no fields belonging to observation type "{obstype}" + are specified for verification.""" + )) + rocoto_config['tasks'].pop(metatask) # # ----------------------------------------------------------------------- @@ -1235,25 +1287,35 @@ def get_location(xcs, fmt, expt_cfg): # # ----------------------------------------------------------------------- # - + # Get list of all top-level tasks and metatasks in the workflow. task_defs = rocoto_config.get('tasks') + all_tasks = [task for task in task_defs] - # Ensemble verification can only be run in ensemble mode - do_ensemble = global_sect["DO_ENSEMBLE"] + # Get list of all valid top-level tasks and metatasks pertaining to ensemble + # verification. + ens_vx_task_defns = load_config_file( + os.path.join(USHdir, os.pardir, "parm", "wflow", "verify_ens.yaml")) + ens_vx_valid_tasks = [task for task in ens_vx_task_defns] + # Get list of all valid top-level tasks and metatasks in the workflow that + # pertain to ensemble verification. + ens_vx_tasks = [task for task in ens_vx_valid_tasks if task in all_tasks] - # Gather all the tasks/metatasks that are defined for verifying - # ensembles - ens_vx_tasks = [task for task in task_defs if "MET_GridStat_vx_ens" in task] + # Get the value of the configuration flag for ensemble mode (DO_ENSEMBLE) + # and ensure that it is set to True if ensemble vx tasks are included in + # the workflow (or vice-versa). + do_ensemble = global_sect["DO_ENSEMBLE"] if (not do_ensemble) and ens_vx_tasks: - task_str = "\n".join(ens_vx_tasks) - raise Exception( - f''' - Ensemble verification can not be run unless running in ensemble mode: - DO_ENSEMBLE = \"{do_ensemble}\" - Ensemble verification tasks: {task_str} - ''' - ) + task_str = " " + "\n ".join(ens_vx_tasks) + msg = dedent(f""" + Ensemble verification can not be run unless running in ensemble mode: + DO_ENSEMBLE = \"{do_ensemble}\" + Ensemble verification tasks: + """) + msg = "".join([msg, task_str, dedent(f""" + Please set DO_ENSEMBLE to True or remove ensemble vx tasks from the + workflow.""")]) + raise Exception(msg) # # ----------------------------------------------------------------------- @@ -1530,13 +1592,25 @@ def dict_find(user_dict, substring): if v is None or v == "": continue vkey = "valid_vals_" + k - if (vkey in cfg_v) and not (v in cfg_v[vkey]): - raise Exception( - f""" - The variable {k}={v} in the user's configuration - does not have a valid value. Possible values are: - {k} = {cfg_v[vkey]}""" - ) + if (vkey in cfg_v): + if (type(v) == list): + if not(all(ele in cfg_v[vkey] for ele in v)): + raise Exception( + dedent(f""" + The variable + {k} = {v} + in the user's configuration has at least one invalid value. Possible values are: + {k} = {cfg_v[vkey]}""" + )) + else: + if not (v in cfg_v[vkey]): + raise Exception( + dedent(f""" + The variable + {k} = {v} + in the user's configuration does not have a valid value. Possible values are: + {k} = {cfg_v[vkey]}""" + )) return expt_config diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index 98fbfd09d9..f839dcb858 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -86,3 +86,5 @@ valid_vals_DO_AQM_GEFS_LBCS: [True, False] valid_vals_DO_AQM_SAVE_AIRNOW_HIST: [True, False] valid_vals_COLDSTART: [True, False] valid_vals_DO_NLDN_LGHT: [True, False] +valid_vals_VX_FIELDS: [ "APCP", "REFC", "RETOP", "SFC", "UPA" ] +valid_vals_VX_APCP_ACCUMS_HRS: [ 1, 3, 6, 24 ] From cc92e5016bbbf48b29f86175ad2ec684451dd893 Mon Sep 17 00:00:00 2001 From: BenjaminBlake-NOAA <52074832+BenjaminBlake-NOAA@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:38:58 -0400 Subject: [PATCH 02/16] [develop] Update hashes of GSI and rrfs_utils (#800) This PR updates the hashes of the GSI and rrfs_utils in Externals.cfg to the versions currently used by the real-time RRFS runs. The build modulefiles and build.ver.wcoss2 file were also updated to include ncdiag and ncio, two additional modules that are required to build the GSI and rrfs_utils. --------- Co-authored-by: Mark Potts --- Externals.cfg | 4 ++-- modulefiles/build_cheyenne_gnu.lua | 2 ++ modulefiles/build_hera_gnu.lua | 2 ++ modulefiles/build_wcoss2_intel.lua | 2 ++ modulefiles/srw_common.lua | 1 + versions/build.ver.wcoss2 | 2 ++ 6 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index e321565065..b1d865e398 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -30,7 +30,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/GSI.git # Specify either a branch name or a hash but not both. #branch = develop -hash = ad84f17 +hash = 4afe6ed local_path = sorc/gsi externals = None required = True @@ -40,7 +40,7 @@ protocol = git repo_url = https://github.com/NOAA-GSL/rrfs_utl.git # Specify either a branch name or a hash but not both. #branch = develop -hash = 8d21f25 +hash = 6cacff5 local_path = sorc/rrfs_utl required = True diff --git a/modulefiles/build_cheyenne_gnu.lua b/modulefiles/build_cheyenne_gnu.lua index da6d887e6e..0504fccbb3 100644 --- a/modulefiles/build_cheyenne_gnu.lua +++ b/modulefiles/build_cheyenne_gnu.lua @@ -27,7 +27,9 @@ load(pathJoin("netcdf", os.getenv("netcdf_ver") or "4.7.4")) load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) load(pathJoin("pio", os.getenv("pio_ver") or "2.5.7")) load(pathJoin("fms", os.getenv("fms_ver") or "2022.04")) +load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) +unsetenv("MKLROOT") setenv("CMAKE_C_COMPILER","mpicc") setenv("CMAKE_CXX_COMPILER","mpicxx") setenv("CMAKE_Fortran_COMPILER","mpif90") diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index 45bdd736a3..888a358549 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -23,7 +23,9 @@ load("srw_common") load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9")) load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) +unsetenv("MKLROOT") setenv("CMAKE_C_COMPILER","mpicc") setenv("CMAKE_CXX_COMPILER","mpicxx") setenv("CMAKE_Fortran_COMPILER","mpif90") diff --git a/modulefiles/build_wcoss2_intel.lua b/modulefiles/build_wcoss2_intel.lua index 6e49f370d3..3f23ffaed9 100644 --- a/modulefiles/build_wcoss2_intel.lua +++ b/modulefiles/build_wcoss2_intel.lua @@ -35,6 +35,8 @@ load(pathJoin("nemsio", os.getenv("nemsio_ver"))) load(pathJoin("sigio", os.getenv("sigio_ver"))) load(pathJoin("sfcio", os.getenv("sfcio_ver"))) load(pathJoin("wrf_io", os.getenv("wrf_io_ver"))) +load(pathJoin("ncdiag", os.getenv("ncdiag_ver"))) +load(pathJoin("ncio", os.getenv("ncio_ver"))) load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) load(pathJoin("bufr", os.getenv("bufr_ver"))) load(pathJoin("nemsiogfs", os.getenv("nemsiogfs_ver"))) diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 9eb529294c..7fa6ef9fea 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -27,5 +27,6 @@ load("sigio/2.3.2") load("w3nco/2.4.1") load("wrf_io/1.2.0") +load("ncdiag/1.0.0") load("ncio/1.1.2") load("wgrib2/2.0.8") diff --git a/versions/build.ver.wcoss2 b/versions/build.ver.wcoss2 index 90f7d56f20..4c972e0a4e 100644 --- a/versions/build.ver.wcoss2 +++ b/versions/build.ver.wcoss2 @@ -33,6 +33,8 @@ export nemsio_ver=2.5.2 export sigio_ver=2.3.2 export sfcio_ver=1.4.1 export wrf_io_ver=1.2.0 +export ncdiag_ver=1.0.0 +export ncio_ver=1.1.2 export wgrib2_ver=2.0.8_wmo export bufr_ver=11.7.0 export nemsiogfs_ver=2.5.3 From 294e18b943d00b3dbf5d45b1e52d3ae65b7393b9 Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Fri, 9 Jun 2023 11:50:47 -0400 Subject: [PATCH 03/16] [develop] Update WM and UPP hashes and minor rearrangement of WE2E coverage tests that fail on certain platforms (#799) * The WM and UPP hashes in the Externals.cfg file were updated to current versions. * The nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km test that is constantly failing on Cheyenne GNU and the grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta test on Hera Intel have been moved to platforms where the tests successfully run. * Minor changes to the DT_ATMOS for the RRFS_CONUS_25km grid were made to allow the WE2E tests run after updating to the current UFS-WM hash. Changes to DT_ATMOS was was also required for for the RRFS_AK_3km grid. --- .cicd/Jenkinsfile | 10 +++++++--- Externals.cfg | 4 ++-- tests/WE2E/machine_suites/coverage.cheyenne.gnu | 2 +- tests/WE2E/machine_suites/coverage.gaea | 1 + tests/WE2E/machine_suites/coverage.hera.intel.nco | 1 - ...m_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot.yaml | 1 + tests/test_python/test_calculate_cost.py | 2 +- ush/predef_grid_params.yaml | 4 ++-- 8 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 7612f799d0..ff0b6914e8 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -13,7 +13,9 @@ pipeline { // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') + // Use the line below to re-enable Gaea + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') // Uncomment the following line to re-enable comprehensive tests @@ -76,7 +78,8 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'cheyenne', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -90,7 +93,8 @@ pipeline { exclude { axis { name 'SRW_PLATFORM' - values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' + // values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { diff --git a/Externals.cfg b/Externals.cfg index b1d865e398..7e433967eb 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = 36d6e16 +hash = e403bb4 local_path = sorc/ufs-weather-model required = True @@ -21,7 +21,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = 22cfb88 +hash = dccb321 local_path = sorc/UPP required = True diff --git a/tests/WE2E/machine_suites/coverage.cheyenne.gnu b/tests/WE2E/machine_suites/coverage.cheyenne.gnu index 47639f3b33..7035bd4ca6 100644 --- a/tests/WE2E/machine_suites/coverage.cheyenne.gnu +++ b/tests/WE2E/machine_suites/coverage.cheyenne.gnu @@ -1,9 +1,9 @@ grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 -nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS diff --git a/tests/WE2E/machine_suites/coverage.gaea b/tests/WE2E/machine_suites/coverage.gaea index d6a43beccf..390bdc038b 100644 --- a/tests/WE2E/machine_suites/coverage.gaea +++ b/tests/WE2E/machine_suites/coverage.gaea @@ -7,3 +7,4 @@ grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot nco_ensemble +nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km diff --git a/tests/WE2E/machine_suites/coverage.hera.intel.nco b/tests/WE2E/machine_suites/coverage.hera.intel.nco index 715033128e..8526a939f1 100644 --- a/tests/WE2E/machine_suites/coverage.hera.intel.nco +++ b/tests/WE2E/machine_suites/coverage.hera.intel.nco @@ -1,7 +1,6 @@ get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200 get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_netcdf_2022040400_ensemble_2mems get_from_HPSS_ics_HRRR_lbcs_RAP -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot.yaml index fb45518bae..fdf9d3333e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot.yaml @@ -24,4 +24,5 @@ task_get_extrn_lbcs: LBC_SPEC_INTVL_HRS: 3 USE_USER_STAGED_EXTRN_FILES: true task_run_fcst: + DT_ATMOS: 180 RESTART_INTERVAL: 1 2 5 diff --git a/tests/test_python/test_calculate_cost.py b/tests/test_python/test_calculate_cost.py index c306fc9276..c259f10067 100644 --- a/tests/test_python/test_calculate_cost.py +++ b/tests/test_python/test_calculate_cost.py @@ -14,4 +14,4 @@ def test_calculate_cost(self): test_dir = os.path.dirname(os.path.abspath(__file__)) USHdir = os.path.join(test_dir, "..", "..", "ush") params = calculate_cost(os.path.join(USHdir, 'config.community.yaml')) - self.assertCountEqual(params, [180, 28689, 180, 28689]) + self.assertCountEqual(params, [150, 28689, 150, 28689]) diff --git a/ush/predef_grid_params.yaml b/ush/predef_grid_params.yaml index a5ab412a26..450e618c15 100644 --- a/ush/predef_grid_params.yaml +++ b/ush/predef_grid_params.yaml @@ -61,7 +61,7 @@ ESGgrid_NY: 131 ESGgrid_PAZI: 0.0 ESGgrid_WIDE_HALO_WIDTH: 6 - DT_ATMOS: 180 + DT_ATMOS: 150 LAYOUT_X: 5 LAYOUT_Y: 2 BLOCKSIZE: 40 @@ -349,7 +349,7 @@ ESGgrid_NY: 1020 ESGgrid_PAZI: 0.0 ESGgrid_WIDE_HALO_WIDTH: 6 - DT_ATMOS: 10 + DT_ATMOS: 30 LAYOUT_X: 30 LAYOUT_Y: 17 BLOCKSIZE: 40 From 8b382cc5854c6303701fb0d3d7729fc25da04753 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Fri, 9 Jun 2023 10:34:27 -0600 Subject: [PATCH 04/16] [develop] Integrate Unified Workflow templater tool (#793) This is the first time a Unified Workflow tool is being integrated into a UFS App. We are starting with the templater tool. It's a direct drop-in replacement for fill_jinja_template.py. This contribution includes the following additional changes: * Updates Externals.cfg to include workflow-tools as an external dependency repo (will be updated to point to a hash once the corresponding branch is merged) * Updates module files for the workflow and the tasks to set PYTHONPATH to the necessary locations for uwtools. * Allows the templater tool to do its own logging and error handling for clarity, instead of SRW callers. * Lints the Python code perturbed by this integration in the spirit of stewardship. * Adds a .pylintrc to help manage some of the project-wide settings that will be needed by SRW to lint appropriately. --------- Co-authored-by: Michael Kavulich --- .github/workflows/python_linter.yaml | 7 +- .github/workflows/python_unittests.yaml | 4 +- .gitignore | 1 + .pylintrc | 624 ++++++++++++++++++ Externals.cfg | 9 + docs/UsersGuide/source/RocotoInfo.rst | 2 +- docs/UsersGuide/source/RunSRW.rst | 2 +- modulefiles/set_pythonpath.lua | 13 + .../tasks/cheyenne/plot_allvars.local.lua | 5 + modulefiles/tasks/cheyenne/python_srw.lua | 2 +- modulefiles/tasks/gaea/plot_allvars.local.lua | 4 + modulefiles/tasks/gaea/python_srw.lua | 2 +- modulefiles/tasks/hera/plot_allvars.local.lua | 4 + modulefiles/tasks/hera/python_srw.lua | 2 +- modulefiles/tasks/jet/plot_allvars.local.lua | 4 + modulefiles/tasks/jet/python_srw.lua | 2 +- modulefiles/tasks/jet/run_vx.local.lua | 4 +- .../tasks/noaacloud/plot_allvars.local.lua | 1 + .../tasks/orion/plot_allvars.local.lua | 5 + modulefiles/tasks/orion/python_srw.lua | 2 +- modulefiles/wflow_cheyenne.lua | 4 +- modulefiles/wflow_gaea.lua | 3 +- modulefiles/wflow_hera.lua | 3 +- modulefiles/wflow_jet.lua | 3 +- modulefiles/wflow_linux.lua | 5 +- modulefiles/wflow_macos.lua | 5 +- modulefiles/wflow_noaacloud.lua | 2 + modulefiles/wflow_odin.lua | 4 +- modulefiles/wflow_orion.lua | 3 +- modulefiles/wflow_singularity.lua | 3 +- modulefiles/wflow_wcoss2.lua | 1 + ...onal_run_met_genensprod_or_ensemblestat.sh | 23 +- ...gional_run_met_gridstat_or_pointstat_vx.sh | 23 +- ...un_met_gridstat_or_pointstat_vx_ensmean.sh | 24 +- ...un_met_gridstat_or_pointstat_vx_ensprob.sh | 22 +- scripts/exregional_run_met_pb2nc_obs.sh | 22 +- scripts/exregional_run_met_pcpcombine.sh | 23 +- tests/test_python/test_python_utils.py | 3 +- .../test_set_FV3nml_ens_stoch_seeds.py | 17 +- ush/create_aqm_rc_file.py | 98 +-- ush/create_diag_table_file.py | 41 +- ush/create_model_configure_file.py | 76 +-- ush/create_nems_configure_file.py | 62 +- ush/fill_jinja_template.py | 302 --------- ush/generate_FV3LAM_wflow.py | 342 ++++------ ush/load_modules_run_task.sh | 12 +- ush/load_modules_wflow.sh | 2 +- ush/set_FV3nml_sfc_climo_filenames.py | 2 +- 48 files changed, 1071 insertions(+), 758 deletions(-) create mode 100644 .pylintrc create mode 100644 modulefiles/set_pythonpath.lua create mode 100644 modulefiles/tasks/cheyenne/plot_allvars.local.lua create mode 100644 modulefiles/tasks/gaea/plot_allvars.local.lua create mode 100644 modulefiles/tasks/hera/plot_allvars.local.lua create mode 100644 modulefiles/tasks/jet/plot_allvars.local.lua create mode 100644 modulefiles/tasks/noaacloud/plot_allvars.local.lua create mode 100644 modulefiles/tasks/orion/plot_allvars.local.lua delete mode 100755 ush/fill_jinja_template.py diff --git a/.github/workflows/python_linter.yaml b/.github/workflows/python_linter.yaml index b53fc50855..b63a2a513c 100644 --- a/.github/workflows/python_linter.yaml +++ b/.github/workflows/python_linter.yaml @@ -33,5 +33,8 @@ jobs: # Run python unittests - name: Lint the test directory run: | - export PYTHONPATH=${PWD}/ush - pylint --min-similarity-lines=15 --ignore-imports=yes tests/test_python/ + ./manage_externals/checkout_externals workflow-tools + export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src + pylint --ignore-imports=yes tests/test_python/ + pylint ush/create_*.py + pylint ush/generate_FV3LAM_wflow.py diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml index 0f213ff6bb..5e491dea6e 100644 --- a/.github/workflows/python_unittests.yaml +++ b/.github/workflows/python_unittests.yaml @@ -37,8 +37,8 @@ jobs: # Run python unittests - name: Run python unittests run: | - ./manage_externals/checkout_externals ufs-weather-model + ./manage_externals/checkout_externals ufs-weather-model workflow-tools # exclude test_retrieve_data that is tested in functional test - export PYTHONPATH=${PWD}/ush + export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src python3 -m unittest -b tests/test_python/*.py diff --git a/.gitignore b/.gitignore index bc3eee8545..a727b940c0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ tests/WE2E/log.* ush/__pycache__/ ush/config.yaml ush/python_utils/__pycache__/ +ush/python_utils/workflow-tools/ ush/*.swp *.swp diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000000..0cb488e3d5 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,624 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.9 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=new + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + logging-fstring-interpolation + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=15 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/Externals.cfg b/Externals.cfg index 7e433967eb..1807bc4b3c 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -62,5 +62,14 @@ hash = 0a86f73 local_path = sorc/AQM-utils required = True +[workflow-tools] +protocol = git +repo_url = https://github.com/ufs-community/workflow-tools +# Specify either a branch name or a hash but not both. +# branch = develop +hash = e1b3b6f +local_path = ush/python_utils/workflow-tools +required = True + [externals_description] schema_version = 1.0.0 diff --git a/docs/UsersGuide/source/RocotoInfo.rst b/docs/UsersGuide/source/RocotoInfo.rst index 7557addc7d..ad0ec024c9 100644 --- a/docs/UsersGuide/source/RocotoInfo.rst +++ b/docs/UsersGuide/source/RocotoInfo.rst @@ -11,7 +11,7 @@ system as the task dependencies allow and runs one instance of the workflow for The SRW App workflow is defined in a Jinja-enabled Rocoto XML template called ``FV3LAM_wflow.xml``, which resides in the ``parm`` directory. When the ``generate_FV3LAM_wflow.py`` -script is run, the ``fill_jinja_template.py`` script is called, and the parameters in the template file +script is run, the ``set_template`` uwtool is called, and the parameters in the template file are filled in. The completed file contains the workflow task names, parameters needed by the job scheduler, and task interdependencies. The generated XML file is then copied to the experiment directory: ``$EXPTDIR/FV3LAM_wflow.xml``. diff --git a/docs/UsersGuide/source/RunSRW.rst b/docs/UsersGuide/source/RunSRW.rst index 7dc69c5149..5bea592809 100644 --- a/docs/UsersGuide/source/RunSRW.rst +++ b/docs/UsersGuide/source/RunSRW.rst @@ -755,7 +755,7 @@ The generated workflow will appear in ``$EXPTDIR``, where ``EXPTDIR=${EXPT_BASED .. _WorkflowGeneration: .. figure:: _static/SRW_regional_workflow_gen.png - :alt: Flowchart of the workflow generation process. Scripts are called in the following order: source_util_funcs.sh (which calls bash_utils), then set_FV3nml_sfc_climo_filenames.py, set_FV3nml_ens_stoch_seeds.py, create_diag_table_file.py, and setup.py. setup.py calls several scripts: set_cycle_dates.py, set_grid_params_GFDLgrid.py, set_grid_params_ESGgrid.py, link_fix.py, set_ozone_param.py, set_thompson_mp_fix_files.py, config_defaults.yaml, config.yaml, and valid_param_vals.yaml. Then, it sets a number of variables, including FIXgsm, TOPO_DIR, and SFC_CLIMO_INPUT_DIR variables. Next, set_predef_grid_params.py is called, and the FIXam and FIXLAM directories are set, along with the forecast input files. The setup script also calls set_extrn_mdl_params.py, sets the GRID_GEN_METHOD with HALO, checks various parameters, and generates shell scripts. Then, the workflow generation script sets up YAML-compliant strings and generates the actual Rocoto workflow XML file from the template file (fill_jinja_template.py). The workflow generation script checks the crontab file and, if applicable, copies certain fix files to the experiment directory. Then, it copies templates of various input files to the experiment directory and sets parameters for the input.nml file. Finally, it generates the workflow. Additional information on each step appears in comments within each script. + :alt: Flowchart of the workflow generation process. Scripts are called in the following order: source_util_funcs.sh (which calls bash_utils), then set_FV3nml_sfc_climo_filenames.py, set_FV3nml_ens_stoch_seeds.py, create_diag_table_file.py, and setup.py. setup.py calls several scripts: set_cycle_dates.py, set_grid_params_GFDLgrid.py, set_grid_params_ESGgrid.py, link_fix.py, set_ozone_param.py, set_thompson_mp_fix_files.py, config_defaults.yaml, config.yaml, and valid_param_vals.yaml. Then, it sets a number of variables, including FIXgsm, TOPO_DIR, and SFC_CLIMO_INPUT_DIR variables. Next, set_predef_grid_params.py is called, and the FIXam and FIXLAM directories are set, along with the forecast input files. The setup script also calls set_extrn_mdl_params.py, sets the GRID_GEN_METHOD with HALO, checks various parameters, and generates shell scripts. Then, the workflow generation script produces a YAML configuration file and generates the actual Rocoto workflow XML file from the template file (by calling uwtools set_template). The workflow generation script checks the crontab file and, if applicable, copies certain fix files to the experiment directory. Then, it copies templates of various input files to the experiment directory and sets parameters for the input.nml file. Finally, it generates the workflow. Additional information on each step appears in comments within each script. *Experiment Generation Description* diff --git a/modulefiles/set_pythonpath.lua b/modulefiles/set_pythonpath.lua new file mode 100644 index 0000000000..e816ed6cfc --- /dev/null +++ b/modulefiles/set_pythonpath.lua @@ -0,0 +1,13 @@ +help([[ +This module sets the PYTHONPATH in the user environment to allow the +workflow tools to be imported +]]) + +whatis([===[Sets paths for using workflow-tools with SRW]===]) + +local mod_path, mod_file = splitFileName(myFileName()) +local uwtools_scripts_path = pathJoin(mod_path, "/../ush/python_utils/workflow-tools") +local uwtools_package_path = pathJoin(mod_path, "/../ush/python_utils/workflow-tools/src/") + +prepend_path("PYTHONPATH", uwtools_scripts_path) +prepend_path("PYTHONPATH", uwtools_package_path) diff --git a/modulefiles/tasks/cheyenne/plot_allvars.local.lua b/modulefiles/tasks/cheyenne/plot_allvars.local.lua new file mode 100644 index 0000000000..2263141a0a --- /dev/null +++ b/modulefiles/tasks/cheyenne/plot_allvars.local.lua @@ -0,0 +1,5 @@ +unload("python") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/cheyenne/python_srw.lua b/modulefiles/tasks/cheyenne/python_srw.lua index 2263141a0a..57e2c2eed0 100644 --- a/modulefiles/tasks/cheyenne/python_srw.lua +++ b/modulefiles/tasks/cheyenne/python_srw.lua @@ -2,4 +2,4 @@ unload("python") prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) -setenv("SRW_ENV", "regional_workflow") +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/tasks/gaea/plot_allvars.local.lua b/modulefiles/tasks/gaea/plot_allvars.local.lua new file mode 100644 index 0000000000..74336ca1cf --- /dev/null +++ b/modulefiles/tasks/gaea/plot_allvars.local.lua @@ -0,0 +1,4 @@ +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/gaea/python_srw.lua b/modulefiles/tasks/gaea/python_srw.lua index 74336ca1cf..c0f454fa71 100644 --- a/modulefiles/tasks/gaea/python_srw.lua +++ b/modulefiles/tasks/gaea/python_srw.lua @@ -1,4 +1,4 @@ prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) -setenv("SRW_ENV", "regional_workflow") +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/tasks/hera/plot_allvars.local.lua b/modulefiles/tasks/hera/plot_allvars.local.lua new file mode 100644 index 0000000000..7934169824 --- /dev/null +++ b/modulefiles/tasks/hera/plot_allvars.local.lua @@ -0,0 +1,4 @@ +prepend_path("MODULEPATH","/scratch1/NCEPDEV/nems/role.epic/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/hera/python_srw.lua b/modulefiles/tasks/hera/python_srw.lua index 7934169824..62ddf7d9e8 100644 --- a/modulefiles/tasks/hera/python_srw.lua +++ b/modulefiles/tasks/hera/python_srw.lua @@ -1,4 +1,4 @@ prepend_path("MODULEPATH","/scratch1/NCEPDEV/nems/role.epic/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) -setenv("SRW_ENV", "regional_workflow") +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/tasks/jet/plot_allvars.local.lua b/modulefiles/tasks/jet/plot_allvars.local.lua new file mode 100644 index 0000000000..ef4f248966 --- /dev/null +++ b/modulefiles/tasks/jet/plot_allvars.local.lua @@ -0,0 +1,4 @@ +prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/jet/python_srw.lua b/modulefiles/tasks/jet/python_srw.lua index ef4f248966..3c7987be18 100644 --- a/modulefiles/tasks/jet/python_srw.lua +++ b/modulefiles/tasks/jet/python_srw.lua @@ -1,4 +1,4 @@ prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) -setenv("SRW_ENV", "regional_workflow") +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/tasks/jet/run_vx.local.lua b/modulefiles/tasks/jet/run_vx.local.lua index 13b6b4b954..750fd7603e 100644 --- a/modulefiles/tasks/jet/run_vx.local.lua +++ b/modulefiles/tasks/jet/run_vx.local.lua @@ -1,3 +1 @@ -append_path("MODULEPATH", "/contrib/anaconda/modulefiles") -load(pathJoin("intel", os.getenv("intel_ver") or "18.0.5.274")) -load(pathJoin("anaconda", os.getenv("anaconda_ver") or "5.3.1")) +load("python_srw") diff --git a/modulefiles/tasks/noaacloud/plot_allvars.local.lua b/modulefiles/tasks/noaacloud/plot_allvars.local.lua new file mode 100644 index 0000000000..602d60842f --- /dev/null +++ b/modulefiles/tasks/noaacloud/plot_allvars.local.lua @@ -0,0 +1 @@ +prepend_path("PATH", "/contrib/EPIC/miniconda3/4.12.0/envs/regional_workflow/bin") diff --git a/modulefiles/tasks/orion/plot_allvars.local.lua b/modulefiles/tasks/orion/plot_allvars.local.lua new file mode 100644 index 0000000000..5b7b0afc57 --- /dev/null +++ b/modulefiles/tasks/orion/plot_allvars.local.lua @@ -0,0 +1,5 @@ +unload("python") +append_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/orion/python_srw.lua b/modulefiles/tasks/orion/python_srw.lua index 5b7b0afc57..c7e25d3ad4 100644 --- a/modulefiles/tasks/orion/python_srw.lua +++ b/modulefiles/tasks/orion/python_srw.lua @@ -2,4 +2,4 @@ unload("python") append_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) -setenv("SRW_ENV", "regional_workflow") +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/wflow_cheyenne.lua b/modulefiles/wflow_cheyenne.lua index 35b7365d43..9a7a37c0b5 100644 --- a/modulefiles/wflow_cheyenne.lua +++ b/modulefiles/wflow_cheyenne.lua @@ -11,12 +11,14 @@ append_path("MODULEPATH","/glade/p/ral/jntp/UFS_SRW_app/modules") load("rocoto") unload("python") + +load("set_pythonpath") prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: - > conda activate regional_workflow + > conda activate workflow_tools ]===]) end diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index e4b726aac1..f623acd15b 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -5,6 +5,7 @@ the NOAA RDHPC machine Gaea whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===]) +load("set_pythonpath") prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) load("rocoto") @@ -14,6 +15,6 @@ setenv("PROJ_LIB", "/lustre/f2/dev/role.epic/contrib/miniconda3/4.12.0/envs/regi if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: - > conda activate regional_workflow + > conda activate workflow_tools ]===]) end diff --git a/modulefiles/wflow_hera.lua b/modulefiles/wflow_hera.lua index efca665dd2..5d6ebeed1d 100644 --- a/modulefiles/wflow_hera.lua +++ b/modulefiles/wflow_hera.lua @@ -6,12 +6,13 @@ the NOAA RDHPC machine Hera whatis([===[Loads libraries needed for running the UFS SRW App on Hera ]===]) load("rocoto") +load("set_pythonpath") prepend_path("MODULEPATH","/scratch1/NCEPDEV/nems/role.epic/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: - > conda activate regional_workflow + > conda activate workflow_tools ]===]) end diff --git a/modulefiles/wflow_jet.lua b/modulefiles/wflow_jet.lua index be11c223bc..5f109429dc 100644 --- a/modulefiles/wflow_jet.lua +++ b/modulefiles/wflow_jet.lua @@ -6,12 +6,13 @@ the NOAA RDHPC machine Jet whatis([===[Loads libraries needed for running the UFS SRW App on Jet ]===]) load("rocoto") +load("set_pythonpath") prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: - > conda activate regional_workflow + > conda activate workflow_tools ]===]) end diff --git a/modulefiles/wflow_linux.lua b/modulefiles/wflow_linux.lua index 6c4cc6949d..3fb5d1123a 100644 --- a/modulefiles/wflow_linux.lua +++ b/modulefiles/wflow_linux.lua @@ -31,9 +31,12 @@ prepend_path("PATH", pathJoin(rocoto_path,"bin")) local srw_path="/home/username/ufs-srweather-app" prepend_path("PATH", pathJoin(srw_path, "ush/rocoto_fake_slurm")) +-- set python path +load("set_pythonpath") + -- display conda activation message if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: - > conda activate regional_workflow + > conda activate workflow_tools ]===]) end diff --git a/modulefiles/wflow_macos.lua b/modulefiles/wflow_macos.lua index d7cf30e0a3..6ee6022b20 100644 --- a/modulefiles/wflow_macos.lua +++ b/modulefiles/wflow_macos.lua @@ -31,10 +31,13 @@ prepend_path("PATH", pathJoin(rocoto_path,"bin")) local srw_path="/Users/username/ufs-srweather-app" prepend_path("PATH", pathJoin(srw_path, "ush/rocoto_fake_slurm")) +-- set python path +load("set_pythonpath") + -- display conda activation message if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda virtual environment: - > conda activate regional_workflow" + > conda activate workflow_tools" ]===]) end diff --git a/modulefiles/wflow_noaacloud.lua b/modulefiles/wflow_noaacloud.lua index 80501f5075..6c8925444d 100644 --- a/modulefiles/wflow_noaacloud.lua +++ b/modulefiles/wflow_noaacloud.lua @@ -7,6 +7,8 @@ whatis([===[Loads libraries needed for running the UFS SRW App on NOAA cloud ]== prepend_path("MODULEPATH","/apps/modules/modulefiles") load("rocoto") +load("set_pythonpath") + prepend_path("MODULEPATH","/contrib/EPIC/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) diff --git a/modulefiles/wflow_odin.lua b/modulefiles/wflow_odin.lua index 7b1b5d8203..f042b00894 100644 --- a/modulefiles/wflow_odin.lua +++ b/modulefiles/wflow_odin.lua @@ -5,6 +5,8 @@ the NSSL machine Odin whatis([===[Loads libraries needed for running the UFS SRW App on Odin ]===]) +load("set_pythonpath") + if mode() == "load" then -- >>> conda initialize >>> -- !! Contents within this block are managed by 'conda init' !! @@ -28,6 +30,6 @@ if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: > conda config --set changeps1 False - > conda activate regional_workflow + > conda activate workflow_tools ]===]) end diff --git a/modulefiles/wflow_orion.lua b/modulefiles/wflow_orion.lua index 78997de1bb..46da63f5d4 100644 --- a/modulefiles/wflow_orion.lua +++ b/modulefiles/wflow_orion.lua @@ -8,6 +8,7 @@ whatis([===[Loads libraries needed for running SRW on Orion ]===]) load("contrib") load("rocoto") load("wget") +load("set_pythonpath") unload("python") append_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles") @@ -15,7 +16,7 @@ load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: - > conda activate regional_workflow + > conda activate workflow_tools ]===]) end diff --git a/modulefiles/wflow_singularity.lua b/modulefiles/wflow_singularity.lua index 3c16a93570..309c5eac23 100644 --- a/modulefiles/wflow_singularity.lua +++ b/modulefiles/wflow_singularity.lua @@ -4,10 +4,11 @@ a singularity container ]]) whatis([===[Loads libraries needed for running the UFS SRW App in a singularity container]===]) +load("set_pythonpath") append_path("MODULEPATH","/opt/hpc-modules/modulefiles/core") load("miniconda3") if mode() == "load" then - execute{cmd="conda activate regional_workflow", modeA={"load"}} + execute{cmd="conda activate workflow_tools", modeA={"load"}} end diff --git a/modulefiles/wflow_wcoss2.lua b/modulefiles/wflow_wcoss2.lua index a061b93323..4212f17493 100644 --- a/modulefiles/wflow_wcoss2.lua +++ b/modulefiles/wflow_wcoss2.lua @@ -7,6 +7,7 @@ whatis([===[Loads libraries needed for running the UFS SRW App on WCOSS2 ]===]) load(pathJoin("intel", os.getenv("intel_ver"))) load(pathJoin("python", os.getenv("python_ver"))) +load("set_pythonpath") prepend_path("MODULEPATH","/apps/ops/test/nco/modulefiles") load(pathJoin("core/rocoto", os.getenv("rocoto_ver"))) diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 49e2dfdd91..9631efdb6d 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -381,25 +381,32 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " + +# Store the settings in a temporary file +tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") +cat > $tmpfile << EOF +$settings +EOF # # Call the python script to generate the METplus configuration file from # the jinja template. # -$USHdir/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ +python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ + -c "${tmpfile}" \ + -i ${metplus_config_tmpl_fp} \ + -o ${metplus_config_fp} || \ print_err_msg_exit "\ -Call to python script fill_jinja_template.py to generate a METplus +Call to workflow-tools templater to generate a METplus configuration file from a jinja template failed. Parameters passed to this script are: Full path to template METplus configuration file: metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" Full path to output METplus configuration file: metplus_config_fp = \"${metplus_config_fp}\" - Jinja settings specified on command line: - settings = -$settings" + Full path to configuration file: + ${tmpfile} +" +rm $tmpfile # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index a6a30671b6..5027b54fcc 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -379,25 +379,32 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " + +# Store the settings in a temporary file +tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") +cat > $tmpfile << EOF +$settings +EOF # # Call the python script to generate the METplus configuration file from # the jinja template. # -$USHdir/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ +python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ + -c "${tmpfile}" \ + -i ${metplus_config_tmpl_fp} \ + -o ${metplus_config_fp} || \ print_err_msg_exit "\ -Call to python script fill_jinja_template.py to generate a METplus +Call to workflow-tools templater to generate a METplus configuration file from a jinja template failed. Parameters passed to this script are: Full path to template METplus configuration file: metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" Full path to output METplus configuration file: metplus_config_fp = \"${metplus_config_fp}\" - Jinja settings specified on command line: - settings = -$settings" + Full path to configuration file: + ${tmpfile} +" +rm $tmpfile # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index c8cb4d9cac..45929fbcb6 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -343,25 +343,33 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " + +# Store the settings in a temporary file +tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") +cat > $tmpfile << EOF +$settings +EOF # # Call the python script to generate the METplus configuration file from # the jinja template. # -$USHdir/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ + +python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ + -c "${tmpfile}" \ + -i ${metplus_config_tmpl_fp} \ + -o ${metplus_config_fp} || \ print_err_msg_exit "\ -Call to python script fill_jinja_template.py to generate a METplus +Call to workflow-tools templater to generate a METplus configuration file from a jinja template failed. Parameters passed to this script are: Full path to template METplus configuration file: metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" Full path to output METplus configuration file: metplus_config_fp = \"${metplus_config_fp}\" - Jinja settings specified on command line: - settings = -$settings" + Full path to configuration file: + ${tmpfile} +" +rm $tmpfile # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index 473a17dcf5..7478f35562 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -305,25 +305,31 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " +# Store the settings in a temporary file +tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") +cat > $tmpfile << EOF +$settings +EOF # # Call the python script to generate the METplus configuration file from # the jinja template. # -$USHdir/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ +python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ + -c "${tmpfile}" \ + -i ${metplus_config_tmpl_fp} \ + -o ${metplus_config_fp} || \ print_err_msg_exit "\ -Call to python script fill_jinja_template.py to generate a METplus +Call to workflow-tools templater to generate a METplus configuration file from a jinja template failed. Parameters passed to this script are: Full path to template METplus configuration file: metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" Full path to output METplus configuration file: metplus_config_fp = \"${metplus_config_fp}\" - Jinja settings specified on command line: - settings = -$settings" + Full path to configuration file: + ${tmpfile} +" +rm $tmpfile # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 95feb701f4..3365e7f8c6 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -270,25 +270,31 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " +# Store the settings in a temporary file +tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") +cat > $tmpfile << EOF +$settings +EOF # # Call the python script to generate the METplus configuration file from # the jinja template. # -$USHdir/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ +python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ + -c ${tmpfile} \ + -i ${metplus_config_tmpl_fp} \ + -o ${metplus_config_fp} || \ print_err_msg_exit "\ -Call to python script fill_jinja_template.py to generate a METplus +Call to workflow-tools templater.py to generate a METplus configuration file from a jinja template failed. Parameters passed to this script are: Full path to template METplus configuration file: metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" Full path to output METplus configuration file: metplus_config_fp = \"${metplus_config_fp}\" - Jinja settings specified on command line: - settings = -$settings" + Full path to configuration file: + ${tmpfile} +" +rm $tmpfile # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index a40bece103..6951158041 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -346,25 +346,32 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " +# Store the settings in a temporary file +tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") +cat > $tmpfile << EOF +$settings +EOF + # # Call the python script to generate the METplus configuration file from # the jinja template. # -$USHdir/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ +python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ + -c ${tmpfile} \ + -i ${metplus_config_tmpl_fp} \ + -o ${metplus_config_fp} || \ print_err_msg_exit "\ -Call to python script fill_jinja_template.py to generate a METplus +Call to workflow-tools templater.py to generate a METplus configuration file from a jinja template failed. Parameters passed to this script are: Full path to template METplus configuration file: metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" Full path to output METplus configuration file: metplus_config_fp = \"${metplus_config_fp}\" - Jinja settings specified on command line: - settings = -$settings" + Full path to configuration file: + ${tmpfile} +" +rm $tmpfile # #----------------------------------------------------------------------- # diff --git a/tests/test_python/test_python_utils.py b/tests/test_python/test_python_utils.py index ea7c903b16..4104d7db93 100644 --- a/tests/test_python/test_python_utils.py +++ b/tests/test_python/test_python_utils.py @@ -168,8 +168,7 @@ def test_import_vars(self): # python util.import_vars(env_vars=env_vars) - # assuming all environments arlready have $PWD set - #pylint: disable= + # assuming all environments already have $PWD set self.assertEqual( os.path.realpath(PWD), #pylint: disable=undefined-variable os.path.realpath(os.getcwd()) diff --git a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py b/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py index 15c7d79223..9a452bc1c2 100644 --- a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py +++ b/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py @@ -21,6 +21,7 @@ class Testing(unittest.TestCase): """ Define the tests """ def test_set_FV3nml_ens_stoch_seeds(self): """ Call the function and make sure it doesn't fail""" + os.chdir(self.mem_dir) set_FV3nml_ens_stoch_seeds(cdate=self.cdate) def setUp(self): @@ -40,18 +41,20 @@ def setUp(self): ) EXPTDIR = self.tmp_dir.name - cp_vrfy( - os.path.join(PARMdir, "input.nml.FV3"), - os.path.join(EXPTDIR, "input.nml_base"), - ) - # Put this in the tmp_dir structure so it gets cleaned up - mem_dir = os.path.join( + self.mem_dir = os.path.join( EXPTDIR, f"{date_to_str(self.cdate,format='%Y%m%d%H')}", "mem2", ) - mkdir_vrfy("-p", mem_dir) + + mkdir_vrfy("-p", self.mem_dir) + cp_vrfy( + os.path.join(PARMdir, "input.nml.FV3"), + os.path.join(EXPTDIR, "input.nml_base"), + ) + + set_env_var("USHdir", USHdir) set_env_var("ENSMEM_INDX", 2) set_env_var("FV3_NML_FN", "input.nml") diff --git a/ush/create_aqm_rc_file.py b/ush/create_aqm_rc_file.py index fa8960f7db..02d8d827e5 100644 --- a/ush/create_aqm_rc_file.py +++ b/ush/create_aqm_rc_file.py @@ -1,25 +1,26 @@ #!/usr/bin/env python3 +""" +Function that creates the config file for running AQM. +""" import os import sys import argparse -from datetime import datetime from textwrap import dedent +import tempfile from python_utils import ( - import_vars, - set_env_var, - print_input_args, + import_vars, + print_input_args, str_to_type, - print_info_msg, - print_err_msg_exit, - lowercase, + print_info_msg, cfg_to_yaml_str, load_shell_config, flatten_dict ) -from fill_jinja_template import fill_jinja_template +# These come from ush/python_utils/workflow-tools +from scripts.templater import set_template def create_aqm_rc_file(cdate, run_dir, init_concentrations): """ Creates an aqm.rc file in the specified run directory @@ -36,7 +37,8 @@ def create_aqm_rc_file(cdate, run_dir, init_concentrations): #import all environment variables import_vars() - + #pylint: disable=undefined-variable + # #----------------------------------------------------------------------- # @@ -56,23 +58,37 @@ def create_aqm_rc_file(cdate, run_dir, init_concentrations): # Extract from cdate the starting year, month, and day of the forecast. # yyyymmdd=cdate.strftime('%Y%m%d') - mm=f"{cdate.month:02d}" - hh=f"{cdate.hour:02d}" + mm=f"{cdate.month:02d}" # pylint: disable=invalid-name + hh=f"{cdate.hour:02d}" # pylint: disable=invalid-name # # Set parameters in the aqm.rc file. # aqm_rc_bio_file_fp=os.path.join(AQM_BIO_DIR, AQM_BIO_FILE) - aqm_fire_file_fn=AQM_FIRE_FILE_PREFIX+"_"+yyyymmdd+"_t"+hh+"z"+AQM_FIRE_FILE_SUFFIX - aqm_rc_fire_file_fp=os.path.join(COMINext, "FIRE_EMISSION", aqm_fire_file_fn) - aqm_dust_file_fn=AQM_DUST_FILE_PREFIX+"_"+PREDEF_GRID_NAME+AQM_DUST_FILE_SUFFIX - aqm_rc_dust_file_fp=os.path.join(AQM_DUST_DIR, aqm_dust_file_fn) - aqm_canopy_file_fn=AQM_CANOPY_FILE_PREFIX+"."+mm+AQM_CANOPY_FILE_SUFFIX - aqm_rc_canopy_file_fp=os.path.join(AQM_CANOPY_DIR, PREDEF_GRID_NAME, aqm_canopy_file_fn) + + # Fire config + aqm_rc_fire_file_fp=os.path.join( + COMINext, + "FIRE_EMISSION", + f"{AQM_FIRE_FILE_PREFIX}_{yyyymmdd}_t{hh}z{AQM_FIRE_FILE_SUFFIX}" + ) + + # Dust config + aqm_rc_dust_file_fp=os.path.join( + AQM_DUST_DIR, + f"{AQM_DUST_FILE_PREFIX}_{PREDEF_GRID_NAME}{AQM_DUST_FILE_SUFFIX}", + ) + + # Canopy config + aqm_rc_canopy_file_fp=os.path.join( + AQM_CANOPY_DIR, + PREDEF_GRID_NAME, + f"{AQM_CANOPY_FILE_PREFIX}.{mm}{AQM_CANOPY_FILE_SUFFIX}", + ) # #----------------------------------------------------------------------- # # Create a multiline variable that consists of a yaml-compliant string - # specifying the values that the jinja variables in the template + # specifying the values that the jinja variables in the template # AQM_RC_TMPL_FN file should be set to. # #----------------------------------------------------------------------- @@ -94,14 +110,14 @@ def create_aqm_rc_file(cdate, run_dir, init_concentrations): "aqm_rc_product_frequency": AQM_RC_PRODUCT_FREQUENCY } settings_str = cfg_to_yaml_str(settings) - + print_info_msg( dedent( f""" The variable \"settings\" specifying values to be used in the \"{AQM_RC_FN}\" file has been set as follows:\n settings =\n\n""" - ) + ) + settings_str, verbose=VERBOSE, ) @@ -113,35 +129,24 @@ def create_aqm_rc_file(cdate, run_dir, init_concentrations): # #----------------------------------------------------------------------- # - try: - fill_jinja_template( + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="aqm_rc_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(settings_str) + tmpfile.seek(0) + set_template( [ - "-q", - "-u", - settings_str, - "-t", - AQM_RC_TMPL_FP, - "-o", + "-q", + "-c", + tmpfile.name, + "-i", + AQM_RC_TMPL_FP, + "-o", aqm_rc_fp, ] ) - except: - print_err_msg_exit( - dedent( - f""" - Call to python script fill_jinja_template.py to create a \"{AQM_RC_FN}\" - file from a jinja2 template failed. Parameters passed to this script are: - Full path to template aqm.rc file: - AQM_RC_TMPL_FP = \"{AQM_RC_TMPL_FP}\" - Full path to output aqm.rc file: - aqm_rc_fp = \"{aqm_rc_fp}\" - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) - return False - return True def parse_args(argv): @@ -178,6 +183,5 @@ def parse_args(argv): create_aqm_rc_file( run_dir=args.run_dir, cdate=str_to_type(args.cdate), - init_concentrations=str_to_type(args.init_concentrations), + init_concentrations=str_to_type(args.init_concentrations), ) - diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 845ce2dc77..3ef9ec3901 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -1,22 +1,27 @@ #!/usr/bin/env python3 +""" +Function to create a diag_table file for the FV3 model using a +template. +""" import os import sys import argparse from textwrap import dedent +import tempfile + from python_utils import ( import_vars, - set_env_var, print_input_args, print_info_msg, - print_err_msg_exit, cfg_to_yaml_str, load_shell_config, flatten_dict, ) -from fill_jinja_template import fill_jinja_template +# These come from ush/python_utils/workflow-tools +from scripts.templater import set_template def create_diag_table_file(run_dir): @@ -33,6 +38,7 @@ def create_diag_table_file(run_dir): # import all environment variables import_vars() + #pylint: disable=undefined-variable # create a diagnostic table file within the specified run directory print_info_msg( f""" @@ -71,27 +77,16 @@ def create_diag_table_file(run_dir): verbose=VERBOSE, ) - # call fill jinja - try: - fill_jinja_template( - ["-q", "-u", settings_str, "-t", DIAG_TABLE_TMPL_FP, "-o", diag_table_fp] - ) - except: - print_err_msg_exit( - dedent( - f""" - Call to python script fill_jinja_template.py to create a '{DIAG_TABLE_FN}' - file from a jinja2 template failed. Parameters passed to this script are: - Full path to template diag table file: - DIAG_TABLE_TMPL_FP = '{DIAG_TABLE_TMPL_FP}' - Full path to output diag table file: - diag_table_fp = '{diag_table_fp}' - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str + with tempfile.NamedTemporaryFile(dir="./", + mode="w+t", + prefix="aqm_rc_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(settings_str) + tmpfile.seek(0) + # set_template does its own error handling + set_template( + ["-c", tmpfile.name, "-i", DIAG_TABLE_TMPL_FP, "-o", diag_table_fp] ) - return False return True diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index f835212e7d..e39fff695e 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -1,30 +1,32 @@ #!/usr/bin/env python3 - +""" +Create a model_configure file for the FV3 forecast model from a +template. +""" import os import sys import argparse -from datetime import datetime from textwrap import dedent +import tempfile from python_utils import ( import_vars, - set_env_var, print_input_args, str_to_type, print_info_msg, - print_err_msg_exit, lowercase, cfg_to_yaml_str, load_shell_config, flatten_dict, ) -from fill_jinja_template import fill_jinja_template +# These come from ush/python_utils/workflow-tools +from scripts.templater import set_template def create_model_configure_file( cdate, fcst_len_hrs, fhrot, run_dir, sub_hourly_post, dt_subhourly_post_mnts, dt_atmos -): + ): #pylint: disable=too-many-arguments """Creates a model configuration file in the specified run directory @@ -45,6 +47,8 @@ def create_model_configure_file( # import all environment variables import_vars() + # pylint: disable=undefined-variable + # # ----------------------------------------------------------------------- # @@ -60,18 +64,6 @@ def create_model_configure_file( verbose=VERBOSE, ) # - # Extract from cdate the starting year, month, day, and hour of the forecast. - # - yyyy = cdate.year - mm = cdate.month - dd = cdate.day - hh = cdate.hour - # - # Set parameters in the model configure file. - # - dot_quilting_dot=f".{lowercase(str(QUILTING))}." - dot_write_dopost=f".{lowercase(str(WRITE_DOPOST))}." - # # ----------------------------------------------------------------------- # # Create a multiline variable that consists of a yaml-compliant string @@ -81,16 +73,16 @@ def create_model_configure_file( # ----------------------------------------------------------------------- # settings = { - "start_year": yyyy, - "start_month": mm, - "start_day": dd, - "start_hour": hh, + "start_year": cdate.year, + "start_month": cdate.month, + "start_day": cdate.day, + "start_hour": cdate.hour, "nhours_fcst": fcst_len_hrs, "fhrot": fhrot, "dt_atmos": DT_ATMOS, "restart_interval": RESTART_INTERVAL, - "write_dopost": dot_write_dopost, - "quilting": dot_quilting_dot, + "write_dopost": f".{lowercase(str(WRITE_DOPOST))}.", + "quilting": f".{lowercase(str(QUILTING))}.", "output_grid": WRTCMP_output_grid, } # @@ -126,8 +118,7 @@ def create_model_configure_file( } ) elif ( - WRTCMP_output_grid == "regional_latlon" - or WRTCMP_output_grid == "rotated_latlon" + WRTCMP_output_grid in ("regional_latlon", "rotated_latlon") ): settings.update( { @@ -203,34 +194,23 @@ def create_model_configure_file( # model_config_fp = os.path.join(run_dir, MODEL_CONFIG_FN) - try: - fill_jinja_template( + with tempfile.NamedTemporaryFile(dir="./", + mode="w+t", + suffix=".yaml", + prefix="model_config_settings.") as tmpfile: + tmpfile.write(settings_str) + tmpfile.seek(0) + # set_template does its own error handling + set_template( [ - "-q", - "-u", - settings_str, - "-t", + "-c", + tmpfile.name, + "-i", MODEL_CONFIG_TMPL_FP, "-o", model_config_fp, ] ) - except: - print_err_msg_exit( - dedent( - f""" - Call to python script fill_jinja_template.py to create a '{MODEL_CONFIG_FN}' - file from a jinja2 template failed. Parameters passed to this script are: - Full path to template model config file: - MODEL_CONFIG_TMPL_FP = '{MODEL_CONFIG_TMPL_FP}' - Full path to output model config file: - model_config_fp = '{model_config_fp}' - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) - return False return True diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index 9d4b9cd2de..7a2ef0723f 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -1,25 +1,27 @@ #!/usr/bin/env python3 +""" +Function to create a NEMS configuration file for the FV3 forecast +model(s) from a template. +""" + import os import sys import argparse -from datetime import datetime +import tempfile from textwrap import dedent from python_utils import ( - import_vars, - set_env_var, - print_input_args, - str_to_type, - print_info_msg, - print_err_msg_exit, - lowercase, + import_vars, + print_input_args, + print_info_msg, cfg_to_yaml_str, load_shell_config, flatten_dict, ) -from fill_jinja_template import fill_jinja_template +# These come from ush/python_utils/workflow-tools +from scripts.templater import set_template def create_nems_configure_file(run_dir): """ Creates a nems configuration file in the specified @@ -35,7 +37,9 @@ def create_nems_configure_file(run_dir): #import all environment variables import_vars() - + + # pylint: disable=undefined-variable + # #----------------------------------------------------------------------- # @@ -56,7 +60,7 @@ def create_nems_configure_file(run_dir): #----------------------------------------------------------------------- # # Create a multiline variable that consists of a yaml-compliant string - # specifying the values that the jinja variables in the template + # specifying the values that the jinja variables in the template # model_configure file should be set to. # #----------------------------------------------------------------------- @@ -69,15 +73,15 @@ def create_nems_configure_file(run_dir): "atm_omp_num_threads": OMP_NUM_THREADS_RUN_FCST, } settings_str = cfg_to_yaml_str(settings) - + print_info_msg( dedent( f""" The variable \"settings\" specifying values to be used in the \"{NEMS_CONFIG_FN}\" file has been set as follows:\n settings =\n\n""" - ) - + settings_str, + ) + + settings_str, verbose=VERBOSE, ) # @@ -88,25 +92,15 @@ def create_nems_configure_file(run_dir): # #----------------------------------------------------------------------- # - try: - fill_jinja_template(["-q", "-u", settings_str, "-t", NEMS_CONFIG_TMPL_FP, "-o", nems_config_fp]) - except: - print_err_msg_exit( - dedent( - f""" - Call to python script fill_jinja_template.py to create the nems.configure - file from a jinja2 template failed. Parameters passed to this script are: - Full path to template nems.configure file: - NEMS_CONFIG_TMPL_FP = \"{NEMS_CONFIG_TMPL_FP}\" - Full path to output nems.configure file: - nems_config_fp = \"{nems_config_fp}\" - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) - return False + # Store the settings in a temporary file + with tempfile.NamedTemporaryFile(dir="./", + mode="w+t", + prefix="nems_config_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(settings_str) + tmpfile.seek(0) + set_template(["-c", tmpfile.name, "-i", NEMS_CONFIG_TMPL_FP, "-o", nems_config_fp]) return True def parse_args(argv): @@ -133,7 +127,5 @@ def parse_args(argv): cfg = flatten_dict(cfg) import_vars(dictionary=cfg) create_nems_configure_file( - run_dir=args.run_dir, + run_dir=args.run_dir, ) - - diff --git a/ush/fill_jinja_template.py b/ush/fill_jinja_template.py deleted file mode 100755 index f810136753..0000000000 --- a/ush/fill_jinja_template.py +++ /dev/null @@ -1,302 +0,0 @@ -#!/usr/bin/env python3 - -""" -This utility fills in a user-supplied Jinja template from either a YAML file, or -command line arguments. - -The user configuration file and commandline arguments should be YAML-formatted. -This script will support a single- or two-level YAML config file. For example: - - 1. expt1: - date_first_cycl: !datetime 2019043000 - date_last_cycl: !datetime 2019050100 - cycl_freq: !!str 12:00:00 - - expt2: - date_first_cycl: !datetime 2019061012 - date_last_cycl: !datetime 2019061212 - cycl_freq: !!str 12:00:00 - - 2. date_first_cycl: !datetime 2019043000 - date_last_cycl: !datetime 2019050100 - cycl_freq: !!str 12:00:00 - - In Case 1, provide the name of the file and the section title, e.g. expt2, - to the -c command line argument. Only provide the name of the file in -c - option if it's configured as in Case 2 above. - - -Supported YAML Tags: - - The script supports additional YAML configuration tags. - - !datetime Converts an input string formatted as YYYYMMDDHH[mm[ss]] to a - Python datetime object - !join Uses os.path.join to join a list as a path. - -Expected behavior: - - - The template file is required. Script fails if not provided. - - Command line arguments in the -u setting override the -c settings. - -""" - -import datetime as dt -import os -import sys - -import argparse -import jinja2 as j2 -from jinja2 import meta -import yaml - - -def join(loader, node): - - """Uses os to join a list as a path.""" - - return os.path.join(*loader.construct_sequence(node)) - - -def to_datetime(loader, node): - - """Converts a date string with format YYYYMMDDHH[MM[SS]] to a datetime - object.""" - - value = loader.construct_scalar(node) - val_len = len(value) - - # Check that the input string contains only numbers and is expected length. - if val_len not in [10, 12, 14] or not value.isnumeric(): - msg = f"{value} does not conform to input format YYYYMMDDHH[MM[SS]]" - raise ValueError(msg) - - # Use a subset of the string corresponding to the input length of the string - # 2 chosen here since Y is a 4 char year. - date_format = "%Y%m%d%H%M%S"[0 : val_len - 2] - - return dt.datetime.strptime(value, date_format) - - -yaml.add_constructor("!datetime", to_datetime, Loader=yaml.SafeLoader) -yaml.add_constructor("!join", join, Loader=yaml.SafeLoader) - - -def file_exists(arg): - - """Checks whether a file exists, and returns the path if it does.""" - - if not os.path.exists(arg): - msg = f"{arg} does not exist!" - raise argparse.ArgumentTypeError(msg) - - return arg - - -def config_exists(arg): - - """ - Checks whether the config file exists and if it contains the input - section. Returns the config as a Python dict. - """ - - if len(arg) > 2: - msg = f"{len(arg)} arguments were provided for config. Only 2 allowed!" - raise argparse.ArgumentTypeError(msg) - - file_name = file_exists(arg[0]) - section_name = arg[1] if len(arg) == 2 else None - - # Load the YAML file into a dictionary - with open(file_name, "r") as fn: - cfg = yaml.load(fn, Loader=yaml.SafeLoader) - - if section_name: - try: - cfg = cfg[section_name] - except KeyError: - msg = f"Section {section_name} does not exist in top level of {file_name}" - raise argparse.ArgumentTypeError(msg) - - return cfg - - -def load_config(arg): - - """ - Check to ensure that the provided config file exists. If it does, load it - with YAML's safe loader and return the resulting dict. - """ - - # Check for existence of file - if not os.path.exists(arg): - msg = f"{arg} does not exist!" - raise argparse.ArgumentTypeError(msg) - - return yaml.safe_load(arg) - - -def load_str(arg): - - """Load a dict string safely using YAML. Return the resulting dict.""" - - return yaml.load(arg, Loader=yaml.SafeLoader) - - -def path_ok(arg): - - """ - Check whether the path to the file exists, and is writeable. Return the path - if it passes all checks, otherwise raise an error. - """ - - # Get the absolute path provided by arg - dir_name = os.path.abspath(os.path.dirname(arg)) - - # Ensure the arg path exists, and is writable. Raise error if not. - if os.path.lexists(dir_name) and os.access(dir_name, os.W_OK): - return arg - - msg = f"{arg} is not a writable path!" - raise argparse.ArgumentTypeError(msg) - - -def parse_args(argv): - - """ - Function maintains the arguments accepted by this script. Please see - Python's argparse documenation for more information about settings of each - argument. - """ - - parser = argparse.ArgumentParser(description="Fill in a Rocoto XML template.") - - # Optional - parser.add_argument( - "-c", - "--config", - help="Full path to a YAML user config file, and a \ - top-level section to use (optional).", - nargs="*", - type=load_config, - ) - parser.add_argument( - "-q", - "--quiet", - action="store_true", - help="Suppress all output", - ) - parser.add_argument( - "-u", - "--user_config", - help="Command-line user config options in YAML-formatted string", - type=load_str, - ) - # Required - parser.add_argument( - "-t", - "--xml_template", - dest="template", - help="Full path to the jinja template", - required=True, - type=file_exists, - ) - parser.add_argument( - "-o", - "--outxml", - dest="outxml", - help="Full path to the output Rocoto XML file.", - required=True, - type=path_ok, - ) - return parser.parse_args(argv) - - -def update_dict(dest, newdict, quiet=False): - - """ - Overwrites all values in dest dictionary section with key/value pairs from - newdict. Does not support multi-layer update. - - Turn off print statements with quiet=True. - - Input: - dest A dict that is to be updated. - newdict A dict containing sections and keys corresponding to - those in dest and potentially additional ones, that will be used to - update the dest dict. - quiet An optional boolean flag to turn off output. - Output: - None - Result: - The dest dict is updated in place. - """ - - if not quiet: - print("*" * 50) - - for key, value in newdict.items(): - if not quiet: - print(f"Overriding {key:>20} = {value}") - - # Set key in dict - dest[key] = value - - if not quiet: - print("*" * 50) - - -def fill_jinja_template(argv, config_dict=None): - - """ - Loads a Jinja template, determines its necessary undefined variables, - retrives them from user supplied settings, and renders the final result. - """ - - # parse args - cla = parse_args(argv) - if cla.config: - cla.config = config_exists(cla.config) - - # Create a Jinja Environment to load the template. - env = j2.Environment(loader=j2.FileSystemLoader(cla.template, - encoding='utf-8')) - template_source = env.loader.get_source(env, "") - template = env.get_template("") - parsed_content = env.parse(template_source) - - # Gather all of the undefined variables in the template. - template_vars = meta.find_undeclared_variables(parsed_content) - - # Read in the config options from the provided (optional) YAML file - cfg = cla.config if cla.config is not None else {} - - if config_dict is not None: - update_dict(cfg, config_dict, quiet=cla.quiet) - - # Update cfg with (optional) command-line entries, overriding those in YAML file - if cla.user_config: - update_dict(cfg, cla.user_config, quiet=cla.quiet) - - # Loop through all the undefined Jinja template variables, and grab the - # required values from the config file. - tvars = {} - for var in template_vars: - - if cfg.get(var, "NULL") == "NULL": - raise KeyError(f"{var} does not exist in user-supplied settings!") - - if not cla.quiet: - print(f"{var:>25}: {cfg.get(var)}") - - tvars[var] = cfg.get(var) - - # Fill in XML template - xml_contents = template.render(**tvars) - with open(cla.outxml, "w") as fn: - fn.write(xml_contents) - - -if __name__ == "__main__": - - fill_jinja_template(sys.argv[1:]) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index d91d21db58..5022bfed4c 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -1,45 +1,46 @@ #!/usr/bin/env python3 +""" +User interface to create an experiment directory consistent with the +user-defined config.yaml file. +""" + +# pylint: disable=invalid-name + import os -import sys -import subprocess import logging -from multiprocessing import Process from textwrap import dedent -from datetime import datetime, timedelta +import sys from python_utils import ( log_info, import_vars, export_vars, - load_config_file, - update_dict, cp_vrfy, ln_vrfy, mkdir_vrfy, mv_vrfy, - run_command, - date_to_str, - define_macos_utilities, create_symlink_to_file, check_for_preexist_dir_file, cfg_to_yaml_str, find_pattern_in_str, - set_env_var, - get_env_var, - lowercase, flatten_dict, ) from setup import setup from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames from get_crontab_contents import add_crontab_line -from fill_jinja_template import fill_jinja_template from set_namelist import set_namelist from check_python_version import check_python_version +# These come from ush/python_utils/workflow-tools +from scripts.templater import set_template -def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", debug: bool = False) -> str: +# pylint: disable=too-many-locals,too-many-branches, too-many-statements +def generate_FV3LAM_wflow( + ushdir, + logfile: str = "log.generate_FV3LAM_wflow", + debug: bool = False) -> str: """Function to setup a forecast experiment and create a workflow (according to the parameters specified in the config file) @@ -101,7 +102,6 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de expt_config["user"]["PARMdir"], wflow_xml_fn, ) - global_var_defns_fp = expt_config["workflow"]["GLOBAL_VAR_DEFNS_FP"] log_info( f""" @@ -114,21 +114,10 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # rocoto_yaml_fp = expt_config["workflow"]["ROCOTO_YAML_FP"] args = ["-o", wflow_xml_fp, - "-t", template_xml_fp, - "-c", rocoto_yaml_fp ] - if not debug: - args.append("-q") - - try: - fill_jinja_template(args) - except: - raise Exception( - dedent( - f""" - Call to fill_jinja_template failed. - """ - ) - ) + "-i", template_xml_fp, + "-c", rocoto_yaml_fp, + ] + set_template(args) # # ----------------------------------------------------------------------- # @@ -168,6 +157,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de import_vars(dictionary=flatten_dict(expt_config)) export_vars(source_dict=flatten_dict(expt_config)) + # pylint: disable=undefined-variable if USE_CRON_TO_RELAUNCH: add_crontab_line() @@ -237,20 +227,20 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # ----------------------------------------------------------------------- # log_info( - f""" + """ Copying templates of various input files to the experiment directory...""", verbose=verbose, ) log_info( - f""" + """ Copying the template data table file to the experiment directory...""", verbose=verbose, ) cp_vrfy(DATA_TABLE_TMPL_FP, DATA_TABLE_FP) log_info( - f""" + """ Copying the template field table file to the experiment directory...""", verbose=verbose, ) @@ -262,7 +252,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # DIR). # log_info( - f""" + """ Copying the CCPP physics suite definition XML file from its location in the forecast model directory structure to the experiment directory...""", verbose=verbose, @@ -274,9 +264,10 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # DIR). # log_info( - f""" - Copying the field dictionary file from its location in the forecast - model directory structure to the experiment directory...""", + """ + Copying the field dictionary file from its location in the + forecast model directory structure to the experiment + directory...""", verbose=verbose, ) cp_vrfy(FIELD_DICT_IN_UWM_FP, FIELD_DICT_FP) @@ -294,7 +285,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de ) # # Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. - # These need to be set in the FV3-LAM Fortran namelist file. They represent + # These need to be set in the FV3-LAM Fortran namelist file. They represent # the number of cell vertices in the x and y directions on the regional # grid. # @@ -323,9 +314,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # Also, may want to set lsm here as well depending on SDF_USES_RUC_LSM. # lsoil = 4 - if (EXTRN_MDL_NAME_ICS == "HRRR" or EXTRN_MDL_NAME_ICS == "RAP") and ( - SDF_USES_RUC_LSM - ): + if EXTRN_MDL_NAME_ICS in ("HRRR", "RAP") and SDF_USES_RUC_LSM: lsoil = 9 if CCPP_PHYS_SUITE == "FV3_GFS_v15_thompson_mynn_lam3km": lsoil = "" @@ -374,9 +363,10 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de "layout": [LAYOUT_X, LAYOUT_Y], "bc_update_interval": LBC_SPEC_INTVL_HRS, }) - if ( CCPP_PHYS_SUITE == "FV3_GFS_2017_gfdl_mp" or - CCPP_PHYS_SUITE == "FV3_GFS_2017_gfdlmp_regional" or - CCPP_PHYS_SUITE == "FV3_GFS_v15p2" ): + if CCPP_PHYS_SUITE in ("FV3_GFS_2017_gfdl_mp", + "FV3_GFS_2017_gfdlmp_regional", + "FV3_GFS_v15p2", + ): if CPL_AQM: fv_core_nml_dict.update({ "dnats": 5 @@ -385,7 +375,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de fv_core_nml_dict.update({ "dnats": 1 }) - elif CCPP_PHYS_SUITE == "FV3_GFS_v16": + elif CCPP_PHYS_SUITE == "FV3_GFS_v16": if CPL_AQM: fv_core_nml_dict.update({ "hord_tr": 8, @@ -417,25 +407,26 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de if CPL_AQM: gfs_physics_nml_dict.update({ - "cplaqm": True, + "cplaqm": True, "cplocn2atm": False, - "fscav_aero": ["aacd:0.0", "acet:0.0", "acrolein:0.0", "acro_primary:0.0", "ald2:0.0", - "ald2_primary:0.0", "aldx:0.0", "benzene:0.0", "butadiene13:0.0", "cat1:0.0", - "cl2:0.0", "clno2:0.0", "co:0.0", "cres:0.0", "cron:0.0", - "ech4:0.0", "epox:0.0", "eth:0.0", "etha:0.0", "ethy:0.0", - "etoh:0.0", "facd:0.0", "fmcl:0.0", "form:0.0", "form_primary:0.0", - "gly:0.0", "glyd:0.0", "h2o2:0.0", "hcl:0.0", "hg:0.0", - "hgiigas:0.0", "hno3:0.0", "hocl:0.0", "hono:0.0", "hpld:0.0", - "intr:0.0", "iole:0.0", "isop:0.0", "ispd:0.0", "ispx:0.0", - "ket:0.0", "meoh:0.0", "mepx:0.0", "mgly:0.0", "n2o5:0.0", - "naph:0.0", "no:0.0", "no2:0.0", "no3:0.0", "ntr1:0.0", - "ntr2:0.0", "o3:0.0", "ole:0.0", "opan:0.0", "open:0.0", - "opo3:0.0", "pacd:0.0", "pan:0.0", "panx:0.0", "par:0.0", - "pcvoc:0.0", "pna:0.0", "prpa:0.0", "rooh:0.0", "sesq:0.0", - "so2:0.0", "soaalk:0.0", "sulf:0.0", "terp:0.0", "tol:0.0", - "tolu:0.0", "vivpo1:0.0", "vlvoo1:0.0", "vlvoo2:0.0", "vlvpo1:0.0", - "vsvoo1:0.0", "vsvoo2:0.0", "vsvoo3:0.0", "vsvpo1:0.0", "vsvpo2:0.0", - "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] + "fscav_aero": [ + "aacd:0.0", "acet:0.0", "acrolein:0.0", "acro_primary:0.0", "ald2:0.0", + "ald2_primary:0.0", "aldx:0.0", "benzene:0.0", "butadiene13:0.0", "cat1:0.0", + "cl2:0.0", "clno2:0.0", "co:0.0", "cres:0.0", "cron:0.0", + "ech4:0.0", "epox:0.0", "eth:0.0", "etha:0.0", "ethy:0.0", + "etoh:0.0", "facd:0.0", "fmcl:0.0", "form:0.0", "form_primary:0.0", + "gly:0.0", "glyd:0.0", "h2o2:0.0", "hcl:0.0", "hg:0.0", + "hgiigas:0.0", "hno3:0.0", "hocl:0.0", "hono:0.0", "hpld:0.0", + "intr:0.0", "iole:0.0", "isop:0.0", "ispd:0.0", "ispx:0.0", + "ket:0.0", "meoh:0.0", "mepx:0.0", "mgly:0.0", "n2o5:0.0", + "naph:0.0", "no:0.0", "no2:0.0", "no3:0.0", "ntr1:0.0", + "ntr2:0.0", "o3:0.0", "ole:0.0", "opan:0.0", "open:0.0", + "opo3:0.0", "pacd:0.0", "pan:0.0", "panx:0.0", "par:0.0", + "pcvoc:0.0", "pna:0.0", "prpa:0.0", "rooh:0.0", "sesq:0.0", + "so2:0.0", "soaalk:0.0", "sulf:0.0", "terp:0.0", "tol:0.0", + "tolu:0.0", "vivpo1:0.0", "vlvoo1:0.0", "vlvoo2:0.0", "vlvpo1:0.0", + "vsvoo1:0.0", "vsvoo2:0.0", "vsvoo3:0.0", "vsvpo1:0.0", "vsvpo2:0.0", + "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] }) settings["gfs_physics_nml"] = gfs_physics_nml_dict @@ -492,7 +483,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de settings_str = cfg_to_yaml_str(settings) log_info( - f""" + """ The variable 'settings' specifying values of the weather model's namelist variables has been set as follows:\n""", verbose=verbose, @@ -511,40 +502,19 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # # ----------------------------------------------------------------------- # - try: - set_namelist( - [ - "-q", - "-n", - FV3_NML_BASE_SUITE_FP, - "-c", - FV3_NML_YAML_CONFIG_FP, - CCPP_PHYS_SUITE, - "-u", - settings_str, - "-o", - FV3_NML_FP, - ] - ) - except: - logging.exception( - dedent( - f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_BASE_SUITE_FP = '{FV3_NML_BASE_SUITE_FP}' - Full path to yaml configuration file for various physics suites: - FV3_NML_YAML_CONFIG_FP = '{FV3_NML_YAML_CONFIG_FP}' - Physics suite to extract from yaml configuration file: - CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) + set_namelist( + [ + "-n", + FV3_NML_BASE_SUITE_FP, + "-c", + FV3_NML_YAML_CONFIG_FP, + CCPP_PHYS_SUITE, + "-u", + settings_str, + "-o", + FV3_NML_FP, + ] + ) # # If not running the TN_MAKE_GRID task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying @@ -581,34 +551,16 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # # populate the namelist file # - try: - set_namelist( - [ - "-q", - "-n", - FV3_NML_FP, - "-u", - settings_str, - "-o", - FV3_NML_CYCSFC_FP, - ] - ) - except: - logging.exception( - dedent( - f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Full path to output namelist file for DA: - FV3_NML_RESTART_FP = '{FV3_NML_CYCSFC_FP}' - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) - + set_namelist( + [ + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_CYCSFC_FP, + ] + ) # # ----------------------------------------------------------------------- # @@ -624,7 +576,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de lupdatebc = False if DO_UPDATE_BC: lupdatebc = False # not ready for setting this to true yet - + settings = {} settings["fv_core_nml"] = { "external_ic": False, @@ -644,33 +596,17 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # # populate the namelist file # - try: - set_namelist( - [ - "-q", - "-n", - FV3_NML_FP, - "-u", - settings_str, - "-o", - FV3_NML_RESTART_FP, - ] - ) - except: - logging.exception( - dedent( - f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Full path to output namelist file for DA: - FV3_NML_RESTART_FP = '{FV3_NML_RESTART_FP}' - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) + set_namelist( + [ + "-q", + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_RESTART_FP, + ] + ) # # ----------------------------------------------------------------------- # @@ -782,65 +718,31 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # #----------------------------------------------------------------------- # - if DO_ENSEMBLE and ( DO_SPP or DO_SPPT or DO_SHUM or DO_SKEB or DO_LSM_SPP): - - try: + if DO_ENSEMBLE and any((DO_SPP, DO_SPPT, DO_SHUM, DO_SKEB, DO_LSM_SPP)): + + set_namelist( + [ + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_STOCH_FP, + ] + ) + + if DO_DACYCLE or DO_ENKFUPDATE: set_namelist( [ "-q", "-n", - FV3_NML_FP, + FV3_NML_RESTART_FP, "-u", settings_str, "-o", - FV3_NML_STOCH_FP, + FV3_NML_RESTART_STOCH_FP, ] ) - except: - logging.exception( - dedent( - f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Full path to output namelist file for stochastics: - FV3_NML_STOCH_FP = '{FV3_NML_STOCH_FP}' - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) - - if DO_DACYCLE or DO_ENKFUPDATE: - try: - set_namelist( - [ - "-q", - "-n", - FV3_NML_RESTART_FP, - "-u", - settings_str, - "-o", - FV3_NML_RESTART_STOCH_FP, - ] - ) - except: - logging.exception( - dedent( - f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Full path to output namelist file for stochastics: - FV3_NML_RESTART_STOCH_FP = '{FV3_NML_RESTART_STOCH_FP}' - Namelist settings specified on command line:\n - settings =\n\n""" - ) - + settings_str - ) - # # ----------------------------------------------------------------------- # @@ -867,6 +769,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de rocotorun_cmd = f"rocotorun -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" rocotostat_cmd = f"rocotostat -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" + # pylint: disable=line-too-long log_info( f""" To launch the workflow, change location to the experiment directory @@ -897,8 +800,10 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de */{CRON_RELAUNCH_INTVL_MNTS} * * * * cd {EXPTDIR} && ./launch_FV3LAM_wflow.sh called_from_cron="TRUE" """ ) + # pylint: enable=line-too-long - # If we got to this point everything was successful: move the log file to the experiment directory. + # If we got to this point everything was successful: move the log + # file to the experiment directory. mv_vrfy(logfile, EXPTDIR) return EXPTDIR @@ -908,7 +813,7 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals """ Sets up logging, printing high-priority (INFO and higher) messages to screen, and printing all messages with detailed timing and routine info in the specified text file. - + If debug = True, print all messages to both screen and log file. """ logging.getLogger().setLevel(logging.DEBUG) @@ -921,7 +826,8 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals logging.getLogger().addHandler(fh) logging.debug(f"Finished setting up debug file logging in {logfile}") - # If there are already multiple handlers, that means generate_FV3LAM_workflow was called from another function. + # If there are already multiple handlers, that means + # generate_FV3LAM_workflow was called from another function. # In that case, do not change the console (print-to-screen) logging. if len(logging.getLogger().handlers) > 1: return @@ -944,7 +850,7 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals # experiment/workflow. try: expt_dir = generate_FV3LAM_wflow(USHdir, wflow_logfile) - except: + except: # pylint: disable=bare-except logging.exception( dedent( f""" @@ -957,18 +863,18 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals """ ) ) - else: - # If no exception, note workflow generation completion - log_info( - f""" - ======================================================================== + sys.exit(1) + + # pylint: disable=undefined-variable + # Note workflow generation completion + log_info( + f""" + ======================================================================== Experiment generation completed. The experiment directory is: EXPTDIR='{EXPTDIR}' - ======================================================================== - """ - ) - - + ======================================================================== + """ + ) diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 8d00a8fe65..a3b8aaca5a 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -98,16 +98,21 @@ set -u # #----------------------------------------------------------------------- # +default_modules_dir="$HOMEdir/modulefiles" machine=$(echo_lowercase $MACHINE) source "${HOMEdir}/etc/lmod-setup.sh" ${machine} +module use "${default_modules_dir}" + if [ "${machine}" != "wcoss2" ]; then - module use "${HOMEdir}/modulefiles" module load "${BUILD_MOD_FN}" || print_err_msg_exit "\ Loading of platform- and compiler-specific module file (BUILD_MOD_FN) for the workflow task specified by task_name failed: task_name = \"${task_name}\" BUILD_MOD_FN = \"${BUILD_MOD_FN}\"" fi + +module load set_pythonpath || print_err_msg_exit "\ + Loading the module to set PYTHONPATH for workflow-tools failed." # #----------------------------------------------------------------------- # @@ -134,9 +139,8 @@ fi # #----------------------------------------------------------------------- # -modules_dir="$HOMEdir/modulefiles/tasks/$machine" +modules_dir="$default_modules_dir/tasks/$machine" modulefile_name="${task_name}" -default_modules_dir="$HOMEdir/modulefiles" # #----------------------------------------------------------------------- # @@ -173,6 +177,8 @@ elif [ -f ${modules_dir}/python_srw.lua ] ; then modules_dir = \"${modules_dir}\"" fi + + module list # Modules that use conda and need an environment activated will set the diff --git a/ush/load_modules_wflow.sh b/ush/load_modules_wflow.sh index 5e7e30e3a7..7631295d76 100755 --- a/ush/load_modules_wflow.sh +++ b/ush/load_modules_wflow.sh @@ -62,7 +62,7 @@ task failed: $has_mu && set +u if [ ! -z $(command -v conda) ]; then - conda activate regional_workflow + conda activate workflow_tools fi $has_mu && set -u diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index cdc4ed1628..bc579151c5 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -60,7 +60,7 @@ def set_FV3nml_sfc_climo_filenames(): # Set the suffix of the surface climatology files. suffix = "tileX.nc" - # create yaml-complaint string + # create yaml-compliant string settings = {} dummy_run_dir = os.path.join(EXPTDIR, "any_cyc") From 44180c7a5bcec3106ad9355941e44e8bb119c100 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:47:57 -0400 Subject: [PATCH 05/16] [develop] Split aqm_all into aqm_prep and aqm_post (#804) * The aqm_all yaml file where all AQM tasks are included is split into two small groups of pre and post processing. * Since the post-processing tasks such as post_stat and bias-correction only work on WCOSS2, this split would make the configuration simpler on other machines. --- parm/wflow/aqm_post.yaml | 77 ++++++++++++++++++++++ parm/wflow/{aqm_all.yaml => aqm_prep.yaml} | 54 --------------- ush/config.aqm.community.yaml | 6 +- ush/config.aqm.nco.realtime.yaml | 6 +- 4 files changed, 81 insertions(+), 62 deletions(-) create mode 100644 parm/wflow/aqm_post.yaml rename parm/wflow/{aqm_all.yaml => aqm_prep.yaml} (76%) diff --git a/parm/wflow/aqm_post.yaml b/parm/wflow/aqm_post.yaml new file mode 100644 index 0000000000..4016c8190c --- /dev/null +++ b/parm/wflow/aqm_post.yaml @@ -0,0 +1,77 @@ +default_aqm_task: &default_aqm + account: '&ACCOUNT;' + attrs: + cycledefs: forecast + maxtries: '2' + envars: &default_vars + GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' + USHdir: '&USHdir;' + PDY: !cycstr "@Y@m@d" + cyc: !cycstr "@H" + nprocs: '{{ parent.nnodes * parent.ppn // 1 }}' + subcyc: !cycstr "@M" + LOGDIR: !cycstr "&LOGDIR;" + SLASH_ENSMEM_SUBDIR: '&SLASH_ENSMEM_SUBDIR;' + native: '{{ platform.SCHED_NATIVE_CMD }}' + nnodes: 1 + nodes: '{{ nnodes }}:ppn={{ ppn }}' + nodesize: "&NCORES_PER_NODE;" + partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' + ppn: 1 + queue: '&QUEUE_DEFAULT;' + walltime: 00:30:00 + +task_pre_post_stat: + <<: *default_aqm + command: '&LOAD_MODULES_RUN_TASK_FP; "pre_post_stat" "&JOBSdir;/JREGIONAL_PRE_POST_STAT"' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + dependency: + or: + datadep: + attrs: + age: 00:00:00:05 + text: !cycstr '&COMIN_DIR;/post_@Y@m@d@H_task_complete.txt' + metataskdep: + attrs: + metatask: run_ens_post + +task_post_stat_o3: + <<: *default_aqm + command: '&LOAD_MODULES_RUN_TASK_FP; "post_stat_o3" "&JOBSdir;/JREGIONAL_POST_STAT_O3"' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: 120G + dependency: + taskdep: + attrs: + task: pre_post_stat + +task_post_stat_pm25: + <<: *default_aqm + command: '&LOAD_MODULES_RUN_TASK_FP; "post_stat_pm25" "&JOBSdir;/JREGIONAL_POST_STAT_PM25"' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: 120G + dependency: + taskdep: + attrs: + task: pre_post_stat + +task_bias_correction_o3: + <<: *default_aqm + command: '&LOAD_MODULES_RUN_TASK_FP; "bias_correction_o3" "&JOBSdir;/JREGIONAL_BIAS_CORRECTION_O3"' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: 120G + dependency: + taskdep: + attrs: + task: pre_post_stat + +task_bias_correction_pm25: + <<: *default_aqm + command: '&LOAD_MODULES_RUN_TASK_FP; "bias_correction_pm25" "&JOBSdir;/JREGIONAL_BIAS_CORRECTION_PM25"' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: 120G + dependency: + taskdep: + attrs: + task: pre_post_stat + diff --git a/parm/wflow/aqm_all.yaml b/parm/wflow/aqm_prep.yaml similarity index 76% rename from parm/wflow/aqm_all.yaml rename to parm/wflow/aqm_prep.yaml index 39dd7c9dfe..43c1e81aa1 100644 --- a/parm/wflow/aqm_all.yaml +++ b/parm/wflow/aqm_prep.yaml @@ -149,57 +149,3 @@ task_aqm_lbcs: attrs: task: make_lbcs_mem000 -task_pre_post_stat: - <<: *default_aqm - command: '&LOAD_MODULES_RUN_TASK_FP; "pre_post_stat" "&JOBSdir;/JREGIONAL_PRE_POST_STAT"' - join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - dependency: - or: - datadep: - attrs: - age: 00:00:00:05 - text: !cycstr '&COMIN_DIR;/post_@Y@m@d@H_task_complete.txt' - metataskdep: - attrs: - metatask: run_ens_post - -task_post_stat_o3: - <<: *default_aqm - command: '&LOAD_MODULES_RUN_TASK_FP; "post_stat_o3" "&JOBSdir;/JREGIONAL_POST_STAT_O3"' - join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - memory: 120G - dependency: - taskdep: - attrs: - task: pre_post_stat - -task_post_stat_pm25: - <<: *default_aqm - command: '&LOAD_MODULES_RUN_TASK_FP; "post_stat_pm25" "&JOBSdir;/JREGIONAL_POST_STAT_PM25"' - join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - memory: 120G - dependency: - taskdep: - attrs: - task: pre_post_stat - -task_bias_correction_o3: - <<: *default_aqm - command: '&LOAD_MODULES_RUN_TASK_FP; "bias_correction_o3" "&JOBSdir;/JREGIONAL_BIAS_CORRECTION_O3"' - join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - memory: 120G - dependency: - taskdep: - attrs: - task: pre_post_stat - -task_bias_correction_pm25: - <<: *default_aqm - command: '&LOAD_MODULES_RUN_TASK_FP; "bias_correction_pm25" "&JOBSdir;/JREGIONAL_BIAS_CORRECTION_PM25"' - join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - memory: 120G - dependency: - taskdep: - attrs: - task: pre_post_stat - diff --git a/ush/config.aqm.community.yaml b/ush/config.aqm.community.yaml index 82713e0ff5..55675bb8ce 100644 --- a/ush/config.aqm.community.yaml +++ b/ush/config.aqm.community.yaml @@ -28,12 +28,8 @@ nco: NET: aqm rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/aqm_all.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' task_aqm_ics_ext: - task_post_stat_o3: - task_post_stat_pm25: - task_bias_correction_o3: - task_bias_correction_pm25: metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 02:00:00 diff --git a/ush/config.aqm.nco.realtime.yaml b/ush/config.aqm.nco.realtime.yaml index 5ee4a0c25e..1a0da22677 100644 --- a/ush/config.aqm.nco.realtime.yaml +++ b/ush/config.aqm.nco.realtime.yaml @@ -10,8 +10,8 @@ workflow: EXPT_SUBDIR: aqm_nco_aqmna13km PREDEF_GRID_NAME: AQM_NA_13km CCPP_PHYS_SUITE: FV3_GFS_v16 - DATE_FIRST_CYCL: '2023051100' - DATE_LAST_CYCL: '2023051118' + DATE_FIRST_CYCL: '2023051600' + DATE_LAST_CYCL: '2023051618' INCR_CYCL_FREQ: 6 FCST_LEN_HRS: -1 FCST_LEN_CYCL: @@ -37,7 +37,7 @@ nco: KEEPDATA: true rocoto: tasks: - taskgroups: '{{ ["parm/wflow/aqm_all.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/aqm_post.yaml"]|include }}' task_get_extrn_lbcs: walltime: 02:00:00 metatask_run_ensemble: From d0bb3c53523b74238a355cc41e6c0edc85a41a3d Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Mon, 12 Jun 2023 10:38:06 -0600 Subject: [PATCH 06/16] [develop] Add option to change vertical coordinate file. (#813) Add a configurable option for users to specify their own vertical coordinate file for the FV3 model. As an alternative to the current one, the RRFS version for 65 vertical levels has been added to parm. --- parm/global_hyblev_fcst_rrfsL65.txt | 68 +++++++++++++++++++ scripts/exregional_make_ics.sh | 2 +- scripts/exregional_make_lbcs.sh | 2 +- .../data_assimilation/config.process_obs.yaml | 6 +- .../config.different_vcoord.yaml | 24 +++++++ ush/config_defaults.yaml | 6 ++ ush/setup.py | 19 ++++++ 7 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 parm/global_hyblev_fcst_rrfsL65.txt create mode 100644 tests/WE2E/test_configs/wflow_features/config.different_vcoord.yaml diff --git a/parm/global_hyblev_fcst_rrfsL65.txt b/parm/global_hyblev_fcst_rrfsL65.txt new file mode 100644 index 0000000000..b1ce286eac --- /dev/null +++ b/parm/global_hyblev_fcst_rrfsL65.txt @@ -0,0 +1,68 @@ + 2 66 + 0.000 1.00000000 + 0.000 0.99787363 + 0.000 0.99547387 + 0.017 0.99276698 + 0.234 0.98971388 + 1.058 0.98626929 + 3.103 0.98238159 + 7.211 0.97799245 + 14.450 0.97303658 + 26.130 0.96744145 + 43.805 0.96112727 + 69.754 0.95400221 + 107.500 0.94595747 + 160.901 0.93687684 + 233.980 0.92663902 + 330.883 0.91511905 + 455.809 0.90219033 + 612.932 0.88772712 + 806.289 0.87160769 + 1039.659 0.85371818 + 1316.413 0.83395695 + 1639.350 0.81223965 + 2010.516 0.78850460 + 2431.016 0.76271850 + 2900.827 0.73488203 + 3418.626 0.70503512 + 3981.638 0.67326135 + 4585.535 0.63969119 + 5224.384 0.60450347 + 5890.665 0.56792486 + 6575.364 0.53022703 + 7268.150 0.49172137 + 7957.623 0.45275150 + 8631.631 0.41368369 + 9277.633 0.37489594 + 9883.087 0.33676625 + 10435.842 0.29966091 + 10924.498 0.26392352 + 11338.722 0.22986547 + 11669.502 0.19775844 + 11909.318 0.16782909 + 12052.236 0.14025627 + 12093.928 0.11517042 + 12031.620 0.09265514 + 11863.989 0.07275035 + 11591.011 0.05545674 + 11213.791 0.04074101 + 10734.376 0.02854147 + 10155.564 0.01877378 + 9480.727 0.01133629 + 8714.452 0.00610706 + 7877.419 0.00279793 + 7004.872 0.00097247 + 6132.608 0.00018806 + 5296.374 0.00000379 + 4530.025 0.00000000 + 3843.468 0.00000000 + 3229.725 0.00000000 + 2681.893 0.00000000 + 2193.544 0.00000000 + 1758.738 0.00000000 + 1372.014 0.00000000 + 1028.378 0.00000000 + 723.285 0.00000000 + 452.613 0.00000000 + 212.637 0.00000000 + 0.000 0.00000000 diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 9c5b995239..f85eefc615 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -537,7 +537,7 @@ settings=" 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, 'orog_dir_target_grid': ${FIXlam}, 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, - 'vcoord_file_target_grid': ${FIXam}/global_hyblev.l65.txt, + 'vcoord_file_target_grid': ${VCOORD_FILE}, 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file}, 'data_dir_input_grid': ${extrn_mdl_staging_dir}, 'atm_files_input_grid': ${fn_atm}, diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 34707686fe..f8071201f5 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -457,7 +457,7 @@ settings=" 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, 'orog_dir_target_grid': ${FIXlam}, 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, - 'vcoord_file_target_grid': ${FIXam}/global_hyblev.l65.txt, + 'vcoord_file_target_grid': ${VCOORD_FILE}, 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file}, 'data_dir_input_grid': ${extrn_mdl_staging_dir}, 'atm_files_input_grid': ${fn_atm}, diff --git a/tests/WE2E/test_configs/data_assimilation/config.process_obs.yaml b/tests/WE2E/test_configs/data_assimilation/config.process_obs.yaml index aca3db917c..c35d95fa09 100644 --- a/tests/WE2E/test_configs/data_assimilation/config.process_obs.yaml +++ b/tests/WE2E/test_configs/data_assimilation/config.process_obs.yaml @@ -10,8 +10,10 @@ workflow: DATE_FIRST_CYCL: '2022072000' DATE_LAST_CYCL: '2022072000' PREEXISTING_DIR_METHOD: rename -task_process_radarref: - RADARREFL_TIMELEVEL: [0] +task_make_ics: + VCOORD_FILE: "{{ user.PARMdir }}/global_hyblev_fcst_rrfsL65.txt" +task_make_lbcs: + VCOORD_FILE: "{{ user.PARMdir }}/global_hyblev_fcst_rrfsL65.txt" rocoto: entities: START_TIME_NSSLMOSAIC: "00:45:00" diff --git a/tests/WE2E/test_configs/wflow_features/config.different_vcoord.yaml b/tests/WE2E/test_configs/wflow_features/config.different_vcoord.yaml new file mode 100644 index 0000000000..16f2d10e74 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.different_vcoord.yaml @@ -0,0 +1,24 @@ +metadata: + description: |- + This test is to ensure that the workflow running in community mode + completes successfully when the RRFS vertical coordinates are chosen. +user: + RUN_ENVIR: community +workflow: + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: RRFS_CONUS_25km + DATE_FIRST_CYCL: '2019070100' + DATE_LAST_CYCL: '2019070100' + FCST_LEN_HRS: 3 + PREEXISTING_DIR_METHOD: rename +task_get_extrn_ics: + EXTRN_MDL_NAME_ICS: FV3GFS + USE_USER_STAGED_EXTRN_FILES: true +task_get_extrn_lbcs: + EXTRN_MDL_NAME_LBCS: FV3GFS + LBC_SPEC_INTVL_HRS: 3 + USE_USER_STAGED_EXTRN_FILES: true +task_make_ics: + VCOORD_FILE: "{{ user.PARMdir }}/global_hyblev_fcst_rrfsL65.txt" +task_make_lbcs: + VCOORD_FILE: "{{ user.PARMdir }}/global_hyblev_fcst_rrfsL65.txt" diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 0087e85c21..7ea9dd501b 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1721,12 +1721,17 @@ task_make_ics: # FV3-LAM grid. This file will be copied later to a new location and name # changed to fvcom.nc # + # VCOORD_FILE: + # Full path to the file used to set the vertical coordinate in FV3. + # This file should be the same in both make_ics and make_lbcs. + # #------------------------------------------------------------------------ # USE_FVCOM: false FVCOM_WCSTART: "cold" FVCOM_DIR: "" FVCOM_FILE: "fvcom.nc" + VCOORD_FILE: "{{ workflow.FIXam }}/global_hyblev.l65.txt" #---------------------------- # MAKE LBCS config parameters @@ -1735,6 +1740,7 @@ task_make_lbcs: KMP_AFFINITY_MAKE_LBCS: "scatter" OMP_NUM_THREADS_MAKE_LBCS: 1 OMP_STACKSIZE_MAKE_LBCS: "1024m" + VCOORD_FILE: "{{ workflow.FIXam }}/global_hyblev.l65.txt" #---------------------------- # FORECAST config parameters diff --git a/ush/setup.py b/ush/setup.py index 6d387c0374..41b0e0bb92 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -657,6 +657,25 @@ def get_location(xcs, fmt, expt_cfg): {data_key} = \"{basedir}\"''' ) + + # Make sure the vertical coordinate file for both make_lbcs and + # make_ics is the same. + if ics_vcoord := expt_config.get("task_make_ics", {}).get("VCOORD_FILE") != \ + (lbcs_vcoord := expt_config.get("task_make_lbcs", {}).get("VCOORD_FILE")): + raise ValueError( + f""" + The VCOORD_FILE must be set to the same value for both the + make_ics task and the make_lbcs task. They are currently + set to: + + make_ics: + VCOORD_FILE: {ics_vcoord} + + make_lbcs: + VCOORD_FILE: {lbcs_vcoord} + """ + ) + # # ----------------------------------------------------------------------- # From 54bd939bf34c04697e11e7bad60f154286bc3e00 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Thu, 15 Jun 2023 09:11:59 -0600 Subject: [PATCH 07/16] [develop] Adding support for FV3_RAP CCPP suite (#811) The release committee has decided on including FV3_RAP as a supported CCPP suite in the SRW for the next release. This PR introduces the necessary changes to the workflow, documentation, and tests to add FV3_RAP as a supported suite in the app. --- docs/UsersGuide/source/Components.rst | 2 +- docs/UsersGuide/source/ConfigWorkflow.rst | 8 +- docs/UsersGuide/source/FAQ.rst | 3 +- docs/UsersGuide/source/LAMGrids.rst | 12 +- docs/UsersGuide/source/Tutorial.rst | 2 +- parm/FV3.input.yml | 25 +- parm/diag_table.FV3_RAP | 380 ++++++++++++++++++ parm/field_table.FV3_RAP | 65 +++ scripts/exregional_make_orog.sh | 2 +- scripts/exregional_run_fcst.sh | 2 +- sorc/CMakeLists.txt | 2 +- tests/WE2E/machine_suites/comprehensive | 6 +- .../machine_suites/comprehensive.cheyenne | 6 +- tests/WE2E/machine_suites/comprehensive.orion | 6 +- .../WE2E/machine_suites/coverage.cheyenne.gnu | 2 +- tests/WE2E/machine_suites/coverage.gaea | 2 +- .../machine_suites/coverage.hera.intel.nco | 2 +- ...3km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml} | 7 +- ...S_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml} | 2 +- ...3km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml} | 2 +- .../verification/config.MET_verification.yaml | 2 +- ush/link_fix.py | 2 +- ush/valid_param_vals.yaml | 3 +- 23 files changed, 510 insertions(+), 35 deletions(-) create mode 100644 parm/diag_table.FV3_RAP create mode 100644 parm/field_table.FV3_RAP rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml => config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml} (86%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml => config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml} (97%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml => config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml} (97%) diff --git a/docs/UsersGuide/source/Components.rst b/docs/UsersGuide/source/Components.rst index 85b72a5289..4bbfb2088d 100644 --- a/docs/UsersGuide/source/Components.rst +++ b/docs/UsersGuide/source/Components.rst @@ -34,7 +34,7 @@ The prognostic atmospheric model in the UFS SRW Application is the Finite-Volume Supported model resolutions in this release include 3-, 13-, and 25-km predefined contiguous U.S. (:term:`CONUS`) domains, each with 127 vertical levels. Preliminary tools for users to define their own domain are also available in the release with full, formal support of these tools to be provided in future releases. The Extended Schmidt Gnomonic (ESG) grid is used with the FV3-LAM, which features relatively uniform grid cells across the entirety of the domain. Additional information about the FV3 dynamical core can be found in the `scientific documentation `__, the `technical documentation `__, and on the `NOAA Geophysical Fluid Dynamics Laboratory website `__. -Interoperable atmospheric physics, along with various land surface model options, are supported through the Common Community Physics Package (CCPP), described `here `__. Atmospheric physics are a set of numerical methods describing small-scale processes such as clouds, turbulence, radiation, and their interactions. There are four physics suites supported as of the SRW App v2.1.0 release. The first is the FV3_RRFS_v1beta physics suite, which is being tested for use in the future operational implementation of the Rapid Refresh Forecast System (:term:`RRFS`) planned for 2023-2024, and the second is an updated version of the physics suite used in the operational Global Forecast System (GFS) v16. Additionally, FV3_WoFS_v0 and FV3_HRRR are supported. A detailed list of CCPP updates since the SRW App v2.0.0 release is available :ref:`here `. A full scientific description of CCPP parameterizations and suites can be found in the `CCPP Scientific Documentation `__, and CCPP technical aspects are described in the `CCPP Technical Documentation `__. The model namelist has many settings beyond the physics options that can optimize various aspects of the model for use with each of the supported suites. Additional information on Stochastic Physics options is available `here `__. +Interoperable atmospheric physics, along with various land surface model options, are supported through the Common Community Physics Package (CCPP), described `here `__. Atmospheric physics are a set of numerical methods describing small-scale processes such as clouds, turbulence, radiation, and their interactions. There are four physics suites supported as of the SRW App v2.1.0 release. The first is the FV3_RRFS_v1beta physics suite, which is being tested for use in the future operational implementation of the Rapid Refresh Forecast System (:term:`RRFS`) planned for 2023-2024, and the second is an updated version of the physics suite used in the operational Global Forecast System (GFS) v16. Additionally, FV3_WoFS_v0, FV3_HRRR, and FV3_RAP are supported. A detailed list of CCPP updates since the SRW App v2.0.0 release is available :ref:`here `. A full scientific description of CCPP parameterizations and suites can be found in the `CCPP Scientific Documentation `__, and CCPP technical aspects are described in the `CCPP Technical Documentation `__. The model namelist has many settings beyond the physics options that can optimize various aspects of the model for use with each of the supported suites. Additional information on Stochastic Physics options is available `here `__. .. note:: SPP is currently only available for specific physics schemes used in the RAP/HRRR physics suite. Users need to be aware of which physics suite definition file (:term:`SDF`) is chosen when turning this option on. Among the supported physics suites, the full set of parameterizations can only be used with the ``FV3_HRRR`` option for ``CCPP_PHYS_SUITE``. diff --git a/docs/UsersGuide/source/ConfigWorkflow.rst b/docs/UsersGuide/source/ConfigWorkflow.rst index 4610c18f32..fc7b5e1391 100644 --- a/docs/UsersGuide/source/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/ConfigWorkflow.rst @@ -310,13 +310,9 @@ CCPP Parameter | ``"FV3_RRFS_v1beta"`` | ``"FV3_HRRR"`` | ``"FV3_WoFS_v0"`` + | ``"FV3_RAP"`` - **Other valid values include:** - - | ``"FV3_GFS_2017_gfdlmp"`` - | ``"FV3_GFS_2017_gfdlmp_regional"`` - | ``"FV3_GFS_v15p2"`` - | ``"FV3_GFS_v15_thompson_mynn_lam3km"`` + Other valid values can be found in the ``ush/valid_param_vals.yaml`` file, but users can not expect full support for these schemes. .. _GridGen: diff --git a/docs/UsersGuide/source/FAQ.rst b/docs/UsersGuide/source/FAQ.rst index 21bb0cd5d0..35d60797ec 100644 --- a/docs/UsersGuide/source/FAQ.rst +++ b/docs/UsersGuide/source/FAQ.rst @@ -27,7 +27,7 @@ See :numref:`Section %s ` and/or :numref:`Section %s `. diff --git a/docs/UsersGuide/source/LAMGrids.rst b/docs/UsersGuide/source/LAMGrids.rst index 2a8854f933..4c69ec0e21 100644 --- a/docs/UsersGuide/source/LAMGrids.rst +++ b/docs/UsersGuide/source/LAMGrids.rst @@ -17,7 +17,7 @@ The SRW App v2.1.0 release includes four predefined limited area model (:term:`L * ``RRFS_CONUS_25km`` * ``SUBCONUS_Ind_3km`` -These four options are provided for flexibility related to compute resources and supported physics options. Other predefined grids are listed :ref:`here `. The high-resolution 3-km :term:`CONUS` grid generally requires more compute power and works well with three of the four supported physics suites (see :numref:`Table %s `). Low-resolution grids (i.e., 13-km and 25-km domains) require less compute power and should generally be used with the fourth supported physics suite: ``FV3_GFS_v16``. +These four options are provided for flexibility related to compute resources and supported physics options. Other predefined grids are listed :ref:`here `. The high-resolution 3-km :term:`CONUS` grid generally requires more compute power and works well with three of the five supported physics suites (see :numref:`Table %s `). Low-resolution grids (i.e., 13-km and 25-km domains) require less compute power and should generally be used with the other supported physics suites: ``FV3_GFS_v16`` and ``FV3_RAP``. .. _GridPhysicsCombos: @@ -39,8 +39,12 @@ These four options are provided for flexibility related to compute resources and | | FV3_WoFS | +-------------------+------------------+ | RRFS_CONUS_13km | FV3_GFS_v16 | + | | | + | | FV3_RAP | +-------------------+------------------+ | RRFS_CONUS_25km | FV3_GFS_v16 | + | | | + | | FV3_RAP | +-------------------+------------------+ In theory, it is possible to run any of the supported physics suites with any of the predefined grids, but the results will be more accurate and meaningful with appropriate grid/physics pairings. @@ -50,7 +54,7 @@ The predefined :term:`CONUS` grids follow the naming convention (e.g., RRFS_CONU Predefined 3-km CONUS Grid ----------------------------- -The 3-km CONUS domain is ideal for running the ``FV3_RRFS_v1beta`` physics suite, since this suite definition file (:term:`SDF`) was specifically created for convection-allowing scales and is the precursor to the operational physics suite that will be used in the RRFS. The 3-km domain can also be used with the ``FV3_HRRR`` and ``FV3_WoFS`` physics suites, which likewise do not include convective parameterizations. In fact, the ``FV3_WoFS`` physics suite is configured to run at 3-km *or less* and could therefore run with even higher-resolution user-defined domains if desired. However, the ``FV3_GFS_v16`` suite generally should *not* be used with the 3-km domain because the cumulus physics used in that physics suite is not configured to run at the 3-km resolution. +The 3-km CONUS domain is ideal for running the ``FV3_RRFS_v1beta`` physics suite, since this suite definition file (:term:`SDF`) was specifically created for convection-allowing scales and is the precursor to the operational physics suite that will be used in the RRFS. The 3-km domain can also be used with the ``FV3_HRRR`` and ``FV3_WoFS`` physics suites, which likewise do not include convective parameterizations. In fact, the ``FV3_WoFS`` physics suite is configured to run at 3-km *or less* and could therefore run with even higher-resolution user-defined domains if desired. However, the ``FV3_GFS_v16`` and ``FV3_RAP`` suites generally should *not* be used with the 3-km domain because the cumulus physics used in those physics suites is not configured to run at the 3-km resolution. .. _RRFS_CONUS_3km: @@ -87,7 +91,7 @@ Predefined 13-km Grid *The boundary of the RRFS_CONUS_13km computational grid (red) and corresponding write-component grid (blue).* -The ``RRFS_CONUS_13km`` grid (:numref:`Fig. %s `) covers the full :term:`CONUS`. This grid is meant to be run with the ``FV3_GFS_v16`` physics suite. The ``FV3_GFS_v16`` physics suite uses convective :term:`parameterizations`, whereas the other supported suites do not. Convective parameterizations are necessary for low-resolution grids because convection occurs on scales smaller than 25-km and 13-km. +The ``RRFS_CONUS_13km`` grid (:numref:`Fig. %s `) covers the full :term:`CONUS`. This grid is meant to be run with the ``FV3_GFS_v16`` or ``FV3_RAP`` physics suites. These suites use convective :term:`parameterizations`, whereas the other supported suites do not. Convective parameterizations are necessary for low-resolution grids because convection occurs on scales smaller than 25-km and 13-km. Predefined 25-km Grid ------------------------ @@ -197,4 +201,4 @@ The following is an example of a code stanza for "NEW_GRID" to be added to ``pre WRTCMP_dy: 25000.0 .. note:: - The process above explains how to create a new *predefined* grid, which can be used more than once. If a user prefers to create a custom grid for one-time use, the variables above can instead be specified in ``config.yaml``, and ``PREDEF_GRID_NAME`` can be set to a null string. In this case, it is not necessary to modify ``valid_param_vals.yaml`` or ``predef_grid_params.yaml``. Users can view an example configuration file for a custom grid `here `__. \ No newline at end of file + The process above explains how to create a new *predefined* grid, which can be used more than once. If a user prefers to create a custom grid for one-time use, the variables above can instead be specified in ``config.yaml``, and ``PREDEF_GRID_NAME`` can be set to a null string. In this case, it is not necessary to modify ``valid_param_vals.yaml`` or ``predef_grid_params.yaml``. Users can view an example configuration file for a custom grid `here `__. diff --git a/docs/UsersGuide/source/Tutorial.rst b/docs/UsersGuide/source/Tutorial.rst index dafcae7025..f96e39ef52 100644 --- a/docs/UsersGuide/source/Tutorial.rst +++ b/docs/UsersGuide/source/Tutorial.rst @@ -484,7 +484,7 @@ Option 2: Compare output from additional physics suites. Users are encouraged to conduct additional experiments using the FV3_HRRR and FV3_WoFS_v0 physics suites. Like FV3_RRFS_v1beta, these physics suites were designed for use with high-resolution grids for storm-scale predictions. Compare them to each other or to the control! -Users may find the difference plots for :term:`updraft helicity` particularly informative. The FV3_GFS_v16 physics suite does not contain updraft helicity output in its ``diag_table`` files, so the difference plot generated in this tutorial is empty. However, high updraft helicity values increase the probability that a storm will become a supercell thunderstorm or a tornado. Comparing the results from two physics suites that measure this parameter can therefore prove insightful. +Users may find the difference plots for :term:`updraft helicity` particularly informative. The FV3_GFS_v16 physics suite does not contain updraft helicity output in its ``diag_table`` files, so the difference plot generated in this tutorial is empty. Observing high values for updraft helicity indicates the presence of a rotating updraft, often the result of a supercell thunderstorm capable of severe weather, including tornadoes. Comparing the results from two physics suites that measure this parameter can therefore prove insightful. .. _fcst2: diff --git a/parm/FV3.input.yml b/parm/FV3.input.yml index 851918e23d..0c50793e8a 100644 --- a/parm/FV3.input.yml +++ b/parm/FV3.input.yml @@ -65,7 +65,7 @@ FV3_WoFS_v0: do_hailcast: True FV3_HRRR: - fv_core_nml: + fv_core_nml: &HRRR_fv_core hord_dp: -5 hord_mt: 5 hord_tm: 5 @@ -92,6 +92,29 @@ FV3_HRRR: lsoil_lsm: 9 sfclay_compute_flux: True +FV3_RAP: + fv_core_nml: + <<: *HRRR_fv_core + gfs_physics_nml: + <<: *RRFS_v1beta_phys + cdmbgwd: [3.5, 1.0] + do_mynnsfclay: True + do_sfcperts: !!python/none + gwd_opt: 3 + do_gsl_drag_ss: True + do_gsl_drag_tofd: True + do_gsl_drag_ls_bl: True + iaer: 5111 + icliq_sw: 2 + iovr: 3 + lsm: 3 + lsoil_lsm: 9 + sfclay_compute_flux: False + do_deep: True + shal_cnv: True + imfdeepcnv: 3 + imfshalcnv: 3 + FV3_GFS_2017_gfdlmp: atmos_model_nml: avg_max_length: 3600.0 diff --git a/parm/diag_table.FV3_RAP b/parm/diag_table.FV3_RAP new file mode 100644 index 0000000000..d87a18720f --- /dev/null +++ b/parm/diag_table.FV3_RAP @@ -0,0 +1,380 @@ +{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional +{{ starttime.strftime("%Y %m %d %H %M %S") }} + +"grid_spec", -1, "months", 1, "days", "time" +"atmos_4xdaily", 6, "hours", 1, "days", "time" +"atmos_static", -1, "hours", 1, "hours", "time" +"fv3_history", 0, "hours", 1, "hours", "time" +"fv3_history2d", 0, "hours", 1, "hours", "time" + +# +#======================= +# ATMOSPHERE DIAGNOSTICS +#======================= +### +# grid_spec +### + "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, + "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, +### +# 4x daily output +### + "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 +#### + "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 +#### +#"dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 +#### + "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 +#### + "dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 +### +# gfs static data +### + "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 + "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 + "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 + "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 + "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 +### +# FV3 variabls needed for NGGPS evaluation +### +"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 + +"gfs_phys", "frzr", "frzr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frzrb", "frzrb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frozr", "frozr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frozrb", "frozrb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tsnowp", "tsnowp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tsnowpb", "tsnowpb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "rhonewsn", "rhonewsn", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 +#"gfs_phys", "refl_10cm", "refl10cm", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt5", "soilt5" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt6", "soilt6" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt7", "soilt7" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt8", "soilt8" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt9", "soilt9" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw5", "soilw5" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw6", "soilw6" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw7", "soilw7" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw8", "soilw8" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw9", "soilw9" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_5", "soill5", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_6", "soill6", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_7", "soill7", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_8", "soill8", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_9", "soill9", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 +# Aerosols (CCN, IN) from Thompson microphysics +"gfs_dyn", "nwfa", "nwfa", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "nifa", "nifa", "fv3_history", "all", .false., "none", 2 +"gfs_sfc", "nwfa2d", "nwfa2d", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "nifa2d", "nifa2d", "fv3_history2d", "all", .false., "none", 2 +# Cloud effective radii from Thompson and WSM6 microphysics +"gfs_phys", "cleffr", "cleffr", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cieffr", "cieffr", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cseffr", "cseffr", "fv3_history", "all", .false., "none", 2 +# Prognostic/diagnostic variables from MYNN +"gfs_dyn", "QC_BL", "qc_bl", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "CLDFRA_BL", "cldfra_bl", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "EL_PBL", "el_pbl", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "QKE", "qke", "fv3_history", "all", .false., "none", 2 +"gfs_sfc", "maxmf", "maxmf", "fv3_history2d", "all", .false., "none", 2 +#"gfs_sfc", "nupdraft", "nupdrafts", "fv3_history2d", "all", .false., "none", 2 +#"gfs_sfc", "ktop_shallow", "ktop_shallow", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zol", "zol", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "flhc", "flhc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "flqc", "flqc", "fv3_history2d", "all", .false., "none", 2 +# Prognostic/diagnostic variables from RUC LSM +"gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 + +# Diagnostic variables for Unified Gravity Wave Physics (UGWP) +"gfs_phys", "dws3dt_ogw", "dws3dt_ogw", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dws3dt_obl", "dws3dt_obl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dws3dt_oss", "dws3dt_oss", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dws3dt_ofd", "dws3dt_ofd", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ldu3dt_ogw", "ldu3dt_ogw", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ldu3dt_obl", "ldu3dt_obl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ldu3dt_oss", "ldu3dt_oss", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ldu3dt_ofd", "ldu3dt_ofd", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ldu3dt_ngw", "ldu3dt_ngw", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ldv3dt_ngw", "ldv3dt_ngw", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ldt3dt_ngw", "ldt3dt_ngw", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "dudt_ogw", "dudt_ogw", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvdt_ogw", "dvdt_ogw", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dudt_obl", "dudt_obl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvdt_obl", "dvdt_obl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dudt_oss", "dudt_oss", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvdt_oss", "dvdt_oss", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dudt_ofd", "dudt_ofd", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvdt_ofd", "dvdt_ofd", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "du_ogwcol", "du_ogwcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dv_ogwcol", "dv_ogwcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "du_oblcol", "du_oblcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dv_oblcol", "dv_oblcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "du_osscol", "du_osscol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dv_osscol", "dv_osscol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "du_ofdcol", "du_ofdcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dv_ofdcol", "dv_ofdcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "du3_ogwcol", "du3_ogwcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dv3_ogwcol", "dv3_ogwcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "du3_oblcol", "du3_oblcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dv3_oblcol", "dv3_oblcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "du3_osscol", "du3_osscol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dv3_osscol", "dv3_osscol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "du3_ofdcol", "du3_ofdcol", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dv3_ofdcol", "dv3_ofdcol", "fv3_history2d", "all", .false., "none", 2 + +#============================================================================================= +# +#====> This file can be used with diag_manager/v2.0a (or higher) <==== +# +# +# FORMATS FOR FILE ENTRIES (not all input values are used) +# ------------------------ +# +#"file_name", output_freq, "output_units", format, "time_units", "long_name", +# +# +#output_freq: > 0 output frequency in "output_units" +# = 0 output frequency every time step +# =-1 output frequency at end of run +# +#output_units = units used for output frequency +# (years, months, days, minutes, hours, seconds) +# +#time_units = units used to label the time axis +# (days, minutes, hours, seconds) +# +# +# FORMAT FOR FIELD ENTRIES (not all input values are used) +# ------------------------ +# +#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing +# +#time_avg = .true. or .false. +# +#packing = 1 double precision +# = 2 float +# = 4 packed 16-bit integers +# = 8 packed 1-byte (not tested?) diff --git a/parm/field_table.FV3_RAP b/parm/field_table.FV3_RAP new file mode 100644 index 0000000000..fbfb0c3399 --- /dev/null +++ b/parm/field_table.FV3_RAP @@ -0,0 +1,65 @@ +# added by FRE: sphum must be present in atmos +# specific humidity for moist runs + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud water mixing ratio + "TRACER", "atmos_mod", "liq_wat" + "longname", "cloud water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic ice water mixing ratio + "TRACER", "atmos_mod", "ice_wat" + "longname", "cloud ice mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic rain water mixing ratio + "TRACER", "atmos_mod", "rainwat" + "longname", "rain water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic snow water mixing ratio + "TRACER", "atmos_mod", "snowwat" + "longname", "snow water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic graupel mixing ratio + "TRACER", "atmos_mod", "graupel" + "longname", "graupel mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud water number concentration + "TRACER", "atmos_mod", "water_nc" + "longname", "cloud liquid water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic cloud ice number concentration + "TRACER", "atmos_mod", "ice_nc" + "longname", "cloud ice water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic rain number concentration + "TRACER", "atmos_mod", "rain_nc" + "longname", "rain number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic ozone mixing ratio tracer + "TRACER", "atmos_mod", "o3mr" + "longname", "ozone mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# water- and ice-friendly aerosols (Thompson) + "TRACER", "atmos_mod", "liq_aero" + "longname", "water-friendly aerosol number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / + "TRACER", "atmos_mod", "ice_aero" + "longname", "ice-friendly aerosol number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic subgrid scale turbulent kinetic energy + "TRACER", "atmos_mod", "sgs_tke" + "longname", "subgrid scale turbulent kinetic energy" + "units", "m2/s2" + "profile_type", "fixed", "surface_value=0.0" / diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 4782ff0001..f595d405a0 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -244,7 +244,7 @@ mv_vrfy "${raw_orog_fp_orig}" "${raw_orog_fp}" # #----------------------------------------------------------------------- # -suites=( "FV3_HRRR" "FV3_GFS_v15_thompson_mynn_lam3km" "FV3_GFS_v17_p8" ) +suites=( "FV3_RAP" "FV3_HRRR" "FV3_GFS_v15_thompson_mynn_lam3km" "FV3_GFS_v17_p8" ) if [[ ${suites[@]} =~ "${CCPP_PHYS_SUITE}" ]] ; then DATA="${DATA:-${OROG_DIR}/temp_orog_data}" mkdir_vrfy -p ${DATA} diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 0728f3de5a..2f4efc4a44 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -195,7 +195,7 @@ create_symlink_to_file target="$target" symlink="$symlink" \ # that the FV3 model is hardcoded to recognize, and those are the names # we use below. # -suites=( "FV3_HRRR" "FV3_GFS_v15_thompson_mynn_lam3km" "FV3_GFS_v17_p8" ) +suites=( "FV3_RAP" "FV3_HRRR" "FV3_GFS_v15_thompson_mynn_lam3km" "FV3_GFS_v17_p8" ) if [[ ${suites[@]} =~ "${CCPP_PHYS_SUITE}" ]] ; then file_ids=( "ss" "ls" ) for file_id in "${file_ids[@]}"; do diff --git a/sorc/CMakeLists.txt b/sorc/CMakeLists.txt index 4d2bb4d872..e8326b2f90 100644 --- a/sorc/CMakeLists.txt +++ b/sorc/CMakeLists.txt @@ -34,7 +34,7 @@ if (BUILD_UFS) if(CPL_AQM) set(CCPP_SUITES "FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v17_p8") else() - set(CCPP_SUITES "FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_regional,FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v17_p8,FV3_RRFS_v1beta,FV3_HRRR,FV3_GFS_v15_thompson_mynn_lam3km,FV3_WoFS_v0") + set(CCPP_SUITES "FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_regional,FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v17_p8,FV3_RRFS_v1beta,FV3_HRRR,FV3_RAP,FV3_GFS_v15_thompson_mynn_lam3km,FV3_WoFS_v0") endif() endif() diff --git a/tests/WE2E/machine_suites/comprehensive b/tests/WE2E/machine_suites/comprehensive index ce5baaff91..dfbac222f2 100644 --- a/tests/WE2E/machine_suites/comprehensive +++ b/tests/WE2E/machine_suites/comprehensive @@ -17,7 +17,7 @@ grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta @@ -28,7 +28,7 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 @@ -47,7 +47,7 @@ grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0 grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 diff --git a/tests/WE2E/machine_suites/comprehensive.cheyenne b/tests/WE2E/machine_suites/comprehensive.cheyenne index 35cc907de8..fa893b5298 100644 --- a/tests/WE2E/machine_suites/comprehensive.cheyenne +++ b/tests/WE2E/machine_suites/comprehensive.cheyenne @@ -7,7 +7,7 @@ grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta @@ -18,7 +18,7 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 @@ -37,7 +37,7 @@ grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0 grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 diff --git a/tests/WE2E/machine_suites/comprehensive.orion b/tests/WE2E/machine_suites/comprehensive.orion index 5d4b3c80f1..5efd810669 100644 --- a/tests/WE2E/machine_suites/comprehensive.orion +++ b/tests/WE2E/machine_suites/comprehensive.orion @@ -9,7 +9,7 @@ grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta @@ -20,7 +20,7 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 @@ -39,7 +39,7 @@ grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0 grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 diff --git a/tests/WE2E/machine_suites/coverage.cheyenne.gnu b/tests/WE2E/machine_suites/coverage.cheyenne.gnu index 7035bd4ca6..79927f7e81 100644 --- a/tests/WE2E/machine_suites/coverage.cheyenne.gnu +++ b/tests/WE2E/machine_suites/coverage.cheyenne.gnu @@ -4,6 +4,6 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS diff --git a/tests/WE2E/machine_suites/coverage.gaea b/tests/WE2E/machine_suites/coverage.gaea index 390bdc038b..25194b2c7d 100644 --- a/tests/WE2E/machine_suites/coverage.gaea +++ b/tests/WE2E/machine_suites/coverage.gaea @@ -1,6 +1,6 @@ community grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR diff --git a/tests/WE2E/machine_suites/coverage.hera.intel.nco b/tests/WE2E/machine_suites/coverage.hera.intel.nco index 8526a939f1..130d68e81c 100644 --- a/tests/WE2E/machine_suites/coverage.hera.intel.nco +++ b/tests/WE2E/machine_suites/coverage.hera.intel.nco @@ -4,7 +4,7 @@ get_from_HPSS_ics_HRRR_lbcs_RAP grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml similarity index 86% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml index e034ccb453..5a01ff753a 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml @@ -6,7 +6,7 @@ metadata: user: RUN_ENVIR: community workflow: - CCPP_PHYS_SUITE: FV3_GFS_v15p2 + CCPP_PHYS_SUITE: FV3_RAP PREDEF_GRID_NAME: RRFS_CONUS_13km DATE_FIRST_CYCL: '2019070100' DATE_LAST_CYCL: '2019070100' @@ -19,3 +19,8 @@ task_get_extrn_lbcs: EXTRN_MDL_NAME_LBCS: FV3GFS LBC_SPEC_INTVL_HRS: 3 USE_USER_STAGED_EXTRN_FILES: true +global: + DO_SHUM: true + DO_SPPT: true + DO_SKEB: true + DO_SPP: true diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml similarity index 97% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml index 41da098f00..6082ecbd20 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml @@ -9,7 +9,7 @@ metadata: user: RUN_ENVIR: community workflow: - CCPP_PHYS_SUITE: FV3_HRRR + CCPP_PHYS_SUITE: FV3_RAP PREDEF_GRID_NAME: RRFS_CONUS_25km DATE_FIRST_CYCL: '2019061518' DATE_LAST_CYCL: '2019061518' diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml similarity index 97% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml index fa3c5ff504..0a52ec4b34 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP.yaml @@ -9,7 +9,7 @@ metadata: user: RUN_ENVIR: community workflow: - CCPP_PHYS_SUITE: FV3_RRFS_v1beta + CCPP_PHYS_SUITE: FV3_RAP PREDEF_GRID_NAME: RRFS_NA_13km DATE_FIRST_CYCL: '2019070100' DATE_LAST_CYCL: '2019070100' diff --git a/tests/WE2E/test_configs/verification/config.MET_verification.yaml b/tests/WE2E/test_configs/verification/config.MET_verification.yaml index 70f8754fb1..2de60fce22 120000 --- a/tests/WE2E/test_configs/verification/config.MET_verification.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification.yaml @@ -1 +1 @@ -../grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml \ No newline at end of file +../grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml \ No newline at end of file diff --git a/ush/link_fix.py b/ush/link_fix.py index 99156019bd..fdd9a65f28 100755 --- a/ush/link_fix.py +++ b/ush/link_fix.py @@ -207,7 +207,7 @@ def link_fix( f"C*{dot_or_uscore}oro_data.tile{tile_rgnl}.halo{nh0}.nc", f"C*{dot_or_uscore}oro_data.tile{tile_rgnl}.halo{nh4}.nc", ] - if ccpp_phys_suite == "FV3_HRRR" or ccpp_phys_suite == "FV3_GFS_v15_thompson_mynn_lam3km" or ccpp_phys_suite == "FV3_GFS_v17_p8": + if ccpp_phys_suite == "FV3_RAP" or ccpp_phys_suite == "FV3_HRRR" or ccpp_phys_suite == "FV3_GFS_v15_thompson_mynn_lam3km" or ccpp_phys_suite == "FV3_GFS_v17_p8": fns += [ f"C*{dot_or_uscore}oro_data_ss.tile{tile_rgnl}.halo{nh0}.nc", f"C*{dot_or_uscore}oro_data_ls.tile{tile_rgnl}.halo{nh0}.nc", diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index f839dcb858..0c7b313325 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -43,7 +43,8 @@ valid_vals_CCPP_PHYS_SUITE: [ "FV3_GFS_v17_p8", "FV3_RRFS_v1beta", "FV3_WoFS_v0", -"FV3_HRRR" +"FV3_HRRR", +"FV3_RAP" ] valid_vals_GFDLgrid_NUM_CELLS: [48, 96, 192, 384, 768, 1152, 3072] valid_vals_EXTRN_MDL_NAME_ICS: ["GSMGFS", "FV3GFS", "GEFS", "GDAS", "RAP", "HRRR", "NAM"] From 4932f02cc74f92c12a67f555a407c7bea2d45df0 Mon Sep 17 00:00:00 2001 From: Yi-Cheng Teng - NOAA <30629225+clouden90@users.noreply.github.com> Date: Wed, 21 Jun 2023 11:34:57 -0400 Subject: [PATCH 08/16] [develop] Introduce test cases from ufs-case-studies platform thru WE2E (#822) The UFS Case Studies Platform provides a set of cases that reveal the forecast challenges of NOAA's operational Global Forecast System (GFS). Here we introduce one of these cases, the 2020 Cold Air Damming, into the UFS SRW App through the WE2E testing framework. A yaml config file is added and moderate modifications are done for exregional_get_extrn_mdl_files.sh. This new function allows users to run any test cases from UFS Case Studies Platform directly through the WE2E framework without need of additional steps (e.g. download ICS/LBCS data from platform first). Users can still modify the yaml file to suit their needs (e.g. increase fcst time, play with different grid resolution or CCPP suite). Additionally, we added CCPP-SCM user and technical guide as a reference in Section 8.2 for users who are interested in running single column model. --- parm/wflow/coldstart.yaml | 2 + scripts/exregional_get_extrn_mdl_files.sh | 30 ++++++++++++- scripts/exregional_make_ics.sh | 21 ++++++++++ scripts/exregional_make_lbcs.sh | 18 ++++++++ tests/WE2E/machine_suites/comprehensive.orion | 1 + tests/WE2E/machine_suites/coverage.orion | 1 + .../ufs_case_studies/config.2020_CAD.yaml | 42 +++++++++++++++++++ ush/machine/gaea.yaml | 1 + ush/valid_param_vals.yaml | 4 +- 9 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index c64b62bbe3..c1935f2160 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -28,6 +28,7 @@ task_get_extrn_ics: join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' memory: 2G nnodes: 1 + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_HPSS;' @@ -55,6 +56,7 @@ task_get_extrn_lbcs: join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' memory: 2G nnodes: 1 + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_HPSS;' diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index e7004f7e2a..5b138eb298 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -62,7 +62,8 @@ if [ "${ICS_OR_LBCS}" = "ICS" ]; then fi fcst_hrs=${TIME_OFFSET_HRS} file_names=${EXTRN_MDL_FILES_ICS[@]} - if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] ; then + if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] \ + || [ ${EXTRN_MDL_NAME} == "UFS-CASE-STUDY" ] ; then file_fmt=$FV3GFS_FILE_FMT_ICS fi input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_ICS:-$EXTRN_MDL_SYSBASEDIR_ICS} @@ -91,7 +92,8 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then fcst_hrs="${first_time} ${last_time} ${LBC_SPEC_INTVL_HRS}" file_names=${EXTRN_MDL_FILES_LBCS[@]} - if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] ; then + if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] \ + || [ ${EXTRN_MDL_NAME} == "UFS-CASE-STUDY" ] ; then file_fmt=$FV3GFS_FILE_FMT_LBCS fi input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} @@ -272,6 +274,30 @@ fi # #----------------------------------------------------------------------- # +# unzip UFS-CASE-STUDY ICS/LBCS files +# +#----------------------------------------------------------------------- +# +if [ "${EXTRN_MDL_NAME}" = "UFS-CASE-STUDY" ]; then + # Look for filenames, if they exist, unzip them + base_path="${EXTRN_MDL_STAGING_DIR}${mem_dir}" + for filename in ${base_path}/*.tar.gz; do + printf "unzip file: ${filename}\n" + tar -zxvf ${filename} --directory ${base_path} + done + # check file naming issue + for filename in ${base_path}/*.nemsio; do + filename=$(basename -- "${filename}") + len=`echo $filename | wc -c` + if [ "${filename:4:4}" != "t${hh}z" ]; then + printf "rename ${filename} to ${filename:0:4}t${hh}z.${filename:4:${len}} \n" + mv ${base_path}/${filename} ${base_path}/${filename:0:4}t${hh}z.${filename:4:${len}} + fi + done +fi +# +#----------------------------------------------------------------------- +# # Restore the shell options saved at the beginning of this script/function. # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index f85eefc615..a110cd1d2c 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -131,6 +131,7 @@ case "${CCPP_PHYS_SUITE}" in varmap_file="GSDphys_var_map.txt" elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ + [ "${EXTRN_MDL_NAME_ICS}" = "UFS-CASE-STUDY" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "GEFS" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "GDAS" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then @@ -391,6 +392,26 @@ case "${EXTRN_MDL_NAME_ICS}" in tg3_from_soil=False ;; +"UFS-CASE-STUDY") + hh="${EXTRN_MDL_CDATE:8:2}" + if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then + external_model="UFS-CASE-STUDY" + input_type="gaussian_nemsio" + tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" + tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" + fn_atm="gfs.t${hh}z.atmanl.nemsio" + fn_sfc="gfs.t${hh}z.sfcanl.nemsio" + convert_nst=True + fi + vgtyp_from_climo=True + sotyp_from_climo=True + vgfrc_from_climo=True + minmax_vgfrc_from_climo=True + lai_from_climo=True + tg3_from_soil=False + unset hh + ;; + "GDAS") if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then input_type="gaussian_nemsio" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index f8071201f5..3dad9bd0be 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -129,6 +129,7 @@ case "${CCPP_PHYS_SUITE}" in varmap_file="GSDphys_var_map.txt" elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ + [ "${EXTRN_MDL_NAME_LBCS}" = "UFS-CASE-STUDY" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "GEFS" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "GDAS" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then @@ -293,6 +294,15 @@ case "${EXTRN_MDL_NAME_LBCS}" in fi ;; +"UFS-CASE-STUDY") + if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then + external_model="UFS-CASE-STUDY" + input_type="gaussian_nemsio" # For FV3GFS data on a Gaussian grid in nemsio format. + tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" + tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" + fi + ;; + "GDAS") if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then input_type="gaussian_nemsio" @@ -389,6 +399,14 @@ for (( ii=0; ii<${num_fhrs}; ii=ii+bcgrpnum10 )); do fn_atm="${EXTRN_MDL_FNS[$i]}" fi ;; + "UFS-CASE-STUDY") + if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then + hh="${EXTRN_MDL_CDATE:8:2}" + fhr_str=$(printf "%03d" ${fhr}) + fn_atm="gfs.t${hh}z.atmf${fhr_str}.nemsio" + unset hh fhr_str + fi + ;; "GDAS") fn_atm="${EXTRN_MDL_FNS[0][$i]}" ;; diff --git a/tests/WE2E/machine_suites/comprehensive.orion b/tests/WE2E/machine_suites/comprehensive.orion index 5efd810669..a547d0b8b8 100644 --- a/tests/WE2E/machine_suites/comprehensive.orion +++ b/tests/WE2E/machine_suites/comprehensive.orion @@ -56,3 +56,4 @@ nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR pregen_grid_orog_sfc_climo specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS specify_template_filenames +2020_CAD diff --git a/tests/WE2E/machine_suites/coverage.orion b/tests/WE2E/machine_suites/coverage.orion index 9771db8084..8364b4ddfc 100644 --- a/tests/WE2E/machine_suites/coverage.orion +++ b/tests/WE2E/machine_suites/coverage.orion @@ -9,3 +9,4 @@ grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0 nco +2020_CAD diff --git a/tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml b/tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml new file mode 100644 index 0000000000..fc50771ca6 --- /dev/null +++ b/tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml @@ -0,0 +1,42 @@ +metadata: + description: |- + This test is to ensure that the workflow running in community mode + completes successfully on the RRFS_CONUS_13km grid using the GFS_v16 + physics suite with ICs and LBCs derived from the UFS-CASE-STUDIES platforms for 2020 Cold Air Damming case. + #Note for runnning this test on Cheyenne: please modify this config as follows + #task_get_extrn_ics: + # EXTRN_MDL_NAME_ICS: FV3GFS + # FV3GFS_FILE_FMT_ICS: nemsio + # USE_USER_STAGED_EXTRN_FILES: true + #task_get_extrn_lbcs: + # EXTRN_MDL_NAME_LBCS: FV3GFS + # LBC_SPEC_INTVL_HRS: 3 + # FV3GFS_FILE_FMT_LBCS: nemsio + # USE_USER_STAGED_EXTRN_FILES: true +user: + RUN_ENVIR: community +platform: + EXTRN_MDL_DATA_STORES: aws +workflow: + CCPP_PHYS_SUITE: FV3_GFS_v16 + PREDEF_GRID_NAME: RRFS_CONUS_13km + DATE_FIRST_CYCL: '2020020312' + DATE_LAST_CYCL: '2020020312' + FCST_LEN_HRS: 6 + PREEXISTING_DIR_METHOD: rename +task_get_extrn_ics: + EXTRN_MDL_NAME_ICS: UFS-CASE-STUDY + FV3GFS_FILE_FMT_ICS: nemsio +task_get_extrn_lbcs: + EXTRN_MDL_NAME_LBCS: UFS-CASE-STUDY + LBC_SPEC_INTVL_HRS: 3 + FV3GFS_FILE_FMT_LBCS: nemsio +rocoto: + tasks: + task_get_extrn_ics: + walltime: 06:00:00 + task_get_extrn_lbcs: + walltime: 06:00:00 + metatask_run_ensemble: + task_make_lbcs_mem#mem#: + walltime: 06:00:00 diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 9653d74c45..0daa1ead20 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -19,6 +19,7 @@ platform: RUN_CMD_SERIAL: time RUN_CMD_UTILS: srun --export=ALL --mpi=pmi2 -n $nprocs SCHED_NATIVE_CMD: -M c3 --export=NONE + SCHED_NATIVE_CMD_HPSS: -M es --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data TEST_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index 0c7b313325..83563c0f62 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -47,8 +47,8 @@ valid_vals_CCPP_PHYS_SUITE: [ "FV3_RAP" ] valid_vals_GFDLgrid_NUM_CELLS: [48, 96, 192, 384, 768, 1152, 3072] -valid_vals_EXTRN_MDL_NAME_ICS: ["GSMGFS", "FV3GFS", "GEFS", "GDAS", "RAP", "HRRR", "NAM"] -valid_vals_EXTRN_MDL_NAME_LBCS: ["GSMGFS", "FV3GFS", "GEFS", "GDAS", "RAP", "HRRR", "NAM"] +valid_vals_EXTRN_MDL_NAME_ICS: ["GSMGFS", "FV3GFS", "UFS-CASE-STUDY", "GEFS", "GDAS", "RAP", "HRRR", "NAM"] +valid_vals_EXTRN_MDL_NAME_LBCS: ["GSMGFS", "FV3GFS", "UFS-CASE-STUDY", "GEFS", "GDAS", "RAP", "HRRR", "NAM"] valid_vals_USE_USER_STAGED_EXTRN_FILES: [True, False] valid_vals_FV3GFS_FILE_FMT_ICS: ["nemsio", "grib2", "netcdf"] valid_vals_FV3GFS_FILE_FMT_LBCS: ["nemsio", "grib2", "netcdf"] From 7edaff8b033ac0b837d566da1092ca38158cadff Mon Sep 17 00:00:00 2001 From: michelleharrold Date: Thu, 22 Jun 2023 11:10:01 -0600 Subject: [PATCH 09/16] [develop]: Updated exregional_get_obs scripts (#833) For more recent dates (June 2022 and beyond), the exregional_get_obs scripts fail because the tar files names on the NOAA HPSS were updated but not updated accordingly in the pull scripts. A simple fix of modifying the logic to account for mapping the valid date to the tar file name have been updated in scripts/exregional_get_obs_ccpa.sh, scripts/exregional_get_obs_mrms.sh, and scripts/exregional_get_obs_ndas.sh. --- scripts/exregional_get_obs_ccpa.sh | 20 ++++++++++++++++---- scripts/exregional_get_obs_mrms.sh | 12 +++++++++--- scripts/exregional_get_obs_ndas.sh | 7 ++++++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/scripts/exregional_get_obs_ccpa.sh b/scripts/exregional_get_obs_ccpa.sh index 9b54e49d56..3ab24a2e3a 100755 --- a/scripts/exregional_get_obs_ccpa.sh +++ b/scripts/exregional_get_obs_ccpa.sh @@ -79,7 +79,7 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do vdd=`echo ${vdate} | cut -c7-8` # day (DD) of valid time vhh=`echo ${vdate} | cut -c9-10` # forecast hour (HH) - vhh_noZero=$(expr ${vhh} + 0) + vhh_noZero=$((10#${vhh})) # Calculate valid date - 1 day vdate_ut_m1=`expr ${vdate_ut} - 86400` @@ -141,18 +141,30 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" fi - if [[ ${vyyyymmdd} -gt 20200217 ]]; then + if [[ ${vyyyymmdd} -gt 20200217 && ${vyyyymmdd} -le 20220618 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_ccpa_prod_ccpa.${vyyyy}${vmm}${vdd}.tar" fi - if [[ ${vyyyymmdd_m1} -gt 20200217 ]]; then + if [[ ${vyyyymmdd_m1} -gt 20200217 && ${vyyyymmdd_p1} -le 20220618 ]]; then TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/com_ccpa_prod_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" fi - if [[ ${vyyyymmdd_p1} -gt 20200217 ]]; then + if [[ ${vyyyymmdd_p1} -gt 20200217 && ${vyyyymmdd_p1} -le 20220618 ]]; then TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/com_ccpa_prod_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" fi + if [[ ${vyyyymmdd} -ge 20220619 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_ccpa_v4.2_ccpa.${vyyyy}${vmm}${vdd}.tar" + fi + + if [[ ${vyyyymmdd_m1} -ge 20220619 ]]; then + TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/com_ccpa_v4.2_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" + fi + + if [[ ${vyyyymmdd_p1} -ge 20220619 ]]; then + TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/com_ccpa_v4.2_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" + fi + # Check if file exists on disk; if not, pull it. ccpa_file="$ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" echo "CCPA FILE:${ccpa_file}" diff --git a/scripts/exregional_get_obs_mrms.sh b/scripts/exregional_get_obs_mrms.sh index 2d4ecac82f..69fcae4aa7 100755 --- a/scripts/exregional_get_obs_mrms.sh +++ b/scripts/exregional_get_obs_mrms.sh @@ -26,6 +26,7 @@ source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} # #----------------------------------------------------------------------- # +set +e mrms_dir=${OBS_DIR}/.. if [[ ! -d "$mrms_dir" ]]; then @@ -107,7 +108,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do # Name of MRMS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until Sept. 2020. if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -lt 20200303 ]]; then - CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" >& /dev/null` + CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar"` Status=$? if [[ ${Status} == 0 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" @@ -122,10 +123,14 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do fi fi - if [[ ${vyyyymmdd} -ge 20200303 ]]; then + if [[ ${vyyyymmdd} -ge 20200303 && ${vyyyymmdd} -lt 20220628 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/dcom_prod_ldmdata_obs.tar" fi + if [[ ${vyyyymmdd} -ge 20220628 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/dcom_ldmdata_obs.tar" + fi + echo "TAR FILE:${TarFile}" TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz\" | awk '{print \$7}'\`" @@ -133,6 +138,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do Status=$? if [[ ${Status} != 0 ]]; then + CurDate="${vyyyy}${vmm}${vdd}" print_err_msg_exit "Bad return status (${Status}) for date \"${CurDate}\".\ Did you forget to run \"module load hpss\"?\ COMMAND: ${TarCommand}" @@ -140,7 +146,7 @@ Did you forget to run \"module load hpss\"?\ if [[ ! -d "$mrms_proc/${vyyyymmdd}" ]]; then mkdir_vrfy -p $mrms_proc/${vyyyymmdd} fi - + hour=0 while [[ ${hour} -le 23 ]]; do echo "hour=${hour}" diff --git a/scripts/exregional_get_obs_ndas.sh b/scripts/exregional_get_obs_ndas.sh index 4249faafca..25f6c2ae95 100755 --- a/scripts/exregional_get_obs_ndas.sh +++ b/scripts/exregional_get_obs_ndas.sh @@ -104,11 +104,16 @@ echo "vhh_noZero=$vhh_noZero" TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` - else + elif [[ ${vyyyymmdd} -gt 20220627 && ${vyyyymmdd} -le 20221129 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_obsproc_v1.0_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` + else + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_obsproc_v1.1_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` fi if [[ ! -d "$ndas_proc" ]]; then From ef12c6fd2b234c89031ef6ec3c52cf4a0c47fcd8 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 22 Jun 2023 15:39:20 -0400 Subject: [PATCH 10/16] [develop] Add a WE2E test for AQM (#840) * Add a WE2E test for AQM (community mode). Please note that this new WE2E test only works on Hera and WCOSS2. --- ...fig.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml | 76 +++++++++++++++++++ ush/machine/hera.yaml | 1 + 2 files changed, 77 insertions(+) create mode 100644 tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml new file mode 100644 index 0000000000..1f0dd8382f --- /dev/null +++ b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml @@ -0,0 +1,76 @@ +metadata: + description: config for SRW-AQM, AQM_NA_13km, community mode, FV3_GFS_v16 physics suite +user: + RUN_ENVIR: community +workflow: + PREDEF_GRID_NAME: AQM_NA_13km + CCPP_PHYS_SUITE: FV3_GFS_v16 + DATE_FIRST_CYCL: '2023021700' + DATE_LAST_CYCL: '2023021706' + INCR_CYCL_FREQ: 6 + FCST_LEN_HRS: -1 + FCST_LEN_CYCL: + - 6 + - 12 + PREEXISTING_DIR_METHOD: rename + DIAG_TABLE_TMPL_FN: diag_table_aqm.FV3_GFS_v16 + FIELD_TABLE_TMPL_FN: field_table_aqm.FV3_GFS_v16 + DO_REAL_TIME: false +nco: + NET: aqm +rocoto: + tasks: + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' + task_aqm_ics_ext: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 02:00:00 +task_get_extrn_ics: + EXTRN_MDL_NAME_ICS: FV3GFS + FV3GFS_FILE_FMT_ICS: netcdf + EXTRN_MDL_ICS_OFFSET_HRS: 0 + USE_USER_STAGED_EXTRN_FILES: true +task_get_extrn_lbcs: + EXTRN_MDL_NAME_LBCS: FV3GFS + LBC_SPEC_INTVL_HRS: 6 + FV3GFS_FILE_FMT_LBCS: netcdf + EXTRN_MDL_LBCS_OFFSET_HRS: 0 + USE_USER_STAGED_EXTRN_FILES: true +task_run_fcst: + DT_ATMOS: 180 + LAYOUT_X: 50 + LAYOUT_Y: 34 + BLOCKSIZE: 16 + RESTART_INTERVAL: 6 + QUILTING: true + PRINT_ESMF: false + DO_FCST_RESTART: false +task_run_post: + POST_OUTPUT_DOMAIN_NAME: 793 + USE_CUSTOM_POST_CONFIG_FILE: false +global: + DO_ENSEMBLE: false + NUM_ENS_MEMBERS: 2 + HALO_BLEND: 0 +cpl_aqm_parm: + CPL_AQM: true + DO_AQM_CHEM_LBCS: true + DO_AQM_GEFS_LBCS: true + DO_AQM_DUST: true + DO_AQM_CANOPY: false + DO_AQM_PRODUCT: true + DO_AQM_SAVE_AIRNOW_HIST: false + DO_AQM_SAVE_FIRE: false + AQM_BIO_FILE: BEIS_RRFScmaq_C775.ncf + AQM_DUST_FILE_PREFIX: FENGSHA_p8_10km_inputs + AQM_DUST_FILE_SUFFIX: .nc + AQM_CANOPY_FILE_PREFIX: gfs.t12z.geo + AQM_CANOPY_FILE_SUFFIX: .canopy_regrid.nc + AQM_FIRE_FILE_PREFIX: Hourly_Emissions_regrid_NA_13km + AQM_FIRE_FILE_SUFFIX: _h72.nc + AQM_RC_FIRE_FREQUENCY: hourly + AQM_LBCS_FILES: am4_bndy.c793.2019.v1.nc + AQM_GEFS_FILE_PREFIX: geaer + NEXUS_GRID_FN: grid_spec_793.nc + NUM_SPLIT_NEXUS: 3 + diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index e1ad70c4f1..7604301fa3 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -53,6 +53,7 @@ cpl_aqm_parm: AQM_GEFS_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/GEFS_DATA NEXUS_INPUT_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/emissions/nexus NEXUS_FIX_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/nexus/fix + NEXUS_GFS_SFC_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/GFS_DATA PT_SRC_BASEDIR: /scratch1/RDARCH/rda-arl-gpu/Barry.Baker/emissions/nexus/NEI2016v1/v2023-01-PT rocoto: From cb1864dff9405bd80a3f50b80074f1491026e249 Mon Sep 17 00:00:00 2001 From: Yi-Cheng Teng - NOAA <30629225+clouden90@users.noreply.github.com> Date: Mon, 26 Jun 2023 08:59:37 -0400 Subject: [PATCH 11/16] [develop] Add ccpp-scm reference (#842) The reference of CCPP-SCM was missing in PR #822. Here, we add it back. --- docs/UsersGuide/source/Components.rst | 2 +- tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/UsersGuide/source/Components.rst b/docs/UsersGuide/source/Components.rst index 4bbfb2088d..b5e03eec85 100644 --- a/docs/UsersGuide/source/Components.rst +++ b/docs/UsersGuide/source/Components.rst @@ -34,7 +34,7 @@ The prognostic atmospheric model in the UFS SRW Application is the Finite-Volume Supported model resolutions in this release include 3-, 13-, and 25-km predefined contiguous U.S. (:term:`CONUS`) domains, each with 127 vertical levels. Preliminary tools for users to define their own domain are also available in the release with full, formal support of these tools to be provided in future releases. The Extended Schmidt Gnomonic (ESG) grid is used with the FV3-LAM, which features relatively uniform grid cells across the entirety of the domain. Additional information about the FV3 dynamical core can be found in the `scientific documentation `__, the `technical documentation `__, and on the `NOAA Geophysical Fluid Dynamics Laboratory website `__. -Interoperable atmospheric physics, along with various land surface model options, are supported through the Common Community Physics Package (CCPP), described `here `__. Atmospheric physics are a set of numerical methods describing small-scale processes such as clouds, turbulence, radiation, and their interactions. There are four physics suites supported as of the SRW App v2.1.0 release. The first is the FV3_RRFS_v1beta physics suite, which is being tested for use in the future operational implementation of the Rapid Refresh Forecast System (:term:`RRFS`) planned for 2023-2024, and the second is an updated version of the physics suite used in the operational Global Forecast System (GFS) v16. Additionally, FV3_WoFS_v0, FV3_HRRR, and FV3_RAP are supported. A detailed list of CCPP updates since the SRW App v2.0.0 release is available :ref:`here `. A full scientific description of CCPP parameterizations and suites can be found in the `CCPP Scientific Documentation `__, and CCPP technical aspects are described in the `CCPP Technical Documentation `__. The model namelist has many settings beyond the physics options that can optimize various aspects of the model for use with each of the supported suites. Additional information on Stochastic Physics options is available `here `__. +Interoperable atmospheric physics, along with various land surface model options, are supported through the Common Community Physics Package (CCPP), described `here `__. Atmospheric physics are a set of numerical methods describing small-scale processes such as clouds, turbulence, radiation, and their interactions. There are four physics suites supported as of the SRW App v2.1.0 release. The first is the FV3_RRFS_v1beta physics suite, which is being tested for use in the future operational implementation of the Rapid Refresh Forecast System (:term:`RRFS`) planned for 2023-2024, and the second is an updated version of the physics suite used in the operational Global Forecast System (GFS) v16. Additionally, FV3_WoFS_v0, FV3_HRRR, and FV3_RAP are supported. A detailed list of CCPP updates since the SRW App v2.0.0 release is available :ref:`here `. A full scientific description of CCPP parameterizations and suites can be found in the `CCPP Scientific Documentation `__, and CCPP technical aspects are described in the `CCPP Technical Documentation `__. The model namelist has many settings beyond the physics options that can optimize various aspects of the model for use with each of the supported suites. Additional information on Stochastic Physics options is available `here `__. Additionally, a CCPP single-column model (`CCPP-SCM `__) option has also been developed as a child repository. Users can refer to the `CCPP Single Column Model User and Technical Guide `__ for more details. This CCPP-SCM user guide contains a Quick Start Guide with instructions for obtaining the code, compiling, and running test cases, which include five standard test cases and two additional FV3 replay cases (refer to section 5.2 in the CCPP-SCM user guide for more details). Moreover, the CCPP-SCM supports a precompiled version in a docker container, allowing it to be easily executed on NOAA's cloud computing platforms without any issues (see section 2.5 in the CCPP-SCM user guide for more details). .. note:: SPP is currently only available for specific physics schemes used in the RAP/HRRR physics suite. Users need to be aware of which physics suite definition file (:term:`SDF`) is chosen when turning this option on. Among the supported physics suites, the full set of parameterizations can only be used with the ``FV3_HRRR`` option for ``CCPP_PHYS_SUITE``. diff --git a/tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml b/tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml index fc50771ca6..71e664e17a 100644 --- a/tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml +++ b/tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml @@ -3,6 +3,7 @@ metadata: This test is to ensure that the workflow running in community mode completes successfully on the RRFS_CONUS_13km grid using the GFS_v16 physics suite with ICs and LBCs derived from the UFS-CASE-STUDIES platforms for 2020 Cold Air Damming case. + #Users can modify date for other test cases listed in the UFS-CASE-STUDIES platforms #Note for runnning this test on Cheyenne: please modify this config as follows #task_get_extrn_ics: # EXTRN_MDL_NAME_ICS: FV3GFS From b3e5e0fd85743e727de20a5b714b7c61b496650b Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:14:19 -0400 Subject: [PATCH 12/16] [develop] Add AQM to SRW user's guide (#846) Add documentation for Air Quality Modeling (AQM) to the user's guide. --- docs/UsersGuide/source/AQM.rst | 175 ++++++++++++++++++++++++--------- 1 file changed, 131 insertions(+), 44 deletions(-) diff --git a/docs/UsersGuide/source/AQM.rst b/docs/UsersGuide/source/AQM.rst index a0b52ca3fe..4c810f0d69 100644 --- a/docs/UsersGuide/source/AQM.rst +++ b/docs/UsersGuide/source/AQM.rst @@ -1,7 +1,7 @@ .. _AQM: ===================================== -Air Quality Modeling (Online-CMAQ) +Air Quality Modeling (SRW-AQM) ===================================== The standard SRW App distribution uses the uncoupled version of the UFS Weather Model (atmosphere-only). However, users have the option to use a coupled version of the SRW App that includes the standard distribution (atmospheric model) plus the Air Quality Model (AQM). @@ -16,7 +16,7 @@ The AQM is a UFS Application that dynamically couples the Community Multiscale A These instructions should work smoothly on Hera and WCOSS2, but users on other systems may need to make additional adjustments. -Quick Start Guide (AQM/Online-CMAQ) +Quick Start Guide (SRW-AQM) ===================================== Download the Code @@ -29,44 +29,10 @@ Clone the ``develop`` branch of the authoritative SRW App repository: git clone -b develop https://github.com/ufs-community/ufs-srweather-app cd ufs-srweather-app -Note that the latest hash of the ``develop`` branch might not be tested with the sample scripts of for AQM. To check out the stable (verified) version for AQM/online-CMAQ, users can check out hash ``#ff6f103``: - -.. code-block:: console - - git checkout ff6f103 - -This hash will then check out the following hashes (as of 03/08/2023) of the external components, which are specified in ``ufs-srweather-app/Externals.cfg``: - -.. _ExternalsAQM: - -.. table:: Externals for Online-CMAQ - - +--------------------+--------------+ - | Component | Hash | - +====================+==============+ - | UFS_UTILS | ca9bed8 | - +--------------------+--------------+ - | ufs-weather-model | e051e0e | - +--------------------+--------------+ - | UPP | 2b2c84a | - +--------------------+--------------+ - | NEXUS | 3842818 | - +--------------------+--------------+ - | AQM-utils | e078c70 | - +--------------------+--------------+ - -Users may replace the hashes above with different ones if they prefer. For example, users can comment out the hash line and uncomment the branch line with a new ``repo_url`` address to use a different branch for development. In the example below, the ``ufs-weather-model`` repository URL has been changed to check out code from a user's personal fork rather than from the authoritative UFS repository. - -.. code-block:: console - - repo_url = https://github.com/chan-hoo/ufs-weather-model - branch = feature/for_example - #hash = ff6f103 - Checkout Externals --------------------- -Along with the components detailed in :numref:`Chapter %s `, the AQM version of the SRW App pulls in the externals listed in :numref:`Table %s `. Users must run the ``checkout_externals`` script to collect (or "check out") the individual components of the SRW App (AQM version) from their respective GitHub repositories. +Users must run the ``checkout_externals`` script to collect (or "check out") the individual components of the SRW App (AQM version) from their respective GitHub repositories. .. code-block:: console @@ -85,7 +51,7 @@ where ```` is ``hera``, or ``wcoss2``. The ``-a`` argument indicates th Building the SRW App with AQM on other machines, including other `Level 1 `__ platforms, is not currently guaranteed to work, and users may have to make adjustments to the modulefiles for their system. -Load the ``regional_workflow`` Environment +Load the ``workflow_tools`` Environment -------------------------------------------- Load the python environment for the workflow: @@ -105,9 +71,9 @@ If the console outputs a message, the user should run the commands specified in .. code-block:: console Please do the following to activate conda: - > conda activate regional_workflow + > conda activate workflow_tools -then the user should run ``conda activate regional_workflow``. Otherwise, the user can continue with configuring the workflow. +then the user should run ``conda activate workflow_tools``. Otherwise, the user can continue with configuring the workflow. .. _AQMConfig: @@ -123,10 +89,6 @@ Users will need to configure their experiment by setting parameters in the ``con Users may prefer to copy the ``config.aqm.nco.realtime.yaml`` for a default "nco" mode experiment instead. -.. note:: - - Additional sample configuration files can be found in the ``online-cmaq`` branch of Chan-Hoo Jeon's (NOAA/NCEP/EMC) ``ufs-srweather-app`` repository fork on `GitHub `__. - Users will need to change the ``MACHINE`` and ``ACCOUNT`` variables in ``config.yaml`` to match their system. They may also wish to adjust other experiment settings. For more information on each task and variable, see :numref:`Chapter %s `. Users may also wish to change :term:`cron`-related parameters in ``config.yaml``. In the ``config.aqm.community.yaml`` file, which was copied into ``config.yaml``, cron is used for automatic submission and resubmission of the workflow: @@ -170,3 +132,128 @@ Users may check experiment status from the experiment directory with either of t # Check the experiment status and relaunch the workflow (for manual jobs) ./launch_FV3LAM_wflow.sh; tail -n 40 log.launch_FV3LAM_wflow + +WE2E Test for AQM +======================= + +Build the app for AQM: + +.. code-block:: console + + ./devbuild.sh -p=hera -a=ATMAQ + + +Add the WE2E test for AQM to the list file: + +.. code-block:: console + + echo "custom_ESGgrid" > my_tests.txt + echo "aqm_grid_AQM_NA13km_suite_GFS_v16" >> my_tests.txt + + +Run the WE2E test: + +.. code-block:: console + + $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q + + + +Additional Tasks for AQM +=============================== + +Structure of SRW-AQM +------------------------- + +The flowchart of the non-DA (data assimilation) SRW-AQM (Air Quality Modeling) is illustrated in :numref:`Figure %s `. Compared to the non-coupled (ATM stand-alone) FV3-LAM, SRW-AQM has additional tasks for pre- and post-processing. For pre-processing, multiple emission data such as NEXUS, fire, and point-source emission are retrieved or created for air quality modeling. Moreover, the chemical initial conditions (ICs) are extracted from the restart files of the previous cycle and added to the existing IC files. The chemical lateral boundary conditions (LBCs) and the GEFS aerosol data are also adeded to the existing LBC files. For post-processing, air quality forecast products for O3 and PM2.5 are generated and the bias-correction technique is applied to improve the accuracy of the results. + +.. _FlowProcAQM: + +.. figure:: https://github.com/ufs-community/ufs-srweather-app/wiki/SRW-AQM_workflow.png + :alt: Flowchart of the SRW-AQM tasks. + + *Workflow structure of SRW-AQM (non-DA)* + + + +Pre-processing Tasks of SRW-AQM +------------------------------------ + +The pre-processing tasks for air quality modeling (AQM) are shown in :numref:`Table %s `. + +.. _TasksPrepAQM: + +.. table:: Tasks for pre-processing of AQM + + +-----------------------+--------------------------------------------------------------------+ + | **Task name** | **Description** | + +=======================+====================================================================+ + | nexus_gfs_sfc | This task retrieves the GFS surface files from the previous cycle | + | | in NRT (Near-Real-Time) or current cycle in retrospective cases. | + | | The surface radiation, soil moisture and temperature fields are | + | | needed for the MEGAN biogenics emissions within nexus_emission. | + +-----------------------+--------------------------------------------------------------------+ + | nexus_emission | This task prepares the run directory with gridded emission inputs, | + | | run nexus to create model ready emission for the given simulation | + | | day, and post processes nexus output to make it more readable. The | + | | task will also split the task into multiple jobs set by the user. | + +-----------------------+--------------------------------------------------------------------+ + | nexus_post_split | This task combines the nexus_emission outputs into a single job. | + +-----------------------+--------------------------------------------------------------------+ + | fire_emission | This tasks is used to convert both satellite-retrieved gas and | + | | aerosol species emissions (RAVE) from mass (kg) to emission rates | + | | (kg/m2/s) and create 3-day hourly model-ready fire emission input | + | | files. | + +-----------------------+--------------------------------------------------------------------+ + | point_source | This task aggregates the anthropogenic point source sectors of the | + | | National Emission Inventory(NEI) into a ready-to-input point-source| + | | emission file based on the weekday/weekend/holiday patterns of each| + | | sector and date/time of the simulation. | + +-----------------------+--------------------------------------------------------------------+ + | aqm_ics | This task creates a chemical initial condition file by using the | + | | previous cycle restart files. | + +-----------------------+--------------------------------------------------------------------+ + | aqm_lbcs | This task adds the chemical lateral boundary condition (LBC) upon | + | | the meteorological lateral boundary condition to form the full-set | + | | ready-to-input LBC for the simulation. It includes two sub-tasks: | + | | the gaseous species LBC and dynamic aerosol LBC. The former adds | + | | static gaseous LBC using monthly mean global data. The latter is | + | | the parallel job, which extracts the GEFS-Aerosol Model's output | + | | along the regional domain, and performs the species conversion | + | | from GOCART aerosols to CMAQ aerosols. | + +-----------------------+--------------------------------------------------------------------+ + + +Post-processing Tasks of SRW-AQM +------------------------------------ + +The post-processing tasks for air quality modeling (AQM) are shown in :numref:`Table %s `. Since the module required to run these tasks is available only on WCOSS2, these tasks should not be defined in the configuration file ``config.yaml`` on other platforms. + +.. _TasksPostAQM: + +.. table:: Tasks for post-processing of AQM + + +-----------------------+--------------------------------------------------------------------+ + | **Task name** | **Description** | + +=======================+====================================================================+ + | pre_post_stat | This task creates surface (i.e., model 1st level) meteorological | + | | and chemical files to support air quality product generation and | + | | generate training data to support bias correction tasks. | + +-----------------------+--------------------------------------------------------------------+ + | post_stat_o3 | This task generates air quality forecast products including hourly | + | | -average and statistical products for O3 (e.g., daily 8-hour | + | | average maximum O3). | + +-----------------------+--------------------------------------------------------------------+ + | post_stat_pm25 | This task generates air quality forecast products including hourly | + | | -average and statistical products for PM2.5 (e.g., 24-hour average | + | | PM2.5). | + +-----------------------+--------------------------------------------------------------------+ + | bias_correction_o3 | This task applies a bias-correction technique (e.g., analog | + | | ensemble) to improve model raw forecast for O3 and generates the | + | | bias-corrected O3 products. | + +-----------------------+--------------------------------------------------------------------+ + | bias_correction_pm25 | This task applies a bias-correction technique (e.g., analog | + | | ensemble) to improve model raw forecast for PM2.5 and generates the| + | | bias-corrected PM2.5 products. | + +-----------------------+--------------------------------------------------------------------+ + From 9b9942f918d5e3ba5b8158b894e43bc44f7185e5 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:59:06 -0400 Subject: [PATCH 13/16] [develop] Bring change of production/AQM.v7 back to develop (#828) Multiple scripts relevant to AQM are updated to meet the NCO standards and EE2 reviewer's comments based on the production/AQM.v7 branch: * The NCO variables such as NET, RUN, envir are renamed with the suffix _default meaning the default value. This is because they should be defined in job cards as can be seen in the NCO standards (pp.4, Table 1). In the current rocoto-based workflow, the job cards do not exist (instead, it uses the XML file). Therefore, they can be set in the configuration file and ush/machine/ file. However, for ecFlow, these variables should be replaced with those in the job cards. (The ecFlow option will be added in a separate PR later). This is done in the ush/job_preamble.sh script which is sourced in the J-job scripts. * NCO wants to define COMIN/COMOUT, COMINmodel, DATAROOT using the compath call available in prod_util. * NCO does not want to add a new directory (COMINext) to COM. The use of COMINext is removed. * NCO wants to use the error handling calls such as 'err_chk' and 'err_exit' in 'prod_util' which is available only on WCOSS2. * NCO uses the YES/NO flags while SRW App uses TRUE/FALSE. To resolve this mismatch, the boolify calls for the NCO variables are added to job_preamble.sh. * The 'pgmerr' function is added to POST_STEP. * The forecast hour in some output file names is changed to 3 digit. * The working directory DATA is defined and created in job_preamble.sh for the NCO mode. Therefore, if-statement for the 'community' mode is added to the J-job scripts. * The KEEPDATA capability does not work correctly for the shared tasks * The input data directories are renamed to DCOMINdata. --- .gitignore | 1 - Externals.cfg | 4 +- jobs/JREGIONAL_AQM_ICS | 8 +- jobs/JREGIONAL_AQM_LBCS | 9 +- jobs/JREGIONAL_BIAS_CORRECTION_O3 | 9 +- jobs/JREGIONAL_BIAS_CORRECTION_PM25 | 9 +- jobs/JREGIONAL_FIRE_EMISSION | 10 +- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 7 +- jobs/JREGIONAL_MAKE_ICS | 8 +- jobs/JREGIONAL_MAKE_LBCS | 8 +- jobs/JREGIONAL_NEXUS_EMISSION | 8 +- jobs/JREGIONAL_NEXUS_GFS_SFC | 21 +- jobs/JREGIONAL_NEXUS_POST_SPLIT | 8 +- jobs/JREGIONAL_POINT_SOURCE | 8 +- jobs/JREGIONAL_POST_STAT_O3 | 9 +- jobs/JREGIONAL_POST_STAT_PM25 | 9 +- jobs/JREGIONAL_PRE_POST_STAT | 7 +- jobs/JREGIONAL_RUN_FCST | 4 +- jobs/JREGIONAL_RUN_POST | 28 +- modulefiles/tasks/wcoss2/python_srw.lua | 2 + parm/aqm.rc | 2 +- parm/wflow/default_workflow.yaml | 10 +- scripts/exregional_aqm_ics.sh | 43 ++-- scripts/exregional_aqm_lbcs.sh | 60 +++-- scripts/exregional_bias_correction_o3.sh | 239 ++++++++++++------ scripts/exregional_bias_correction_pm25.sh | 182 +++++++------ scripts/exregional_fire_emission.sh | 136 ++++++---- scripts/exregional_get_extrn_mdl_files.sh | 14 +- scripts/exregional_make_ics.sh | 75 +++--- scripts/exregional_make_lbcs.sh | 80 +++--- scripts/exregional_nexus_emission.sh | 92 +++++-- scripts/exregional_nexus_gfs_sfc.sh | 69 +++-- scripts/exregional_nexus_post_split.sh | 42 ++- scripts/exregional_point_source.sh | 21 +- scripts/exregional_post_stat_o3.sh | 83 +++--- scripts/exregional_post_stat_pm25.sh | 76 +++--- scripts/exregional_pre_post_stat.sh | 11 - scripts/exregional_run_fcst.sh | 111 ++++++-- scripts/exregional_run_post.sh | 23 +- tests/WE2E/run_WE2E_tests.py | 4 +- ...fig.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml | 2 +- ush/config.aqm.community.yaml | 2 +- ush/config.aqm.nco.realtime.yaml | 12 +- ush/config.nco.yaml | 6 +- ush/config_defaults.yaml | 123 ++++----- ush/create_aqm_rc_file.py | 10 +- ush/job_preamble.sh | 116 +++++++-- ush/load_modules_run_task.sh | 15 +- ush/machine/hera.yaml | 13 +- ush/machine/wcoss2.yaml | 18 +- ush/setup.py | 32 +-- ush/valid_param_vals.yaml | 2 +- 52 files changed, 1193 insertions(+), 708 deletions(-) diff --git a/.gitignore b/.gitignore index a727b940c0..b6da1c53a0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ fix/ include/ lib/ share/ -modulefiles/extrn_comp_build/ sorc/*/ tests/WE2E/WE2E_tests_*.yaml tests/WE2E/*.txt diff --git a/Externals.cfg b/Externals.cfg index 1807bc4b3c..1338e4a6af 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -49,7 +49,7 @@ protocol = git repo_url = https://github.com/noaa-oar-arl/NEXUS # Specify either a branch name or a hash but not both. #branch = develop -hash = 3842818 +hash = 6a7a994 local_path = sorc/arl_nexus required = True @@ -58,7 +58,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/AQM-utils # Specify either a branch name or a hash but not both. #branch = develop -hash = 0a86f73 +hash = 694a139 local_path = sorc/AQM-utils required = True diff --git a/jobs/JREGIONAL_AQM_ICS b/jobs/JREGIONAL_AQM_ICS index b03f22dc2a..ce3e539cc9 100755 --- a/jobs/JREGIONAL_AQM_ICS +++ b/jobs/JREGIONAL_AQM_ICS @@ -77,8 +77,12 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_AQM_ICS}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_AQM_LBCS b/jobs/JREGIONAL_AQM_LBCS index 0b675a388f..7f13d12fa1 100755 --- a/jobs/JREGIONAL_AQM_LBCS +++ b/jobs/JREGIONAL_AQM_LBCS @@ -77,8 +77,13 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_AQM_LBCS}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi + # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_BIAS_CORRECTION_O3 b/jobs/JREGIONAL_BIAS_CORRECTION_O3 index 32b12accd5..6586d4427c 100755 --- a/jobs/JREGIONAL_BIAS_CORRECTION_O3 +++ b/jobs/JREGIONAL_BIAS_CORRECTION_O3 @@ -60,7 +60,14 @@ This is the J-job script for the task that runs BIAS-CORRECTION-O3. # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_BIAS_CORRECTION_O3}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi + +mkdir_vrfy -p ${COMOUTwmo} export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}" diff --git a/jobs/JREGIONAL_BIAS_CORRECTION_PM25 b/jobs/JREGIONAL_BIAS_CORRECTION_PM25 index 27c6728e9c..a0c0ba1b4a 100755 --- a/jobs/JREGIONAL_BIAS_CORRECTION_PM25 +++ b/jobs/JREGIONAL_BIAS_CORRECTION_PM25 @@ -60,7 +60,14 @@ This is the J-job script for the task that runs BIAS-CORRECTION-PM25. # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_BIAS_CORRECTION_PM25}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi + +mkdir_vrfy -p ${COMOUTwmo} export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}" diff --git a/jobs/JREGIONAL_FIRE_EMISSION b/jobs/JREGIONAL_FIRE_EMISSION index 95b53ca4ff..ee540cfa6e 100755 --- a/jobs/JREGIONAL_FIRE_EMISSION +++ b/jobs/JREGIONAL_FIRE_EMISSION @@ -90,8 +90,12 @@ fi # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_FIRE_EMISSION}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # #----------------------------------------------------------------------- # @@ -99,7 +103,7 @@ mkdir_vrfy -p "${DATA}" # #----------------------------------------------------------------------- # -export FIRE_EMISSION_STAGING_DIR="${COMINext}/FIRE_EMISSION" +export FIRE_EMISSION_STAGING_DIR="${FIRE_EMISSION_STAGING_DIR:-${COMIN}/FIRE_EMISSION}" mkdir_vrfy -p "${FIRE_EMISSION_STAGING_DIR}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 7ef397b738..718e920d65 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -30,7 +30,7 @@ # . $USHdir/source_util_funcs.sh source_config_for_task "task_get_extrn_ics|task_get_extrn_lbcs" ${GLOBAL_VAR_DEFNS_FP} -. $USHdir/job_preamble.sh +. $USHdir/job_preamble.sh "TRUE" # #----------------------------------------------------------------------- # @@ -221,8 +221,7 @@ esac #----------------------------------------------------------------------- # if [ $RUN_ENVIR = "nco" ]; then - export EXTRN_MDL_STAGING_DIR="${COMINext}" - mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}" + export EXTRN_MDL_STAGING_DIR="${EXTRN_MDL_STAGING_DIR:-${DATA}}" else export EXTRN_MDL_STAGING_DIR="${COMIN}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}" @@ -245,7 +244,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -job_postamble +job_postamble "FALSE" # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index ffefa18ed5..679a9392bb 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -72,8 +72,12 @@ mkdir_vrfy -p "${INPUT_DATA_NWGES}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ $RUN_ENVIR = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_MAKE_ICS}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index 7b5c7f2622..35aba70846 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -71,8 +71,12 @@ mkdir_vrfy -p "${INPUT_DATA_NWGES}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_MAKE_LBCS}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_NEXUS_EMISSION b/jobs/JREGIONAL_NEXUS_EMISSION index b916fabf25..0f1a00d1a6 100755 --- a/jobs/JREGIONAL_NEXUS_EMISSION +++ b/jobs/JREGIONAL_NEXUS_EMISSION @@ -75,8 +75,12 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_NEXUS_EMISSION_${nspt}}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_NEXUS_GFS_SFC b/jobs/JREGIONAL_NEXUS_GFS_SFC index 184f6b7681..5930762106 100755 --- a/jobs/JREGIONAL_NEXUS_GFS_SFC +++ b/jobs/JREGIONAL_NEXUS_GFS_SFC @@ -17,7 +17,7 @@ # . $USHdir/source_util_funcs.sh source_config_for_task "cpl_aqm_parm|task_nexus_gfs_sfc" ${GLOBAL_VAR_DEFNS_FP} -. $USHdir/job_preamble.sh +. $USHdir/job_preamble.sh "TRUE" # #----------------------------------------------------------------------- # @@ -91,21 +91,16 @@ fi # #----------------------------------------------------------------------- # -# Create the directory where the RAVE fire emission files should be stored -# -#----------------------------------------------------------------------- -# -export GFS_SFC_STAGING_DIR="${COMINext}/GFS_SFC" -mkdir_vrfy -p "${GFS_SFC_STAGING_DIR}" -# -#----------------------------------------------------------------------- -# # Set the run directory # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_NEXUS_GFS_SFC}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # #----------------------------------------------------------------------- # @@ -123,7 +118,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -job_postamble +job_postamble "FALSE" # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_NEXUS_POST_SPLIT b/jobs/JREGIONAL_NEXUS_POST_SPLIT index 83054ab10b..fc21421f8d 100755 --- a/jobs/JREGIONAL_NEXUS_POST_SPLIT +++ b/jobs/JREGIONAL_NEXUS_POST_SPLIT @@ -67,8 +67,12 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_NEXUS_POST_SPLIT}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_POINT_SOURCE b/jobs/JREGIONAL_POINT_SOURCE index 2164d822a8..d3a02b7534 100755 --- a/jobs/JREGIONAL_POINT_SOURCE +++ b/jobs/JREGIONAL_POINT_SOURCE @@ -66,8 +66,12 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -mkdir_vrfy -p "${DATA}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_POINT_SOURCE}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_POST_STAT_O3 b/jobs/JREGIONAL_POST_STAT_O3 index 99cdaeb789..af8652e05d 100755 --- a/jobs/JREGIONAL_POST_STAT_O3 +++ b/jobs/JREGIONAL_POST_STAT_O3 @@ -60,7 +60,14 @@ This is the J-job script for the task that runs POST-STAT-O3. # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_POST_STAT_O3}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi + +mkdir_vrfy -p ${COMOUTwmo} export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}" # diff --git a/jobs/JREGIONAL_POST_STAT_PM25 b/jobs/JREGIONAL_POST_STAT_PM25 index b0f1b2bfab..81ec818075 100755 --- a/jobs/JREGIONAL_POST_STAT_PM25 +++ b/jobs/JREGIONAL_POST_STAT_PM25 @@ -60,7 +60,14 @@ This is the J-job script for the task that runs POST-UPP-STAT. # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_POST_STAT_PM25}" +if [ "${RUN_ENVIR}" = "community" ]; then + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi + +mkdir_vrfy -p ${COMOUTwmo} export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}" # diff --git a/jobs/JREGIONAL_PRE_POST_STAT b/jobs/JREGIONAL_PRE_POST_STAT index 7611c6bd60..41cb25d6a0 100755 --- a/jobs/JREGIONAL_PRE_POST_STAT +++ b/jobs/JREGIONAL_PRE_POST_STAT @@ -60,7 +60,12 @@ This is the J-job script for the task that runs POST-UPP-STAT. # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_PRE_POST_STAT}" + check_for_preexist_dir_file "$DATA" "delete" + mkdir_vrfy -p $DATA + cd_vrfy $DATA +fi # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index e2501c37e4..383739e172 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -66,6 +66,7 @@ the specified cycle. if [ $RUN_ENVIR = "nco" ]; then export INPUT_DATA="${COMIN}" else + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT" fi # @@ -75,7 +76,6 @@ fi # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" mkdir_vrfy -p ${DATA}/INPUT mkdir_vrfy -p ${DATA}/RESTART # @@ -96,7 +96,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -job_postamble +job_postamble "FALSE" # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index ef783f34da..459431522f 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -56,14 +56,6 @@ This is the J-job script for the task that runs the post-processor (UPP) on the output files corresponding to a specified forecast hour. ========================================================================" # -#----------------------------------------------------------------------- -# -# Set the run directory. -# -#----------------------------------------------------------------------- -# -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" -# # If SUB_HOURLY_POST is not set to "TRUE", ensure that the forecast # minutes (fmn) are set to "00". This is necessary in order to pass # "fmn" into the post ex-script for the calculation of post_time. @@ -90,10 +82,11 @@ fi # #----------------------------------------------------------------------- # -if [ "${RUN_ENVIR}" != "nco" ]; then +if [ "${RUN_ENVIR}" = "community" ]; then + DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" export COMOUT="${DATA}/postprd" + mkdir_vrfy -p "${COMOUT}" fi -mkdir_vrfy -p "${COMOUT}" if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then export DATA_FHR="${DATA:-$COMOUT}/$fhr$fmn" @@ -143,9 +136,11 @@ if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} )) FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]} - fcst_len_hrs=$( printf "%03d" "${FCST_LEN_HRS}" ) - if [ "${fhr}" = "${fcst_len_hrs}" ]; then - touch "${COMIN}/post_${PDY}${cyc}_task_complete.txt" + if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then + fcst_len_hrs=$( printf "%03d" "${FCST_LEN_HRS}" ) + if [ "${fhr}" = "${fcst_len_hrs}" ]; then + touch "${COMIN}/post_${PDY}${cyc}_task_complete.txt" + fi fi fi # @@ -155,7 +150,12 @@ fi # #----------------------------------------------------------------------- # -job_postamble +fcst_len_hrs=$( printf "%03d" "${FCST_LEN_HRS}" ) +if [ "${fhr}" = "${fcst_len_hrs}" ]; then + job_postamble "TRUE" +else + job_postamble +fi # #----------------------------------------------------------------------- # diff --git a/modulefiles/tasks/wcoss2/python_srw.lua b/modulefiles/tasks/wcoss2/python_srw.lua index 519f1cdf4a..0a3bc3207b 100644 --- a/modulefiles/tasks/wcoss2/python_srw.lua +++ b/modulefiles/tasks/wcoss2/python_srw.lua @@ -1,3 +1,5 @@ +load(pathJoin("PrgEnv-intel", os.getenv("PrgEnv_intel_ver"))) +load(pathJoin("craype", os.getenv("craype_ver"))) load(pathJoin("intel", os.getenv("intel_ver"))) load(pathJoin("python", os.getenv("python_ver"))) load(pathJoin("prod_util", os.getenv("prod_util_ver"))) diff --git a/parm/aqm.rc b/parm/aqm.rc index ad2c85940a..3d2ad32711 100644 --- a/parm/aqm.rc +++ b/parm/aqm.rc @@ -172,7 +172,7 @@ bio_format: netcdf bio_file: {{ aqm_rc_bio_file_fp }} bio_frequency: static bio_period: summer -bio_speciation_file: {{ aqm_bio_dir }}/gspro_biogenics_1mar2017.txt +bio_speciation_file: {{ dcominbio }}/gspro_biogenics_1mar2017.txt bio_speciation_profile: B10C6 bio_species:: AVG_NOAG_GROW 1.00000 AVG_NOAG_GROW gmN/hr diff --git a/parm/wflow/default_workflow.yaml b/parm/wflow/default_workflow.yaml index 77231f8136..a2e20174ff 100644 --- a/parm/wflow/default_workflow.yaml +++ b/parm/wflow/default_workflow.yaml @@ -5,25 +5,25 @@ rocoto: entities: ACCOUNT: '{{ user.ACCOUNT }}' CCPA_OBS_DIR: '{{ platform.CCPA_OBS_DIR }}' - COMIN_DIR: '{% if user.RUN_ENVIR == "nco" %}{{"{}/{}.@Y@m@d/@H".format(nco.COMIN_BASEDIR,nco.RUN)}}{% else %}{{"{}/@Y@m@d@H".format(workflow.EXPTDIR)}}{% endif %}' + COMIN_DIR: '{% if user.RUN_ENVIR == "nco" %}{{"{}/{}.@Y@m@d/@H".format(nco.COMIN_BASEDIR,nco.RUN_default)}}{% else %}{{"{}/@Y@m@d@H".format(workflow.EXPTDIR)}}{% endif %}' COMINgfs: '{{ platform.get("COMINgfs") }}' - FCST_DIR: '{% if user.RUN_ENVIR == "nco" %}{{"{}/run_fcst_mem#mem#.{}_@Y@m@d@H".format(nco.DATAROOT,workflow.WORKFLOW_ID)}}{% else %}{{"{}/@Y@m@d@H".format(workflow.EXPTDIR)}}{% endif %}' + FCST_DIR: '{% if user.RUN_ENVIR == "nco" %}{{"{}/run_fcst_mem#mem#.{}_@Y@m@d@H".format(nco.DATAROOT_default,workflow.WORKFLOW_ID)}}{% else %}{{"{}/@Y@m@d@H".format(workflow.EXPTDIR)}}{% endif %}' GLOBAL_VAR_DEFNS_FP: '{{ workflow.GLOBAL_VAR_DEFNS_FP }}' JOBSdir: '{{ user.JOBSdir }}' LOAD_MODULES_RUN_TASK_FP: '{{ workflow.LOAD_MODULES_RUN_TASK_FP }}' - LOGDIR: '{% if user.RUN_ENVIR == "nco" %}{{"{}/@Y@m@d".format(nco.LOGBASEDIR)}}{% else %}{{nco.LOGBASEDIR }}{% endif %}' + LOGDIR: '{% if user.RUN_ENVIR == "nco" %}{{"{}/@Y@m@d".format(nco.LOGBASEDIR_default)}}{% else %}{{nco.LOGBASEDIR_default }}{% endif %}' LOGEXT: '{% if user.RUN_ENVIR == "nco" %}{{".{}.log".format(workflow.WORKFLOW_ID)}}{% else %}{{".log"}}{% endif %}' MRMS_OBS_DIR: '{{ platform.MRMS_OBS_DIR }}' NCORES_PER_NODE: '{{ platform.NCORES_PER_NODE }}' NDAS_OBS_DIR: '{{ platform.NDAS_OBS_DIR }}' - NET: '{{ nco.NET }}' + NET: '{{ nco.NET_default }}' PARTITION_DEFAULT: '{{ platform.get("PARTITION_DEFAULT") }}' PARTITION_FCST: '{{ platform.get("PARTITION_FCST") }}' PARTITION_HPSS: '{{ platform.get("PARTITION_HPSS") }}' QUEUE_DEFAULT: '{{ platform.get("QUEUE_DEFAULT") }}' QUEUE_FCST: '{{ platform.get("QUEUE_FCST") }}' QUEUE_HPSS: '{{ platform.get("QUEUE_HPSS") }}' - RUN: '{{ nco.RUN }}' + RUN: '{{ nco.RUN_default }}' SCRIPTSdir: '{{ user.SCRIPTSdir }}' SLASH_ENSMEM_SUBDIR: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% else %}{{ "/" }}{% endif %}' USHdir: '{{ user.USHdir }}' diff --git a/scripts/exregional_aqm_ics.sh b/scripts/exregional_aqm_ics.sh index 73fd27cfb0..c59bf87f81 100755 --- a/scripts/exregional_aqm_ics.sh +++ b/scripts/exregional_aqm_ics.sh @@ -72,8 +72,12 @@ if [ ! -r ${fv_tracer_file} ]; then print_info_msg " Tracer file found: \"${fv_tracer_file}\"" else - print_err_msg_exit "\ - No suitable tracer restart file found." + message_txt="No suitable tracer restart file found." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2"]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi fi fi fi @@ -81,16 +85,6 @@ fi # #----------------------------------------------------------------------- # -# Move to work directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_AQM_ICS" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA -# -#----------------------------------------------------------------------- -# # Add air quality tracer variables from previous cycle's restart output # to atmosphere's initial condition file according to the steps below: # @@ -112,13 +106,28 @@ print_info_msg " cp_vrfy ${gfs_ic_file} ${wrk_ic_file} python3 ${HOMEdir}/sorc/AQM-utils/python_utils/add_aqm_ics.py --fv_tracer_file "${fv_tracer_file}" --wrk_ic_file "${wrk_ic_file}" +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to python script \"add_aqm_ics.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi -ncatted -a checksum,,d,s, tmp1.nc || print_err_msg_exit "\ -Call to NCATTED returned with nonzero exit code." - -mv_vrfy tmp1.nc ${gfs_ic_file} +ncatted -a checksum,,d,s, tmp1.nc +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to NCATTED returned with nonzero exit code." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi -rm_vrfy gfs.nc +cp_vrfy tmp1.nc ${gfs_ic_file} unset fv_tracer_file unset wrk_ic_file diff --git a/scripts/exregional_aqm_lbcs.sh b/scripts/exregional_aqm_lbcs.sh index ccf09f52e2..24bac24ad7 100755 --- a/scripts/exregional_aqm_lbcs.sh +++ b/scripts/exregional_aqm_lbcs.sh @@ -73,16 +73,6 @@ fi # #----------------------------------------------------------------------- # -# Move to working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_AQM_LBCS" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA -# -#----------------------------------------------------------------------- -# # Add chemical LBCS # #----------------------------------------------------------------------- @@ -107,20 +97,33 @@ if [ ${DO_AQM_CHEM_LBCS} = "TRUE" ]; then ext_lbcs_file=${AQM_LBCS_FILES} chem_lbcs_fn=${ext_lbcs_file///${mm}} - chem_lbcs_fp=${AQM_LBCS_DIR}/${chem_lbcs_fn} + chem_lbcs_fp=${DCOMINchem_lbcs}/${chem_lbcs_fn} if [ -f ${chem_lbcs_fp} ]; then #Copy the boundary condition file to the current location cp_vrfy ${chem_lbcs_fp} . else - print_err_msg_exit "\ -The chemical LBC files do not exist: - CHEM_BOUNDARY_CONDITION_FILE = \"${chem_lbcs_fp}\"" + message_txt="The chemical LBC files do not exist: + CHEM_BOUNDARY_CONDITION_FILE = \"${chem_lbcs_fp}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi fi for hr in 0 ${LBC_SPEC_FCST_HRS[@]}; do fhr=$( printf "%03d" "${hr}" ) if [ -r ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fhr}.nc ]; then ncks -A ${chem_lbcs_fn} ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fhr}.nc + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to NCKS returned with nonzero exit code." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi fi done @@ -152,7 +155,7 @@ if [ ${DO_AQM_GEFS_LBCS} = "TRUE" ]; then if [ "${DO_REAL_TIME}" = "TRUE" ]; then AQM_MOFILE_FP="${COMINgefs}/gefs.${yyyymmdd}/${AQM_GEFS_FILE_CYC}/chem/sfcsig/${AQM_MOFILE_FN}" else - AQM_MOFILE_FP="${AQM_GEFS_DIR}/${yyyymmdd}/${AQM_GEFS_FILE_CYC}/${AQM_MOFILE_FN}" + AQM_MOFILE_FP="${DCOMINgefs}/${yyyymmdd}/${AQM_GEFS_FILE_CYC}/${AQM_MOFILE_FN}" fi # Check if GEFS aerosol files exist @@ -161,8 +164,17 @@ if [ ${DO_AQM_GEFS_LBCS} = "TRUE" ]; then fhr=$( printf "%03d" "${hr_mod}" ) AQM_MOFILE_FHR_FP="${AQM_MOFILE_FP}${fhr}.nemsio" if [ ! -e "${AQM_MOFILE_FHR_FP}" ]; then - print_err_msg_exit "The GEFS file (AQM_MOFILE_FHR_FP) for LBCs does not exist: - AQM_MOFILE_FHR_FP = \"${AQM_MOFILE_FHR_FP}\"" + message_txt="The GEFS file (AQM_MOFILE_FHR_FP) for LBCs of \"${cycle}\" does not exist: + AQM_MOFILE_FHR_FP = \"${AQM_MOFILE_FHR_FP}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + message_warning="WARNING: ${message_txt}" + print_info_msg "${message_warning}" + if [ ! -z "${maillist}" ]; then + echo "${message_warning}" | mail.py $maillist + fi + else + print_err_msg_exit "${message_txt}" + fi fi done @@ -214,11 +226,17 @@ Please ensure that you've built this executable." #---------------------------------------------------------------------- # PREP_STEP - eval ${RUN_CMD_AQMLBC} ${exec_fp} ${REDIRECT_OUT_ERR} || \ - print_err_msg_exit "\ -Call to executable (exec_fp) to generate chemical and GEFS LBCs -file for RRFS-CMAQ failed: + eval ${RUN_CMD_AQMLBC} ${exec_fp} ${REDIRECT_OUT_ERR} + export err=$? + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk + else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable (exec_fp) to generate chemical and +GEFS LBCs file for RRFS-CMAQ failed: exec_fp = \"${exec_fp}\"" + fi + fi POST_STEP print_info_msg " diff --git a/scripts/exregional_bias_correction_o3.sh b/scripts/exregional_bias_correction_o3.sh index 82a099066f..79ed056f59 100755 --- a/scripts/exregional_bias_correction_o3.sh +++ b/scripts/exregional_bias_correction_o3.sh @@ -73,18 +73,6 @@ else All executables will be submitted with command \'${RUN_CMD_SERIAL}\'." fi -# -#----------------------------------------------------------------------- -# -# Move to the working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_BIAS_CORRECTION_O3" -rm_vrfy -rf "$DATA" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA - yyyy=${PDY:0:4} yyyymm=${PDY:0:6} yyyy_m1=${PDYm1:0:4} @@ -92,8 +80,8 @@ yyyymm_m1=${PDYm1:0:6} yyyy_m2=${PDYm2:0:4} yyyymm_m2=${PDYm2:0:6} yyyy_m3=${PDYm3:0:4} -yyyymm_m3=${PDYm3:0:6} - +yyyymm_m3=${PDYm3:0:6} + # #----------------------------------------------------------------------- # @@ -115,26 +103,53 @@ fi # STEP 1: Retrieve AIRNOW observation data #----------------------------------------------------------------------------- -# Link the historical airnow data mkdir_vrfy -p "${DATA}/data" -ln_vrfy -sf ${AQM_AIRNOW_HIST_DIR}/bcdata* "${DATA}/data" -if [ -d "${DATA}/data/bcdata.${yyyymm}" ]; then - rm_vrfy -rf "${DATA}/data/bcdata.${yyyymm}" - mkdir_vrfy -p "${DATA}/data/bcdata.${yyyymm}" - cp_vrfy -rL "${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/airnow" "${DATA}/data/bcdata.${yyyymm}" - cp_vrfy -rL "${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated" "${DATA}/data/bcdata.${yyyymm}" -fi -# Retrieve real-time airnow data for the last three days -if [ "${DO_REAL_TIME}" = "TRUE" ]; then - mkdir -p ${DATA}/data/bcdata.${yyyymm_m1}/airnow/${yyyy_m1}/${PDYm1}/b008 - mkdir -p ${DATA}/data/bcdata.${yyyymm_m2}/airnow/${yyyy_m2}/${PDYm2}/b008 - mkdir -p ${DATA}/data/bcdata.${yyyymm_m3}/airnow/${yyyy_m3}/${PDYm3}/b008 +# Retrieve real-time airnow data for the last three days and convert them into netcdf. +# In the following for-loop, pdym stands for previous (m) day of the present day (PDY) +# in the NCO standards, i.e. PDYm1: 1day ago, PDYm2: 2days ago, PDYm3: 3days ago +for i_pdym in {1..3}; do + case $i_pdym in + 1) + cvt_yyyy="${yyyy_m1}" + cvt_yyyymm="${yyyymm_m1}" + cvt_pdy="${PDYm1}" + ;; + 2) + cvt_yyyy="${yyyy_m2}" + cvt_yyyymm="${yyyymm_m2}" + cvt_pdy="${PDYm2}" + ;; + 3) + cvt_yyyy="${yyyy_m3}" + cvt_yyyymm="${yyyymm_m3}" + cvt_pdy="${PDYm3}" + ;; + esac + + cvt_input_dir="${DATA}/data/bcdata.${cvt_yyyymm}/airnow/csv" + cvt_output_dir="${DATA}/data/bcdata.${cvt_yyyymm}/airnow/netcdf" + cvt_input_fn="HourlyAQObs_YYYYMMDDHH.dat" + cvt_output_fn="HourlyAQObs.YYYYMMDD.nc" + cvt_input_fp="${cvt_input_dir}/YYYY/YYYYMMDD/${cvt_input_fn}" + cvt_output_fp="${cvt_output_dir}/YYYY/YYYYMMDD/${cvt_output_fn}" + + mkdir_vrfy -p "${cvt_input_dir}/${cvt_yyyy}/${cvt_pdy}" + mkdir_vrfy -p "${cvt_output_dir}/${cvt_yyyy}/${cvt_pdy}" + cp_vrfy ${DCOMINairnow}/${cvt_pdy}/airnow/HourlyAQObs_${cvt_pdy}*.dat "${cvt_input_dir}/${cvt_yyyy}/${cvt_pdy}" - cp_vrfy ${COMINairnow}/${PDYm1}/b008/xx021 ${DATA}/data/bcdata.${yyyymm_m1}/airnow/${yyyy_m1}/${PDYm1}/b008 - cp_vrfy ${COMINairnow}/${PDYm2}/b008/xx021 ${DATA}/data/bcdata.${yyyymm_m2}/airnow/${yyyy_m2}/${PDYm2}/b008 - cp_vrfy ${COMINairnow}/${PDYm3}/b008/xx021 ${DATA}/data/bcdata.${yyyymm_m3}/airnow/${yyyy_m3}/${PDYm3}/b008 -fi + PREP_STEP + eval ${RUN_CMD_SERIAL} ${EXECdir}/convert_airnow_csv ${cvt_input_fp} ${cvt_output_fp} ${cvt_pdy} ${cvt_pdy} ${REDIRECT_OUT_ERR} + export err=$? + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk + else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run CONVERT_AIRNOW_CSV returned with nonzero exit code." + fi + fi + POST_STEP +done #----------------------------------------------------------------------------- # STEP 2: Extracting PM2.5, O3, and met variables from CMAQ input and outputs @@ -174,48 +189,90 @@ mkdir_vrfy -p ${DATA}/data/site-lists.interp mkdir_vrfy -p ${DATA}/out/ozone/${yyyy} mkdir_vrfy -p ${DATA}/data/bcdata.${yyyymm}/interpolated/ozone/${yyyy} -cp_vrfy ${PARMaqm_utils}/bias_correction/sites.valid.ozone.20220724.12z.list ${DATA}/data/site-lists.interp +cp_vrfy ${PARMaqm_utils}/bias_correction/sites.valid.ozone.20230331.12z.list ${DATA}/data/site-lists.interp cp_vrfy ${PARMaqm_utils}/bias_correction/aqm.t12z.chem_sfc.f000.nc ${DATA}/data/coords cp_vrfy ${PARMaqm_utils}/bias_correction/config.interp.ozone.7-vars_${id_domain}.${cyc}z ${DATA} PREP_STEP -eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_interpolate config.interp.ozone.7-vars_${id_domain}.${cyc}z ${cyc}z ${PDY} ${PDY} ${REDIRECT_OUT_ERR} || print_err_msg_exit "Call to executable to run AQM_BIAS_INTERPOLATE returned with nonzero exit code." +eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_interpolate config.interp.ozone.7-vars_${id_domain}.${cyc}z ${cyc}z ${PDY} ${PDY} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_BIAS_INTERPOLATE returned with nonzero exit code." + fi +fi POST_STEP cp_vrfy ${DATA}/out/ozone/${yyyy}/*nc ${DATA}/data/bcdata.${yyyymm}/interpolated/ozone/${yyyy} if [ "${DO_AQM_SAVE_AIRNOW_HIST}" = "TRUE" ]; then - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated/ozone/${yyyy} - cp_vrfy ${DATA}/out/ozone/${yyyy}/*nc ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated/ozone/${yyyy} - - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/airnow/${yyyy}/${PDY}/b008 - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m1}/airnow/${yyyy_m1}/${PDYm1}/b008 - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m2}/airnow/${yyyy_m2}/${PDYm2}/b008 - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m3}/airnow/${yyyy_m3}/${PDYm3}/b008 - cp_vrfy ${COMINairnow}/${PDY}/b008/xx021 ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/airnow/${yyyy}/${PDY}/b008 - cp_vrfy ${COMINairnow}/${PDYm1}/b008/xx021 ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m1}/airnow/${yyyy_m1}/${PDYm1}/b008 - cp_vrfy ${COMINairnow}/${PDYm2}/b008/xx021 ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m2}/airnow/${yyyy_m2}/${PDYm2}/b008 - cp_vrfy ${COMINairnow}/${PDYm3}/b008/xx021 ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m3}/airnow/${yyyy_m3}/${PDYm3}/b008 - - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/grid/${cyc}z/${PDY} - cp_vrfy ${COMIN}/${NET}.${cycle}.*sfc*.nc ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/grid/${cyc}z/${PDY} + mkdir_vrfy -p ${COMOUTbicor}/bcdata.${yyyymm}/interpolated/ozone/${yyyy} + cp_vrfy ${DATA}/out/ozone/${yyyy}/*nc ${COMOUTbicor}/bcdata.${yyyymm}/interpolated/ozone/${yyyy} + + for i_pdym in {0..3}; do + case $i_pdym in + 0) + cvt_yyyy="${yyyy}" + cvt_yyyymm="${yyyymm}" + cvt_pdy="${PDY}" + ;; + 1) + cvt_yyyy="${yyyy_m1}" + cvt_yyyymm="${yyyymm_m1}" + cvt_pdy="${PDYm1}" + ;; + 2) + cvt_yyyy="${yyyy_m2}" + cvt_yyyymm="${yyyymm_m2}" + cvt_pdy="${PDYm2}" + ;; + 3) + cvt_yyyy="${yyyy_m3}" + cvt_yyyymm="${yyyymm_m3}" + cvt_pdy="${PDYm3}" + ;; + esac + # CSV and NetCDF files + mkdir_vrfy -p ${COMOUTbicor}/bcdata.${cvt_yyyymm}/airnow/csv/${cvt_yyyy}/${cvt_pdy} + mkdir_vrfy -p ${COMOUTbicor}/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy} + if [ "${i_pdym}" != "0" ]; then + cp_vrfy ${DCOMINairnow}/${cvt_pdy}/airnow/HourlyAQObs_${cvt_pdy}*.dat ${COMOUTbicor}/bcdata.${cvt_yyyymm}/airnow/csv/${cvt_yyyy}/${cvt_pdy} + cp_vrfy ${DATA}/data/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy}/HourlyAQObs.${cvt_pdy}.nc ${COMOUTbicor}/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy} + fi + done + mkdir_vrfy -p ${COMOUTbicor}/bcdata.${yyyymm}/grid/${cyc}z/${PDY} + cp_vrfy ${COMIN}/${NET}.${cycle}.*sfc*.nc ${COMOUTbicor}/bcdata.${yyyymm}/grid/${cyc}z/${PDY} fi #----------------------------------------------------------------------------- # STEP 4: Performing Bias Correction for Ozone #----------------------------------------------------------------------------- +rm_vrfy -rf ${DATA}/data/bcdata* + +ln_vrfy -sf ${COMINbicor}/bcdata* "${DATA}/data" + mkdir_vrfy -p ${DATA}/data/sites cp_vrfy ${PARMaqm_utils}/bias_correction/config.ozone.bias_corr_${id_domain}.${cyc}z ${DATA} PREP_STEP -eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_correct config.ozone.bias_corr_${id_domain}.${cyc}z ${cyc}z ${BC_STDAY} ${PDY} ${REDIRECT_OUT_ERR} || print_err_msg_exit "Call to executable to run AQM_BIAS_CORRECT returned with nonzero exit code." +eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_correct config.ozone.bias_corr_${id_domain}.${cyc}z ${cyc}z ${BC_STDAY} ${PDY} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_BIAS_CORRECT returned with nonzero exit code." + fi +fi POST_STEP cp_vrfy ${DATA}/out/ozone.corrected* ${COMIN} if [ "${cyc}" = "12" ]; then - cp_vrfy ${DATA}/sites/sites.valid.ozone.${PDY}.${cyc}z.list ${DATA} + cp_vrfy ${DATA}/data/sites/sites.valid.ozone.${PDY}.${cyc}z.list ${DATA} fi #----------------------------------------------------------------------------- @@ -236,8 +293,15 @@ EOF1 # convert from netcdf to grib2 format PREP_STEP -eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_bias_cor_grib2 ${PDY} ${cyc} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ -Call to executable to run AQM_POST_BIAS_COR_GRIB2 returned with nonzero exit code." +eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_bias_cor_grib2 ${PDY} ${cyc} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_POST_BIAS_COR_GIRB2 returned with nonzero exit code." + fi +fi POST_STEP cp_vrfy ${DATA}/${NET}.${cycle}.awpozcon*bc*.grib2 ${COMOUT} @@ -264,10 +328,10 @@ EOF1 flag_run_bicor_max=yes # 06z needs b.nc to find current day output from 04Z to 06Z if [ "${cyc}" = "06" ]; then - if [ -s ${COMIN_PDY}/00/ozone.corrected.${PDY}.00z.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/00/ozone.corrected.${PDY}.00z.nc b.nc - elif [ -s ${COMIN_PDYm1}/12/ozone.corrected.${PDYm1}.12z.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/ozone.corrected.${PDYm1}.12z.nc b.nc + if [ -s ${COMIN}/../00/ozone.corrected.${PDY}.00z.nc ]; then + ln_vrfy -sf ${COMIN}/../00/ozone.corrected.${PDY}.00z.nc b.nc + elif [ -s ${COMINm1}/12/ozone.corrected.${PDYm1}.12z.nc ]; then + ln_vrfy -sf ${COMINm1}/12/ozone.corrected.${PDYm1}.12z.nc b.nc chk=0 else flag_run_bicor_max=no @@ -276,20 +340,20 @@ EOF1 if [ "${cyc}" = "12" ]; then # 12z needs b.nc to find current day output from 04Z to 06Z - if [ -s ${COMIN_PDY}/00/ozone.corrected.${PDY}.00z.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/00/ozone.corrected.${PDY}.00z.nc b.nc - elif [ -s ${COMIN_PDYm1}/12/ozone.corrected.${PDYm1}.12z.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/ozone.corrected.${PDYm1}.12z.nc b.nc + if [ -s ${COMIN}/../00/ozone.corrected.${PDY}.00z.nc ]; then + ln_vrfy -sf ${COMIN}/../00/ozone.corrected.${PDY}.00z.nc b.nc + elif [ -s ${COMINm1}/12/ozone.corrected.${PDYm1}.12z.nc ]; then + ln_vrfy -sf ${COMINm1}/12/ozone.corrected.${PDYm1}.12z.nc b.nc chk=0 else flag_run_bicor_max=no fi # 12z needs c.nc to find current day output from 07Z to 12z - if [ -s ${COMIN_PDY}/06/ozone.corrected.${PDY}.06z.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/06/ozone.corrected.${PDY}.06z.nc c.nc - elif [ -s ${COMIN_PDYm1}/12/ozone.corrected.${PDYm1}.12z.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/ozone.corrected.${PDYm1}.12z.nc c.nc + if [ -s ${COMIN}/../06/ozone.corrected.${PDY}.06z.nc ]; then + ln_vrfy -sf ${COMIN}/../06/ozone.corrected.${PDY}.06z.nc c.nc + elif [ -s ${COMINm1}/12/ozone.corrected.${PDYm1}.12z.nc ]; then + ln_vrfy -sf ${COMINm1}/12/ozone.corrected.${PDYm1}.12z.nc c.nc chk1=0 else flag_run_bicor_max=no @@ -301,8 +365,15 @@ EOF1 # write out grib2 format #------------------------------------------------- PREP_STEP - eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_maxi_bias_cor_grib2 ${PDY} ${cyc} ${chk} ${chk1} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ - Call to executable to run AQM_POST_MAXI_BIAS_COR_GRIB2 returned with nonzero exit code." + eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_maxi_bias_cor_grib2 ${PDY} ${cyc} ${chk} ${chk1} ${REDIRECT_OUT_ERR} + export err=$? + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk + else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_POST_MAXI_BIAS_COR_GRIB2 returned with nonzero exit code." + fi + fi POST_STEP # split into max_1h and max_8h files and copy to grib227 @@ -316,7 +387,7 @@ EOF1 cp_vrfy ${DATA}/${NET}.${cycle}.max_*hr_o3_bc.*.grib2 ${COMOUT} - if [ "$SENDDBN" = "YES" ]; then + if [ "$SENDDBN" = "TRUE" ]; then ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUT}/${NET}.${cycle}.max_1hr_o3_bc.227.grib2 ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUT}/${NET}.${cycle}.max_8hr_o3_bc.227.grib2 fi @@ -340,13 +411,13 @@ EOF1 tocgrib2super < ${PARMaqm_utils}/wmo/grib2_aqm-${hr}hro3-maxi.${cycle}.227 done - # Post Files to COMOUT - cp_vrfy awpaqm.${cycle}.*o3-max-bc.227.grib2 ${COMOUT} + # Post Files to COMOUTwmo + cp_vrfy awpaqm.${cycle}.*o3-max-bc.227.grib2 ${COMOUTwmo} # Distribute Data - if [ "${SENDDBN_NTC}" = "YES" ] ; then - ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.1ho3-max-bc.227.grib2 - ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.8ho3-max-bc.227.grib2 + if [ "${SENDDBN_NTC}" = "TRUE" ] ; then + ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUTwmo}/awpaqm.${cycle}.1ho3-max-bc.227.grib2 + ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUTwmo}/awpaqm.${cycle}.8ho3-max-bc.227.grib2 fi fi fi @@ -356,17 +427,17 @@ rm_vrfy -rf tmpfile fhr=01 while [ "${fhr}" -le "${FCST_LEN_HRS}" ]; do - fhr2d=$( printf "%02d" "${fhr}" ) + fhr3d=$( printf "%03d" "${fhr}" ) - cp_vrfy ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr2d}.${id_domain}.grib2 ${COMOUT} + cp_vrfy ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr3d}.${id_domain}.grib2 ${COMOUT} # create GRIB file to convert to grid 227 then to GRIB2 for NDFD - cat ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr2d}.${id_domain}.grib2 >> tmpfile + cat ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr3d}.${id_domain}.grib2 >> tmpfile if [ "${fhr}" -le "07" ]; then - cat ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr2d}.${id_domain}.grib2 >> tmpfile.1hr + cat ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr3d}.${id_domain}.grib2 >> tmpfile.1hr else - wgrib2 ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr2d}.${id_domain}.grib2 -d 1 -append -grib tmpfile.1hr - wgrib2 ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr2d}.${id_domain}.grib2 -d 2 -append -grib tmpfile.8hr + wgrib2 ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr3d}.${id_domain}.grib2 -d 1 -append -grib tmpfile.1hr + wgrib2 ${DATA}/${NET}.${cycle}.awpozcon_bc.f${fhr3d}.${id_domain}.grib2 -d 2 -append -grib tmpfile.8hr fi (( fhr=fhr+1 )) done @@ -391,7 +462,7 @@ if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then cp_vrfy ${NET}.${cycle}.ave_8hr_o3_bc.227.grib2 ${COMOUT} fi -if [ "${SENDDBN}" = "YES" ] ; then +if [ "${SENDDBN}" = "TRUE" ] ; then ${DBNROOT}/bin/dbn_alert MODEL AQM_CONC ${job} ${COMOUT}/${NET}.${cycle}.ave_1hr_o3_bc.227.grib2 if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then ${DBNROOT}/bin/dbn_alert MODEL AQM_CONC ${job} ${COMOUT}/${NET}.${cycle}.ave_8hr_o3_bc.227.grib2 @@ -438,14 +509,14 @@ if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then export FORT51=awpaqm.${cycle}.${hr}ho3-max-bc.227.grib2 tocgrib2super < ${PARMaqm_utils}/wmo/grib2_aqm-${hr}hro3_bc-maxi.${cycle}.227 - # Post Files to COMOUT - cp_vrfy awpaqm.${cycle}.${hr}ho3-bc.227.grib2 ${COMOUT} - cp_vrfy awpaqm.${cycle}.${hr}ho3-max-bc.227.grib2 ${COMOUT} + # Post Files to COMOUTwmo + cp_vrfy awpaqm.${cycle}.${hr}ho3-bc.227.grib2 ${COMOUTwmo} + cp_vrfy awpaqm.${cycle}.${hr}ho3-max-bc.227.grib2 ${COMOUTwmo} # Distribute Data - if [ "${SENDDBN}" = "YES" ]; then - ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.${hr}ho3-bc.227.grib2 - ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.${hr}ho3-max-bc.227.grib2 + if [ "${SENDDBN}" = "TRUE" ]; then + ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUTwmo}/awpaqm.${cycle}.${hr}ho3-bc.227.grib2 + ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUTwmo}/awpaqm.${cycle}.${hr}ho3-max-bc.227.grib2 fi done fi diff --git a/scripts/exregional_bias_correction_pm25.sh b/scripts/exregional_bias_correction_pm25.sh index 2e49e9554b..05348ecbc8 100755 --- a/scripts/exregional_bias_correction_pm25.sh +++ b/scripts/exregional_bias_correction_pm25.sh @@ -73,18 +73,6 @@ else All executables will be submitted with command \'${RUN_CMD_SERIAL}\'." fi -# -#----------------------------------------------------------------------- -# -# Move to the working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_BIAS_CORRECTION_PM25" -rm_vrfy -rf $DATA -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA - yyyy=${PDY:0:4} yyyymm=${PDY:0:6} yyyy_m1=${PDYm1:0:4} @@ -115,26 +103,53 @@ fi # STEP 1: Retrieve AIRNOW observation data #----------------------------------------------------------------------------- -# Link the historical airnow data mkdir_vrfy -p "${DATA}/data" -ln_vrfy -sf ${AQM_AIRNOW_HIST_DIR}/bcdata* "${DATA}/data" -if [ -d "${DATA}/data/bcdata.${yyyymm}" ]; then - rm_vrfy -rf "${DATA}/data/bcdata.${yyyymm}" - mkdir_vrfy -p "${DATA}/data/bcdata.${yyyymm}" - cp_vrfy -rL "${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/airnow" "${DATA}/data/bcdata.${yyyymm}" - cp_vrfy -rL "${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated" "${DATA}/data/bcdata.${yyyymm}" -fi - -# Retrieve real-time airnow data for the last three days -if [ "${DO_REAL_TIME}" = "TRUE" ]; then - mkdir -p ${DATA}/data/bcdata.${yyyymm_m1}/airnow/${yyyy_m1}/${PDYm1}/b008 - mkdir -p ${DATA}/data/bcdata.${yyyymm_m2}/airnow/${yyyy_m2}/${PDYm2}/b008 - mkdir -p ${DATA}/data/bcdata.${yyyymm_m3}/airnow/${yyyy_m3}/${PDYm3}/b008 - cp_vrfy ${COMINairnow}/${PDYm1}/b008/xx031 ${DATA}/data/bcdata.${yyyymm_m1}/airnow/${yyyy_m1}/${PDYm1}/b008 - cp_vrfy ${COMINairnow}/${PDYm2}/b008/xx031 ${DATA}/data/bcdata.${yyyymm_m2}/airnow/${yyyy_m2}/${PDYm2}/b008 - cp_vrfy ${COMINairnow}/${PDYm3}/b008/xx031 ${DATA}/data/bcdata.${yyyymm_m3}/airnow/${yyyy_m3}/${PDYm3}/b008 -fi +# Retrieve real-time airnow data for the last three days. +# In the following for-loop, pdym stands for previous (m) day of the present day (PDY) +# in the NCO standards, i.e. PDYm1: 1day ago, PDYm2: 2days ago, PDYm3: 3days ago +for i_pdym in {1..3}; do + case $i_pdym in + 1) + cvt_yyyy="${yyyy_m1}" + cvt_yyyymm="${yyyymm_m1}" + cvt_pdy="${PDYm1}" + ;; + 2) + cvt_yyyy="${yyyy_m2}" + cvt_yyyymm="${yyyymm_m2}" + cvt_pdy="${PDYm2}" + ;; + 3) + cvt_yyyy="${yyyy_m3}" + cvt_yyyymm="${yyyymm_m3}" + cvt_pdy="${PDYm3}" + ;; + esac + + cvt_input_dir="${DATA}/data/bcdata.${cvt_yyyymm}/airnow/csv" + cvt_output_dir="${DATA}/data/bcdata.${cvt_yyyymm}/airnow/netcdf" + cvt_input_fn="HourlyAQObs_YYYYMMDDHH.dat" + cvt_output_fn="HourlyAQObs.YYYYMMDD.nc" + cvt_input_fp="${cvt_input_dir}/YYYY/YYYYMMDD/${cvt_input_fn}" + cvt_output_fp="${cvt_output_dir}/YYYY/YYYYMMDD/${cvt_output_fn}" + + mkdir_vrfy -p "${cvt_input_dir}/${cvt_yyyy}/${cvt_pdy}" + mkdir_vrfy -p "${cvt_output_dir}/${cvt_yyyy}/${cvt_pdy}" + cp_vrfy ${DCOMINairnow}/${cvt_pdy}/airnow/HourlyAQObs_${cvt_pdy}*.dat "${cvt_input_dir}/${cvt_yyyy}/${cvt_pdy}" + + PREP_STEP + eval ${RUN_CMD_SERIAL} ${EXECdir}/convert_airnow_csv ${cvt_input_fp} ${cvt_output_fp} ${cvt_pdy} ${cvt_pdy} ${REDIRECT_OUT_ERR} + export err=$? + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk + else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run CONVERT_AIRNOW_CSV returned with nonzero exit code." + fi + fi + POST_STEP +done #----------------------------------------------------------------------------- # STEP 2: Extracting PM2.5, O3, and met variables from CMAQ input and outputs @@ -174,34 +189,37 @@ mkdir_vrfy -p ${DATA}/data/site-lists.interp mkdir_vrfy -p ${DATA}/out/pm25/${yyyy} mkdir_vrfy -p ${DATA}/data/bcdata.${yyyymm}/interpolated/pm25/${yyyy} -cp_vrfy ${PARMaqm_utils}/bias_correction/sites.valid.pm25.20220724.12z.list ${DATA}/data/site-lists.interp +cp_vrfy ${PARMaqm_utils}/bias_correction/sites.valid.pm25.20230331.12z.list ${DATA}/data/site-lists.interp cp_vrfy ${PARMaqm_utils}/bias_correction/aqm.t12z.chem_sfc.f000.nc ${DATA}/data/coords cp_vrfy ${PARMaqm_utils}/bias_correction/config.interp.pm2.5.5-vars_${id_domain}.${cyc}z ${DATA} PREP_STEP -eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_interpolate config.interp.pm2.5.5-vars_${id_domain}.${cyc}z ${cyc}z ${PDY} ${PDY} ${REDIRECT_OUT_ERR} || print_err_msg_exit "Call to executable to run AQM_BIAS_INTERPOLATE returned with nonzero exit code." +eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_interpolate config.interp.pm2.5.5-vars_${id_domain}.${cyc}z ${cyc}z ${PDY} ${PDY} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run CONVERT_AIRNOW_CSV returned with nonzero exit code." + fi +fi POST_STEP cp_vrfy ${DATA}/out/pm25/${yyyy}/*nc ${DATA}/data/bcdata.${yyyymm}/interpolated/pm25/${yyyy} if [ "${DO_AQM_SAVE_AIRNOW_HIST}" = "TRUE" ]; then - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated/pm25/${yyyy} - cp_vrfy ${DATA}/out/pm25/${yyyy}/*nc ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated/pm25/${yyyy} - - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/airnow/${yyyy}/${PDY}/b008 - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m1}/airnow/${yyyy_m1}/${PDYm1}/b008 - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m2}/airnow/${yyyy_m2}/${PDYm2}/b008 - mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m3}/airnow/${yyyy_m3}/${PDYm3}/b008 - cp_vrfy ${COMINairnow}/${PDY}/b008/xx031 ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/airnow/${yyyy}/${PDY}/b008 - cp_vrfy ${COMINairnow}/${PDYm1}/b008/xx031 ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m1}/airnow/${yyyy_m1}/${PDYm1}/b008 - cp_vrfy ${COMINairnow}/${PDYm2}/b008/xx031 ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m2}/airnow/${yyyy_m2}/${PDYm2}/b008 - cp_vrfy ${COMINairnow}/${PDYm3}/b008/xx031 ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm_m3}/airnow/${yyyy_m3}/${PDYm3}/b008 + mkdir_vrfy -p ${COMOUTbicor}/bcdata.${yyyymm}/interpolated/pm25/${yyyy} + cp_vrfy ${DATA}/out/pm25/${yyyy}/*nc ${COMOUTbicor}/bcdata.${yyyymm}/interpolated/pm25/${yyyy} fi #----------------------------------------------------------------------- # STEP 4: Performing Bias Correction for PM2.5 #----------------------------------------------------------------------- +rm_vrfy -rf ${DATA}/data/bcdata* + +ln_vrfy -sf ${COMINbicor}/bcdata* "${DATA}/data" + mkdir_vrfy -p ${DATA}/data/sites cp_vrfy ${PARMaqm_utils}/bias_correction/config.pm2.5.bias_corr_${id_domain}.${cyc}z ${DATA} @@ -209,13 +227,21 @@ cp_vrfy ${PARMaqm_utils}/bias_correction/site_blocking.pm2.5.2021.0427.2-sites.t cp_vrfy ${PARMaqm_utils}/bias_correction/bias_thresholds.pm2.5.2015.1030.32-sites.txt ${DATA} PREP_STEP -eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_correct config.pm2.5.bias_corr_${id_domain}.${cyc}z ${cyc}z ${BC_STDAY} ${PDY} ${REDIRECT_OUT_ERR} || print_err_msg_exit "Call to executable to run AQM_BIAS_CORRECT returned with nonzero exit code." +eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_correct config.pm2.5.bias_corr_${id_domain}.${cyc}z ${cyc}z ${BC_STDAY} ${PDY} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_BIAS_CORRECT returned with nonzero exit code." + fi +fi POST_STEP cp_vrfy $DATA/out/pm2.5.corrected* ${COMIN} if [ "${cyc}" = "12" ]; then - cp_vrfy ${DATA}/sites/sites.valid.pm25.${PDY}.${cyc}z.list ${DATA} + cp_vrfy ${DATA}/data/sites/sites.valid.pm25.${PDY}.${cyc}z.list ${DATA} fi #------------------------------------------------------------------------ @@ -235,12 +261,19 @@ id_gribdomain=${id_domain} EOF1 PREP_STEP -eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_bias_cor_grib2 ${PDY} ${cyc} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ -Call to executable to run AQM_POST_BIAS_COR_GRIB2 returned with nonzero exit code." +eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_bias_cor_grib2 ${PDY} ${cyc} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_POST_BIAS_COR_GRIB2 returned with nonzero exit code." + fi +fi POST_STEP cp_vrfy ${DATA}/${NET}.${cycle}.pm25*bc*.grib2 ${COMOUT} -if [ "$SENDDBN" = "YES" ]; then +if [ "$SENDDBN" = "TRUE" ]; then $DBNROOT/bin/dbn_alert MODEL AQM_PM ${job} ${COMOUT} fi @@ -266,10 +299,10 @@ EOF1 flag_run_bicor_max=yes # 06z needs b.nc to find current day output from 04Z to 06Z if [ "${cyc}" = "06" ]; then - if [ -s ${COMIN_PDY}/00/pm2.5.corrected.${PDY}.00z.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/00/pm2.5.corrected.${PDY}.00z.nc b.nc - elif [ -s ${COMIN_PDYm1}/12/pm2.5.corrected.${PDYm1}.12z.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/pm2.5.corrected.${PDYm1}.12z.nc b.nc + if [ -s ${COMIN}/../00/pm2.5.corrected.${PDY}.00z.nc ]; then + ln_vrfy -sf ${COMIN}/../00/pm2.5.corrected.${PDY}.00z.nc b.nc + elif [ -s ${COMINm1}/12/pm2.5.corrected.${PDYm1}.12z.nc ]; then + ln_vrfy -sf ${COMINm1}/12/pm2.5.corrected.${PDYm1}.12z.nc b.nc chk=0 else flag_run_bicor_max=no @@ -278,20 +311,20 @@ EOF1 if [ "${cyc}" = "12" ]; then # 12z needs b.nc to find current day output from 04Z to 06Z - if [ -s ${COMIN_PDY}/00/pm2.5.corrected.${PDY}.00z.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/00/pm2.5.corrected.${PDY}.00z.nc b.nc - elif [ -s ${COMIN_PDYm1}/12/pm2.5.corrected.${PDYm1}.12z.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/pm2.5.corrected.${PDYm1}.12z.nc b.nc + if [ -s ${COMIN}/../00/pm2.5.corrected.${PDY}.00z.nc ]; then + ln_vrfy -sf ${COMIN}/../00/pm2.5.corrected.${PDY}.00z.nc b.nc + elif [ -s ${COMINm1}/12/pm2.5.corrected.${PDYm1}.12z.nc ]; then + ln_vrfy -sf ${COMINm1}/12/pm2.5.corrected.${PDYm1}.12z.nc b.nc chk=0 else flag_run_bicor_max=no fi # 12z needs c.nc to find current day output from 07Z to 12z - if [ -s ${COMIN_PDY}/06/pm2.5.corrected.${PDY}.06z.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/06/pm2.5.corrected.${PDY}.06z.nc c.nc - elif [ -s ${COMIN_PDYm1}/12/pm2.5.corrected.${PDYm1}.12z.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/pm2.5.corrected.${PDYm1}.12z.nc c.nc + if [ -s ${COMIN}/../06/pm2.5.corrected.${PDY}.06z.nc ]; then + ln_vrfy -sf ${COMIN}/../06/pm2.5.corrected.${PDY}.06z.nc c.nc + elif [ -s ${COMINm1}/12/pm2.5.corrected.${PDYm1}.12z.nc ]; then + ln_vrfy -sf ${COMINm1}/12/pm2.5.corrected.${PDYm1}.12z.nc c.nc chk1=0 else flag_run_bicor_max=no @@ -302,8 +335,15 @@ EOF1 # write out grib2 format #------------------------------------------------- PREP_STEP - eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_maxi_bias_cor_grib2 ${PDY} ${cyc} ${chk} ${chk1} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ - Call to executable to run AQM_POST_MAXI_BIAS_COR_GRIB2 returned with nonzero exit code." + eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_maxi_bias_cor_grib2 ${PDY} ${cyc} ${chk} ${chk1} ${REDIRECT_OUT_ERR} + export err=$? + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk + else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_POST_MAXI_BIAS_COR_GRIB2 returned with nonzero exit code." + fi + fi POST_STEP # split into two files: one for 24hr_ave and one for 1h_max @@ -330,7 +370,7 @@ EOF1 cp_vrfy ${NET}.${cycle}.max_1hr_pm25_bc.227.grib2 ${COMOUT} cp_vrfy ${NET}.${cycle}.ave_24hr_pm25_bc.227.grib2 ${COMOUT} - if [ "${SENDDBN}" = "YES" ]; then + if [ "${SENDDBN}" = "TRUE" ]; then ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUT}/${NET}.${cycle}.max_1hr_pm25_bc.227.grib2 ${DBNROOT}/bin/dbn_alert MODEL AQM_PM ${job} ${COMOUT}/${NET}.${cycle}.ave_24hr_pm25_bc.227.grib2 fi @@ -338,8 +378,8 @@ fi fhr=01 while [ "${fhr}" -le "${FCST_LEN_HRS}" ]; do - fhr2d=$( printf "%02d" "${fhr}" ) - cat ${DATA}/${NET}.${cycle}.pm25_bc.f${fhr2d}.${id_domain}.grib2 >> tmpfile_pm25_bc + fhr3d=$( printf "%03d" "${fhr}" ) + cat ${DATA}/${NET}.${cycle}.pm25_bc.f${fhr3d}.${id_domain}.grib2 >> tmpfile_pm25_bc (( fhr=fhr+1 )) done @@ -348,7 +388,7 @@ wgrib2 tmpfile_pm25_bc -set_grib_type c3b -new_grid_winds earth -new_grid ${grid cp_vrfy tmpfile_pm25_bc ${COMOUT}/${NET}.${cycle}.ave_1hr_pm25_bc.${id_domain}.grib2 cp_vrfy ${NET}.${cycle}.grib2_pm25_bc.227 ${COMOUT}/${NET}.${cycle}.ave_1hr_pm25_bc.227.grib2 -if [ "${SENDDBN}" = "YES" ]; then +if [ "${SENDDBN}" = "TRUE" ]; then ${DBNROOT}/bin/dbn_alert MODEL AQM_PM ${job} ${COMOUT}/${NET}.${cycle}.ave_1hr_pm25_bc.227.grib2 fi @@ -410,13 +450,13 @@ if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then export FORT51=awpaqm.${cycle}.24hr-pm25-ave-bc.227.grib2 tocgrib2super < ${PARMaqm_utils}/wmo/grib2_aqm_ave_24hrpm25_bc_awp.${cycle}.227 - # Post Files to COMOUT - cp_vrfy awpaqm.${cycle}.1hpm25-bc.227.grib2 ${COMOUT} - cp_vrfy awpaqm.${cycle}.daily-1hr-pm25-max-bc.227.grib2 ${COMOUT} - cp_vrfy awpaqm.${cycle}.24hr-pm25-ave-bc.227.grib2 ${COMOUT} + # Post Files to COMOUTwmo + cp_vrfy awpaqm.${cycle}.1hpm25-bc.227.grib2 ${COMOUTwmo} + cp_vrfy awpaqm.${cycle}.daily-1hr-pm25-max-bc.227.grib2 ${COMOUTwmo} + cp_vrfy awpaqm.${cycle}.24hr-pm25-ave-bc.227.grib2 ${COMOUTwmo} # Distribute Data - if [ "${SENDDBN_NTC}" = "YES" ] ; then + if [ "${SENDDBN_NTC}" = "TRUE" ] ; then ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.1hpm25-bc.227.grib2 ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.daily-1hr-pm25-max-bc.227.grib2 ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.24hr-pm25-ave-bc.227.grib2 diff --git a/scripts/exregional_fire_emission.sh b/scripts/exregional_fire_emission.sh index 4605d52486..a2e9f9b44d 100755 --- a/scripts/exregional_fire_emission.sh +++ b/scripts/exregional_fire_emission.sh @@ -49,17 +49,6 @@ data files. # #----------------------------------------------------------------------- # -# Move to the FIRE EMISSION working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_FIRE_EMISSION" -rm_vrfy -rf $DATA -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA -# -#----------------------------------------------------------------------- -# # Set up variables for call to retrieve_data.py # #----------------------------------------------------------------------- @@ -67,13 +56,10 @@ cd_vrfy $DATA yyyymmdd=${FIRE_FILE_CDATE:0:8} hh=${FIRE_FILE_CDATE:8:2} -CDATE_md1=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 24 hours" "+%Y%m%d%H" ) -CDATE_mh3=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 3 hours" "+%Y%m%d%H" ) -yyyymmdd_mh3=${CDATE_mh3:0:8} -hh_mh3=${CDATE_mh3:8:2} -CDATE_mh2=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 2 hours" "+%Y%m%d%H" ) CDATE_mh1=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 1 hours" "+%Y%m%d%H" ) +yyyymmdd_mh1=${CDATE_mh1:0:8} +hh_mh1=${CDATE_mh1:8:2} # #----------------------------------------------------------------------- # @@ -84,64 +70,122 @@ CDATE_mh1=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 1 hours" "+%Y%m%d% aqm_fire_file_fn="${AQM_FIRE_FILE_PREFIX}_${yyyymmdd}_t${hh}z${AQM_FIRE_FILE_SUFFIX}" # Check if the fire file exists in the designated directory -if [ -e "${AQM_FIRE_DIR}/${yyyymmdd}/${aqm_fire_file_fn}" ]; then - cp_vrfy "${AQM_FIRE_DIR}/${yyyymmdd}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}" +if [ -e "${DCOMINfire}/${yyyymmdd}/${aqm_fire_file_fn}" ]; then + cp_vrfy "${DCOMINfire}/${yyyymmdd}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}" else # Copy raw data - for ihr in {0..21}; do - download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_mh3} ${hh_mh3} UTC - $ihr hours" "+%Y%m%d%H" ) + for ihr in {0..23}; do + download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_mh1} ${hh_mh1} UTC - $ihr hours" "+%Y%m%d%H" ) FILE_13km="Hourly_Emissions_13km_${download_time}00_${download_time}00.nc" - if [ -e "${AQM_FIRE_DIR}/RAVE_raw_new/${FILE_13km}" ]; then - ln_vrfy -sf "${AQM_FIRE_DIR}/RAVE_raw_new/Hourly_Emissions_13km_${download_time}00_${download_time}00.nc" . - elif [ -d "${AQM_FIRE_DIR}/${CDATE_md1}" ]; then - echo "${FILE_13km} does not exist. Replacing with the file of previous date ..." - yyyymmdd_dn=${download_time:0:8} - hh_dn=${download_time:8:2} - missing_download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_dn} ${hh_dn} UTC - 24 hours" "+%Y%m%d%H" ) - ln_vrfy -sf "${AQM_FIRE_DIR}/${CDATE_md1}/Hourly_Emissions_13km_${missing_download_time}00_${missing_download_time}00.nc" "Hourly_Emissions_13km_${download_time}00_${download_time}00.nc" + yyyymmdd_dn=${download_time:0:8} + hh_dn=${download_time:8:2} + missing_download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_dn} ${hh_dn} UTC - 24 hours" "+%Y%m%d%H" ) + yyyymmdd_dn_md1=${missing_download_time:0:8} + FILE_13km_md1=Hourly_Emissions_13km_${missing_download_time}00_${missing_download_time}00.nc + if [ -e "${DCOMINfire}/${yyyymmdd_dn}/rave/${FILE_13km}" ]; then + cp_vrfy "${DCOMINfire}/${yyyymmdd_dn}/rave/${FILE_13km}" . + elif [ -e "${DCOMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1}" ]; then + echo "WARNING: ${FILE_13km} does not exist. Replacing with the file of previous date ..." + cp_vrfy "${DCOMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1}" "${FILE_13km}" else - print_err_msg_exit "RAVE raw data files do not exist." + message_txt="Fire Emission RAW data does not exist: + FILE_13km_md1 = \"${FILE_13km_md1}\" + DCOMINfire = \"${DCOMINfire}\"" + + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + cp_vrfy "${DCOMINfire}/Hourly_Emissions_13km_dummy.nc" "${FILE_13km}" + message_warning="WARNING: ${message_txt}. Replacing with the dummy file :: AQM RUN SOFT FAILED." + print_info_msg "${message_warning}" + if [ ! -z "${maillist}" ]; then + echo "${message_warning}" | mail.py $maillist + fi + else + print_err_msg_exit "${message_txt}" + fi fi done - ncks -O -h --mk_rec_dmn time Hourly_Emissions_13km_${download_time}00_${download_time}00.nc temp.nc || print_err_msg_exit "\ -Call to NCKS returned with nonzero exit code." + ncks -O -h --mk_rec_dmn time Hourly_Emissions_13km_${download_time}00_${download_time}00.nc temp.nc + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to NCKS returned with nonzero exit code." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi mv_vrfy temp.nc Hourly_Emissions_13km_${download_time}00_${download_time}00.nc - # Extra times - cp_vrfy Hourly_Emissions_13km_${CDATE_mh3}00_${CDATE_mh3}00.nc Hourly_Emissions_13km_${CDATE_mh2}00_${CDATE_mh2}00.nc - cp_vrfy Hourly_Emissions_13km_${CDATE_mh3}00_${CDATE_mh3}00.nc Hourly_Emissions_13km_${CDATE_mh1}00_${CDATE_mh1}00.nc - - ncrcat -h Hourly_Emissions_13km_*.nc Hourly_Emissions_13km_${yyyymmdd}0000_${yyyymmdd}2300.t${cyc}z.nc || print_err_msg_exit "\ -Call to NCRCAT returned with nonzero exit code." + ncrcat -h Hourly_Emissions_13km_*.nc Hourly_Emissions_13km_${yyyymmdd}0000_${yyyymmdd}2300.t${cyc}z.nc + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to NCRCAT returned with nonzero exit code." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi input_fire="${DATA}/Hourly_Emissions_13km_${yyyymmdd}0000_${yyyymmdd}2300.t${cyc}z.nc" output_fire="${DATA}/Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_new24.t${cyc}z.nc" python3 ${HOMEdir}/sorc/AQM-utils/python_utils/RAVE_remake.allspecies.aqmna13km.g793.py --date "${yyyymmdd}" --cyc "${hh}" --input_fire "${input_fire}" --output_fire "${output_fire}" + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to python script \"RAVE_remake.allspecies.py\" returned with nonzero exit code." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi - ncks --mk_rec_dmn Time Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_new24.t${cyc}z.nc -o Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc || print_err_msg_exit "\ -Call to NCKS returned with nonzero exit code." + ncks --mk_rec_dmn Time Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_new24.t${cyc}z.nc -o Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to NCKS returned with nonzero exit code." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi - ncrcat Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc ${aqm_fire_file_fn} || print_err_msg_exit "\ -Call to NCRCAT returned with nonzero exit code." + ncrcat Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc ${aqm_fire_file_fn} + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to NCRCAT returned with nonzero exit code." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi # Copy the final fire emission file to STAGING_DIR cp_vrfy "${DATA}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}" # Archive the final fire emission file to disk and HPSS if [ "${DO_AQM_SAVE_FIRE}" = "TRUE" ]; then - mkdir -p "${AQM_FIRE_DIR}/${yyyymmdd}" - cp_vrfy "${DATA}/${aqm_fire_file_fn}" "${AQM_FIRE_DIR}/${yyyymmdd}" + cp "${DATA}/${aqm_fire_file_fn}" ${DCOMINfire} hsi_log_fn="log.hsi_put.${yyyymmdd}_${hh}" - hsi put ${aqm_fire_file_fn} : ${AQM_FIRE_ARCHV_DIR}/${aqm_fire_file_fn} >& ${hsi_log_fn} || \ - print_err_msg_exit "\ -htar file writing operation (\"hsi put ...\") failed. Check the log + hsi put ${aqm_fire_file_fn} : ${AQM_FIRE_ARCHV_DIR}/${aqm_fire_file_fn} >& ${hsi_log_fn} + export err=$? + if [ $err -ne 0 ]; then + message_txt="htar file writing operation (\"hsi put ...\") failed. Check the log file hsi_log_fn in the DATA directory for details: DATA = \"${DATA}\" hsi_log_fn = \"${hsi_log_fn}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi fi fi # diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 5b138eb298..6152a62716 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -192,12 +192,20 @@ python3 -u ${USHdir}/retrieve_data.py \ --summary_file ${EXTRN_DEFNS} \ $additional_flags" -$cmd || print_err_msg_exit "\ -Call to retrieve_data.py failed with a non-zero exit status. - +$cmd +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to retrieve_data.py failed with a non-zero exit status. The command was: ${cmd} " + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index a110cd1d2c..98867afc79 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -84,7 +84,7 @@ fi #----------------------------------------------------------------------- # if [ $RUN_ENVIR = "nco" ]; then - extrn_mdl_staging_dir="${COMINext}${SLASH_ENSMEM_SUBDIR}" + extrn_mdl_staging_dir="${DATAROOT}/get_extrn_ics.${share_pid}${SLASH_ENSMEM_SUBDIR}" extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_ICS}.ICS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" else extrn_mdl_staging_dir="${COMIN}/${EXTRN_MDL_NAME_ICS}/for_ICS${SLASH_ENSMEM_SUBDIR}" @@ -94,16 +94,6 @@ fi # #----------------------------------------------------------------------- # -# -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_ICS" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA -# -#----------------------------------------------------------------------- -# # Set physics-suite-dependent variable mapping table needed in the FORTRAN # namelist file that the chgres_cube executable will read in. # @@ -140,10 +130,14 @@ case "${CCPP_PHYS_SUITE}" in ;; # *) - print_err_msg_exit "\ -The variable \"varmap_file\" has not yet been specified for this physics -suite (CCPP_PHYS_SUITE): + message_txt="The variable \"varmap_file\" has not yet been specified for +this physics suite (CCPP_PHYS_SUITE): CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi ;; # esac @@ -495,10 +489,14 @@ case "${EXTRN_MDL_NAME_ICS}" in ;; *) - print_err_msg_exit "\ -External-model-dependent namelist variables have not yet been specified + message_txt="External-model-dependent namelist variables have not yet been specified for this external IC model (EXTRN_MDL_NAME_ICS): EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi ;; esac @@ -522,11 +520,15 @@ hh="${EXTRN_MDL_CDATE:8:2}" exec_fn="chgres_cube" exec_fp="$EXECdir/${exec_fn}" if [ ! -f "${exec_fp}" ]; then - print_err_msg_exit "\ -The executable (exec_fp) for generating initial conditions on the FV3-LAM -native grid does not exist: + message_txt="The executable (exec_fp) for generating initial conditions +on the FV3-LAM native grid does not exist: exec_fp = \"${exec_fp}\" Please ensure that you've built this executable." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi fi # #----------------------------------------------------------------------- @@ -592,16 +594,23 @@ settings=" # Call the python script to create the namelist file. # nml_fn="fort.41" -${USHdir}/set_namelist.py -q -u "$settings" -o ${nml_fn} || \ - print_err_msg_exit "\ -Call to python script set_namelist.py to set the variables in the namelist -file read in by the ${exec_fn} executable failed. Parameters passed to -this script are: +${USHdir}/set_namelist.py -q -u "$settings" -o ${nml_fn} +err=$? +if [ $err -ne 0 ]; then + message_txt="Call to python script set_namelist.py to set the variables +in the namelist file read in by the ${exec_fn} executable failed. Parameters +passed to this script are: Name of output namelist file: nml_fn = \"${nml_fn}\" Namelist settings specified on command line (these have highest precedence): settings = $settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #----------------------------------------------------------------------- # @@ -660,21 +669,29 @@ if [ "${USE_FVCOM}" = "TRUE" ]; then fvcom_exec_fp="$EXECdir/${fvcom_exec_fn}" fvcom_time="${DATE_FIRST_CYCL:0:4}-${DATE_FIRST_CYCL:4:2}-${DATE_FIRST_CYCL:6:2}T${DATE_FIRST_CYCL:8:2}:00:00.000000" if [ ! -f "${fvcom_exec_fp}" ]; then - print_err_msg_exit "\ -The executable (fvcom_exec_fp) for processing FVCOM data onto FV3-LAM -native grid does not exist: + message_txt="The executable (fvcom_exec_fp) for processing FVCOM data +onto FV3-LAM native grid does not exist: fvcom_exec_fp = \"${fvcom_exec_fp}\" Please ensure that you've built this executable." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}"\ + else + print_err_msg_exit "${message_txt}" + fi fi cp_vrfy ${fvcom_exec_fp} ${INPUT_DATA}/. fvcom_data_fp="${FVCOM_DIR}/${FVCOM_FILE}" if [ ! -f "${fvcom_data_fp}" ]; then - print_err_msg_exit "\ -The file or path (fvcom_data_fp) does not exist: + message_txt="The file or path (fvcom_data_fp) does not exist: fvcom_data_fp = \"${fvcom_data_fp}\" Please check the following user defined variables: FVCOM_DIR = \"${FVCOM_DIR}\" FVCOM_FILE= \"${FVCOM_FILE}\" " + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi fi cp_vrfy ${fvcom_data_fp} ${INPUT_DATA}/fvcom.nc diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 3dad9bd0be..b0311c626d 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -82,7 +82,7 @@ fi #----------------------------------------------------------------------- # if [ $RUN_ENVIR = "nco" ]; then - extrn_mdl_staging_dir="${COMINext}${SLASH_ENSMEM_SUBDIR}" + extrn_mdl_staging_dir="${DATAROOT}/get_extrn_lbcs.${share_pid}${SLASH_ENSMEM_SUBDIR}" extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_LBCS}.LBCS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" else extrn_mdl_staging_dir="${COMIN}/${EXTRN_MDL_NAME_LBCS}/for_LBCS${SLASH_ENSMEM_SUBDIR}" @@ -92,16 +92,6 @@ fi # #----------------------------------------------------------------------- # -# -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_LBCS" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA -# -#----------------------------------------------------------------------- -# # Set physics-suite-dependent variable mapping table needed in the FORTRAN # namelist file that the chgres_cube executable will read in. # @@ -138,10 +128,14 @@ case "${CCPP_PHYS_SUITE}" in ;; # *) - print_err_msg_exit "\ -The variable \"varmap_file\" has not yet been specified for this physics -suite (CCPP_PHYS_SUITE): + message_txt="The variable \"varmap_file\" has not yet been specified +for this physics suite (CCPP_PHYS_SUITE): CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi ;; # esac @@ -337,10 +331,14 @@ case "${EXTRN_MDL_NAME_LBCS}" in ;; *) - print_err_msg_exit "\ -External-model-dependent namelist variables have not yet been specified -for this external LBC model (EXTRN_MDL_NAME_LBCS): + message_txt="External-model-dependent namelist variables have not yet been +specified for this external LBC model (EXTRN_MDL_NAME_LBCS): EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi ;; esac @@ -354,11 +352,15 @@ esac exec_fn="chgres_cube" exec_fp="$EXECdir/${exec_fn}" if [ ! -f "${exec_fp}" ]; then - print_err_msg_exit "\ -The executable (exec_fp) for generating initial conditions on the FV3-LAM -native grid does not exist: + message_txt="The executable (exec_fp) for generating initial conditions +on the FV3-LAM native grid does not exist: exec_fp = \"${exec_fp}\" Please ensure that you've built this executable." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi fi # #----------------------------------------------------------------------- @@ -423,10 +425,14 @@ for (( ii=0; ii<${num_fhrs}; ii=ii+bcgrpnum10 )); do fn_grib2="${EXTRN_MDL_FNS[$i]}" ;; *) - print_err_msg_exit "\ -The external model output file name to use in the chgres_cube FORTRAN name- -list file has not specified for this external LBC model (EXTRN_MDL_NAME_LBCS): + message_txt="The external model output file name to use in the chgres_cube +FORTRAN namelist file has not specified for this external LBC model (EXTRN_MDL_NAME_LBCS): EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi ;; esac # @@ -498,16 +504,23 @@ settings=" # Call the python script to create the namelist file. # nml_fn="fort.41" - ${USHdir}/set_namelist.py -q -u "$settings" -o ${nml_fn} || \ - print_err_msg_exit "\ -Call to python script set_namelist.py to set the variables in the namelist -file read in by the ${exec_fn} executable failed. Parameters passed to -this script are: + ${USHdir}/set_namelist.py -q -u "$settings" -o ${nml_fn} + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to python script set_namelist.py to set the variables +in the namelist file read in by the ${exec_fn} executable failed. Parameters +passed to this script are: Name of output namelist file: nml_fn = \"${nml_fn}\" Namelist settings specified on command line (these have highest precedence): settings = $settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi # #----------------------------------------------------------------------- # @@ -524,8 +537,13 @@ $settings" # forecast task. # PREP_STEP - eval ${RUN_CMD_UTILS} ${exec_fp} ${REDIRECT_OUT_ERR} || \ - print_err_msg_exit "\ + eval ${RUN_CMD_UTILS} ${exec_fp} ${REDIRECT_OUT_ERR} + export err=$? + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk + else + if [ $err -ne 0 ]; then + print_err_msg_exit "\ Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) file for the FV3-LAM for forecast hour fhr failed: exec_fp = \"${exec_fp}\" @@ -535,6 +553,8 @@ The external model from which the LBCs files are to be generated is: The external model files that are inputs to the executable (exec_fp) are located in the following directory: extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" + fi + fi POST_STEP # # Move LBCs file for the current lateral boundary update time to the LBCs diff --git a/scripts/exregional_nexus_emission.sh b/scripts/exregional_nexus_emission.sh index 2c9915f304..8c0f904b94 100755 --- a/scripts/exregional_nexus_emission.sh +++ b/scripts/exregional_nexus_emission.sh @@ -79,13 +79,9 @@ fi # #----------------------------------------------------------------------- # -DATA="${DATA}/tmp_NEXUS/${nspt}" -mkdir_vrfy -p "$DATA" - DATAinput="${DATA}/input" mkdir_vrfy -p "$DATAinput" -cd_vrfy $DATA # #----------------------------------------------------------------------- # @@ -94,9 +90,15 @@ cd_vrfy $DATA #----------------------------------------------------------------------- # USE_GFS_SFC="FALSE" -if [ -d "${COMINext}/GFS_SFC" ]; then - if [ "$(ls -A ${COMINext}/GFS_SFC)" ]; then - ln_vrfy -sf "${COMINext}/GFS_SFC" . +if [ "${RUN_ENVIR}" = "nco" ]; then + GFS_SFC_INPUT="${DATAROOT}/nexus_gfs_sfc.${share_pid}" +else + GFS_SFC_INPUT="${COMIN}/GFS_SFC" +fi + +if [ -d "${GFS_SFC_INPUT}" ]; then + if [ "$(ls -A ${GFS_SFC_INPUT})" ]; then + ln -sf "${GFS_SFC_INPUT}" "GFS_SFC" USE_GFS_SFC="TRUE" fi fi @@ -149,10 +151,10 @@ else start_del_hr=$(( len_per_split * nspt )) start_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + ${start_del_hr} hours " "+%Y%m%d%H" ) if [ "${nsptp}" = "${NUM_SPLIT_NEXUS}" ];then - end_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H" ) + end_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + $(expr $FCST_LEN_HRS + 1) hours" "+%Y%m%d%H" ) else end_del_hr=$(( len_per_split * nsptp )) - end_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + ${end_del_hr} hours" "+%Y%m%d%H" ) + end_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + $(expr $end_del_hr + 1) hours" "+%Y%m%d%H" ) fi fi # @@ -186,6 +188,15 @@ NEXUS_INPUT_BASE_DIR=${NEXUS_INPUT_DIR} # modify time configuration file # python3 ${ARL_NEXUS_DIR}/utils/python/nexus_time_parser.py -f ${DATA}/HEMCO_sa_Time.rc -s $start_date -e $end_date +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to python script \"nexus_time_parser.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #--------------------------------------------------------------------- @@ -193,6 +204,15 @@ python3 ${ARL_NEXUS_DIR}/utils/python/nexus_time_parser.py -f ${DATA}/HEMCO_sa_T # set the root directory to the temporary directory # python3 ${ARL_NEXUS_DIR}/utils/python/nexus_root_parser.py -f ${DATA}/NEXUS_Config.rc -d ${DATAinput} +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to python script \"nexus_root_parser.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #---------------------------------------------------------------------- @@ -203,7 +223,26 @@ if [ "${NEI2016}" = "TRUE" ]; then #NEI2016 mkdir_vrfy -p ${DATAinput}/NEI2016v1/v2022-07 mkdir_vrfy -p ${DATAinput}/NEI2016v1/v2022-07/${mm} python3 ${ARL_NEXUS_DIR}/utils/python/nexus_nei2016_linker.py --src_dir ${NEXUS_INPUT_BASE_DIR} --date ${yyyymmdd} --work_dir ${DATAinput} -v "v2022-07" + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to python script \"nexus_nei2016_linker.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi + python3 ${ARL_NEXUS_DIR}/utils/python/nexus_nei2016_control_tilefix.py -f ${DATA}/NEXUS_Config.rc -t ${DATA}/HEMCO_sa_Time.rc # -d ${yyyymmdd} + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to python script \"nexus_nei2016_control_tilefix.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi fi if [ "${TIMEZONES}" = "TRUE" ]; then # TIME ZONES @@ -265,6 +304,15 @@ fi if [ "${USE_GFS_SFC}" = "TRUE" ]; then # GFS INPUT mkdir_vrfy -p ${DATAinput}/GFS_SFC python3 ${ARL_NEXUS_DIR}/utils/python/nexus_gfs_bio.py -i ${DATA}/GFS_SFC/gfs.t??z.sfcf???.nc -o ${DATA}/GFS_SFC_MEGAN_INPUT.nc + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to python script \"nexus_gfs_bio.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi fi # @@ -275,20 +323,34 @@ fi #----------------------------------------------------------------------- # PREP_STEP -eval ${RUN_CMD_NEXUS} ${EXECdir}/nexus -c NEXUS_Config.rc -r grid_spec.nc -o NEXUS_Expt_split.nc ${REDIRECT_OUT_ERR} || \ -print_err_msg_exit "\ -Call to execute nexus standalone for the FV3LAM failed." +eval ${RUN_CMD_NEXUS} ${EXECdir}/nexus -c NEXUS_Config.rc -r grid_spec.nc -o NEXUS_Expt_split.nc ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to execute nexus standalone for the FV3LAM failed." + fi +fi POST_STEP # #----------------------------------------------------------------------- # -# Move NEXUS output to INPUT_DATA directory. +# Make NEXUS output pretty and move to INPUT_DATA directory. # #----------------------------------------------------------------------- # -mv_vrfy ${DATA}/NEXUS_Expt_split.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.${nspt}.nc - +python3 ${ARL_NEXUS_DIR}/utils/python/make_nexus_output_pretty.py --src ${DATA}/NEXUS_Expt_split.nc --grid ${DATA}/grid_spec.nc -o ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.${nspt}.nc -t ${DATA}/HEMCO_sa_Time.rc +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to python script \"make_nexus_output_pretty.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "wcoss2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_nexus_gfs_sfc.sh b/scripts/exregional_nexus_gfs_sfc.sh index 46ed3b15ef..01207e5666 100755 --- a/scripts/exregional_nexus_gfs_sfc.sh +++ b/scripts/exregional_nexus_gfs_sfc.sh @@ -48,12 +48,6 @@ data files from disk or HPSS. # #----------------------------------------------------------------------- # -DATA="${DATA}/tmp_GFS_SFC" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA -# -#----------------------------------------------------------------------- -# # Set up variables for call to retrieve_data.py # #----------------------------------------------------------------------- @@ -68,10 +62,11 @@ if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} )) FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]} fi +fcst_len_hrs_offset=$(( FCST_LEN_HRS + TIME_OFFSET_HRS )) # #----------------------------------------------------------------------- # -# Retrieve GFS surface files to GFS_SFC_STAGING_DIR +# Retrieve GFS surface files # #----------------------------------------------------------------------- # @@ -89,17 +84,31 @@ GFS_SFC_DATA_INTVL="3" # copy files from local directory if [ -d ${GFS_SFC_LOCAL_DIR} ]; then gfs_sfc_fn="gfs.t${hh}z.sfcanl.nc" - cp_vrfy "${GFS_SFC_LOCAL_DIR}/${gfs_sfc_fn}" ${GFS_SFC_STAGING_DIR} - for fhr in $(seq -f "%03g" 0 ${GFS_SFC_DATA_INTVL} ${FCST_LEN_HRS}); do + relative_link_flag="FALSE" + gfs_sfc_fp="${GFS_SFC_LOCAL_DIR}/${gfs_sfc_fn}" + create_symlink_to_file target="${gfs_sfc_fp}" symlink="${gfs_sfc_fn}" \ + relative="${relative_link_flag}" + + for fhr in $(seq -f "%03g" 0 ${GFS_SFC_DATA_INTVL} ${fcst_len_hrs_offset}); do gfs_sfc_fn="gfs.t${hh}z.sfcf${fhr}.nc" if [ -e "${GFS_SFC_LOCAL_DIR}/${gfs_sfc_fn}" ]; then - cp_vrfy "${GFS_SFC_LOCAL_DIR}/${gfs_sfc_fn}" ${GFS_SFC_STAGING_DIR} + gfs_sfc_fp="${GFS_SFC_LOCAL_DIR}/${gfs_sfc_fn}" + create_symlink_to_file target="${gfs_sfc_fp}" symlink="${gfs_sfc_fn}" \ + relative="${relative_link_flag}" else - print_err_msg_exit "\ -sfc file does not exist in the directory: + message_txt="SFC file for nexus emission for \"${cycle}\" does not exist in the directory: GFS_SFC_LOCAL_DIR = \"${GFS_SFC_LOCAL_DIR}\" gfs_sfc_fn = \"${gfs_sfc_fn}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + message_warning="WARNING: ${message_txt}" + print_info_msg "${message_warning}" + if [ ! -z "${maillist}" ]; then + echo "${message_warning}" | mail.py $maillist + fi + else + print_err_msg_exit "${message_txt}" + fi fi done @@ -121,8 +130,8 @@ else gfs_sfc_tar_fp="${GFS_SFC_TAR_DIR}/${gfs_sfc_tar_fn}" gfs_sfc_fns=("gfs.t${hh}z.sfcanl.nc") gfs_sfc_fps="./${GFS_SFC_TAR_SUB_DIR}/gfs.t${hh}z.sfcanl.nc" - if [ "${FCST_LEN_HRS}" -lt "40" ]; then - ARCHV_LEN_HRS="${FCST_LEN_HRS}" + if [ "${fcst_len_hrs_offset}" -lt "40" ]; then + ARCHV_LEN_HRS="${fcst_len_hrs_offset}" else ARCHV_LEN_HRS="39" fi @@ -131,27 +140,43 @@ else gfs_sfc_fps+=" ./${GFS_SFC_TAR_SUB_DIR}/gfs.t${hh}z.sfcf${fhr}.nc" done - # Retrieve data from A file up to FCST_LEN_HRS=39 + # Retrieve data from A file up to fcst_len_hrs_offset=39 htar -tvf ${gfs_sfc_tar_fp} PREP_STEP - htar -xvf ${gfs_sfc_tar_fp} ${gfs_sfc_fps} ${REDIRECT_OUT_ERR} || \ - print_err_msg_exit "htar file reading operation (\"htar -xvf ...\") failed." + htar -xvf ${gfs_sfc_tar_fp} ${gfs_sfc_fps} ${REDIRECT_OUT_ERR} + export err=$? + if [ $err -ne 0 ]; then + message_txt="htar file reading operation (\"htar -xvf ...\") failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi POST_STEP - # Retireve data from B file when FCST_LEN_HRS>=40 - if [ "${FCST_LEN_HRS}" -ge "40" ]; then + # Retireve data from B file when fcst_len_hrs_offset>=40 + if [ "${fcst_len_hrs_offset}" -ge "40" ]; then gfs_sfc_tar_fn="${GFS_SFC_TAR_FN_PREFIX}.${yyyymmdd}_${hh}.${GFS_SFC_TAR_FN_SUFFIX_B}" gfs_sfc_tar_fp="${GFS_SFC_TAR_DIR}/${gfs_sfc_tar_fn}" gfs_sfc_fns=() gfs_sfc_fps="" - for fhr in $(seq -f "%03g" 42 ${GFS_SFC_DATA_INTVL} ${FCST_LEN_HRS}); do + for fhr in $(seq -f "%03g" 42 ${GFS_SFC_DATA_INTVL} ${fcst_len_hrs_offset}); do gfs_sfc_fns+="gfs.t${hh}z.sfcf${fhr}.nc" gfs_sfc_fps+=" ./${GFS_SFC_TAR_SUB_DIR}/gfs.t${hh}z.sfcf${fhr}.nc" done htar -tvf ${gfs_sfc_tar_fp} PREP_STEP - htar -xvf ${gfs_sfc_tar_fp} ${gfs_sfc_fps} ${REDIRECT_OUT_ERR} || \ - print_err_msg_exit "htar file reading operation (\"htar -xvf ...\") failed." + htar -xvf ${gfs_sfc_tar_fp} ${gfs_sfc_fps} ${REDIRECT_OUT_ERR} + export err=$? + if [ $err -ne 0 ]; then + message_txt="htar file reading operation (\"htar -xvf ...\") failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi POST_STEP fi # Link retrieved files to staging directory diff --git a/scripts/exregional_nexus_post_split.sh b/scripts/exregional_nexus_post_split.sh index 054b825d0c..a2e2a63b2a 100755 --- a/scripts/exregional_nexus_post_split.sh +++ b/scripts/exregional_nexus_post_split.sh @@ -52,17 +52,6 @@ This is the ex-script for the task that runs NEXUS. #----------------------------------------------------------------------- # eval ${PRE_TASK_CMDS} -# -#----------------------------------------------------------------------- -# -# Move to the NEXUS working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_NEXUS_POST_SPLIT" -mkdir_vrfy -p "$DATA" - -cd_vrfy $DATA mm="${PDY:4:2}" dd="${PDY:6:2}" @@ -94,18 +83,16 @@ if [ "${NUM_SPLIT_NEXUS}" = "01" ]; then cp_vrfy ${COMIN}/NEXUS/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.${nspt}.nc ${DATA}/NEXUS_Expt_combined.nc else python3 ${ARL_NEXUS_DIR}/utils/python/concatenate_nexus_post_split.py "${COMIN}/NEXUS/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.*.nc" "${DATA}/NEXUS_Expt_combined.nc" + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to python script \"concatenate_nexus_post_split.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi fi - -# -#----------------------------------------------------------------------- -# -# make nexus output pretty -# -#----------------------------------------------------------------------- -# -python3 ${ARL_NEXUS_DIR}/utils/python/nexus_time_parser.py -f ${DATA}/HEMCO_sa_Time.rc -s $start_date -e $end_date - -python3 ${ARL_NEXUS_DIR}/utils/python/make_nexus_output_pretty.py --src ${DATA}/NEXUS_Expt_combined.nc --grid ${DATA}/grid_spec.nc -o ${DATA}/NEXUS_Expt_pretty.nc -t ${DATA}/HEMCO_sa_Time.rc # #----------------------------------------------------------------------- @@ -114,7 +101,16 @@ python3 ${ARL_NEXUS_DIR}/utils/python/make_nexus_output_pretty.py --src ${DATA}/ # #----------------------------------------------------------------------- # -python3 ${ARL_NEXUS_DIR}/utils/combine_ant_bio.py ${DATA}/NEXUS_Expt_pretty.nc ${DATA}/NEXUS_Expt.nc +python3 ${ARL_NEXUS_DIR}/utils/combine_ant_bio.py "${DATA}/NEXUS_Expt_combined.nc" ${DATA}/NEXUS_Expt.nc +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to python script \"NEXUS_Expt_pretty.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #----------------------------------------------------------------------- diff --git a/scripts/exregional_point_source.sh b/scripts/exregional_point_source.sh index a3be0256c4..8eee83337c 100755 --- a/scripts/exregional_point_source.sh +++ b/scripts/exregional_point_source.sh @@ -61,16 +61,6 @@ fi nstep=$(( FCST_LEN_HRS+1 )) yyyymmddhh="${PDY}${cyc}" -# -#----------------------------------------------------------------------- -# -# Move to the working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_PT_SOURCE" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA # #----------------------------------------------------------------------- # @@ -78,7 +68,7 @@ cd_vrfy $DATA # #----------------------------------------------------------------------- # -PT_SRC_PRECOMB="${PT_SRC_BASEDIR}" +PT_SRC_PRECOMB="${DCOMINpt_src}" # #----------------------------------------------------------------------- # @@ -88,6 +78,15 @@ PT_SRC_PRECOMB="${PT_SRC_BASEDIR}" # if [ ! -s "${DATA}/pt-${yyyymmddhh}.nc" ]; then python3 ${HOMEdir}/sorc/AQM-utils/python_utils/stack-pt-merge.py -s ${yyyymmddhh} -n ${nstep} -i ${PT_SRC_PRECOMB} + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to python script \"stack-pt-merge.py\" failed." + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi fi # Move to COMIN diff --git a/scripts/exregional_post_stat_o3.sh b/scripts/exregional_post_stat_o3.sh index 555d43b82a..266cfe67f9 100755 --- a/scripts/exregional_post_stat_o3.sh +++ b/scripts/exregional_post_stat_o3.sh @@ -63,16 +63,6 @@ else All executables will be submitted with command \'${RUN_CMD_SERIAL}\'." fi -# -#----------------------------------------------------------------------- -# -# Move to the working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_POST_STAT_O3" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA # #----------------------------------------------------------------------- # @@ -98,8 +88,15 @@ EOF1 # convert from netcdf to grib2 format PREP_STEP -eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_grib2 ${PDY} ${cyc} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ -Call to executable to run AQM_POST_GRIB2 returned with nonzero exit code." +eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_grib2 ${PDY} ${cyc} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_POST_GRIB2 returned with nonzero exit code." + fi +fi POST_STEP if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then @@ -110,13 +107,13 @@ fi fhr=01 while [ ${fhr} -le ${FCST_LEN_HRS} ]; do - fhr9=$( printf "%02d" "${fhr}" ) + fhr3d=$( printf "%03d" "${fhr}" ) - if [ "${fhr9}" -le "07" ]; then - cat ${DATA}/${NET}.${cycle}.awpozcon.f${fhr9}.${id_domain}.grib2 >> ${NET}.${cycle}.1ho3.${id_domain}.grib2 + if [ "${fhr3d}" -le "07" ]; then + cat ${DATA}/${NET}.${cycle}.awpozcon.f${fhr3d}.${id_domain}.grib2 >> ${NET}.${cycle}.1ho3.${id_domain}.grib2 else - wgrib2 ${DATA}/${NET}.${cycle}.awpozcon.f${fhr9}.${id_domain}.grib2 -d 1 -append -grib ${NET}.${cycle}.1ho3.${id_domain}.grib2 - wgrib2 ${DATA}/${NET}.${cycle}.awpozcon.f${fhr9}.${id_domain}.grib2 -d 2 -append -grib ${NET}.${cycle}.8ho3.${id_domain}.grib2 + wgrib2 ${DATA}/${NET}.${cycle}.awpozcon.f${fhr3d}.${id_domain}.grib2 -d 1 -append -grib ${NET}.${cycle}.1ho3.${id_domain}.grib2 + wgrib2 ${DATA}/${NET}.${cycle}.awpozcon.f${fhr3d}.${id_domain}.grib2 -d 2 -append -grib ${NET}.${cycle}.8ho3.${id_domain}.grib2 fi (( fhr=fhr+1 )) done @@ -150,9 +147,12 @@ for grid in 227 196 198;do export FORT51=awpaqm.${cycle}.${hr}ho3.${grid}.grib2 tocgrib2super < ${PARMaqm_utils}/wmo/grib2_aqm_ave_${hr}hr_o3-awpozcon.${cycle}.${grid} done - for var in 1ho3 8ho3 awpozcon;do + for var in 1ho3 8ho3;do + cp_vrfy ${DATA}/${NET}.${cycle}.${var}*grib2 ${COMOUT} + cp_vrfy ${DATA}/awpaqm.${cycle}.${var}*grib2 ${COMOUTwmo} + done + for var in awpozcon;do cp_vrfy ${DATA}/${NET}.${cycle}.${var}*grib2 ${COMOUT} - cp_vrfy ${DATA}/awpaqm.${cycle}.${var}*grib2 ${COMOUT} done else for var in 1ho3 awpozcon;do @@ -185,10 +185,10 @@ EOF1 ## 06z needs b.nc to find current day output from 04Z to 06Z if [ "${cyc}" = "06" ]; then - if [ -s ${COMIN_PDY}/00/${NET}.t00z.chem_sfc.nc ]; then - ln_vrfy -s ${COMIN_PDY}/00/${NET}.t00z.chem_sfc.nc b.nc - elif [ -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc ]; then - ln_vrfy -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc b.nc + if [ -s ${COMIN}/../00/${NET}.t00z.chem_sfc.nc ]; then + ln_vrfy -s ${COMIN}/../00/${NET}.t00z.chem_sfc.nc b.nc + elif [ -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc ]; then + ln_vrfy -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc b.nc chk=0 else flag_run_bicor_max=no @@ -197,20 +197,20 @@ EOF1 if [ "${cyc}" = "12" ]; then ## 12z needs b.nc to find current day output from 04Z to 06Z - if [ -s ${COMIN_PDY}/00/${NET}.t00z.chem_sfc.nc ]; then - ln_vrfy -s ${COMIN_PDY}/00/${NET}.t00z.chem_sfc.nc b.nc - elif [ -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc ]; then - ln_vrfy -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc b.nc + if [ -s ${COMIN}/../00/${NET}.t00z.chem_sfc.nc ]; then + ln_vrfy -s ${COMIN}/../00/${NET}.t00z.chem_sfc.nc b.nc + elif [ -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc ]; then + ln_vrfy -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc b.nc chk=0 else flag_run_bicor_max=no fi ## 12z needs c.nc to find current day output from 07Z to 12z - if [ -s ${COMIN_PDY}/06/${NET}.t06z.chem_sfc.nc ]; then - ln_vrfy -s ${COMIN_PDY}/06/${NET}.t06z.chem_sfc.nc c.nc - elif [ -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc ]; then - ln_vrfy -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc c.nc + if [ -s ${COMIN}/../06/${NET}.t06z.chem_sfc.nc ]; then + ln_vrfy -s ${COMIN}/../06/${NET}.t06z.chem_sfc.nc c.nc + elif [ -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc ]; then + ln_vrfy -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc c.nc chk1=0 else flag_run_bicor_max=no @@ -218,8 +218,15 @@ EOF1 fi PREP_STEP - eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_maxi_grib2 ${PDY} ${cyc} ${chk} ${chk1} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ - Call to executable to run AQM_POST_MAXI_GRIB2 returned with nonzero exit code." + eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_maxi_grib2 ${PDY} ${cyc} ${chk} ${chk1} ${REDIRECT_OUT_ERR} + export err=$? + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk + else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_POST_MAXI_GRIB2 returned with nonzero exit code." + fi + fi POST_STEP # split into max_1h and max_8h files and copy to grib227 @@ -237,7 +244,7 @@ EOF1 wgrib2 ${NET}.${cycle}.max_1hr_o3.${id_domain}.grib2 -set_grib_type c3b -new_grid_winds earth -new_grid ${!gg} ${NET}.${cycle}.max_1hr_o3.${grid}.grib2 cp_vrfy ${DATA}/${NET}.${cycle}.max_*hr_o3.*.grib2 ${COMOUT} - if [ "$SENDDBN" = "YES" ]; then + if [ "$SENDDBN" = "TRUE" ]; then ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUT}/${NET}.${cycle}.max_1hr_o3.${grid}.grib2 ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUT}/${NET}.${cycle}.max_8hr_o3.${grid}.grib2 fi @@ -260,10 +267,10 @@ EOF1 tocgrib2super < ${PARMaqm_utils}/wmo/grib2_aqm-${hr}hro3-maxi.${cycle}.${grid} done - cp_vrfy awpaqm.${cycle}.*o3-max.${grid}.grib2 ${COMOUT} - if [ "${SENDDBN_NTC}" = "YES" ]; then - ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.1ho3-max.${grid}.grib2 - ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.8ho3-max.${grid}.grib2 + cp_vrfy awpaqm.${cycle}.*o3-max.${grid}.grib2 ${COMOUTwmo} + if [ "${SENDDBN_NTC}" = "TRUE" ]; then + ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUTwmo}/awpaqm.${cycle}.1ho3-max.${grid}.grib2 + ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUTwmo}/awpaqm.${cycle}.8ho3-max.${grid}.grib2 fi done fi diff --git a/scripts/exregional_post_stat_pm25.sh b/scripts/exregional_post_stat_pm25.sh index cc3131b3da..47ee326fb4 100755 --- a/scripts/exregional_post_stat_pm25.sh +++ b/scripts/exregional_post_stat_pm25.sh @@ -63,16 +63,6 @@ else All executables will be submitted with command \'${RUN_CMD_SERIAL}\'." fi -# -#----------------------------------------------------------------------- -# -# Move to the working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_POST_STAT_PM25" -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA # #----------------------------------------------------------------------- # @@ -102,8 +92,15 @@ EOF1 # convert from netcdf to grib2 format PREP_STEP -eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_grib2 ${PDY} ${cyc} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ -Call to executable to run AQM_POST_GRIB2 returned with nonzero exit code." +eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_grib2 ${PDY} ${cyc} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] || [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_POST_GRIB2 returned with nonzero exit code." + fi +fi POST_STEP cat ${NET}.${cycle}.pm25.*.${id_domain}.grib2 >> ${NET}.${cycle}.1hpm25.${id_domain}.grib2 @@ -139,13 +136,13 @@ if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then export FORT51=awpaqm.${cycle}.1hpm25.${grid}.grib2 tocgrib2super < ${PARMaqm_utils}/wmo/grib2_aqm_1hpm25.${cycle}.${grid} - # Post Files to COMOUT - cp_vrfy awpaqm.${cycle}.1hpm25.${grid}.grib2 ${COMOUT} + # Post Files to COMOUTwmo + cp_vrfy awpaqm.${cycle}.1hpm25.${grid}.grib2 ${COMOUTwmo} # Distribute Data - if [ "${SENDDBN_NTC}" = "YES" ] ; then - ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.1hpm25.${grid}.grib2 - ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUT}/awpaqm.${cycle}.daily-1hr-pm25-max.${grid}.grib2 + if [ "${SENDDBN_NTC}" = "TRUE" ] ; then + ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUTwmo}/awpaqm.${cycle}.1hpm25.${grid}.grib2 + ${DBNROOT}/bin/dbn_alert ${DBNALERT_TYPE} ${NET} ${job} ${COMOUTwmo}/awpaqm.${cycle}.daily-1hr-pm25-max.${grid}.grib2 fi done fi @@ -173,10 +170,10 @@ EOF1 flag_run_bicor_max=yes # 06z needs b.nc to find current day output from 04Z to 06Z if [ "${cyc}" = "06" ]; then - if [ -s ${COMIN_PDY}/00/${NET}.t00z.chem_sfc.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/00/${NET}.t00z.chem_sfc.nc b.nc - elif [ -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc b.nc + if [ -s ${COMIN}/../00/${NET}.t00z.chem_sfc.nc ]; then + ln_vrfy -sf ${COMIN}/../00/${NET}.t00z.chem_sfc.nc b.nc + elif [ -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc ]; then + ln_vrfy -sf ${COMINm1}/12/${NET}.t12z.chem_sfc.nc b.nc chk=0 else flag_run_bicor_max=no @@ -185,20 +182,20 @@ EOF1 if [ "${cyc}" = "12" ]; then # 12z needs b.nc to find current day output from 04Z to 06Z - if [ -s ${COMIN_PDY}/00/${NET}.t00z.chem_sfc.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/00/${NET}.t00z.chem_sfc.nc b.nc - elif [ -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/${NET}.${PDYm1}.t12z.chem_sfc.nc b.nc + if [ -s ${COMIN}/../00/${NET}.t00z.chem_sfc.nc ]; then + ln_vrfy -sf ${COMIN}/../00/${NET}.t00z.chem_sfc.nc b.nc + elif [ -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc ]; then + ln_vrfy -sf ${COMINm1}/12/${NET}.${PDYm1}.t12z.chem_sfc.nc b.nc chk=0 else flag_run_bicor_max=no fi # 12z needs c.nc to find current day output from 07Z to 12z - if [ -s ${COMIN_PDY}/06/${NET}.t06z.chem_sfc.nc ]; then - ln_vrfy -sf ${COMIN_PDY}/06/${NET}.t06z.chem_sfc.nc c.nc - elif [ -s ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc ]; then - ln_vrfy -sf ${COMIN_PDYm1}/12/${NET}.t12z.chem_sfc.nc c.nc + if [ -s ${COMIN}/../06/${NET}.t06z.chem_sfc.nc ]; then + ln_vrfy -sf ${COMIN}/../06/${NET}.t06z.chem_sfc.nc c.nc + elif [ -s ${COMINm1}/12/${NET}.t12z.chem_sfc.nc ]; then + ln_vrfy -sf ${COMINm1}/12/${NET}.t12z.chem_sfc.nc c.nc chk1=0 else flag_run_bicor_max=no @@ -206,8 +203,15 @@ EOF1 fi PREP_STEP - eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_maxi_grib2 ${PDY} ${cyc} ${chk} ${chk1} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ - Call to executable to run AQM_POST_MAXI_GRIB2 returned with nonzero exit code." + eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_post_maxi_grib2 ${PDY} ${cyc} ${chk} ${chk1} ${REDIRECT_OUT_ERR} + export err=$? + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk + else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run AQM_POST_MAXI_GRIB2 returned with nonzero exit code." + fi + fi POST_STEP wgrib2 ${NET}_pm25_24h_ave.${id_domain}.grib2 |grep "PMTF" | wgrib2 -i ${NET}_pm25_24h_ave.${id_domain}.grib2 -grib ${NET}.${cycle}.ave_24hr_pm25.${id_domain}.grib2 @@ -260,12 +264,12 @@ EOF1 cp_vrfy ${DATA}/${NET}.${cycle}.ave_24hr_pm25*.grib2 ${COMOUT} cp_vrfy ${DATA}/${NET}.${cycle}.max_1hr_pm25*.grib2 ${COMOUT} - cp_vrfy awpaqm.${cycle}.daily-1hr-pm25-max.${grid}.grib2 ${COMOUT} - cp_vrfy awpaqm.${cycle}.24hr-pm25-ave.${grid}.grib2 ${COMOUT} + cp_vrfy awpaqm.${cycle}.daily-1hr-pm25-max.${grid}.grib2 ${COMOUTwmo} + cp_vrfy awpaqm.${cycle}.24hr-pm25-ave.${grid}.grib2 ${COMOUTwmo} - if [ "$SENDDBN" = "YES" ]; then - ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUT}/${NET}.${cycle}.ave_24hr_pm25.${grid}.grib2 - ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUT}/${NET}.${cycle}.max_1hr_pm25.${grid}.grib2 + if [ "$SENDDBN" = "TRUE" ]; then + ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUTwmo}/${NET}.${cycle}.ave_24hr_pm25.${grid}.grib2 + ${DBNROOT}/bin/dbn_alert MODEL AQM_MAX ${job} ${COMOUTwmo}/${NET}.${cycle}.max_1hr_pm25.${grid}.grib2 fi done fi diff --git a/scripts/exregional_pre_post_stat.sh b/scripts/exregional_pre_post_stat.sh index a78ce9e2f5..33bbd75a41 100755 --- a/scripts/exregional_pre_post_stat.sh +++ b/scripts/exregional_pre_post_stat.sh @@ -52,17 +52,6 @@ This is the ex-script for the task that runs POST-UPP-STAT. #----------------------------------------------------------------------- # eval ${PRE_TASK_CMDS} -# -#----------------------------------------------------------------------- -# -# Move to the working directory -# -#----------------------------------------------------------------------- -# -DATA="${DATA}/tmp_PRE_POST_STAT" -rm_vrfy -r $DATA -mkdir_vrfy -p "$DATA" -cd_vrfy $DATA if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then cyc_mod=$(( ${cyc} - ${DATE_FIRST_CYCL:8:2} )) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 2f4efc4a44..736a2fc115 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -97,10 +97,7 @@ print_info_msg "$VERBOSE" " Creating links in the INPUT subdirectory of the current run directory to the grid and (filtered) orography files ..." - # Create links to fix files in the FIXlam directory. - - cd_vrfy ${DATA}/INPUT # @@ -514,11 +511,19 @@ if [ "${DO_FCST_RESTART}" = "TRUE" ] && [ "$(ls -A ${DATA}/RESTART )" ]; then python3 $USHdir/update_input_nml.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --run_dir "${DATA}" \ - --restart || print_err_msg_exit "\ -Call to function to update the FV3 input.nml file for restart for the -current cycle's (cdate) run directory (DATA) failed: + --restart + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to function to update the FV3 input.nml file for restart +for the current cycle's (cdate) run directory (DATA) failed: cdate = \"${CDATE}\" DATA = \"${DATA}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi # Check that restart files exist at restart_interval file_ids=( "coupler.res" "fv_core.res.nc" "fv_core.res.tile1.nc" "fv_srf_wnd.res.tile1.nc" "fv_tracer.res.tile1.nc" "phy_data.nc" "sfc_data.nc" ) @@ -579,12 +584,19 @@ if [ "${CPL_AQM}" = "TRUE" ]; then --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --cdate "$CDATE" \ --run-dir "${DATA}" \ - --init_concentrations "${init_concentrations}" \ - || print_err_msg_exit "\ -Call to function to create an aqm.rc file for the current + --init_concentrations "${init_concentrations}" + export err=$? + if [ $err -ne 0 ]; then + message_txt="Call to function to create an aqm.rc file for the current cycle's (cdate) run directory (DATA) failed: cdate = \"${CDATE}\" DATA = \"${DATA}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi + fi fi # @@ -603,11 +615,19 @@ python3 $USHdir/create_model_configure_file.py \ --run-dir "${DATA}" \ --sub-hourly-post "${SUB_HOURLY_POST}" \ --dt-subhourly-post-mnts "${DT_SUBHOURLY_POST_MNTS}" \ - --dt-atmos "${DT_ATMOS}" || print_err_msg_exit "\ -Call to function to create a model configuration file for the current -cycle's (cdate) run directory (DATA) failed: + --dt-atmos "${DT_ATMOS}" +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to function to create a model configuration file +for the current cycle's (cdate) run directory (DATA) failed: cdate = \"${CDATE}\" DATA = \"${DATA}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #----------------------------------------------------------------------- # @@ -618,14 +638,23 @@ cycle's (cdate) run directory (DATA) failed: # python3 $USHdir/create_diag_table_file.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ - --run-dir "${DATA}" || print_err_msg_exit "\ -Call to function to create a diag table file for the current cycle's -(cdate) run directory (DATA) failed: + --run-dir "${DATA}" +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to function to create a diag table file for the current +cycle's (cdate) run directory (DATA) failed: DATA = \"${DATA}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #----------------------------------------------------------------------- # -# Pre-generate symlinks to forecast output in DATA +# Pre-generate symlink to forecast RESTART in DATA for early start of +# the next cycle # #----------------------------------------------------------------------- # @@ -643,11 +672,18 @@ fi # python3 $USHdir/create_nems_configure_file.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ - --run-dir "${DATA}" \ - || print_err_msg_exit "\ -Call to function to create a NEMS configuration file for the current -cycle's (cdate) run directory (DATA) failed: + --run-dir "${DATA}" +export err=$? +if [ $err -ne 0 ]; then + message_txt="Call to function to create a NEMS configuration file for +the current cycle's (cdate) run directory (DATA) failed: DATA = \"${DATA}\"" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #----------------------------------------------------------------------- # @@ -660,9 +696,15 @@ cycle's (cdate) run directory (DATA) failed: #----------------------------------------------------------------------- # PREP_STEP -eval ${RUN_CMD_FCST} ${FV3_EXEC_FP} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ -Call to executable to run FV3-LAM forecast returned with nonzero exit -code." +eval ${RUN_CMD_FCST} ${FV3_EXEC_FP} ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run FV3-LAM forecast returned with nonzero exit code." + fi +fi POST_STEP # #----------------------------------------------------------------------- @@ -677,15 +719,28 @@ POST_STEP # if [ "${CPL_AQM}" = "TRUE" ]; then if [ "${RUN_ENVIR}" = "nco" ]; then - rm_vrfy -rf "${COMIN}/RESTART" + if [ -d "${COMIN}/RESTART" ] && [ "$(ls -A ${DATA}/RESTART)" ]; then + rm_vrfy -rf "${COMIN}/RESTART" + fi if [ "$(ls -A ${DATA}/RESTART)" ]; then - mv_vrfy ${DATA}/RESTART ${COMIN} - ln_vrfy -sf ${COMIN}/RESTART ${DATA}/RESTART + cp_vrfy -Rp ${DATA}/RESTART ${COMIN} fi fi - mv_vrfy ${DATA}/${AQM_RC_PRODUCT_FN} ${COMOUT}/${NET}.${cycle}${dot_ensmem}.${AQM_RC_PRODUCT_FN} - + cp_vrfy -p ${DATA}/${AQM_RC_PRODUCT_FN} ${COMOUT}/${NET}.${cycle}${dot_ensmem}.${AQM_RC_PRODUCT_FN} + + fhr_ct=0 + fhr=0 + while [ $fhr -le ${FCST_LEN_HRS} ]; do + fhr_ct=$(printf "%03d" $fhr) + source_dyn="${DATA}/dynf${fhr_ct}.nc" + source_phy="${DATA}/phyf${fhr_ct}.nc" + target_dyn="${COMIN}/${NET}.${cycle}${dot_ensmem}.dyn.f${fhr_ct}.nc" + target_phy="${COMIN}/${NET}.${cycle}${dot_ensmem}.phy.f${fhr_ct}.nc" + [ -f ${source_dyn} ] && cp_vrfy -p ${source_dyn} ${target_dyn} + [ -f ${source_phy} ] && cp_vrfy -p ${source_phy} ${target_phy} + (( fhr=fhr+1 )) + done fi # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 3e1494822f..76a1196329 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -216,9 +216,16 @@ print_info_msg "$VERBOSE" " Starting post-processing for fhr = $fhr hr..." PREP_STEP -eval ${RUN_CMD_POST} ${EXECdir}/upp.x < itag ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ -Call to executable to run post for forecast hour $fhr returned with non- -zero exit code." +eval ${RUN_CMD_POST} ${EXECdir}/upp.x < itag ${REDIRECT_OUT_ERR} +export err=$? +if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_chk +else + if [ $err -ne 0 ]; then + print_err_msg_exit "Call to executable to run post for forecast hour $fhr +returned with non-zero exit code." + fi +fi POST_STEP # #----------------------------------------------------------------------- @@ -287,18 +294,8 @@ for fid in "${fids[@]}"; do fi done -# Move phy and dyn files to COMIN only for AQM -if [ "${CPL_AQM}" = "TRUE" ]; then - mv_vrfy ${dyn_file} ${COMIN}/${NET}.${cycle}${dot_ensmem}.dyn.f${fhr}.nc - mv_vrfy ${phy_file} ${COMIN}/${NET}.${cycle}${dot_ensmem}.phy.f${fhr}.nc -fi - rm_vrfy -rf ${DATA_FHR} -# Delete the forecast directory -if [ $RUN_ENVIR = "nco" ] && [ $KEEPDATA = "FALSE" ]; then - rm -rf $DATAFCST -fi # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index f83073a735..4a2e6babe0 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -154,11 +154,11 @@ def run_we2e_tests(homedir, args) -> None: if run_envir == "nco": if 'nco' not in test_cfg: test_cfg['nco'] = dict() - test_cfg['nco'].update({"model_ver": "we2e"}) + test_cfg['nco'].update({"model_ver_default": "we2e"}) if args.opsroot: if 'nco' not in test_cfg: test_cfg['nco'] = dict() - test_cfg['nco'].update({"OPSROOT": args.opsroot}) + test_cfg['nco'].update({"OPSROOT_default": args.opsroot}) # if platform section was not in input config, initialize as empty dict if 'platform' not in test_cfg: test_cfg['platform'] = dict() diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml index 1f0dd8382f..789c5e9674 100644 --- a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml @@ -17,7 +17,7 @@ workflow: FIELD_TABLE_TMPL_FN: field_table_aqm.FV3_GFS_v16 DO_REAL_TIME: false nco: - NET: aqm + NET_default: aqm rocoto: tasks: taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' diff --git a/ush/config.aqm.community.yaml b/ush/config.aqm.community.yaml index 55675bb8ce..30e391edf2 100644 --- a/ush/config.aqm.community.yaml +++ b/ush/config.aqm.community.yaml @@ -25,7 +25,7 @@ workflow: FIELD_TABLE_TMPL_FN: field_table_aqm.FV3_GFS_v16 DO_REAL_TIME: false nco: - NET: aqm + NET_default: aqm rocoto: tasks: taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' diff --git a/ush/config.aqm.nco.realtime.yaml b/ush/config.aqm.nco.realtime.yaml index 1a0da22677..f2299eacc9 100644 --- a/ush/config.aqm.nco.realtime.yaml +++ b/ush/config.aqm.nco.realtime.yaml @@ -29,12 +29,12 @@ workflow: COLDSTART: false WARMSTART_CYCLE_DIR: /path/to/restart/dir nco: - envir: prod - NET: aqm - model_ver: v7.0 - RUN: aqm_nco_aqmna13km - OPSROOT: /path/to/custom/opsroot - KEEPDATA: true + envir_default: prod + NET_default: aqm + model_ver_default: v7.0 + RUN_default: aqm + OPSROOT_default: /path/to/custom/opsroot + KEEPDATA_default: true rocoto: tasks: taskgroups: '{{ ["parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/aqm_post.yaml"]|include }}' diff --git a/ush/config.nco.yaml b/ush/config.nco.yaml index 0019cf989c..8339f5d9a6 100644 --- a/ush/config.nco.yaml +++ b/ush/config.nco.yaml @@ -17,9 +17,9 @@ workflow: VERBOSE: true COMPILER: intel nco: - NET: rrfs - model_ver: v1.0 - RUN: rrfs_test + NET_default: rrfs + model_ver_default: v1.0 + RUN_default: rrfs_test task_get_extrn_ics: EXTRN_MDL_NAME_ICS: FV3GFS FV3GFS_FILE_FMT_ICS: grib2 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 7ea9dd501b..5a83c090e9 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -430,23 +430,6 @@ platform: #----------------------------------------------------------------------- # EXTRN_MDL_DATA_STORES: "" - # - #----------------------------------------------------------------------- - # - # COMINgfs: - # Path to the real-time GFS data - # - # COMINgefs: - # Path to the real-time GEFS data - # - # COMINairnow: - # Path to the real-time AIRNOW observation data - # - #----------------------------------------------------------------------- - # - COMINgfs: "" - COMINgefs: "" - COMINairnow: "/path/to/real/time/airnow/data" #----------------------------- # WORKFLOW config parameters #----------------------------- @@ -986,7 +969,7 @@ workflow: #----------------------------------------------------------------------- # # COLDSTART: - # Flag turning on/off warm start + # Flag turning on/off warm start of the first cycle # # WARMSTART_CYCLE_DIR: # Path to the directory where RESTART dir is located for warm start @@ -1004,7 +987,11 @@ nco: #----------------------------------------------------------------------- # # Set variables that are only used in NCO mode (i.e. when RUN_ENVIR is - # set to "nco"). Definitions: + # set to "nco"). All variables have the suffix [_default] meaning the default value. + # This is because they are supposed to be defined in job cards for the + # production using ecFlow. + # + # Definitions: # # envir, NET, model_ver, RUN: # Standard environment variables defined in the NCEP Central Operations WCOSS @@ -1042,43 +1029,34 @@ nco: # #----------------------------------------------------------------------- # - envir: "para" - NET: "rrfs" - RUN: "rrfs" - model_ver: "v1.0.0" + envir_default: "para" + NET_default: "rrfs" + RUN_default: "rrfs" + model_ver_default: "v1.0.0" - OPSROOT: '{{ workflow.EXPT_BASEDIR }}/../nco_dirs' - COMROOT: '{{ OPSROOT }}/com' - PACKAGEROOT: '{{ OPSROOT }}/packages' - DATAROOT: '{{ OPSROOT }}/tmp' - DCOMROOT: '{{ OPSROOT }}/dcom' - LOGBASEDIR: '{% if user.RUN_ENVIR == "nco" %}{{ [OPSROOT, "output"]|path_join }}{% else %}{{ [workflow.EXPTDIR, "log"]|path_join }}{% endif %}' - EXTROOT: '{{ OPSROOT }}/ext' - COMIN_BASEDIR: '{{ COMROOT }}/{{ NET }}/{{ model_ver }}' - COMOUT_BASEDIR: '{{ COMROOT }}/{{ NET }}/{{ model_ver }}' + OPSROOT_default: '{{ workflow.EXPT_BASEDIR }}/../nco_dirs' + COMROOT_default: '{{ OPSROOT_default }}/com' + DATAROOT_default: '{{ OPSROOT_default }}/tmp' + DCOMROOT_default: '{{ OPSROOT_default }}/dcom' + LOGBASEDIR_default: '{% if user.RUN_ENVIR == "nco" %}{{ [OPSROOT_default, "output"]|path_join }}{% else %}{{ [workflow.EXPTDIR, "log"]|path_join }}{% endif %}' + COMIN_BASEDIR: '{{ COMROOT_default }}/{{ NET_default }}/{{ model_ver_default }}' + COMOUT_BASEDIR: '{{ COMROOT_default }}/{{ NET_default }}/{{ model_ver_default }}' # # New additions from RRFS_dev1 # - NWGES: '{{ OPSROOT }}/nwges' + NWGES: '{{ OPSROOT_default }}/nwges' NWGES_BASEDIR: '{{ NWGES }}' - # - #----------------------------------------------------------------------- - # - # The following are also described in the NCO doc above - # - #----------------------------------------------------------------------- - # - DBNROOT: "" - SENDECF: false - SENDDBN: false - SENDDBN_NTC: false - SENDCOM: false - SENDWEB: false - KEEPDATA: true - MAILTO: "" - MAILCC: "" + DBNROOT_default: "" + SENDECF_default: false + SENDDBN_default: false + SENDDBN_NTC_default: false + SENDCOM_default: false + SENDWEB_default: false + KEEPDATA_default: true + MAILTO_default: "" + MAILCC_default: "" #---------------------------------- # GSI namelist parameters @@ -2529,13 +2507,13 @@ cpl_aqm_parm: # AQM_CONFIG_DIR: # Configuration directory for AQM # - # AQM_BIO_DIR: + # DCOMINbio: # Path to the directory containing AQM bio files # # AQM_BIO_FILE: # File name of AQM BIO file # - # AQM_DUST_DIR: + # DCOMINdust: # Path to the directory containing AQM dust file # # AQM_DUST_FILE_PREFIX: @@ -2544,7 +2522,7 @@ cpl_aqm_parm: # AQM_DUST_FILE_SUFFIX: # Suffix and extension of AQM dust file # - # AQM_CANOPY_DIR: + # DCOMINcanopy: # Path to the directory containing AQM canopy files # # AQM_CANOPY_FILE_PREFIX: @@ -2553,7 +2531,7 @@ cpl_aqm_parm: # AQM_CANOPY_FILE_SUFFIX: # Suffix and extension of AQM CANOPY file # - # AQM_FIRE_DIR: + # DCOMINfire: # Path to the directory containing AQM fire files # # AQM_FIRE_FILE_PREFIX: @@ -2574,13 +2552,13 @@ cpl_aqm_parm: # AQM_RC_PRODUCT_FREQUENCY: # Frequency of AQM output products # - # AQM_LBCS_DIR: + # DCOMINchem_lbc: # Path to the directory containing chemical LBC files # # AQM_LBCS_FILES: # File name of chemical LBCs # - # AQM_GEFS_DIR: + # DCOMINgefs: # Path to the directory containing GEFS aerosol LBC files # # AQM_GEFS_FILE_PREFIX: @@ -2612,11 +2590,19 @@ cpl_aqm_parm: # NEXUS_GFS_SFC_ARCHV_DIR: # Path to archive directory for gfs surface files on HPSS # - # PT_SRC_BASEDIR: + # DCOMINpt_src: # Parent directory containing point source files # - # AQM_AIRNOW_HIST_DIR: - # Path to the directory where the historical AIRNOW data are located + # DCOMINairnow: + # Path to the directory containing AIRNOW observation data + # + # COMINbicor: + # Path of reading in historical training data for biascorrection + # + # COMOUTbicor: + # Path to save the current cycle's model output and AirNow obs as + # training data for future use $COMINbicor and $COMOUTbicor can be + # distuigshed by the ${yyyy}${mm}$dd under the same location # #----------------------------------------------------------------------- # @@ -2630,19 +2616,26 @@ cpl_aqm_parm: DO_AQM_SAVE_AIRNOW_HIST: false DO_AQM_SAVE_FIRE: false + DCOMINbio_default: "" + DCOMINdust_default: "/path/to/dust/dir" + DCOMINcanopy_default: "/path/to/canopy/dir" + DCOMINfire_default: "" + DCOMINchem_lbcs_default: "" + DCOMINgefs_default: "" + DCOMINpt_src_default: "/path/to/point/source/base/directory" + DCOMINairnow_default: "/path/to/airnow/obaservation/data" + COMINbicor: "/path/to/historical/airnow/data/dir" + COMOUTbicor: "/path/to/historical/airnow/data/dir" + AQM_CONFIG_DIR: "" - AQM_BIO_DIR: "" AQM_BIO_FILE: "BEIS_SARC401.ncf" - AQM_DUST_DIR: "/path/to/dust/dir" AQM_DUST_FILE_PREFIX: "FENGSHA_p8_10km_inputs" AQM_DUST_FILE_SUFFIX: ".nc" - AQM_CANOPY_DIR: "/path/to/canopy/dir" AQM_CANOPY_FILE_PREFIX: "gfs.t12z.geo" AQM_CANOPY_FILE_SUFFIX: ".canopy_regrid.nc" - AQM_FIRE_DIR: "" AQM_FIRE_FILE_PREFIX: "GBBEPx_C401GRID.emissions_v003" AQM_FIRE_FILE_SUFFIX: ".nc" AQM_FIRE_FILE_OFFSET_HRS: 0 @@ -2652,10 +2645,8 @@ cpl_aqm_parm: AQM_RC_PRODUCT_FN: "aqm.prod.nc" AQM_RC_PRODUCT_FREQUENCY: "hourly" - AQM_LBCS_DIR: "" AQM_LBCS_FILES: "gfs_bndy_chen_.tile7.000.nc" - AQM_GEFS_DIR: "" AQM_GEFS_FILE_PREFIX: "geaer" AQM_GEFS_FILE_CYC: "" @@ -2667,10 +2658,6 @@ cpl_aqm_parm: NEXUS_GFS_SFC_DIR: "" NEXUS_GFS_SFC_ARCHV_DIR: "/NCEPPROD/hpssprod/runhistory" - PT_SRC_BASEDIR: "/path/to/point/source/base/directory" - - AQM_AIRNOW_HIST_DIR: "/path/to/historical/airnow/data/dir" - rocoto: attrs: "" cycledefs: "" diff --git a/ush/create_aqm_rc_file.py b/ush/create_aqm_rc_file.py index 02d8d827e5..acf675a3b9 100644 --- a/ush/create_aqm_rc_file.py +++ b/ush/create_aqm_rc_file.py @@ -63,24 +63,24 @@ def create_aqm_rc_file(cdate, run_dir, init_concentrations): # # Set parameters in the aqm.rc file. # - aqm_rc_bio_file_fp=os.path.join(AQM_BIO_DIR, AQM_BIO_FILE) + aqm_rc_bio_file_fp=os.path.join(DCOMINbio, AQM_BIO_FILE) # Fire config aqm_rc_fire_file_fp=os.path.join( - COMINext, + COMIN, "FIRE_EMISSION", f"{AQM_FIRE_FILE_PREFIX}_{yyyymmdd}_t{hh}z{AQM_FIRE_FILE_SUFFIX}" ) # Dust config aqm_rc_dust_file_fp=os.path.join( - AQM_DUST_DIR, + DCOMINdust, f"{AQM_DUST_FILE_PREFIX}_{PREDEF_GRID_NAME}{AQM_DUST_FILE_SUFFIX}", ) # Canopy config aqm_rc_canopy_file_fp=os.path.join( - AQM_CANOPY_DIR, + DCOMINcanopy, PREDEF_GRID_NAME, f"{AQM_CANOPY_FILE_PREFIX}.{mm}{AQM_CANOPY_FILE_SUFFIX}", ) @@ -101,7 +101,7 @@ def create_aqm_rc_file(cdate, run_dir, init_concentrations): "aqm_config_dir": AQM_CONFIG_DIR, "init_concentrations": init_concentrations, "aqm_rc_bio_file_fp": aqm_rc_bio_file_fp, - "aqm_bio_dir": AQM_BIO_DIR, + "dcominbio": DCOMINbio, "aqm_rc_fire_file_fp": aqm_rc_fire_file_fp, "aqm_rc_fire_frequency": AQM_RC_FIRE_FREQUENCY, "aqm_rc_dust_file_fp": aqm_rc_dust_file_fp, diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 4c0ce2ed60..e243f31b37 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -4,6 +4,7 @@ #----------------------------------------------------------------------- # # If requested to share data with next task, override jobid +# When an argument exists with this script, a shared job id will be created. # #----------------------------------------------------------------------- # @@ -12,6 +13,77 @@ if [ $# -ne 0 ]; then export pid=$share_pid export jobid=${job}.${pid} fi + +# +#----------------------------------------------------------------------- +# +# Set NCO standard environment variables +# +#----------------------------------------------------------------------- +# +export envir="${envir:-${envir_default}}" +export NET="${NET:-${NET_default}}" +export RUN="${RUN:-${RUN_default}}" +export model_ver="${model_ver:-${model_ver_default}}" +export COMROOT="${COMROOT:-${COMROOT_default}}" +export DATAROOT="${DATAROOT:-${DATAROOT_default}}" +export DCOMROOT="${DCOMROOT:-${DCOMROOT_default}}" +export LOGBASEDIR="${LOGBASEDIR:-${LOGBASEDIR_default}}" + +export DBNROOT="${DBNROOT:-${DBNROOT_default}}" +export SENDECF="${SENDECF:-${SENDECF_default}}" +export SENDDBN="${SENDDBN:-${SENDDBN_default}}" +export SENDDBN_NTC="${SENDDBN_NTC:-${SENDDBN_NTC_default}}" +export SENDCOM="${SENDCOM:-${SENDCOM_default}}" +export SENDWEB="${SENDWEB:-${SENDWEB_default}}" +export KEEPDATA="${KEEPDATA:-${KEEPDATA_default}}" +export MAILTO="${MAILTO:-${MAILTO_default}}" +export MAILCC="${MAILCC:-${MAILCC_default}}" + +if [ "${RUN_ENVIR}" = "nco" ]; then + if [ "${MACHINE}" = "WCOSS2" ]; then + [[ "$WORKFLOW_MANAGER" = "rocoto" ]] && export COMROOT=$COMROOT + export COMIN="${COMIN:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDY}/${cyc})}" + export COMOUT="${COMOUT:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDY}/${cyc})}" + export COMINm1="${COMINm1:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDYm1})}" + export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}" + export COMINgefs="${COMINgefs:-$(compath.py ${envir}/gefs/${gefs_ver})}" + else + export COMIN="${COMIN_BASEDIR}/${RUN}.${PDY}/${cyc}" + export COMOUT="${COMOUT_BASEDIR}/${RUN}.${PDY}/${cyc}" + export COMINm1="${COMIN_BASEDIR}/${RUN}.${PDYm1}" + fi +else + export COMIN="${COMIN_BASEDIR}/${PDY}${cyc}" + export COMOUT="${COMOUT_BASEDIR}/${PDY}${cyc}" + export COMINm1="${COMIN_BASEDIR}/${RUN}.${PDYm1}" +fi +export COMOUTwmo="${COMOUTwmo:-${COMOUT}/wmo}" + +export DCOMINbio="${DCOMINbio:-${DCOMINbio_default}}" +export DCOMINdust="${DCOMINdust:-${DCOMINdust_default}}" +export DCOMINcanopy="${DCOMINcanopy:-${DCOMINcanopy_default}}" +export DCOMINfire="${DCOMINfire:-${DCOMINfire_default}}" +export DCOMINchem_lbcs="${DCOMINchem_lbcs:-${DCOMINchem_lbcs_default}}" +export DCOMINgefs="${DCOMINgefs:-${DCOMINgefs_default}}" +export DCOMINpt_src="${DCOMINpt_src:-${DCOMINpt_src_default}}" +export DCOMINairnow="${DCOMINairnow:-${DCOMINairnow_default}}" + +# +#----------------------------------------------------------------------- +# +# Change YES/NO (NCO standards; job card) to TRUE/FALSE (workflow standards) +# for NCO environment variables +# +#----------------------------------------------------------------------- +# +export KEEPDATA=$(boolify "${KEEPDATA}") +export SENDCOM=$(boolify "${SENDCOM}") +export SENDDBN=$(boolify "${SENDDBN}") +export SENDDBN_NTC=$(boolify "${SENDDBN_NTC}") +export SENDECF=$(boolify "${SENDECF}") +export SENDWEB=$(boolify "${SENDWEB}") + # #----------------------------------------------------------------------- # @@ -72,6 +144,7 @@ if [ "${RUN_ENVIR}" = "nco" ]; then export pgmerr="${DATA}/errfile" export REDIRECT_OUT_ERR=">>${pgmout} 2>${pgmerr}" export pgmout_lines=1 + export pgmerr_lines=1 function PREP_STEP() { export pgm="$(basename ${0})" @@ -94,6 +167,11 @@ if [ "${RUN_ENVIR}" = "nco" ]; then pgmout_line=$( wc -l $pgmout ) pgmout_lines=$((pgmout_lines + 1)) fi + if [ -f $pgmerr ]; then + tail -n +${pgmerr_lines} $pgmerr + pgmerr_line=$( wc -l $pgmerr ) + pgmerr_lines=$((pgmerr_lines + 1)) + fi } else export pgmout= @@ -108,24 +186,6 @@ else fi export -f PREP_STEP export -f POST_STEP -# -#----------------------------------------------------------------------- -# -# Set COMIN / COMOUT -# -#----------------------------------------------------------------------- -# -if [ "${RUN_ENVIR}" = "nco" ]; then - export COMIN="${COMIN_BASEDIR}/${RUN}.${PDY}/${cyc}" - export COMOUT="${COMOUT_BASEDIR}/${RUN}.${PDY}/${cyc}" - export COMINext="${EXTROOT}/${RUN}.${PDY}/${cyc}" -else - export COMIN="${COMIN_BASEDIR}/${PDY}${cyc}" - export COMOUT="${COMOUT_BASEDIR}/${PDY}${cyc}" - export COMINext="${EXTROOT}/${PDY}${cyc}" -fi -export COMIN_PDY="${COMIN_BASEDIR}/${RUN}.${PDY}" -export COMIN_PDYm1="${COMIN_BASEDIR}/${RUN}.${PDYm1}" # #----------------------------------------------------------------------- @@ -135,7 +195,7 @@ export COMIN_PDYm1="${COMIN_BASEDIR}/${RUN}.${PDYm1}" # #----------------------------------------------------------------------- # -if [ "${RUN_ENVIR}" = "nco" ]; then +if [ "${RUN_ENVIR}" = "nco" ] && [ "${WORKFLOW_MANAGER}" != "ecflow" ]; then __EXPTLOG=${EXPTDIR}/log mkdir_vrfy -p ${__EXPTLOG} for i in ${LOGDIR}/*.${WORKFLOW_ID}.log; do @@ -147,15 +207,27 @@ fi #----------------------------------------------------------------------- # # Add a postamble function +# When an argument exists, the working directory will not be removed +# even with KEEPDATA: false. +# Only when an argument is TRUE, the existing working directories in +# the tmp directory will be removed. # #----------------------------------------------------------------------- # function job_postamble() { # Remove temp directory - if [ "${RUN_ENVIR}" = "nco" ] && [ $# -eq 0 ]; then - cd ${DATAROOT} - [[ $KEEPDATA = "FALSE" ]] && rm -rf $DATA + if [ "${RUN_ENVIR}" = "nco" ] && [ "${KEEPDATA}" = "FALSE" ]; then + cd ${DATAROOT} + # Remove current data directory + if [ $# -eq 0 ]; then + rm -rf $DATA + # Remove all current and shared data directories + elif [ "$1" = "TRUE" ]; then + rm -rf $DATA + share_pid="${WORKFLOW_ID}_${PDY}${cyc}" + rm -rf *${share_pid} + fi fi # Print exit message diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index a3b8aaca5a..fcd64d6b64 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -100,7 +100,9 @@ set -u # default_modules_dir="$HOMEdir/modulefiles" machine=$(echo_lowercase $MACHINE) -source "${HOMEdir}/etc/lmod-setup.sh" ${machine} +if [ "${WORKFLOW_MANAGER}" != "ecflow" ]; then + source "${HOMEdir}/etc/lmod-setup.sh" ${machine} +fi module use "${default_modules_dir}" if [ "${machine}" != "wcoss2" ]; then @@ -177,8 +179,6 @@ elif [ -f ${modules_dir}/python_srw.lua ] ; then modules_dir = \"${modules_dir}\"" fi - - module list # Modules that use conda and need an environment activated will set the @@ -210,7 +210,13 @@ print_info_msg "$VERBOSE" " Launching J-job (jjob_fp) for task \"${task_name}\" ... jjob_fp = \"${jjob_fp}\" " -exec "${jjob_fp}" + +if [ "${WORKFLOW_MANAGER}" = "ecflow" ]; then + /bin/bash "${jjob_fp}" +else + exec "${jjob_fp}" +fi + # #----------------------------------------------------------------------- # @@ -221,4 +227,3 @@ exec "${jjob_fp}" # { restore_shell_opts; } > /dev/null 2>&1 - diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 7604301fa3..121e3a91ef 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -45,12 +45,13 @@ task_get_da_obs: cpl_aqm_parm: AQM_CONFIG_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/epa/data - AQM_BIO_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/bio - AQM_DUST_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/FENGSHA - AQM_CANOPY_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/canopy - AQM_FIRE_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/RAVE_fire - AQM_LBCS_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/LBCS/AQM_NA13km_AM4_v1 - AQM_GEFS_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/GEFS_DATA + DCOMINbio_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/bio + DCOMINdust_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/FENGSHA + DCOMINcanopy_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/canopy + DCOMINfire_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/RAVE_fire + DCOMINchem_lbcs_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/LBCS/AQM_NA13km_AM4_v1 + DCOMINgefs_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/GEFS_DATA + DCOMINpt_src_default: /scratch1/RDARCH/rda-arl-gpu/Barry.Baker/emissions/nexus/NEI2016v1/v2023-01-PT NEXUS_INPUT_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/emissions/nexus NEXUS_FIX_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/nexus/fix NEXUS_GFS_SFC_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/GFS_DATA diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index 2703451bf1..6da06b7288 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -48,17 +48,19 @@ data: HRRR: compath.py ${envir}/hrrr/${hrrr_ver}/hrrr.${PDYext}/conus cpl_aqm_parm: AQM_CONFIG_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/epa/data - AQM_BIO_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/bio - AQM_DUST_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/FENGSHA - AQM_CANOPY_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/canopy - AQM_FIRE_DIR: /lfs/h2/emc/physics/noscrub/kai.wang/RAVE_fire/RAVE_NA_NRT - AQM_LBCS_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/LBCS/AQM_NA13km_AM4_v1 - AQM_GEFS_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/GEFS_DATA - AQM_AIRNOW_HIST_DIR: /lfs/h2/emc/physics/noscrub/jianping.huang/Bias_correction/aqmv7.0 + DCOMINbio_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/bio + DCOMINdust_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/FENGSHA + DCOMINcanopy_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/canopy + DCOMINfire_default: /lfs/h1/ops/dev/dcom + DCOMINchem_lbcs_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/LBCS/AQM_NA13km_AM4_v1 + DCOMINgefs_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/GEFS_DATA + DCOMINpt_src_default: /lfs/h2/emc/physics/noscrub/Youhua.Tang/nei2016v1-pt/v2023-01-PT + DCOMINairnow_default: /lfs/h1/ops/prod/dcom + COMINbicor: /lfs/h2/emc/physics/noscrub/jianping.huang/Bias_correction/aqmv7.0.81 + COMOUTbicor: /lfs/h2/emc/physics/noscrub/jianping.huang/Bias_correction/aqmv7.0.81 NEXUS_INPUT_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/nexus_emissions NEXUS_FIX_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/nexus/fix NEXUS_GFS_SFC_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/GFS_DATA - PT_SRC_BASEDIR: /lfs/h2/emc/physics/noscrub/Youhua.Tang/nei2016v1-pt/v2023-01-PT rocoto: tasks: diff --git a/ush/setup.py b/ush/setup.py index 41b0e0bb92..637e3b541c 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1160,14 +1160,12 @@ def get_location(xcs, fmt, expt_cfg): # running in community mode, we set these paths to the experiment # directory. nco_vars = [ - "opsroot", - "comroot", - "packageroot", - "dataroot", - "dcomroot", + "opsroot_default", + "comroot_default", + "dataroot_default", + "dcomroot_default", "comin_basedir", "comout_basedir", - "extroot", ] nco_config = expt_config["nco"] @@ -1177,30 +1175,28 @@ def get_location(xcs, fmt, expt_cfg): nco_config[nco_var.upper()] = exptdir # Use env variables for NCO variables and create NCO directories - if run_envir == "nco": - + workflow_manager = expt_config["platform"].get("WORKFLOW_MANAGER") + if run_envir == "nco" and workflow_manager == "rocoto": for nco_var in nco_vars: envar = os.environ.get(nco_var) if envar is not None: nco_config[nco_var.upper()] = envar - mkdir_vrfy(f' -p "{nco_config.get("OPSROOT")}"') - mkdir_vrfy(f' -p "{nco_config.get("COMROOT")}"') - mkdir_vrfy(f' -p "{nco_config.get("PACKAGEROOT")}"') - mkdir_vrfy(f' -p "{nco_config.get("DATAROOT")}"') - mkdir_vrfy(f' -p "{nco_config.get("DCOMROOT")}"') - mkdir_vrfy(f' -p "{nco_config.get("EXTROOT")}"') + mkdir_vrfy(f' -p "{nco_config.get("OPSROOT_default")}"') + mkdir_vrfy(f' -p "{nco_config.get("COMROOT_default")}"') + mkdir_vrfy(f' -p "{nco_config.get("DATAROOT_default")}"') + mkdir_vrfy(f' -p "{nco_config.get("DCOMROOT_default")}"') # Update the rocoto string for the fcst output location if # running an ensemble in nco mode if global_sect["DO_ENSEMBLE"]: rocoto_config["entities"]["FCST_DIR"] = \ - "{{ nco.DATAROOT }}/run_fcst_mem#mem#.{{ workflow.WORKFLOW_ID }}_@Y@m@d@H" + "{{ nco.DATAROOT_default }}/run_fcst_mem#mem#.{{ workflow.WORKFLOW_ID }}_@Y@m@d@H" - if nco_config["DBNROOT"]: - mkdir_vrfy(f' -p "{nco_config["DBNROOT"]}"') + if nco_config["DBNROOT_default"] and workflow_manager == "rocoto": + mkdir_vrfy(f' -p "{nco_config["DBNROOT_default"]}"') - mkdir_vrfy(f' -p "{nco_config.get("LOGBASEDIR")}"') + mkdir_vrfy(f' -p "{nco_config.get("LOGBASEDIR_default")}"') # create experiment dir mkdir_vrfy(f' -p "{exptdir}"') diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index 83563c0f62..0af5f3c4ac 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -7,7 +7,7 @@ valid_vals_DEBUG: [True, False] valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "JET", "ODIN", "CHEYENNE", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"] valid_vals_FCST_MODEL: ["ufs-weather-model"] -valid_vals_WORKFLOW_MANAGER: ["rocoto", "none"] +valid_vals_WORKFLOW_MANAGER: ["rocoto", "ecflow", "none"] valid_vals_PREDEF_GRID_NAME: [ "RRFS_CONUS_25km", "RRFS_CONUS_13km", From 8ced03fea308ee78ce88843fb78709abf0167a41 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Fri, 30 Jun 2023 07:00:06 -0600 Subject: [PATCH 14/16] [develop] Change Jenkins test to build with "all" target. (#832) Extends the Jenkins build so that RRFS components may be added to coverage tests. Also adds the process_obs test to the Jet coverage suite. Jet is chosen here since most of manual testing and development is done on Hera, it should help diversify the platforms GSL is most interested in. --- .cicd/scripts/srw_build.sh | 2 +- tests/WE2E/machine_suites/coverage.jet | 1 + tests/build.sh | 7 ++----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh index ca81576963..7c315655ab 100755 --- a/.cicd/scripts/srw_build.sh +++ b/.cicd/scripts/srw_build.sh @@ -27,7 +27,7 @@ fi # Build and install cd ${workspace}/tests set +e -./build.sh ${platform} ${SRW_COMPILER} +./build.sh ${platform} ${SRW_COMPILER} all build_exit=$? set -e cd - diff --git a/tests/WE2E/machine_suites/coverage.jet b/tests/WE2E/machine_suites/coverage.jet index 7d75fce4b2..75e0261d38 100644 --- a/tests/WE2E/machine_suites/coverage.jet +++ b/tests/WE2E/machine_suites/coverage.jet @@ -9,3 +9,4 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR +process_obs diff --git a/tests/build.sh b/tests/build.sh index 01248c80f2..eccfca8830 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -95,11 +95,8 @@ if [ $components = "all" ]; then # Array of all optional GSI executables built #----------------------------------------------------------------------- executables_created+=( enkf.x \ - gsi.x \ - nc_diag_cat.x \ - ncdiag_cat_serial.x \ - test_nc_unlimdims.x ) - + gsi.x ) + #----------------------------------------------------------------------- # Array of all optional rrfs_utl executables built #----------------------------------------------------------------------- From cff6f5c6e48bb04033f49e05bd170072d1f2f590 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 5 Jul 2023 12:30:15 -0400 Subject: [PATCH 15/16] [develop] Change conda env for AQM on Hera to shared one and fix input data issue on WCOSS2 (#844) * Change the python environments for AQM to the conda in the shared location on Hera and Orion. * Add the input data for AQM fire emission (TEST_AQM_INPUT_BASEDIR) to the shared directory for WE2E. * Fix the missing source attribute in AQM_ICS on WCOSS2 (Cactus). --- modulefiles/tasks/hera/aqm_ics.local.lua | 2 +- modulefiles/tasks/hera/fire_emission.local.lua | 2 +- modulefiles/tasks/hera/miniconda_online-cmaq.lua | 5 ----- modulefiles/tasks/hera/miniconda_regional_workflow_cmaq.lua | 4 ++++ modulefiles/tasks/hera/nexus_emission.local.lua | 2 +- modulefiles/tasks/hera/nexus_post_split.local.lua | 2 +- modulefiles/tasks/hera/point_source.local.lua | 2 +- modulefiles/tasks/orion/aqm_ics.local.lua | 2 +- modulefiles/tasks/orion/fire_emission.local.lua | 2 ++ modulefiles/tasks/orion/miniconda_online-cmaq.lua | 4 ---- .../tasks/orion/miniconda_regional_workflow_cmaq.lua | 4 ++++ modulefiles/tasks/orion/nexus_emission.local.lua | 2 +- modulefiles/tasks/orion/nexus_post_split.local.lua | 2 +- modulefiles/tasks/orion/point_source.local.lua | 2 +- tests/WE2E/run_WE2E_tests.py | 5 ++++- ush/config_defaults.yaml | 1 + ush/load_modules_run_task.sh | 6 ------ ush/machine/cheyenne.yaml | 1 + ush/machine/hera.yaml | 1 + ush/machine/orion.yaml | 1 + ush/machine/wcoss2.yaml | 1 + 21 files changed, 28 insertions(+), 25 deletions(-) delete mode 100644 modulefiles/tasks/hera/miniconda_online-cmaq.lua create mode 100644 modulefiles/tasks/hera/miniconda_regional_workflow_cmaq.lua create mode 100644 modulefiles/tasks/orion/fire_emission.local.lua delete mode 100644 modulefiles/tasks/orion/miniconda_online-cmaq.lua create mode 100644 modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua diff --git a/modulefiles/tasks/hera/aqm_ics.local.lua b/modulefiles/tasks/hera/aqm_ics.local.lua index 836582f847..4202cbf232 100644 --- a/modulefiles/tasks/hera/aqm_ics.local.lua +++ b/modulefiles/tasks/hera/aqm_ics.local.lua @@ -1,2 +1,2 @@ -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) diff --git a/modulefiles/tasks/hera/fire_emission.local.lua b/modulefiles/tasks/hera/fire_emission.local.lua index 0260ca57cf..ef04528a3f 100644 --- a/modulefiles/tasks/hera/fire_emission.local.lua +++ b/modulefiles/tasks/hera/fire_emission.local.lua @@ -1,3 +1,3 @@ load("hpss") -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) diff --git a/modulefiles/tasks/hera/miniconda_online-cmaq.lua b/modulefiles/tasks/hera/miniconda_online-cmaq.lua deleted file mode 100644 index 9712c40628..0000000000 --- a/modulefiles/tasks/hera/miniconda_online-cmaq.lua +++ /dev/null @@ -1,5 +0,0 @@ -prepend_path("MODULEPATH", "/contrib/miniconda3/modulefiles") -load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) - -setenv("AQM_ENV_FP", "/scratch2/NCEPDEV/naqfc/RRFS_CMAQ/PY_VENV") -setenv("AQM_ENV", "online-cmaq") diff --git a/modulefiles/tasks/hera/miniconda_regional_workflow_cmaq.lua b/modulefiles/tasks/hera/miniconda_regional_workflow_cmaq.lua new file mode 100644 index 0000000000..c0094e0ab2 --- /dev/null +++ b/modulefiles/tasks/hera/miniconda_regional_workflow_cmaq.lua @@ -0,0 +1,4 @@ +prepend_path("MODULEPATH","/scratch1/NCEPDEV/nems/role.epic/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow_cmaq") diff --git a/modulefiles/tasks/hera/nexus_emission.local.lua b/modulefiles/tasks/hera/nexus_emission.local.lua index 027881a685..5a7b0cece6 100644 --- a/modulefiles/tasks/hera/nexus_emission.local.lua +++ b/modulefiles/tasks/hera/nexus_emission.local.lua @@ -1,2 +1,2 @@ load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hera/nexus_post_split.local.lua b/modulefiles/tasks/hera/nexus_post_split.local.lua index 836582f847..4202cbf232 100644 --- a/modulefiles/tasks/hera/nexus_post_split.local.lua +++ b/modulefiles/tasks/hera/nexus_post_split.local.lua @@ -1,2 +1,2 @@ -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) diff --git a/modulefiles/tasks/hera/point_source.local.lua b/modulefiles/tasks/hera/point_source.local.lua index 0ef3de3b66..07cacc8ce2 100644 --- a/modulefiles/tasks/hera/point_source.local.lua +++ b/modulefiles/tasks/hera/point_source.local.lua @@ -1 +1 @@ -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/orion/aqm_ics.local.lua b/modulefiles/tasks/orion/aqm_ics.local.lua index 027881a685..5a7b0cece6 100644 --- a/modulefiles/tasks/orion/aqm_ics.local.lua +++ b/modulefiles/tasks/orion/aqm_ics.local.lua @@ -1,2 +1,2 @@ load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/orion/fire_emission.local.lua b/modulefiles/tasks/orion/fire_emission.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/orion/fire_emission.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/orion/miniconda_online-cmaq.lua b/modulefiles/tasks/orion/miniconda_online-cmaq.lua deleted file mode 100644 index 3b01515fb6..0000000000 --- a/modulefiles/tasks/orion/miniconda_online-cmaq.lua +++ /dev/null @@ -1,4 +0,0 @@ -load(pathJoin("miniconda", os.getenv("miniconda_ver") or "4.12.0")) - -setenv("SRW_ENV", "/work/noaa/fv3-cam/RRFS_CMAQ/PY_VENV") -setenv("SRW_ENV", "online-cmaq") diff --git a/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua b/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua new file mode 100644 index 0000000000..9d84332bd5 --- /dev/null +++ b/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua @@ -0,0 +1,4 @@ +prepend_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow_cmaq") diff --git a/modulefiles/tasks/orion/nexus_emission.local.lua b/modulefiles/tasks/orion/nexus_emission.local.lua index 027881a685..5a7b0cece6 100644 --- a/modulefiles/tasks/orion/nexus_emission.local.lua +++ b/modulefiles/tasks/orion/nexus_emission.local.lua @@ -1,2 +1,2 @@ load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/orion/nexus_post_split.local.lua b/modulefiles/tasks/orion/nexus_post_split.local.lua index 027881a685..5a7b0cece6 100644 --- a/modulefiles/tasks/orion/nexus_post_split.local.lua +++ b/modulefiles/tasks/orion/nexus_post_split.local.lua @@ -1,2 +1,2 @@ load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/orion/point_source.local.lua b/modulefiles/tasks/orion/point_source.local.lua index 0ef3de3b66..07cacc8ce2 100644 --- a/modulefiles/tasks/orion/point_source.local.lua +++ b/modulefiles/tasks/orion/point_source.local.lua @@ -1 +1 @@ -load("miniconda_online-cmaq") +load("miniconda_regional_workflow_cmaq") diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index 4a2e6babe0..3ef3649efd 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -178,7 +178,6 @@ def run_we2e_tests(homedir, args) -> None: if args.verbose_tests: test_cfg['workflow'].update({"VERBOSE": args.verbose_tests}) - logging.debug(f"Overwriting WE2E-test-specific settings for test \n{test_name}\n") if 'task_get_extrn_ics' in test_cfg: @@ -191,6 +190,10 @@ def run_we2e_tests(homedir, args) -> None: if 'verification' in test_cfg: logging.debug(test_cfg['verification']) + if 'cpl_aqm_parm' in test_cfg: + test_aqm_input_basedir = machine_defaults['platform']['TEST_AQM_INPUT_BASEDIR'] + test_cfg['cpl_aqm_parm']['DCOMINfire_default'] = f"{test_aqm_input_basedir}/RAVE_fire" + logging.debug(f"Writing updated config.yaml for test {test_name}\n"\ "based on specified command-line arguments:\n") logging.debug(cfg_to_yaml_str(test_cfg)) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 5a83c090e9..39f76501ac 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -372,6 +372,7 @@ platform: #----------------------------------------------------------------------- # TEST_EXTRN_MDL_SOURCE_BASEDIR: "" + TEST_AQM_INPUT_BASEDIR: "" TEST_PREGEN_BASEDIR: "" TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: "" TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: "" diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index fcd64d6b64..d3806b279c 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -192,12 +192,6 @@ if [ -n "${SRW_ENV:-}" ] ; then set -u fi -if [ -n "${AQM_ENV:-}" ] ; then - set +u - source "${AQM_ENV_FP}/${AQM_ENV}/bin/activate" - set -u -fi - # #----------------------------------------------------------------------- # diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml index c738a772fc..c0e6a3566c 100644 --- a/ush/machine/cheyenne.yaml +++ b/ush/machine/cheyenne.yaml @@ -21,6 +21,7 @@ platform: RUN_CMD_AQMLBC: mpirun -np ${NUMTS} PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data + TEST_AQM_INPUT_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/aqm_data TEST_PREGEN_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/FV3LAM_pregen TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 121e3a91ef..75258583e4 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -25,6 +25,7 @@ platform: SCHED_NATIVE_CMD: --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/input_model_data + TEST_AQM_INPUT_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/aqm_data TEST_PREGEN_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/FV3LAM_pregen TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 14221024f3..8393661e45 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -25,6 +25,7 @@ platform: SCHED_NATIVE_CMD: --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/input_model_data + TEST_AQM_INPUT_BASEDIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/aqm_data TEST_PREGEN_BASEDIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/FV3LAM_pregen FIXaer: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/fix/fix_aer FIXgsi: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/fix/fix_gsi diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index 6da06b7288..062b00099f 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -22,6 +22,7 @@ platform: SCHED_NATIVE_CMD: -l place=excl PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/input_model_data + TEST_AQM_INPUT_BASEDIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/aqm_data TEST_PREGEN_BASEDIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/FV3LAM_pregen FIXaer: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/fix/fix_aer FIXgsm: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/fix/fix_am From f9696e1bcc4295556e679e67e4ab2fb133dde2a8 Mon Sep 17 00:00:00 2001 From: Yi-Cheng Teng - NOAA <30629225+clouden90@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:31:00 -0400 Subject: [PATCH 16/16] [develop] Bugfix/metric (#835) Bug fix for forecast metric case due to recently merged verification PRs. --- .cicd/scripts/srw_metric_example.sh | 12 ++++++------ ...Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.cicd/scripts/srw_metric_example.sh b/.cicd/scripts/srw_metric_example.sh index 3560d99254..7505334ba0 100755 --- a/.cicd/scripts/srw_metric_example.sh +++ b/.cicd/scripts/srw_metric_example.sh @@ -50,15 +50,15 @@ conda activate regional_workflow set -e -u # build srw -cd ${WORKSPACE}/tests +cd ${workspace}/tests ./build.sh ${platform,,} ${SRW_COMPILER} -cd ${WORKSPACE} +cd ${workspace} # run test [[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir} -cd ${WORKSPACE}/tests/WE2E +cd ${workspace}/tests/WE2E ./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${SRW_PROJECT} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q -cd ${WORKSPACE} +cd ${workspace} # run skill-score check # first load MET env variables @@ -70,8 +70,8 @@ source ${we2e_experiment_base_dir}/${we2e_test_name}/var_defns.sh # It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. # In this example, skill score index is a weighted average of 16 skill scores of RMSE statistics for wind speed, dew point temperature, # temperature, and pressure at lowest level in the atmosphere over 48 hour lead time. -cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/mem000/metprd/PointStat/*.stat ${WORKSPACE}/Indy-Severe-Weather/metprd/point_stat/ -${MET_INSTALL_DIR}/${MET_BIN_EXEC}/stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${WORKSPACE}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.out +cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/ +${MET_INSTALL_DIR}/${MET_BIN_EXEC}/stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.out # check skill-score.out cat skill-score.out diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml index 44cf2ae52b..05af923cc5 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml @@ -16,7 +16,7 @@ workflow: PREEXISTING_DIR_METHOD: rename rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml"]|include }}' metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 04:00:00