diff --git a/jobs/JREGIONAL_GET_OBS_NOHRSC b/jobs/JREGIONAL_GET_OBS_NOHRSC deleted file mode 100755 index cc892951fd..0000000000 --- a/jobs/JREGIONAL_GET_OBS_NOHRSC +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# This script pulls NOHRSC observation data for comparison to the model for -# the requested accumulations. Supported accumulations: 01h, 03h, and 06h. -# NOTE: Accumulation is currently hardcoded to 06h. -# The verification uses MET/pcp-combine to sum 06h files into -# desired accumulations. -# -#----------------------------------------------------------------------- -# - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -. $USHdir/job_preamble.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the J-job script for the task that pulls NOHRSC observation data -for verification purposes. -========================================================================" - -# -# -#----------------------------------------------------------------------- -# -# Call the ex-script for this J-job and pass to it the necessary varia- -# bles. -# -#----------------------------------------------------------------------- -# -$SCRIPTSdir/exregional_get_obs_nohrsc.sh || \ -print_err_msg_exit "\ -Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." -# -#----------------------------------------------------------------------- -# -# Run job postamble. -# -#----------------------------------------------------------------------- -# -job_postamble -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/parm/data_locations.yml b/parm/data_locations.yml index fe785ccd14..68f43ba2d0 100644 --- a/parm/data_locations.yml +++ b/parm/data_locations.yml @@ -479,6 +479,22 @@ NDAS_obs: obs: - "./nam.t{hh}z.prepbufr.tm*.nr" +NOHRSC_obs: + hpss: + protocol: htar + archive_format: tar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + archive_file_names: + - "dcom_prod_{yyyy}{mm}{dd}.tar" + - "dcom_{yyyy}{mm}{dd}.tar" + file_names: + obs: + - "sfav2_CONUS_*h_{yyyy}{mm}{dd}{hh}_grid184.grb2" + archive_internal_dir: + - ./wgrbbul/nohrsc_snowfall/ + ########################### ########################### ####### Fix Files ######### diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml index f85bedf50d..e46474c361 100644 --- a/parm/wflow/verify_pre.yaml +++ b/parm/wflow/verify_pre.yaml @@ -37,10 +37,11 @@ task_get_obs_ccpa: task_get_obs_nohrsc: <<: *default_task_verify_pre - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_NOHRSC"' + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_VERIF_OBS"' envars: <<: *default_vars OBS_DIR: '&NOHRSC_OBS_DIR;' + OBTYPE: 'NOHRSC' FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' queue: "&QUEUE_HPSS;" diff --git a/scripts/exregional_get_obs_nohrsc.sh b/scripts/exregional_get_obs_nohrsc.sh deleted file mode 100755 index 12ec224fce..0000000000 --- a/scripts/exregional_get_obs_nohrsc.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# This script retrieves and organizes the NOHRSC data into a more intuitive structure: -# A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. -# NOTE: Accumulation is currently hardcoded to 06h and 24h which are available every 6 and 12 hours, respectively. -# -#----------------------------------------------------------------------- -# - -# Top-level NOHRSC directory -nohrsc_dir=${OBS_DIR}/.. -if [[ ! -d "$nohrsc_dir" ]]; then - mkdir_vrfy -p $nohrsc_dir -fi - -# NOHRSC data from HPSS -nohrsc_raw=$nohrsc_dir/raw -if [[ ! -d "$nohrsc_raw" ]]; then - mkdir_vrfy -p $nohrsc_raw -fi - -# Reorganized NOHRSC location -nohrsc_proc=$nohrsc_dir/proc -if [[ ! -d "$nohrsc_proc" ]]; then - mkdir_vrfy -p $nohrsc_proc -fi - -# Accumulation is for accumulation of NOHRSC data to pull (hardcoded to 06h, see note above.) -#accum=${ACCUM} -accum=06 - -# Initialization -yyyymmdd=${PDY} -hh=${cyc} - -init=${CDATE}${hh} - -fhr_last=`echo ${FHR} | awk '{ print $NF }'` - -# Forecast length -fcst_length=${fhr_last} - -current_fcst=$accum -while [[ ${current_fcst} -le ${fcst_length} ]]; do - # Calculate valid date info - fcst_sec=`expr ${current_fcst} \* 3600` # convert forecast lead hour to seconds - yyyy=`echo ${init} | cut -c1-4` # year (YYYY) of initialization time - mm=`echo ${init} | cut -c5-6` # month (MM) of initialization time - dd=`echo ${init} | cut -c7-8` # day (DD) of initialization time - hh=`echo ${init} | cut -c9-10` # hour (HH) of initialization time - init_ut=`$DATE_UTIL -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time - vdate_ut=`expr ${init_ut} + ${fcst_sec}` # calculate current forecast time in universal time - vdate=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time - vyyyymmdd=`echo ${vdate} | cut -c1-8` # forecast time (YYYYMMDD) - vyyyy=`echo ${vdate} | cut -c1-4` # year (YYYY) of valid time - vmm=`echo ${vdate} | cut -c5-6` # month (MM) of valid time - vdd=`echo ${vdate} | cut -c7-8` # day (DD) of valid time - vhh=`echo ${vdate} | cut -c9-10` # forecast hour (HH) - - # Create necessary raw and prop directories - if [[ ! -d "$nohrsc_raw/${vyyyymmdd}" ]]; then - mkdir_vrfy -p $nohrsc_raw/${vyyyymmdd} - fi - - if [[ ! -d "$nohrsc_proc/${vyyyymmdd}" ]]; then - mkdir_vrfy -p $nohrsc_proc/${vyyyymmdd} - fi - - # Name of NOHRSC tar file on HPSS is for files after Mar. 2020. - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/dcom_prod_${vyyyy}${vmm}${vdd}.tar" - - accum_noZero=$(( 10#$accum )) - nohrsc_file="$nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_06h_${vyyyymmdd}${vhh}_grid184.grb2" - echo "NOHRSC FILE:${nohrsc_file}" - if [[ ! -f "${nohrsc_file}" ]]; then - - if (( ${current_fcst} % 6 == 0 )) ; then - cd_vrfy $nohrsc_raw/${vyyyymmdd} - # Pull NOHRSC data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"sfav2_CONUS_6h_${vyyyymmdd}${vhh}_grid184.grb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "sfav2_CONUS_6h_${vyyyymmdd}${vhh}_grid184.grb2" | awk '{print \$7}'` - - cp_vrfy $nohrsc_raw/${vyyyymmdd}/wgrbbul/nohrsc_snowfall/sfav2_CONUS_6h_${vyyyymmdd}${vhh}_grid184.grb2 $nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_06h_${vyyyymmdd}${vhh}_grid184.grb2 - fi - - if (( ${current_fcst} % 12 == 0 )) && (( ${current_fcst} >= 24 )) ; then - cd_vrfy $nohrsc_raw/${vyyyymmdd} - # Pull NOHRSC data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2" | awk '{print \$7}'` - - cp_vrfy $nohrsc_raw/${vyyyymmdd}/wgrbbul/nohrsc_snowfall/sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2 $nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2 - fi - fi - - # Increment to next forecast hour - current_fcst=$((${current_fcst} + 06)) - echo "Current fcst hr=${current_fcst}" - -done -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/scripts/exregional_get_verif_obs.sh b/scripts/exregional_get_verif_obs.sh index 2feff10e95..8788f87c53 100755 --- a/scripts/exregional_get_verif_obs.sh +++ b/scripts/exregional_get_verif_obs.sh @@ -24,12 +24,12 @@ set -x # # This script performs several important tasks for preparing data for # verification tasks. Depending on the value of the environment variable -# OBTYPE=(CCPA|MRMS|NDAS), the script will prepare that particular data +# OBTYPE=(CCPA|MRMS|NDAS|NOHRSC), the script will prepare that particular data # set. # # If data is not available on disk (in the location specified by -# CCPA_OBS_DIR, MRMS_OBS_DIR, or NDAS_OBS_DIR respectively), the script -# attempts to retrieve the data from HPSS using the retrieve_data.py +# CCPA_OBS_DIR, MRMS_OBS_DIR, NDAS_OBS_DIR, or NOHRSC_OBS_DIR respectively), +# the script attempts to retrieve the data from HPSS using the retrieve_data.py # script. Depending on the data set, there are a few strange quirks and/or # bugs in the way data is organized; see in-line comments for details. # @@ -91,6 +91,23 @@ set -x # # If data is retrieved from HPSS, it will automatically staged by this # this script. +# +# +# NOHRSC +# ---------- +# If data is available on disk, it must be in the following +# directory structure and file name conventions expected by verification +# tasks: +# +# {NOHRSC_OBS_DIR}/{YYYYMMDD}/sfav2_CONUS_{AA}h_{YYYYMMDD}{HH}_grid184.grb2 +# +# where AA is the 2-digit accumulation duration in hours: 06 or 24 +# +# METplus is configured to verify snowfall using 06- and 24-h accumulated +# snowfall from 6- and 12-hourly NOHRSC files, respectively. +# +# If data is retrieved from HPSS, it will automatically staged by this +# this script. #----------------------------------------------------------------------- # Create and enter top-level obs directory (so temporary data from HPSS won't collide with other tasks) @@ -356,15 +373,67 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do fi fi + # Retrieve NOHRSC observations + elif [[ ${OBTYPE} == "NOHRSC" ]]; then + # Reorganized NOHRSC location (no need for raw data dir) + nohrsc_proc=${OBS_DIR} + nohrsc06h_file="$nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_06h_${vyyyymmdd}${vhh}_grid184.grb2" + nohrsc24h_file="$nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2" + retrieve=0 + # If 24-hour files should be available (at 00z and 12z) then look for both files + # Otherwise just look for 6hr file + if (( ${current_fcst} % 12 == 0 )) && (( ${current_fcst} >= 24 )) ; then + if [ ! -f "${nohrsc06h_file}" ] || [ ! -f "${nohrsc24h_file}" ] ; then + retrieve=1 + else + echo "NOHRSC 6h accumulation file exists: ${nohrsc06h_file}" + echo "NOHRSC 24h accumulation file exists: ${nohrsc24h_file}" + fi + elif (( ${current_fcst} % 6 == 0 )) ; then + if [ ! -f "${nohrsc06h_file}" ]; then + retrieve=1 + else + echo "NOHRSC 6h accumulation file exists: ${nohrsc06h_file}" + fi + fi + if [ $retrieve == 1 ]; then + if [[ ! -d "$nohrsc_proc/${vyyyymmdd}" ]]; then + mkdir -p $nohrsc_proc/${vyyyymmdd} + fi + # Pull NOHRSC data from HPSS; script will retrieve all files so only call once + cmd=" + python3 -u ${USHdir}/retrieve_data.py \ + --debug \ + --file_set obs \ + --config ${PARMdir}/data_locations.yml \ + --cycle_date ${vyyyymmdd}${vhh} \ + --data_stores hpss \ + --data_type NOHRSC_obs \ + --output_path $nohrsc_proc/${vyyyymmdd} \ + --summary_file ${logfile}" + + echo "CALLING: ${cmd}" + + $cmd || print_err_msg_exit "\ + Could not retrieve NOHRSC data from HPSS + + The following command exited with a non-zero exit status: + ${cmd} +" + # 6-hour forecast needs to be renamed + mv $nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_6h_${vyyyymmdd}${vhh}_grid184.grb2 ${nohrsc06h_file} + else + echo "Will not retrieve from HPSS" + fi else print_err_msg_exit "\ - Invalid OBTYPE specified for script; valid options are CCPA, MRMS, and NDAS + Invalid OBTYPE specified for script; valid options are CCPA, MRMS, NDAS, and NOHRSC " - fi + fi # Increment to next forecast hour # Increment to next forecast hour echo "Finished fcst hr=${current_fcst}" current_fcst=$((${current_fcst} + 1)) diff --git a/tests/WE2E/test_configs/verification/config.custom_ESGgrid_Great_Lakes_snow_8km.yaml b/tests/WE2E/test_configs/verification/config.custom_ESGgrid_Great_Lakes_snow_8km.yaml index e1410d003e..64d9e02d9e 100644 --- a/tests/WE2E/test_configs/verification/config.custom_ESGgrid_Great_Lakes_snow_8km.yaml +++ b/tests/WE2E/test_configs/verification/config.custom_ESGgrid_Great_Lakes_snow_8km.yaml @@ -60,7 +60,9 @@ task_run_post: POST_OUTPUT_DOMAIN_NAME: custom_ESGgrid_Michigan_Ontario verification: VX_FCST_MODEL_NAME: Michigan_Ontario_snow_8km + VX_FIELDS: [ "APCP", "REFC", "RETOP", "SFC", "UPA", "ASNOW" ] platform: CCPA_OBS_DIR: '{{ workflow.EXPTDIR }}/CCPA_obs' MRMS_OBS_DIR: '{{ workflow.EXPTDIR }}/MRMS_obs' NDAS_OBS_DIR: '{{ workflow.EXPTDIR }}/NDAS_obs' + NOHRSC_OBS_DIR: '{{ workflow.EXPTDIR }}/NOHRSC_obs' diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 1cec64d4ae..78e1b7c0fb 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -248,35 +248,13 @@ platform: # CCPA_OBS_DIR: # User-specified location of the directory where CCPA hourly # precipitation files used by METplus are located (or, if - # retrieved by the workflow, where they will be placed). See comments in file + # retrieved by the workflow, where they will be placed). See comments # in file scripts/exregional_get_verif_obs.sh for more details about # files and directory structure, as well as important caveats about # errors in the metadata and file names. # NOTE: This should not be set equal to other *OBS_DIR variables, # otherwise unexpected results and data loss may occur # - # NOHRSC_OBS_DIR: - # User-specified location of top-level directory where NOHRSC 6- and 24-h hour - # snowfall accumulation files used by METplus are located. This parameter - # needs to be set for both user-provided observations and for - # observations that are retrieved from the NOAA HPSS (if the user has - # access) via the get_obs_nohrsc task (activated in workflow by - # including the parm/wflow/verify_pre.yaml task set in the wflow config). - # In the case of pulling observations directly from NOAA HPSS, the - # data retrieved will be placed in this directory. Please note, this - # path must be defind as /full-path-to-obs/nohrsc/proc. METplus is - # configured to verify 06- and 24-h accumulated snowfall - # precipitation using 6- and 12-hourly NOHRSC files, respectively. - # METplus configuration files require the use of predetermined - # directory structure and file names. Therefore, if the NOHRSC files are - # user provided, they need to follow the anticipated naming structure: - # - # {YYYYMMDD}/sfav2_CONUS_{AA}h_{YYYYMMDD}{HH}_grid184.grb2 - # - # where AA is the 2-digit accumulation duration, YYYY is the 4-digit - # valid year, MM the 2-digit valid month, DD the 2-digit valid day - # of the month, and HH the 2-digit valid hour of the day. - # # MRMS_OBS_DIR: # User-specified location of the directory where MRMS composite # reflectivity and echo top files used by METplus are located (or, if @@ -295,6 +273,15 @@ platform: # NOTE: This should not be set equal to other *OBS_DIR variables, # otherwise unexpected results and data loss may occur # + # NOHRSC_OBS_DIR: + # User-specified location of top-level directory where NOHRSC 6- and + # 24-hour snowfall accumulation files used by METplus are located (or, + # if retrieved by the workflow, where they will be placed). See comments + # in file scripts/exregional_get_verif_obs.sh for more details about + # files and directory structure + # NOTE: This should not be set equal to other *OBS_DIR variables, + # otherwise unexpected results and data loss may occur + # #----------------------------------------------------------------------- # MET_INSTALL_DIR: ""