additional update from production

ufs-community · Jun 27, 2023 · f8e479e · f8e479e
1 parent e0f79fa
commit f8e479e
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 94 deletions.
diff --git a/Externals.cfg b/Externals.cfg
@@ -49,7 +49,7 @@ protocol = git
 repo_url = https://github.com/noaa-oar-arl/NEXUS
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 3842818
+hash = 6a7a994
 local_path = sorc/arl_nexus
 required = True
 
@@ -58,7 +58,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/AQM-utils
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 17c7957
+hash = 694a139
 local_path = sorc/AQM-utils
 required = True
 

diff --git a/scripts/exregional_bias_correction_o3.sh b/scripts/exregional_bias_correction_o3.sh
@@ -103,21 +103,12 @@ fi
 # STEP 1: Retrieve AIRNOW observation data
 #-----------------------------------------------------------------------------
 
-# Link the historical airnow data
 mkdir_vrfy -p "${DATA}/data"
-ln_vrfy -sf ${AQM_AIRNOW_HIST_DIR}/bcdata* "${DATA}/data"
-if [ -d "${DATA}/data/bcdata.${yyyymm}" ]; then
-  rm_vrfy -rf "${DATA}/data/bcdata.${yyyymm}"
-  mkdir_vrfy -p "${DATA}/data/bcdata.${yyyymm}"
-  cp_vrfy -rL "${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/airnow" "${DATA}/data/bcdata.${yyyymm}"
-  cp_vrfy -rL "${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated" "${DATA}/data/bcdata.${yyyymm}" 
-fi
 
 # Retrieve real-time airnow data for the last three days and convert them into netcdf.
 # In the following for-loop, pdym stands for previous (m) day of the present day (PDY)
 # in the NCO standards, i.e. PDYm1: 1day ago, PDYm2: 2days ago, PDYm3: 3days ago
-if [ "${DO_REAL_TIME}" = "TRUE" ]; then
-  for i_pdym in {1..3}; do
+for i_pdym in {1..3}; do
     case $i_pdym in
       1)
         cvt_yyyy="${yyyy_m1}"
@@ -158,8 +149,7 @@ if [ "${DO_REAL_TIME}" = "TRUE" ]; then
       fi
     fi
     POST_STEP
-  done     
-fi
+done     
 
 #-----------------------------------------------------------------------------
 # STEP 2:  Extracting PM2.5, O3, and met variables from CMAQ input and outputs
@@ -218,8 +208,8 @@ POST_STEP
 cp_vrfy ${DATA}/out/ozone/${yyyy}/*nc ${DATA}/data/bcdata.${yyyymm}/interpolated/ozone/${yyyy}
 
 if [ "${DO_AQM_SAVE_AIRNOW_HIST}" = "TRUE" ]; then
-  mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated/ozone/${yyyy}
-  cp_vrfy ${DATA}/out/ozone/${yyyy}/*nc ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated/ozone/${yyyy}
+  mkdir_vrfy -p ${COMOUTbicor}/bcdata.${yyyymm}/interpolated/ozone/${yyyy}
+  cp_vrfy ${DATA}/out/ozone/${yyyy}/*nc ${COMOUTbicor}/bcdata.${yyyymm}/interpolated/ozone/${yyyy}
 
   for i_pdym in {0..3}; do
     case $i_pdym in
@@ -245,15 +235,15 @@ if [ "${DO_AQM_SAVE_AIRNOW_HIST}" = "TRUE" ]; then
         ;;
     esac
     # CSV and NetCDF files
-    mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${cvt_yyyymm}/airnow/csv/${cvt_yyyy}/${cvt_pdy}
-    mkdir_vrfy -p ${AQM_AIRNOW_HIST_DIR}/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy}
+    mkdir_vrfy -p ${COMOUTbicor}/bcdata.${cvt_yyyymm}/airnow/csv/${cvt_yyyy}/${cvt_pdy}
+    mkdir_vrfy -p ${COMOUTbicor}/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy}
     if [ "${i_pdym}" != "0" ]; then
-      cp_vrfy ${DCOMINairnow}/${cvt_pdy}/airnow/HourlyAQObs_${cvt_pdy}*.dat ${AQM_AIRNOW_HIST_DIR}/bcdata.${cvt_yyyymm}/airnow/csv/${cvt_yyyy}/${cvt_pdy}
-      cp_vrfy ${DATA}/data/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy}/HourlyAQObs.${cvt_pdy}.nc ${AQM_AIRNOW_HIST_DIR}/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy}  
+      cp_vrfy ${DCOMINairnow}/${cvt_pdy}/airnow/HourlyAQObs_${cvt_pdy}*.dat ${COMOUTbicor}/bcdata.${cvt_yyyymm}/airnow/csv/${cvt_yyyy}/${cvt_pdy}
+      cp_vrfy ${DATA}/data/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy}/HourlyAQObs.${cvt_pdy}.nc ${COMOUTbicor}/bcdata.${cvt_yyyymm}/airnow/netcdf/${cvt_yyyy}/${cvt_pdy}  
     fi
   done
-  mkdir_vrfy -p  ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/grid/${cyc}z/${PDY}
-  cp_vrfy ${COMIN}/${NET}.${cycle}.*sfc*.nc ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/grid/${cyc}z/${PDY}
+  mkdir_vrfy -p  ${COMOUTbicor}/bcdata.${yyyymm}/grid/${cyc}z/${PDY}
+  cp_vrfy ${COMIN}/${NET}.${cycle}.*sfc*.nc ${COMOUTbicor}/bcdata.${yyyymm}/grid/${cyc}z/${PDY}
 fi
 
 #-----------------------------------------------------------------------------
@@ -262,7 +252,7 @@ fi
 
 rm_vrfy -rf ${DATA}/data/bcdata*
 
-ln_vrfy -sf ${AQM_AIRNOW_HIST_DIR}/bcdata* "${DATA}/data"
+ln_vrfy -sf ${COMINbicor}/bcdata* "${DATA}/data"
 
 mkdir_vrfy -p ${DATA}/data/sites
 cp_vrfy ${PARMaqm_utils}/bias_correction/config.ozone.bias_corr_${id_domain}.${cyc}z ${DATA}

diff --git a/scripts/exregional_bias_correction_pm25.sh b/scripts/exregional_bias_correction_pm25.sh
@@ -103,21 +103,12 @@ fi
 # STEP 1: Retrieve AIRNOW observation data
 #-----------------------------------------------------------------------------
 
-# Link the historical airnow data
 mkdir_vrfy -p "${DATA}/data"
-ln_vrfy -sf ${AQM_AIRNOW_HIST_DIR}/bcdata* "${DATA}/data"
-if [ -d "${DATA}/data/bcdata.${yyyymm}" ]; then
-  rm_vrfy -rf "${DATA}/data/bcdata.${yyyymm}"
-  mkdir_vrfy -p "${DATA}/data/bcdata.${yyyymm}"
-  cp_vrfy -rL "${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/airnow" "${DATA}/data/bcdata.${yyyymm}"
-  cp_vrfy -rL "${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated" "${DATA}/data/bcdata.${yyyymm}"
-fi
 
 # Retrieve real-time airnow data for the last three days.
 # In the following for-loop, pdym stands for previous (m) day of the present day (PDY)
 # in the NCO standards, i.e. PDYm1: 1day ago, PDYm2: 2days ago, PDYm3: 3days ago
-if [ "${DO_REAL_TIME}" = "TRUE" ]; then
-  for i_pdym in {1..3}; do
+for i_pdym in {1..3}; do
     case $i_pdym in
       1)
         cvt_yyyy="${yyyy_m1}"
@@ -158,8 +149,7 @@ if [ "${DO_REAL_TIME}" = "TRUE" ]; then
       fi
     fi
     POST_STEP
-  done
-fi
+done
 
 #-----------------------------------------------------------------------------
 # STEP 2:  Extracting PM2.5, O3, and met variables from CMAQ input and outputs
@@ -218,8 +208,8 @@ POST_STEP
 cp_vrfy ${DATA}/out/pm25/${yyyy}/*nc ${DATA}/data/bcdata.${yyyymm}/interpolated/pm25/${yyyy}
 
 if [ "${DO_AQM_SAVE_AIRNOW_HIST}" = "TRUE" ]; then
-  mkdir_vrfy -p  ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated/pm25/${yyyy}
-  cp_vrfy ${DATA}/out/pm25/${yyyy}/*nc ${AQM_AIRNOW_HIST_DIR}/bcdata.${yyyymm}/interpolated/pm25/${yyyy}
+  mkdir_vrfy -p  ${COMOUTbicor}/bcdata.${yyyymm}/interpolated/pm25/${yyyy}
+  cp_vrfy ${DATA}/out/pm25/${yyyy}/*nc ${COMOUTbicor}/bcdata.${yyyymm}/interpolated/pm25/${yyyy}
 fi
 
 #-----------------------------------------------------------------------
@@ -228,7 +218,7 @@ fi
 
 rm_vrfy -rf ${DATA}/data/bcdata*
 
-ln_vrfy -sf ${AQM_AIRNOW_HIST_DIR}/bcdata* "${DATA}/data"
+ln_vrfy -sf ${COMINbicor}/bcdata* "${DATA}/data"
 
 mkdir_vrfy -p ${DATA}/data/sites
 

diff --git a/scripts/exregional_fire_emission.sh b/scripts/exregional_fire_emission.sh
@@ -56,12 +56,10 @@ data files.
 yyyymmdd=${FIRE_FILE_CDATE:0:8}
 hh=${FIRE_FILE_CDATE:8:2}
 
-CDATE_mh3=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 3 hours" "+%Y%m%d%H" )
-CDATE_mh2=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 2 hours" "+%Y%m%d%H" )
 CDATE_mh1=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 1 hours" "+%Y%m%d%H" )
 
-yyyymmdd_mh3=${CDATE_mh3:0:8}
-hh_mh3=${CDATE_mh3:8:2}
+yyyymmdd_mh1=${CDATE_mh1:0:8}
+hh_mh1=${CDATE_mh1:8:2}
 #
 #-----------------------------------------------------------------------
 #
@@ -76,26 +74,26 @@ if [ -e "${DCOMINfire}/${yyyymmdd}/${aqm_fire_file_fn}" ]; then
   cp_vrfy "${DCOMINfire}/${yyyymmdd}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}"
 else
   # Copy raw data 
-  for ihr in {0..21}; do
-    download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_mh3} ${hh_mh3} UTC - $ihr hours" "+%Y%m%d%H" )
+  for ihr in {0..23}; do
+    download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_mh1} ${hh_mh1} UTC - $ihr hours" "+%Y%m%d%H" )
     FILE_13km="Hourly_Emissions_13km_${download_time}00_${download_time}00.nc"
     yyyymmdd_dn=${download_time:0:8}
     hh_dn=${download_time:8:2}
     missing_download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_dn} ${hh_dn} UTC - 24 hours" "+%Y%m%d%H" )
     yyyymmdd_dn_md1=${missing_download_time:0:8}
     FILE_13km_md1=Hourly_Emissions_13km_${missing_download_time}00_${missing_download_time}00.nc
-    if [ -e "${DCOMINfire}/RAVE_raw_new/${yyyymmdd_dn}/${FILE_13km}" ]; then
-      ln -sf "${DCOMINfire}/RAVE_raw_new/${yyyymmdd_dn}/${FILE_13km}" .
-    elif [ -e "${DCOMINfire}/RAVE_raw_new/${yyyymmdd_dn_md1}/${FILE_13km_md1}" ]; then
+    if [ -e "${DCOMINfire}/${yyyymmdd_dn}/rave/${FILE_13km}" ]; then
+      cp_vrfy "${DCOMINfire}/${yyyymmdd_dn}/rave/${FILE_13km}" .
+    elif [ -e "${DCOMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1}" ]; then
       echo "WARNING: ${FILE_13km} does not exist. Replacing with the file of previous date ..."
-      ln -sf "${DCOMINfire}/RAVE_raw_new/${yyyymmdd_dn_md1}/${FILE_13km_md1}" "${FILE_13km}"
+      cp_vrfy "${DCOMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1}" "${FILE_13km}"
     else
       message_txt="Fire Emission RAW data does not exist:
   FILE_13km_md1 = \"${FILE_13km_md1}\"
   DCOMINfire = \"${DCOMINfire}\""
 
       if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then
-        ln -sf  "${DCOMINfire}/Hourly_Emissions_13km_dummy.nc" "${FILE_13km}"
+        cp_vrfy "${DCOMINfire}/Hourly_Emissions_13km_dummy.nc" "${FILE_13km}"
         message_warning="WARNING: ${message_txt}. Replacing with the dummy file :: AQM RUN SOFT FAILED."
         print_info_msg "${message_warning}"
         if [ ! -z "${maillist}" ]; then
@@ -120,10 +118,6 @@ else
 
   mv_vrfy temp.nc Hourly_Emissions_13km_${download_time}00_${download_time}00.nc
 
-  # Extra times
-  cp_vrfy Hourly_Emissions_13km_${CDATE_mh3}00_${CDATE_mh3}00.nc Hourly_Emissions_13km_${CDATE_mh2}00_${CDATE_mh2}00.nc
-  cp_vrfy Hourly_Emissions_13km_${CDATE_mh3}00_${CDATE_mh3}00.nc Hourly_Emissions_13km_${CDATE_mh1}00_${CDATE_mh1}00.nc
-
   ncrcat -h Hourly_Emissions_13km_*.nc Hourly_Emissions_13km_${yyyymmdd}0000_${yyyymmdd}2300.t${cyc}z.nc
   export err=$?
   if [ $err -ne 0 ]; then

diff --git a/scripts/exregional_nexus_emission.sh b/scripts/exregional_nexus_emission.sh
@@ -151,10 +151,10 @@ else
   start_del_hr=$(( len_per_split * nspt ))
   start_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + ${start_del_hr} hours " "+%Y%m%d%H" )
   if [ "${nsptp}" = "${NUM_SPLIT_NEXUS}" ];then
-    end_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H" )
+	  end_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + $(expr $FCST_LEN_HRS + 1) hours" "+%Y%m%d%H" )
   else
     end_del_hr=$(( len_per_split * nsptp ))
-    end_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + ${end_del_hr} hours" "+%Y%m%d%H" )
+    end_date=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + $(expr $end_del_hr + 1) hours" "+%Y%m%d%H" )
   fi
 fi
 #
@@ -337,12 +337,20 @@ POST_STEP
 #
 #-----------------------------------------------------------------------
 #
-# Move NEXUS output to INPUT_DATA directory.
+# Make NEXUS output pretty and move to INPUT_DATA directory.
 #
 #-----------------------------------------------------------------------
 #
-mv_vrfy ${DATA}/NEXUS_Expt_split.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.${nspt}.nc
-
+python3 ${ARL_NEXUS_DIR}/utils/python/make_nexus_output_pretty.py --src ${DATA}/NEXUS_Expt_split.nc --grid ${DATA}/grid_spec.nc -o ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.${nspt}.nc -t ${DATA}/HEMCO_sa_Time.rc
+export err=$?
+if [ $err -ne 0 ]; then
+  message_txt="Call to python script \"make_nexus_output_pretty.py\" failed."
+  if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "wcoss2" ]; then
+    err_exit "${message_txt}"
+  else
+    print_err_msg_exit "${message_txt}"
+  fi
+fi
 #
 #-----------------------------------------------------------------------
 #

diff --git a/scripts/exregional_nexus_post_split.sh b/scripts/exregional_nexus_post_split.sh
@@ -94,43 +94,14 @@ else
   fi
 fi
 
-#
-#-----------------------------------------------------------------------
-#
-# make nexus output pretty
-#
-#-----------------------------------------------------------------------
-#
-python3 ${ARL_NEXUS_DIR}/utils/python/nexus_time_parser.py -f ${DATA}/HEMCO_sa_Time.rc -s $start_date -e $end_date
-export err=$?
-if [ $err -ne 0 ]; then
-  message_txt="Call to python script \"nexus_time_parser.py\" failed."
-  if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then
-    err_exit "${message_txt}"
-  else
-    print_err_msg_exit "${message_txt}"
-  fi
-fi
-
-python3 ${ARL_NEXUS_DIR}/utils/python/make_nexus_output_pretty.py --src ${DATA}/NEXUS_Expt_combined.nc --grid ${DATA}/grid_spec.nc -o ${DATA}/NEXUS_Expt_pretty.nc -t ${DATA}/HEMCO_sa_Time.rc
-export err=$?
-if [ $err -ne 0 ]; then
-  message_txt="Call to python script \"make_nexus_output_pretty.py\" failed."
-  if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then
-    err_exit "${message_txt}"
-  else
-    print_err_msg_exit "${message_txt}"
-  fi
-fi
-
 #
 #-----------------------------------------------------------------------
 #
 # run MEGAN NCO script
 #
 #-----------------------------------------------------------------------
 #
-python3 ${ARL_NEXUS_DIR}/utils/combine_ant_bio.py ${DATA}/NEXUS_Expt_pretty.nc ${DATA}/NEXUS_Expt.nc
+python3 ${ARL_NEXUS_DIR}/utils/combine_ant_bio.py "${DATA}/NEXUS_Expt_combined.nc" ${DATA}/NEXUS_Expt.nc
 export err=$?
 if [ $err -ne 0 ]; then
   message_txt="Call to python script \"NEXUS_Expt_pretty.py\" failed."

diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml
@@ -2596,8 +2596,13 @@ cpl_aqm_parm:
   # DCOMINairnow:
   # Path to the directory containing AIRNOW observation data
   #
-  # AQM_AIRNOW_HIST_DIR:
-  # Path to the directory where the historical AIRNOW data are located
+  # COMINbicor:
+  # Path of reading in historical training data for biascorrection 
+  #
+  # COMOUTbicor:
+  # Path to save the current cycle's model output and AirNow obs as 
+  # training data for future use $COMINbicor and $COMOUTbicor can be 
+  # distuigshed by the ${yyyy}${mm}$dd under the same location
   #
   #-----------------------------------------------------------------------
   #
@@ -2619,6 +2624,8 @@ cpl_aqm_parm:
   DCOMINgefs_default: ""
   DCOMINpt_src_default: "/path/to/point/source/base/directory"
   DCOMINairnow_default: "/path/to/airnow/obaservation/data"
+  COMINbicor: "/path/to/historical/airnow/data/dir"
+  COMOUTbicor: "/path/to/historical/airnow/data/dir"
 
   AQM_CONFIG_DIR: ""
   AQM_BIO_FILE: "BEIS_SARC401.ncf"
@@ -2651,8 +2658,6 @@ cpl_aqm_parm:
   NEXUS_GFS_SFC_DIR: ""
   NEXUS_GFS_SFC_ARCHV_DIR: "/NCEPPROD/hpssprod/runhistory"
 
-  AQM_AIRNOW_HIST_DIR: "/path/to/historical/airnow/data/dir"
-
 rocoto:
   attrs: ""
   cycledefs: ""

diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml
@@ -51,12 +51,13 @@ cpl_aqm_parm:
   DCOMINbio_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/bio
   DCOMINdust_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/FENGSHA
   DCOMINcanopy_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/canopy
-  DCOMINfire_default: /lfs/h2/emc/physics/noscrub/kai.wang/RAVE_fire/RAVE_NA_NRT
+  DCOMINfire_default: /lfs/h1/ops/dev/dcom
   DCOMINchem_lbcs_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/LBCS/AQM_NA13km_AM4_v1
   DCOMINgefs_default: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/GEFS_DATA
   DCOMINpt_src_default: /lfs/h2/emc/physics/noscrub/Youhua.Tang/nei2016v1-pt/v2023-01-PT
   DCOMINairnow_default: /lfs/h1/ops/prod/dcom
-  AQM_AIRNOW_HIST_DIR: /lfs/h2/emc/physics/noscrub/jianping.huang/Bias_correction/aqmv7.0
+  COMINbicor: /lfs/h2/emc/physics/noscrub/jianping.huang/Bias_correction/aqmv7.0.81
+  COMOUTbicor: /lfs/h2/emc/physics/noscrub/jianping.huang/Bias_correction/aqmv7.0.81
   NEXUS_INPUT_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/nexus_emissions
   NEXUS_FIX_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/nexus/fix
   NEXUS_GFS_SFC_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/GFS_DATA