From 876868267dbe42da7585df642fb5a81938143ca2 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Wed, 3 Jul 2024 12:09:40 -0400 Subject: [PATCH] GFSv16.3.16 - add GDAS WDQMS (WIGOS Data Quality Monitoring System) job (#2746) The GFSv16.3.16 operational upgrade adds a job for the GDAS WDQMS (WIGOS Data Quality Monitoring System) data, which provides 6-hourly summaries of observation reports. The GFSv16.3.16 was implemented into operations during the 12z gfs cycle on July 2nd 2024. Refs #2389 --------- Co-authored-by: Rahul Mahajan Co-authored-by: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- docs/Release_Notes.gfs.v16.3.16.md | 141 ++++ ecf/defs/gfs_v16_3.def | 8 + .../analysis/create/jenkfgdas_diag.ecf | 2 +- .../analysis/recenter/jenkfgdas_sfc.ecf | 2 +- .../analysis/jgdas_atmos_analysis_diag.ecf | 2 +- .../analysis/jgdas_atmos_analysis_wdqms.ecf | 61 ++ .../gdas/atmos/gempak/jgdas_atmos_gempak.ecf | 2 +- .../gempak/jgdas_atmos_gempak_meta_ncdc.ecf | 2 +- .../dump/jgdas_atmos_tropcy_qc_reloc.ecf | 2 +- .../prep/jgdas_atmos_emcsfc_sfc_prep.ecf | 2 +- .../atmos/post/jgdas_atmos_post_manager.ecf | 2 +- .../jgdas_atmos_chgres_forenkf.ecf | 2 +- .../gdas/atmos/verf/jgdas_atmos_verfozn.ecf | 2 +- .../gdas/atmos/verf/jgdas_atmos_verfrad.ecf | 2 +- .../gdas/atmos/verf/jgdas_atmos_vminmon.ecf | 2 +- .../gdas/wave/init/jgdas_wave_init.ecf | 2 +- .../gdas/wave/post/jgdas_wave_postpnt.ecf | 2 +- .../gdas/wave/post/jgdas_wave_postsbs.ecf | 2 +- .../gdas/wave/prep/jgdas_wave_prep.ecf | 2 +- .../gfs/atmos/gempak/jgfs_atmos_gempak.ecf | 2 +- .../atmos/gempak/jgfs_atmos_gempak_meta.ecf | 2 +- .../gempak/jgfs_atmos_gempak_ncdc_upapgif.ecf | 2 +- .../gempak/jgfs_atmos_npoess_pgrb2_0p5deg.ecf | 2 +- .../gempak/jgfs_atmos_pgrb2_spec_gempak.ecf | 2 +- .../dump/jgfs_atmos_tropcy_qc_reloc.ecf | 2 +- .../prep/jgfs_atmos_emcsfc_sfc_prep.ecf | 2 +- .../atmos/post/jgfs_atmos_post_manager.ecf | 2 +- .../jgfs_atmos_awips_master.ecf | 2 +- .../awips_g2/jgfs_atmos_awips_g2_master.ecf | 2 +- .../bulletins/jgfs_atmos_fbwind.ecf | 2 +- .../grib2_wafs/jgfs_atmos_wafs_blending.ecf | 2 +- .../jgfs_atmos_wafs_blending_0p25.ecf | 2 +- .../grib2_wafs/jgfs_atmos_wafs_grib2.ecf | 2 +- .../grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf | 2 +- .../grib_wafs/jgfs_atmos_wafs_master.ecf | 2 +- .../post_processing/jgfs_atmos_wafs_gcip.ecf | 2 +- .../gfs/atmos/verf/jgfs_atmos_vminmon.ecf | 2 +- .../gfs/wave/gempak/jgfs_wave_gempak.ecf | 2 +- ecf/scripts/gfs/wave/init/jgfs_wave_init.ecf | 2 +- .../gfs/wave/post/jgfs_wave_postsbs.ecf | 2 +- .../gfs/wave/post/jgfs_wave_prdgen_bulls.ecf | 2 +- .../wave/post/jgfs_wave_prdgen_gridded.ecf | 2 +- ecf/scripts/gfs/wave/prep/jgfs_wave_prep.ecf | 2 +- jobs/JGDAS_ATMOS_ANALYSIS_WDQMS | 74 ++ jobs/rocoto/wdqms.sh | 13 + parm/config/config.base.emc.dyn | 1 + parm/config/config.base.nco.static | 1 + parm/config/config.resources.emc.dyn | 8 + parm/config/config.resources.nco.static | 8 + parm/config/config.wdqms | 14 + .../transfer/transfer_gfs_gdas_gdas_misc.list | 41 - parm/transfer/transfer_gfs_gfs_misc.list | 43 - scripts/exgdas_atmos_analysis_wdqms.sh | 113 +++ ush/rocoto/setup_workflow.py | 17 + ush/wdqms.py | 775 ++++++++++++++++++ versions/run.ver | 4 +- 56 files changed, 1276 insertions(+), 126 deletions(-) create mode 100644 docs/Release_Notes.gfs.v16.3.16.md create mode 100755 ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_wdqms.ecf create mode 100755 jobs/JGDAS_ATMOS_ANALYSIS_WDQMS create mode 100755 jobs/rocoto/wdqms.sh create mode 100755 parm/config/config.wdqms delete mode 100644 parm/transfer/transfer_gfs_gfs_misc.list create mode 100755 scripts/exgdas_atmos_analysis_wdqms.sh create mode 100755 ush/wdqms.py diff --git a/docs/Release_Notes.gfs.v16.3.16.md b/docs/Release_Notes.gfs.v16.3.16.md new file mode 100644 index 0000000000..3d3ab7c15a --- /dev/null +++ b/docs/Release_Notes.gfs.v16.3.16.md @@ -0,0 +1,141 @@ +GFS V16.3.16 RELEASE NOTES + +------- +PRELUDE +------- + +This upgrade adds a new GDAS job - `gdaswdqms`. It generates 6-hourly summaries of observation reports, which contain information on time, location, station ID, and quality (usage flag and innovations) from GDAS analysis. The objective of this new job is to support the WMO WDQMS (WIGOS Data Quality Monitoring System) project and ensure that WMO observational data and products are reliable and correspond to agreed-upon needs. + +The 6-hourly summaries include surface (synop), upper-air, and surface marine (ship and buoy) observations. + +IMPLEMENTATION INSTRUCTIONS +--------------------------- + +The NOAA VLab and the NOAA-EMC and NCAR organization spaces on GitHub are used to manage the GFS code. The SPA(s) handling the GFS implementation need to have permissions to clone VLab Gerrit repositories and the private NCAR UPP_GTG repository. All NOAA-EMC organization repositories are publicly readable and do not require access permissions. Please proceed with the following steps to install the package on WCOSS2: + +```bash +cd $PACKAGEROOT +mkdir gfs.v16.3.16 +cd gfs.v16.3.16 +git clone -b EMC-v16.3.16 https://github.com/NOAA-EMC/global-workflow.git . +cd sorc +./checkout.sh -o +``` + +The checkout script extracts the following GFS components: + +| Component | Tag | POC | +| --------- | ----------- | ----------------- | +| MODEL | GFS.v16.3.1 | Jun.Wang@noaa.gov | +| GLDAS | gldas_gfsv16_release.v.2.1.0 | Helin.Wei@noaa.gov | +| GSI | gfsda.v16.3.12 | Andrew.Collard@noaa.gov | +| UFS_UTILS | ops-gfsv16.3.0 | George.Gayno@noaa.gov | +| POST | upp_v8.3.0 | Wen.Meng@noaa.gov | +| WAFS | gfs_wafs.v6.3.2 | Yali.Mao@noaa.gov | + +To build all the GFS components, execute: +```bash +./build_all.sh +``` +The `build_all.sh` script compiles all GFS components. Runtime output from the build for each package is written to log files in directory logs. To build an individual program, for instance, gsi, use `build_gsi.sh`. + +Next, link the executables, fix files, parm files, etc in their final respective locations by executing: +```bash +./link_fv3gfs.sh nco wcoss2 +``` + +Lastly, link the ecf scripts by moving back up to the ecf folder and executing: +```bash +cd ../ecf +./setup_ecf_links.sh +``` +VERSION FILE CHANGES +-------------------- + +* `versions/run.ver` - change `version=v16.3.16` and `gfs_ver=v16.3.16` + +SORC CHANGES +------------ + +* No changes from GFS v16.3.15 + +JOBS CHANGES +------------ + +* Added a new `j-job` `JGDAS_ATMOS_ANALYSIS_WDQMS` + +PARM/CONFIG CHANGES +------------------- + +* Added a `config.wdqms` file for the new j-job added. +* Added new configuration parameter `DO_WDQMS` in the following configuration files + - `config.base.emc.dyn` (default to NO) + - `config.base.nco.static` (default to YES) +* Added resource parameters for `WDQMS` job in the following resource configuration files + - `config.resource.emc.dyn` + - `config.resource.nco.static` + +SCRIPT CHANGES +-------------- + +* Added `exscript` `exgdas_atmos_analysis_wdqms.sh` +* Added a utility in `ush` `wdqms.py` called by the new job. +* Added `ecf` script `jgdas_atmos_analysis_wdqms.ecf` and updated the `ecflow` suite definition files. + +FIX CHANGES +----------- + +* No changes from GFS v16.3.15 + +MODULE CHANGES +-------------- + +* New job loads `python` module in the ecf script. + +CHANGES TO FILE SIZES +--------------------- + +* No changes of existing file sizes from GFS v16.3.15 +* The `gdaswdqms` job creates three observation quality reports in CSV format for the following data types: + - Upper air (~0.5 MB) + - Marine (-6.5 MB) + - Synop (~ 22 MB) + +ENVIRONMENT AND RESOURCE CHANGES +-------------------------------- + +* Computing resource added for the `gdaswdqms` job for each gdas cycle: The new job `gdaswdqms` requests a single (1) core with `48GB` memory for 20 minutes + +PRE-IMPLEMENTATION TESTING REQUIREMENTS +--------------------------------------- + +* Which production jobs should be tested as part of this implementation? + * WDQMS +* Does this change require a 30-day evaluation? + * No + +DISSEMINATION INFORMATION +------------------------- + +* No changes from GFS v16.3.15 + +HPSS ARCHIVE +------------ + +* No changes from GFS v16.3.15 + +JOB DEPENDENCIES AND FLOW DIAGRAM +--------------------------------- + +* Added a new job `gdaswdqms` after `gdasanaldiag` with dependency on the completion of `gdasanaldiag` job. + +DOCUMENTATION +------------- + +* No changes from GFS v16.3.15 + +PREPARED BY +----------- +Kate.Friedman@noaa.gov +Rahul.Mahajan@noaa.gov +Emily.Liu@noaa.gov diff --git a/ecf/defs/gfs_v16_3.def b/ecf/defs/gfs_v16_3.def index 512000813a..883259b9e9 100644 --- a/ecf/defs/gfs_v16_3.def +++ b/ecf/defs/gfs_v16_3.def @@ -2251,6 +2251,8 @@ suite gfs_v16_3 trigger ./jgdas_atmos_analysis == complete task jgdas_atmos_analysis_diag trigger ./jgdas_atmos_analysis == complete + task jgdas_atmos_analysis_wdqms + trigger ./jgdas_atmos_analysis_diag == complete endfamily family post task jgdas_atmos_post_manager @@ -4876,6 +4878,8 @@ suite gfs_v16_3 trigger ./jgdas_atmos_analysis == complete task jgdas_atmos_analysis_diag trigger ./jgdas_atmos_analysis == complete + task jgdas_atmos_analysis_wdqms + trigger ./jgdas_atmos_analysis_diag == complete endfamily family post task jgdas_atmos_post_manager @@ -7501,6 +7505,8 @@ suite gfs_v16_3 trigger ./jgdas_atmos_analysis == complete task jgdas_atmos_analysis_diag trigger ./jgdas_atmos_analysis == complete + task jgdas_atmos_analysis_wdqms + trigger ./jgdas_atmos_analysis_diag == complete endfamily family post task jgdas_atmos_post_manager @@ -10126,6 +10132,8 @@ suite gfs_v16_3 trigger ./jgdas_atmos_analysis == complete task jgdas_atmos_analysis_diag trigger ./jgdas_atmos_analysis == complete + task jgdas_atmos_analysis_wdqms + trigger ./jgdas_atmos_analysis_diag == complete endfamily family post task jgdas_atmos_post_manager diff --git a/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_diag.ecf b/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_diag.ecf index af5adf0fdc..86bf12980a 100755 --- a/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_diag.ecf +++ b/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_diag.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:06:00 #PBS -l select=1:mpiprocs=48:ompthreads=1:ncpus=48:mem=48GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf b/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf index be8d6d81a9..8a7f813ae9 100755 --- a/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf +++ b/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:06:00 #PBS -l select=1:mpiprocs=80:ompthreads=1:ncpus=80:mem=60GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_diag.ecf b/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_diag.ecf index 214a838e54..09045cc667 100755 --- a/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_diag.ecf +++ b/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_diag.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 #PBS -l select=1:mpiprocs=96:ompthreads=1:ncpus=96:mem=48GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true export model=gfs diff --git a/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_wdqms.ecf b/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_wdqms.ecf new file mode 100755 index 0000000000..dba2a57f54 --- /dev/null +++ b/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_wdqms.ecf @@ -0,0 +1,61 @@ +#PBS -S /bin/bash +#PBS -N %RUN%_atmos_analysis_wdqms_%CYC% +#PBS -j oe +#PBS -q %QUEUE% +#PBS -A %PROJ%-%PROJENVIR% +#PBS -l walltime=00:20:00 +#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=48GB +#PBS -l place=vscatter:shared +#PBS -l debug=true + +export model=gfs +%include +%include + +set -x + +export NET=%NET:gfs% +export RUN=%RUN% +export CDUMP=%RUN% + +############################################################ +# Load modules +############################################################ +module load PrgEnv-intel/${PrgEnv_intel_ver} +module load craype/${craype_ver} +module load intel/${intel_ver} +module load cray-mpich/${cray_mpich_ver} +module load cray-pals/${cray_pals_ver} +module load python/${python_ver} +module list + +############################################################# +# WCOSS environment settings +############################################################# +export cyc=%CYC% +export cycle=t%CYC%z + +############################################################ +# CALL executable job script here +############################################################ +${HOMEgfs}/jobs/JGDAS_ATMOS_ANALYSIS_WDQMS +if [ $? -ne 0 ]; then + ecflow_client --msg="***JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT ***" + ecflow_client --abort + exit +fi + +%include +%manual +###################################################################### +# Purpose: To execute the job that creates the WDQMS products +###################################################################### + +###################################################################### +# Job specific troubleshooting instructions: +# see generic troubleshoot manual page +# +###################################################################### + +# include manual page below +%end diff --git a/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak.ecf b/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak.ecf index 0bc2d76455..039ca56852 100755 --- a/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak.ecf +++ b/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:15:00 #PBS -l select=1:ncpus=2:mpiprocs=2:mem=4GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak_meta_ncdc.ecf b/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak_meta_ncdc.ecf index 9d66f4bda1..61f7f0a17f 100755 --- a/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak_meta_ncdc.ecf +++ b/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak_meta_ncdc.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/obsproc/dump/jgdas_atmos_tropcy_qc_reloc.ecf b/ecf/scripts/gdas/atmos/obsproc/dump/jgdas_atmos_tropcy_qc_reloc.ecf index 2dd0bdf06c..5322c363ac 100755 --- a/ecf/scripts/gdas/atmos/obsproc/dump/jgdas_atmos_tropcy_qc_reloc.ecf +++ b/ecf/scripts/gdas/atmos/obsproc/dump/jgdas_atmos_tropcy_qc_reloc.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:05:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/obsproc/prep/jgdas_atmos_emcsfc_sfc_prep.ecf b/ecf/scripts/gdas/atmos/obsproc/prep/jgdas_atmos_emcsfc_sfc_prep.ecf index 7e3282bc95..e0ab513b33 100755 --- a/ecf/scripts/gdas/atmos/obsproc/prep/jgdas_atmos_emcsfc_sfc_prep.ecf +++ b/ecf/scripts/gdas/atmos/obsproc/prep/jgdas_atmos_emcsfc_sfc_prep.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:08:00 #PBS -l select=1:ncpus=1:mem=2GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/post/jgdas_atmos_post_manager.ecf b/ecf/scripts/gdas/atmos/post/jgdas_atmos_post_manager.ecf index 1da24c0d46..55d7932aaf 100755 --- a/ecf/scripts/gdas/atmos/post/jgdas_atmos_post_manager.ecf +++ b/ecf/scripts/gdas/atmos/post/jgdas_atmos_post_manager.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=01:15:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/post_processing/jgdas_atmos_chgres_forenkf.ecf b/ecf/scripts/gdas/atmos/post_processing/jgdas_atmos_chgres_forenkf.ecf index 78d2d99a1b..fbb53fca14 100755 --- a/ecf/scripts/gdas/atmos/post_processing/jgdas_atmos_chgres_forenkf.ecf +++ b/ecf/scripts/gdas/atmos/post_processing/jgdas_atmos_chgres_forenkf.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 #PBS -l select=1:ncpus=3:mpiprocs=3:ompthreads=1:mem=200GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfozn.ecf b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfozn.ecf index 5af9636402..ff4910a277 100755 --- a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfozn.ecf +++ b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfozn.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:05:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfrad.ecf b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfrad.ecf index 46dbe52cf4..28fdd7f266 100755 --- a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfrad.ecf +++ b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfrad.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:20:00 #PBS -l select=1:ncpus=1:mem=5GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_vminmon.ecf b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_vminmon.ecf index 57e3d91a29..f4a1a748f2 100755 --- a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_vminmon.ecf +++ b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_vminmon.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:05:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/wave/init/jgdas_wave_init.ecf b/ecf/scripts/gdas/wave/init/jgdas_wave_init.ecf index 8321494bed..74a0f2806a 100755 --- a/ecf/scripts/gdas/wave/init/jgdas_wave_init.ecf +++ b/ecf/scripts/gdas/wave/init/jgdas_wave_init.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 #PBS -l select=1:mpiprocs=11:ompthreads=1:ncpus=11:mem=2GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/wave/post/jgdas_wave_postpnt.ecf b/ecf/scripts/gdas/wave/post/jgdas_wave_postpnt.ecf index 717e1def63..15adbc8a69 100755 --- a/ecf/scripts/gdas/wave/post/jgdas_wave_postpnt.ecf +++ b/ecf/scripts/gdas/wave/post/jgdas_wave_postpnt.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:12:00 #PBS -l select=4:mpiprocs=50:ompthreads=1:ncpus=50:mem=10GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/wave/post/jgdas_wave_postsbs.ecf b/ecf/scripts/gdas/wave/post/jgdas_wave_postsbs.ecf index 1f8a36e3d3..c323c32753 100755 --- a/ecf/scripts/gdas/wave/post/jgdas_wave_postsbs.ecf +++ b/ecf/scripts/gdas/wave/post/jgdas_wave_postsbs.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:20:00 #PBS -l select=1:mpiprocs=8:ompthreads=1:ncpus=8:mem=10GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gdas/wave/prep/jgdas_wave_prep.ecf b/ecf/scripts/gdas/wave/prep/jgdas_wave_prep.ecf index 2b699e4778..5ccacc4958 100755 --- a/ecf/scripts/gdas/wave/prep/jgdas_wave_prep.ecf +++ b/ecf/scripts/gdas/wave/prep/jgdas_wave_prep.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 #PBS -l select=1:mpiprocs=5:ompthreads=1:ncpus=5:mem=100GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak.ecf index 863f1323aa..2a384546d7 100755 --- a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak.ecf +++ b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=03:00:00 #PBS -l select=1:ncpus=28:mpiprocs=28:mem=2GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_meta.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_meta.ecf index 1ff3933042..81f8e14864 100755 --- a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_meta.ecf +++ b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_meta.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=03:00:00 #PBS -l select=1:ncpus=23:mpiprocs=23:mem=2GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_ncdc_upapgif.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_ncdc_upapgif.ecf index 928796b4d3..e7cbbab8cc 100755 --- a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_ncdc_upapgif.ecf +++ b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_ncdc_upapgif.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=02:00:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_npoess_pgrb2_0p5deg.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_npoess_pgrb2_0p5deg.ecf index 9a9d483e5a..6f1d6b3ba5 100755 --- a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_npoess_pgrb2_0p5deg.ecf +++ b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_npoess_pgrb2_0p5deg.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=02:00:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_pgrb2_spec_gempak.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_pgrb2_spec_gempak.ecf index 250fd6e981..3eb0596993 100755 --- a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_pgrb2_spec_gempak.ecf +++ b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_pgrb2_spec_gempak.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/obsproc/dump/jgfs_atmos_tropcy_qc_reloc.ecf b/ecf/scripts/gfs/atmos/obsproc/dump/jgfs_atmos_tropcy_qc_reloc.ecf index 2dd0bdf06c..5322c363ac 100755 --- a/ecf/scripts/gfs/atmos/obsproc/dump/jgfs_atmos_tropcy_qc_reloc.ecf +++ b/ecf/scripts/gfs/atmos/obsproc/dump/jgfs_atmos_tropcy_qc_reloc.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:05:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/obsproc/prep/jgfs_atmos_emcsfc_sfc_prep.ecf b/ecf/scripts/gfs/atmos/obsproc/prep/jgfs_atmos_emcsfc_sfc_prep.ecf index bb0bcf8db7..f0a1a3346f 100755 --- a/ecf/scripts/gfs/atmos/obsproc/prep/jgfs_atmos_emcsfc_sfc_prep.ecf +++ b/ecf/scripts/gfs/atmos/obsproc/prep/jgfs_atmos_emcsfc_sfc_prep.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:07:00 #PBS -l select=1:ncpus=1:mem=2GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post/jgfs_atmos_post_manager.ecf b/ecf/scripts/gfs/atmos/post/jgfs_atmos_post_manager.ecf index d2e315bcef..fc22e941bc 100755 --- a/ecf/scripts/gfs/atmos/post/jgfs_atmos_post_manager.ecf +++ b/ecf/scripts/gfs/atmos/post/jgfs_atmos_post_manager.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=04:00:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/awips_20km_1p0/jgfs_atmos_awips_master.ecf b/ecf/scripts/gfs/atmos/post_processing/awips_20km_1p0/jgfs_atmos_awips_master.ecf index 866feeb320..cb1ddbe70c 100755 --- a/ecf/scripts/gfs/atmos/post_processing/awips_20km_1p0/jgfs_atmos_awips_master.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/awips_20km_1p0/jgfs_atmos_awips_master.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 #PBS -l select=1:ncpus=1:mem=3GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/awips_g2/jgfs_atmos_awips_g2_master.ecf b/ecf/scripts/gfs/atmos/post_processing/awips_g2/jgfs_atmos_awips_g2_master.ecf index 1a07668e2d..b45add609c 100755 --- a/ecf/scripts/gfs/atmos/post_processing/awips_g2/jgfs_atmos_awips_g2_master.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/awips_g2/jgfs_atmos_awips_g2_master.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:05:00 #PBS -l select=1:ncpus=1:mem=3GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/bulletins/jgfs_atmos_fbwind.ecf b/ecf/scripts/gfs/atmos/post_processing/bulletins/jgfs_atmos_fbwind.ecf index de293dcbf5..069cb62683 100755 --- a/ecf/scripts/gfs/atmos/post_processing/bulletins/jgfs_atmos_fbwind.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/bulletins/jgfs_atmos_fbwind.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:05:00 #PBS -l select=1:ncpus=1:mem=4GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending.ecf b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending.ecf index 963c681932..72e69281b1 100755 --- a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 #PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf index 8866e16b90..0b743f7d63 100755 --- a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 #PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=15GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf index fbf2f64b68..267e4313b3 100755 --- a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 #PBS -l select=1:mpiprocs=18:ompthreads=1:ncpus=18:mem=80GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf index 42ff28d997..442742bc61 100755 --- a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 #PBS -l select=1:mpiprocs=39:ompthreads=1:ncpus=39:mem=200GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/grib_wafs/jgfs_atmos_wafs_master.ecf b/ecf/scripts/gfs/atmos/post_processing/grib_wafs/jgfs_atmos_wafs_master.ecf index 6dec906669..7e56ea1b9e 100755 --- a/ecf/scripts/gfs/atmos/post_processing/grib_wafs/jgfs_atmos_wafs_master.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/grib_wafs/jgfs_atmos_wafs_master.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:05:00 #PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf b/ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf index 264db37908..dceb8d72bd 100755 --- a/ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 #PBS -l select=1:mpiprocs=2:ompthreads=1:ncpus=2:mem=50GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/atmos/verf/jgfs_atmos_vminmon.ecf b/ecf/scripts/gfs/atmos/verf/jgfs_atmos_vminmon.ecf index 8a6f034c92..4eb9d4e585 100755 --- a/ecf/scripts/gfs/atmos/verf/jgfs_atmos_vminmon.ecf +++ b/ecf/scripts/gfs/atmos/verf/jgfs_atmos_vminmon.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:05:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/wave/gempak/jgfs_wave_gempak.ecf b/ecf/scripts/gfs/wave/gempak/jgfs_wave_gempak.ecf index 2cb7f75949..199f68adeb 100755 --- a/ecf/scripts/gfs/wave/gempak/jgfs_wave_gempak.ecf +++ b/ecf/scripts/gfs/wave/gempak/jgfs_wave_gempak.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=02:00:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/wave/init/jgfs_wave_init.ecf b/ecf/scripts/gfs/wave/init/jgfs_wave_init.ecf index 8321494bed..74a0f2806a 100755 --- a/ecf/scripts/gfs/wave/init/jgfs_wave_init.ecf +++ b/ecf/scripts/gfs/wave/init/jgfs_wave_init.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 #PBS -l select=1:mpiprocs=11:ompthreads=1:ncpus=11:mem=2GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/wave/post/jgfs_wave_postsbs.ecf b/ecf/scripts/gfs/wave/post/jgfs_wave_postsbs.ecf index d34fdf94f8..d398c490ce 100755 --- a/ecf/scripts/gfs/wave/post/jgfs_wave_postsbs.ecf +++ b/ecf/scripts/gfs/wave/post/jgfs_wave_postsbs.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=03:00:00 #PBS -l select=1:mpiprocs=8:ompthreads=1:ncpus=8:mem=10GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/wave/post/jgfs_wave_prdgen_bulls.ecf b/ecf/scripts/gfs/wave/post/jgfs_wave_prdgen_bulls.ecf index 3e039d88e4..9f30289093 100755 --- a/ecf/scripts/gfs/wave/post/jgfs_wave_prdgen_bulls.ecf +++ b/ecf/scripts/gfs/wave/post/jgfs_wave_prdgen_bulls.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:20:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/wave/post/jgfs_wave_prdgen_gridded.ecf b/ecf/scripts/gfs/wave/post/jgfs_wave_prdgen_gridded.ecf index 2f40944928..192f8cd98e 100755 --- a/ecf/scripts/gfs/wave/post/jgfs_wave_prdgen_gridded.ecf +++ b/ecf/scripts/gfs/wave/post/jgfs_wave_prdgen_gridded.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=02:00:00 #PBS -l select=1:ncpus=1:mem=1GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/ecf/scripts/gfs/wave/prep/jgfs_wave_prep.ecf b/ecf/scripts/gfs/wave/prep/jgfs_wave_prep.ecf index dbf5be906f..363c090ce0 100755 --- a/ecf/scripts/gfs/wave/prep/jgfs_wave_prep.ecf +++ b/ecf/scripts/gfs/wave/prep/jgfs_wave_prep.ecf @@ -5,7 +5,7 @@ #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 #PBS -l select=1:mpiprocs=65:ompthreads=1:ncpus=65:mem=150GB -#PBS -l place=vscatter:shared +#PBS -l place=vscatter #PBS -l debug=true model=gfs diff --git a/jobs/JGDAS_ATMOS_ANALYSIS_WDQMS b/jobs/JGDAS_ATMOS_ANALYSIS_WDQMS new file mode 100755 index 0000000000..7c3721c644 --- /dev/null +++ b/jobs/JGDAS_ATMOS_ANALYSIS_WDQMS @@ -0,0 +1,74 @@ +#!/bin/bash + +set -x +export PS4='$SECONDS + ' +date + +############################## +# Source relevant config files +############################## +configs="base wdqms" +export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} +for config in ${configs}; do + source "${EXPDIR}/config.${config}" + status=$? + (( status != 0 )) && exit "${status}" +done + +############################################################### +source "${BASE_ENV}/${machine}.env" wdqms +status=$? +(( status != 0 )) && exit "${status}" + + +########################################################## +# obtain unique process id (pid) and make temp directory +########################################################## +export DATA=${DATA:-${DATAROOT}/${jobid:?}} +mkdir -p "${DATA}" +cd "${DATA}" + +###################################### +# Set up the cycle variable +###################################### +export cycle=${cycle:-t${cyc}z} + +########################################### +# Run setpdy and initialize PDY variables +########################################### +setpdy.sh +source ./PDY + +############################################## +# Define COM directories +############################################## +export COMIN="${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos" +export COMOUT="${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos/wdqms" + +mkdir -m 775 -p "${COMOUT}" + +######################################################## +# Execute the script for generating WDQMS products +"${HOMEgfs}/scripts/exgdas_atmos_analysis_wdqms.sh" +export err=$? +err_chk + +######################################################## + +echo "JOB ${job:-} HAS COMPLETED NORMALLY!" + +############################################ +# print exec I/O output +############################################ +if [ -e "${pgmout:-}" ] ; then + cat "${pgmout}" +fi + +################################### +# Remove temp directories +################################### +if [[ "${KEEPDATA}" != "YES" ]] ; then + rm -rf "${DATA}" +fi + +date diff --git a/jobs/rocoto/wdqms.sh b/jobs/rocoto/wdqms.sh new file mode 100755 index 0000000000..0dbe821df1 --- /dev/null +++ b/jobs/rocoto/wdqms.sh @@ -0,0 +1,13 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +(( status != 0 )) && exit "${status}" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGDAS_ATMOS_ANALYSIS_WDQMS" +status=$? +exit "${status}" diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 2e7cdee387..00ac698807 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -52,6 +52,7 @@ export DO_BUFRSND="YES" # BUFR sounding products export DO_GEMPAK="NO" # GEMPAK products export DO_AWIPS="NO" # AWIPS products export WAFSF="NO" # WAFS products +export DO_WDQMS="NO" # Generate WDQMS products # NO for retrospective parallel; YES for real-time parallel # arch.sh uses REALTIME for MOS. Need to set REALTIME=YES diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static index 1dbcbd0e2f..b0b06e6cd9 100755 --- a/parm/config/config.base.nco.static +++ b/parm/config/config.base.nco.static @@ -50,6 +50,7 @@ export DO_BUFRSND="YES" # BUFR sounding products export DO_GEMPAK="YES" # GEMPAK products export DO_AWIPS="YES" # AWIPS products export WAFSF="YES" # WAFS products +export DO_WDQMS="YES" # Generate WDQMS products # NO for retrospective parallel; YES for real-time parallel export REALTIME="YES" diff --git a/parm/config/config.resources.emc.dyn b/parm/config/config.resources.emc.dyn index 6ad784473f..1dc073abaa 100755 --- a/parm/config/config.resources.emc.dyn +++ b/parm/config/config.resources.emc.dyn @@ -176,6 +176,14 @@ elif [ $step = "analdiag" ]; then if [[ "$npe_node_analdiag" -gt "$npe_node_max" ]]; then export npe_node_analdiag=$npe_node_max ; fi export memory_analdiag="48GB" +elif [ $step = "wdqms" ]; then + + export wtime_wdqms="00:20:00" + export npe_wdqms=1 + export nth_wdqms=1 + export npe_node_wdqms=1 + export memory_wdqms="48GB" + elif [ $step = "gldas" ]; then export wtime_gldas="00:10:00" diff --git a/parm/config/config.resources.nco.static b/parm/config/config.resources.nco.static index d7fd7d2975..50bfef8810 100755 --- a/parm/config/config.resources.nco.static +++ b/parm/config/config.resources.nco.static @@ -144,6 +144,14 @@ elif [ $step = "analdiag" ]; then export npe_node_analdiag=$npe_analdiag export memory_analdiag="48GB" +elif [ $step = "wdqms" ]; then + + export wtime_wdqms="00:20:00" + export npe_wdqms=1 + export nth_wdqms=1 + export npe_node_wdqms=1 + export memory_wdqms="48GB" + elif [ $step = "gldas" ]; then export wtime_gldas="00:10:00" diff --git a/parm/config/config.wdqms b/parm/config/config.wdqms new file mode 100755 index 0000000000..d5118a2cbe --- /dev/null +++ b/parm/config/config.wdqms @@ -0,0 +1,14 @@ +#!/bin/bash + +########## config.wdqms ########## +# WDQMS steps specific + +echo "BEGIN: config.wdqms" + +# Get task specific resources +source "${EXPDIR}/config.resources" wdqms + +# WDQMS utility script +export WDQMSPY="${HOMEgfs}/ush/wdqms.py" + +echo "END: config.wdqms" diff --git a/parm/transfer/transfer_gfs_gdas_gdas_misc.list b/parm/transfer/transfer_gfs_gdas_gdas_misc.list index 3d7fe47a88..9418e0c006 100644 --- a/parm/transfer/transfer_gfs_gdas_gdas_misc.list +++ b/parm/transfer/transfer_gfs_gdas_gdas_misc.list @@ -33,44 +33,3 @@ _COMROOT_/gfs/_SHORTVER_/syndat/ B 180 -_COMROOT_/gfs/_SHORTVER_/gdascounts/ -+ /data_counts._MONPREV_/*** -- * -B 16000000 - -_COMROOT_/gfs/_SHORTVER_/gdascounts/ -+ /data_counts._MONCUR_/*** -- * -B 16000000 - -_COMROOT_/gfs/_SHORTVER_/gdascounts/ -+ /satcounts._MONPREV_/*** -- * -B 16000000 - -_COMROOT_/gfs/_SHORTVER_/gdascounts/ -+ /satcounts._MONCUR_/*** -- * -B 16000000 - -_COMROOT_/gfs/_SHORTVER_/sdm_rtdm/ -+ /obcount_30day/ -+ /obcount_30day/gdas/ -+ /obcount_30day/gdas/gdas._PDYm1_/*** -+ /obcount_30day/gdas/gdas._PDY_/*** -- * -B 2000000 - -_COMROOT_/gfs/_SHORTVER_/sdm_rtdm/ -+ /avgdata/ -+ /avgdata/obcount_30davg.gdas._MONPREV_ -+ /avgdata/obcount_30davg.gdas.current -- * -B 256000 - -_COMROOT_/gfs/_SHORTVER_/gdascounts/ -+ /index.shtml -+ /index_backup.shtml -- * -B 6 - diff --git a/parm/transfer/transfer_gfs_gfs_misc.list b/parm/transfer/transfer_gfs_gfs_misc.list deleted file mode 100644 index 32f002d1e7..0000000000 --- a/parm/transfer/transfer_gfs_gfs_misc.list +++ /dev/null @@ -1,43 +0,0 @@ -# This file specifies the directories to be tranatmfered and, optionally, the files within -# those directories to include or exclude. If one directory is specified per line, it -# will be used as both the source and destination. If two directories are specified per -# line, separated by one or more spaces, the first will be used as the source and the -# second the destination. Directories that begin with "com/" will be resolved using -# the compath.py utility. Rules may be placed below each directory or directory pair -# and must begin with one of the following characters: -# - exclude, specifies an exclude pattern -# + include, specifies an include pattern -# . merge, specifies a merge-file to read for more rules -# : dir-merge, specifies a per-directory merge-file -# H hide, specifies a pattern for hiding files from the tranatmfer -# S show, files that match the pattern are not hidden -# P protect, specifies a pattern for protecting files from deletion -# R risk, files that match the pattern are not protected -# ! clear, clears the current include/exclude list (takes no arg) -# B bytes, relative size of the path in relation to the other paths in the list -# D delete, delete extraneous files from destination directories (takes no arg) -# E encrypt, enables data encryption [two cores should be allocated] (takes no arg) -# W whole files, copy whole files rather than use delta-xfer algorithm (takes no arg) (v2.2.3+) -# T two-way syncronization will update both sides with latest changes (takes no arg) -# Z compress data as it is sent, accepts optional compression level argument (1-9) -# Rules higher in the list take precedence over lower ones. By default, all files in a -# directory are included, so if no exclude patterns match that file, it will be -# tranatmferred. - -_COMROOT_/gfs/_SHORTVER_/sdm_rtdm/ -+ /avgdata/ -+ /avgdata/obcount_30davg.gfs._MONPREV_ -+ /avgdata/obcount_30davg.gfs.current -- * -B 256000 - - -_COMROOT_/gfs/_SHORTVER_/sdm_rtdm/ -+ /obcount_30day/ -+ /obcount_30day/gfs/ -+ /obcount_30day/gfs/gfs._PDYm1_/*** -+ /obcount_30day/gfs/gfs._PDY_/*** -- * -B 2000000 - - diff --git a/scripts/exgdas_atmos_analysis_wdqms.sh b/scripts/exgdas_atmos_analysis_wdqms.sh new file mode 100755 index 0000000000..73ccfca45a --- /dev/null +++ b/scripts/exgdas_atmos_analysis_wdqms.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -x + +################################################################################ +# UNIX Script Documentation Block +# Name: exgdas_atmos_analysis_wdqms.sh +# Author: Rahul Mahajan +# Org: NCEP/EMC +# Abstract: This script unpacks GSI diagnostic files, runs them through a python +# script ush/wdqms.py, and creates CSV files. +# These CSV files contain observation information, residual statistics, +# and data use in the assimilation for certain observation sub-types +# History Log: +# 2024-04-01 Rahul Mahajan Initial Version. +# Usage: +# Input Files: +# ${RUN}.t${cyc}z.cnvstat +# Output Files: +# ${RUN}.t${cyc}z.${otype}.csv +# where: otype = synop | temp | marine +# Condition codes: +# == 0 : success +# != 0 : non-fatal error encountered while generating output file +################################################################################ + +# Input GSI diagnostic file containing inputs to wdqms.py +CNVSTAT="${RUN}.t${cyc}z.cnvstat" + +# Input files from CNVSTAT fed to wdqms.py +INPUT_LIST=("diag_conv_ps_ges.${PDY}${cyc}.nc4" \ + "diag_conv_t_ges.${PDY}${cyc}.nc4" \ + "diag_conv_q_ges.${PDY}${cyc}.nc4" \ + "diag_conv_uv_ges.${PDY}${cyc}.nc4") + +# Observation types being processed by wdqms.py +OTYPES=(SYNOP TEMP MARINE) + +################################################################################ +echo "Begin job ${job:-}" + +#------------------------------------------------------------------------------- +# Enter working directory +cd "${DATA}" || ( echo "FATAL ERROR: Unable to cd into '${DATA}', ABORT!"; exit 2 ) + +#------------------------------------------------------------------------------- +# Copy cnvstat file from COMIN to DATA, untar and gunzip input files for wdqms.py +# These should always be available +cp "${COMIN}/${CNVSTAT}" . +export err=$? +(( err != 0 )) && ( msg="FATAL ERROR: Unable to copy '${CNVSTAT}' from '${COMIN}', ABORT!"; err_exit "${msg}" ) +for diagfile in "${INPUT_LIST[@]}"; do + tar -xvf "${CNVSTAT}" "${diagfile}.gz" + export err=$? + (( err != 0 )) && ( msg="FATAL ERROR: Unable to extract '${diagfile}.gz' from '${CNVSTAT}', ABORT!"; err_exit "${msg}" ) + gunzip "${diagfile}.gz" + export err=$? + (( err != 0 )) && ( msg="FATAL ERROR: Unable to gunzip '${diagfile}.gz', ABORT!"; err_exit "${msg}" ) +done + +#------------------------------------------------------------------------------- +# Loop over observation types, produce CSV files +# Copy CSV files to COMOUT +# Issue DBN alerts +# Issue warnings if wdqms.py fails for any reason +# These do not need to be a FATAL ERROR, but developers should be notified +error=0 +for otype in "${OTYPES[@]}"; do + + echo "Processing ... ${otype}" + + #============================================================================= + # Process with wdqms.py + ${WDQMSPY} -i ${INPUT_LIST[@]} -t "${otype}" -o "${DATA}" + rc=$? + if (( rc != 0 )); then + echo "WARNING: wdqms.py failed to process observation type '${otype}'" + error=$((error + 1)) + fi + #============================================================================= + + #============================================================================= + # Copy to COMOUT if wdqms.py created the output file + csvfile="NCEP_${otype}_${PDY}_${cyc}.csv" + csvfileout="${RUN}.t${cyc}z.${otype,,}.csv" + if [[ -f "${csvfile}" ]]; then + cp "./${csvfile}" "${COMOUT}/${csvfileout}" || ( echo "WARNING: Unable to copy '${csvfile}' to '${COMOUT}/${csvfileout}'" ) + else + echo "WARNING: wdqms.py failed to create csvfile '${csvfile}'" + error=$((error + 1)) + fi + #============================================================================= + + #============================================================================= + # Send DBN alerts + if [[ "${SENDDBN:-}" == "YES" ]]; then + if [[ -f "${COMOUT}/${csvfileout}" ]]; then + "${DBNROOT}/bin/dbn_alert" MODEL "${RUN^^}_WDQMS" "${job}" "${COMOUT}/${csvfileout}" + fi + fi + #============================================================================= + +done # for otype +#------------------------------------------------------------------------------- + +#------------------------------------------------------------------------------- +if (( error == 0 )); then + echo "Job completed normally." +else + echo "WARNING: Job completed with non-fatal errors." +fi +################################################################################ + +exit 0 diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 27d021030b..bb3568b082 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -64,6 +64,7 @@ def main(): steps = steps + wav_steps if _base.get('DO_WAVE', 'NO') == 'YES' else steps steps = steps + wav_steps_gempak if _base.get('DO_GEMPAK', 'NO') == 'YES' else steps steps = steps + wav_steps_awips if _base.get('DO_AWIPS', 'NO') == 'YES' else steps + steps = steps + ['wdqms'] if _base.get('DO_WDQMS', 'NO') == 'YES' else steps dict_configs = wfu.source_configs(configs, steps) @@ -243,6 +244,7 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): do_wafs = base.get('WAFSF', 'NO').upper() do_metp = base.get('DO_METP', 'NO').upper() do_gldas = base.get('DO_GLDAS', 'NO').upper() + do_wdqms = base.get('DO_WDQMS', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() reservation = base.get('RESERVATION', 'NONE').upper() @@ -252,6 +254,8 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): if cdump in ['gdas']: tasks += ['analdiag'] + if do_wdqms in ['Y', 'YES']: + tasks += ['wdqms'] if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: tasks += ['gldas'] if cdump in ['gdas'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GDAS', 'BOTH']: @@ -411,6 +415,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): do_wafs = base.get('WAFSF', 'NO').upper() do_metp = base.get('DO_METP', 'NO').upper() do_gldas = base.get('DO_GLDAS', 'NO').upper() + do_wdqms = base.get('DO_WDQMS', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() dumpsuffix = base.get('DUMP_SUFFIX', '') @@ -522,6 +527,18 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dict_tasks[f'{cdump}analdiag'] = task + # wdqms + if cdump in ['gdas'] and do_wdqms in ['Y', 'YES']: + deps1 = [] + dep_dict = {'type': 'task', 'name': f'{cdump}analdiag'} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) + + task = wfu.create_wf_task('wdqms', cdump=cdump, envar=envars, dependency=dependencies1) + dict_tasks[f'{cdump}wdqms'] = task + # gldas if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: deps1 = [] diff --git a/ush/wdqms.py b/ush/wdqms.py new file mode 100755 index 0000000000..a7844c0f19 --- /dev/null +++ b/ush/wdqms.py @@ -0,0 +1,775 @@ +#!/usr/bin/env python3 + +import sys +import os +import argparse +import logging +import pandas as pd +import numpy as np +from netCDF4 import Dataset +from pathlib import Path + + +class WDQMS: + + def __init__(self, inputfiles, wdqms_type, outdir, + loglvl=logging.INFO): + + # Start logging + logging.basicConfig(filename='file.log', + filemode='w', + level=loglvl, + format='%(levelname)s:%(message)s') + + self.wdqms_type = wdqms_type + self.outdir = outdir + + logging.info("Working in wdqms()") + + # Create dataframes from GSI diag files + logging.info('Creating dataframe from GSI diag files ...') + + self.wdqms_type_dict = { + 'SYNOP': { + 'df_type': self._create_conv_df, + 'obs_types': [181, 187, 281, 287], + 'variable_ids': {'ps': 110, 'q': 58, 't': 39, 'u': 41, 'v': 42} + }, + 'TEMP': { + 'df_type': self._create_sondes_df, + 'obs_types': [120, 220], + 'variable_ids': {'ps': 110, 'q': 29, 't': 2, 'u': 3, 'v': 4} + }, + 'MARINE': { + 'df_type': self._create_conv_df, + 'obs_types': [180, 183, 280, 282, 284], + 'variable_ids': {'ps': 110, 'q': 58, 't': 39, 'u': 41, 'v': 42} + } + } + + df_list = [] + + for file in inputfiles: + logging.info(f'Working on {file} ...') + logging.info(f'Reading gsi diag ... ') + + df = self._read_gsi_diag(file) + df_list.append(df) + + df_total = pd.concat(df_list) + + logging.info('Files successfully read into dataframe!') + + # Grab data specific to WDQMS type + df_total = self._wdqms_type_requirements(df_total) + + # Grab actual datetimes from datetime + timedelta + df_total = self._get_datetimes(df_total) + + # Drop duplicates + df_total = df_total.drop_duplicates() + + # Create temporary dataframe of only temperature and q values + tq_condition = [self.wdqms_type_dict[self.wdqms_type]['variable_ids']['t'], + self.wdqms_type_dict[self.wdqms_type]['variable_ids']['q']] + + tq_df = df_total.loc[df_total['var_id'].isin(tq_condition)] + no_tq_df = df_total.loc[~df_total['var_id'].isin(tq_condition)] + + # Adjust relative humidity data + tq_df = self._genqsat(tq_df) + + # Merge the non t and q values back into returned t and q dataframe + df_total = pd.concat([tq_df, no_tq_df]) + + # Add Status Flag column + df_total = self._create_status_flag(df_total) + df_total['StatusFlag'] = df_total['StatusFlag'].astype(int) + + # Sort by Station ID + df_total['Station_ID'] = df_total['Station_ID'].astype(str) + df_total = df_total.sort_values('Station_ID') + + logging.info(f'Creating dataframe for {self.wdqms_type} type ...') + + # Create dataframe for appropriate WDQMS type + output_df = self.wdqms_type_dict[self.wdqms_type]['df_type'](df_total) + + # Get str datetime + self.datetime = inputfiles[0].split('/')[-1].split('.')[-2] + + out_filename = self._df_to_csv(output_df) + + logging.info(f"Success! Output file saved to: {out_filename}") + logging.info("Exiting ...") + sys.exit() + + return + + def _wdqms_type_requirements(self, df): + """ + Filter dataframe to only include specific observation types. + """ + logging.info("Working in wdqms_type_requirements()") + logging.debug(f"WDQMS Type: {self.wdqms_type}") + logging.debug(f"Total observations for {self.wdqms_type} before filter: {len(df)}") + + obs_types = self.wdqms_type_dict[self.wdqms_type]['obs_types'] + + df = df.loc[(df['Observation_Type'].isin(obs_types))] + + if self.wdqms_type in ['SYNOP', 'MARINE']: + # Only include -3 > val >= 3 to avoid overlapping in cycles + df = df.loc[df['Time'] != -3.] + + # Remove bad background departures for each variable + df.loc[(df['var_id'] == 110) & (df['Obs_Minus_Forecast_adjusted'].abs() > 200), + 'Obs_Minus_Forecast_adjusted'] = -999.9999 + df.loc[(df['var_id'] != 110) & (df['Obs_Minus_Forecast_adjusted'].abs() > 500), + 'Obs_Minus_Forecast_adjusted'] = -999.9999 + + if self.wdqms_type in ['TEMP']: + # Only include assimilated data as per WDQMS requirement document + df = df.loc[df['Analysis_Use_Flag'] == 1] + + logging.debug(f"Total observations for {self.wdqms_type} after filter: {len(df)}") + logging.info("Exiting wdqms_type_requirements()") + + return df + + def _get_datetimes(self, df): + """ + Use 'Datetime' and 'Time' columns to create new datetime and + separate into new columns: 'YYYYMMDD' and 'HHMMSS' + Args: + df : (df) pandas dataframe populated with data from GSI + diagnostic files + Returns: + df: (df) the same dataframe read in with new columns: + 'YYYYMMDD' and 'HHMMSS' + """ + logging.info("Working in get_datetimes()") + + # Convert 'Datetime' column from str to datetime + dates = pd.to_datetime(df['Datetime'], format='%Y%m%d%H') + # Converts 'Time' column to time delta in hours + hrs = pd.to_timedelta(df['Time'], unit='hours') + # Actual datetime of ob adding datetime and timedelta in hours + new_dt = dates + hrs + + df['yyyymmdd'] = new_dt.dt.strftime('%Y%m%d') + df['HHMMSS'] = new_dt.dt.strftime('%H%M%S') + + logging.info("Exiting get_datetimes()") + + return df + + def _create_status_flag(self, df): + """ + Create Status Flag based on the values from Prep_QC_Mark, + Prep_Use_Flag, and Analysis_Use_Flag. + Args: + df: (df) pandas dataframe populated with data from GSI + diagnostic files + Returns: + df: (df) the same dataframe read in with a new column: 'StatusFlag' + """ + logging.info("Working in create_status_flag()") + + # Create 'StatusFlag' column and fill with nans + df['StatusFlag'] = np.nan + + # Obs used by GSI, Status_Flag=0 + df.loc[(df['Prep_QC_Mark'] <= 8) & ( + df['Analysis_Use_Flag'] == 1), 'StatusFlag'] = 0 + + # Obs rejected by GSI, Status_Flag=0 + df.loc[(df['Prep_QC_Mark'] <= 8) & ( + df['Analysis_Use_Flag'] == -1), 'StatusFlag'] = 2 + + # Obs never used by GSI, Status_Flag=3 + df.loc[(df['Prep_QC_Mark'] > 8) & ( + df['Prep_Use_Flag'] >= 100), 'StatusFlag'] = 3 + + # Obs is flagged for non-use by the analysis, Status_Flag=3 + df.loc[df['Prep_QC_Mark'] >= 15, 'StatusFlag'] = 3 + + # Obs rejected by SDM or CQCPROF, Status_Flag=7 + df.loc[(df['Prep_QC_Mark'] >= 12) & ( + df['Prep_QC_Mark'] <= 14), 'StatusFlag'] = 7 + + # Fill those that do not fit a condition with -999 + df.loc[df['StatusFlag'].isnull(), 'StatusFlag'] = -999 + + logging.debug("Status Flag Counts:") + logging.debug(f"{df['StatusFlag'].value_counts()}") + logging.info("Exiting create_status_flag()") + + return df + + def _round_column(self, df, col): + """ + Round column numbers to 4 decimal places. + Input: + df: dataframe with information + col: column name to round data + Output: + df: dataframe with changed values in provided column + """ + logging.debug("Working in round_column()") + + df[col] = df[col].map('{:,.4f}'.format) + + logging.debug("Exiting round_column()") + + return df + + def _create_conv_df(self, df): + """ + Create dataframe for conventional data. + """ + logging.info("Working in create_conv_df") + + # Add center_id + df['Centre_id'] = 'NCEP' + df['CodeType'] = 999 + + # Remove unnecessary columns + df.drop(['Observation_Type', 'Pressure', 'Time', 'Prep_QC_Mark', + 'Prep_Use_Flag', 'Analysis_Use_Flag', 'Datetime'], + axis=1, inplace=True) + + # Rename columns + df = df.rename({'Obs_Minus_Forecast_adjusted': 'Bg_dep', + 'Latitude': 'latitude', + 'Station_ID': 'Station_id'}, axis=1) + + # ordered columns + cols = ['Station_id', 'yyyymmdd', 'HHMMSS', 'latitude', 'Longitude', + 'StatusFlag', 'Centre_id', 'var_id', 'Bg_dep', 'CodeType'] + + df = df[cols] + df = df.reset_index(drop=True) + + # Round given columns to four decimal places + for col in ['latitude', 'Longitude', 'Bg_dep']: + df = self._round_column(df, col) + + logging.info("Exiting create_conv_df()") + + return df + + def _create_sondes_df(self, df): + """ + Create dataframe for sondes. + """ + logging.info("Working in create_sondes_df()") + + stn_ids = df['Station_ID'].unique() + + df_list = [] + + # Loop through stations and create individual dataframes + # that grabs average stats from surface, troposphere, and + # stratosphere + for stn in stn_ids: + logging.debug(f"Station ID: {stn}") + + d = { + 'var_id': [], + 'Mean_Bg_dep': [], + 'Std_Bg_dep': [], + 'Levels': [], + 'LastRepLevel': [], + 'StatusFlag': [] + } + + surf_lat = None + surf_lon = None + + # Temporary dataframe of specific station data + tmp = df.loc[df['Station_ID'] == stn] + + # Add pressure info if available + if 110 in tmp['var_id'].unique(): + logging.debug(f"Variable: p") + + mean_bg_dep = tmp['Obs_Minus_Forecast_adjusted'].loc[tmp['var_id'] == 110].values[0] + std_bg_dep = 0 + level = 'Surf' + last_rep_lvl = -999.99 + status_flag = tmp['StatusFlag'].loc[tmp['var_id'] == 110].values[0] + + d['var_id'].append(110) + d['Mean_Bg_dep'].append(mean_bg_dep) + # cannot compute std w/ one value so set to 0 + d['Std_Bg_dep'].append(std_bg_dep) + d['Levels'].append(level) + d['LastRepLevel'].append(last_rep_lvl) + d['StatusFlag'].append(status_flag) + + # surface lat and lon if exists + surf_lat = tmp['Latitude'].loc[tmp['var_id'] == 110].values[0] + surf_lon = tmp['Longitude'].loc[tmp['var_id'] == 110].values[0] + + logging.debug("Mean_Bg_dep, Std_Bg_dep, Levels, LastRepLevel, StatusFlag") + logging.debug(f"{mean_bg_dep}, {std_bg_dep}, {level}, {last_rep_lvl}, {status_flag}") + + # Get unique variable ID's and remove 110 (surface pressure) + var_ids = sorted(tmp['var_id'].unique()) + var_ids.remove(110) if 110 in var_ids else var_ids + + for var in var_ids: + logging.debug(f"Variable: {var}") + + # Surface + + # Find max pressure of the surface 110 value + surf_p_max = tmp['Pressure'].loc[tmp['var_id'] == 110].max() + + if (110 in tmp['var_id'].unique() and + var in tmp['var_id'].loc[tmp['Pressure'] == surf_p_max].unique()): + + surf_tmp = tmp.loc[(tmp['Pressure'] == surf_p_max) & + (tmp['var_id'] == var)] + + surf_omf = surf_tmp['Obs_Minus_Forecast_adjusted'].values.mean( + ) + surf_std = 0 # cannot compute std w/ one value so set to 0 + level = 'Surf' + last_rep_lvl = -999.99 + + # If at least one ob is used, we report the lowest Status Flag. + # Although it does not represent the whole column, it is what is + # required by the WDQMS team. + status_flag = surf_tmp['StatusFlag'].min() + + d['var_id'].append(var) + d['Mean_Bg_dep'].append(surf_omf) + d['Std_Bg_dep'].append(surf_std) + d['Levels'].append(level) + d['LastRepLevel'].append(last_rep_lvl) + d['StatusFlag'].append(status_flag) + + logging.debug("Mean_Bg_dep, Std_Bg_dep, Levels, LastRepLevel, StatusFlag") + logging.debug(f"{surf_omf}, {surf_std}, {level}, {last_rep_lvl}, {status_flag}") + + # Troposphere + trop_tmp = tmp.loc[(tmp['var_id'] == var) & + (tmp['Pressure'] >= 100)] + + if len(trop_tmp) > 0: + trop_avg_omf = trop_tmp['Obs_Minus_Forecast_adjusted'].mean( + ) + trop_std_omf = trop_tmp['Obs_Minus_Forecast_adjusted'].std( + ) + level = 'Trop' + # Get lowest p for entire atmosphere + last_rep_lvl = tmp['Pressure'].min() + + # If at least one ob is used, we report the lowest Status Flag. + # Although it does not represent the whole column, it is what is + # required by the WDQMS team. + status_flag = trop_tmp['StatusFlag'].min() + + d['var_id'].append(var) + d['Mean_Bg_dep'].append(trop_avg_omf) + d['Std_Bg_dep'].append(trop_std_omf) + d['Levels'].append(level) + d['LastRepLevel'].append(last_rep_lvl) + d['StatusFlag'].append(status_flag) + + logging.debug("Mean_Bg_dep, Std_Bg_dep, Levels, LastRepLevel, StatusFlag") + logging.debug(f"{trop_avg_omf}, {trop_std_omf}, {level}, {last_rep_lvl}, {status_flag}") + + # Stratosphere + stra_tmp = tmp.loc[(tmp['var_id'] == var) & + (tmp['Pressure'] < 100)] + + if len(stra_tmp) > 0: + stra_avg_omf = stra_tmp['Obs_Minus_Forecast_adjusted'].mean( + ) + stra_std_omf = 0 if len( + stra_tmp == 1) else stra_tmp['Obs_Minus_Forecast_adjusted'].std() + level = 'Stra' + # Get lowest p for entire atmosphere + last_rep_lvl = tmp['Pressure'].min() + + # If at least one ob is used, we report the lowest Status Flag. + # Although it does not represent the whole column, it is what is + # required by the WDQMS team. + status_flag = stra_tmp['StatusFlag'].min() + + d['var_id'].append(var) + d['Mean_Bg_dep'].append(stra_avg_omf) + d['Std_Bg_dep'].append(stra_std_omf) + d['Levels'].append(level) + d['LastRepLevel'].append(last_rep_lvl) + d['StatusFlag'].append(status_flag) + + logging.debug("Mean_Bg_dep, Std_Bg_dep, Levels, LastRepLevel, StatusFlag") + logging.debug(f"{stra_avg_omf}, {stra_std_omf}, {level}, {last_rep_lvl}, {status_flag}") + + sub_df = pd.DataFrame.from_dict(d) + sub_df['Station_id'] = stn + # Add lats and lons + lat = surf_lat if surf_lat else tmp['Latitude'].value_counts( + ).index[0] + lon = surf_lon if surf_lon else tmp['Longitude'].value_counts( + ).index[0] + sub_df['latitude'] = lat + sub_df['Longitude'] = lon + # add datetime + str_datetime = str(tmp['Datetime'].values[0]) + sub_df['yyyymmdd'] = str_datetime[:-2] + sub_df['HHMMSS'] = str_datetime[-2:] + '0000' + + df_list.append(sub_df) + + df = pd.concat(df_list) + df['Centre_id'] = 'NCEP' + df['CodeType'] = 999 + df = df.dropna() + + # Ordered columns + cols = ['Station_id', 'yyyymmdd', 'HHMMSS', 'latitude', 'Longitude', + 'StatusFlag', 'Centre_id', 'var_id', 'Mean_Bg_dep', 'Std_Bg_dep', + 'Levels', 'LastRepLevel', 'CodeType'] + + df = df[cols] + df = df.reset_index(drop=True) + + # Round given columns to four decimal places + for col in ['latitude', 'Longitude', 'Mean_Bg_dep', 'Std_Bg_dep', 'LastRepLevel']: + df = self._round_column(df, col) + + logging.info("Exiting create_sondes_df()") + + return df + + def _genqsat(self, df): + """ + Calculates new background departure values for specific humidity (q) + by calculating saturation specific humidity from corresponding temperature + and pressure values. + + bg_dep = (q_obs/qsat_obs)-(q_ges/qsat_ges) + + q_obs : measured q obs + qsat_obs : calculated saturation q + q_ges : q_obs minus q background error from GSI diagnostic file + qsat_ges : calculated saturation q using temperature obs minus + temperature background error from GSI diagnostic file + + Args: + df : (df) pandas dataframe populated with t and q data from GSI + diagnostic files + wdqms_type : (str) wdqms type file being created + Returns: + df: (df) the same dataframe read in with new background + departure values for humidity data + """ + logging.info("Working in genqstat()") + + # Get variable type specific to WDQMS type + q_id = self.wdqms_type_dict[self.wdqms_type]['variable_ids']['q'] + t_id = self.wdqms_type_dict[self.wdqms_type]['variable_ids']['t'] + + df_list = [] + + # Groupby Station_ID + for stn, stn_df in df.groupby('Station_ID'): + + # Filter the dataframes + q_df = stn_df[stn_df['var_id'] == q_id] + t_df = stn_df[stn_df['var_id'] == t_id] + + # Make sure there are no duplicates in q_df and t_df before merging + columns_to_compare = ['Station_ID', 'Latitude', 'Longitude', 'Pressure', 'Time'] + q_df = q_df.drop_duplicates(subset=columns_to_compare) + t_df = t_df.drop_duplicates(subset=columns_to_compare) + + # Merge dataframes on common keys using an inner join + merged_df = pd.merge(q_df, t_df, on=['Station_ID', 'Latitude', 'Longitude', 'Pressure', 'Time'], suffixes=('_q', '_t'), how='inner') + + # Calculate needed values + q_obs = merged_df['Observation_q'].to_numpy() * 1.0e6 + q_ges = (merged_df['Observation_q'].to_numpy() - merged_df['Obs_Minus_Forecast_adjusted_q'].to_numpy()) * 1.0e6 + t_obs = merged_df['Observation_t'].to_numpy() - 273.16 + t_ges = (merged_df['Observation_t'].to_numpy() - merged_df['Obs_Minus_Forecast_adjusted_t'].to_numpy()) - 273.16 + pressure = merged_df['Pressure'].to_numpy() + + qsat_obs = self._temp_2_saturation_specific_humidity(pressure, t_obs) + qsat_ges = self._temp_2_saturation_specific_humidity(pressure, t_ges) + + # Calculate background departure + bg_dep = (q_obs / qsat_obs) - (q_ges / qsat_ges) + + # Grab conditions from merged_df + station_ids = merged_df['Station_ID'] + pressure_vals = merged_df['Pressure'] + time_vals = merged_df['Time'] + latitude = merged_df['Latitude'] + longitude = merged_df['Longitude'] + conditions = (q_df['Station_ID'].isin(station_ids)) & \ + (q_df['Pressure'].isin(pressure_vals)) & \ + (q_df['Time'].isin(time_vals)) & \ + (q_df['Latitude'].isin(latitude)) & \ + (q_df['Longitude'].isin(longitude)) + + # Update the background departure values for q_df + q_df = q_df.loc[conditions] + q_df['Obs_Minus_Forecast_adjusted'] = bg_dep + + df_list.append(pd.concat([t_df, q_df])) + + df = pd.concat(df_list) + + logging.info("Exiting genqstat()") + + return df + + def _temp_2_saturation_specific_humidity(self, pres, tsen): + """ + Uses pressure and temperature arrays to calculate saturation + specific humidity. + Args: + pres: (array) array of pressure obs + tsen: (array) array of temperature obs in Celsius + Returns: + qsat_array: (array) corresponding calculated sat. spec. humidity + """ + logging.debug("Working in temp_2_saturation_specific_humidity()") + + ttp = 2.7316e2 # temperature at h2o triple point (K) + psat = 6.1078e1 # pressure at h2o triple point (Pa) + cvap = 1.8460e3 # specific heat of h2o vapor (J/kg/K) + csol = 2.1060e3 # specific heat of solid h2o (ice)(J/kg/K) + hvap = 2.5000e6 # latent heat of h2o condensation (J/kg) + hfus = 3.3358e5 # latent heat of h2o fusion (J/kg) + rd = 2.8705e2 + rv = 4.6150e2 + cv = 7.1760e2 + cliq = 4.1855e3 + + dldt = cvap - cliq + dldti = cvap - csol + hsub = hvap + hfus + tmix = ttp - 20. + xa = -(dldt / rv) + xai = -(dldti / rv) + xb = xa + hvap / (rv * ttp) + xbi = xai + hsub / (rv * ttp) + eps = rd / rv + omeps = 1.0 - eps + + tdry = tsen + ttp + tdry = np.array([1.0e-8 if np.abs(t) < 1.0e-8 else t for t in tdry]) + + tr = ttp / tdry + + qsat_array = [] + + # Loop through temperatures and appropriate indexes to solve qsat + for idx, t in enumerate(tdry): + # Get correct estmax and es values based on conditions + if t >= ttp: + estmax = psat * (tr[idx]**xa) * np.exp(xb * (1.0 - tr[idx])) + es = estmax + elif t < tmix: + estmax = psat * (tr[idx]**xa) * np.exp(xbi * (1.0 - tr[idx])) + es = estmax + else: + w = (t - tmix) / (ttp - tmix) + estmax = w * psat * (tr[idx]**xa) * np.exp(xb * (1.0 - tr[idx])) \ + + (1.0 - w) * psat * (tr[idx]**xai) * \ + np.exp(xbi * (1.0 - tr[idx])) + + es = w * psat * (tr[idx]**xa) * np.exp(xb * (1.0 - tr[idx])) \ + + (1.0 - w) * psat * (tr[idx]**xai) * \ + np.exp(xbi * (1.0 - tr[idx])) + + pw = pres[idx] + esmax = pw + + esmax = np.min([esmax, estmax]) + es2 = np.min([es, esmax]) + + qsat = eps * es2 / ((pw * 10.0) - (omeps * es2)) + qsat2 = qsat * 1e6 + + qsat_array.append(qsat2) + + logging.debug("Exiting temp_2_saturation_specific_humidity()") + + return np.array(qsat_array) + + def _read_gsi_diag(self, file): + """ + Reads the data from the conventional diagnostic file during + initialization into a pandas dataframe. + Args: + file : (str) netCDF GSI diagnostic file + Returns: + df : (dataframe) pandas dataframe populated with data from + netCDF GSI diagnostic file + """ + logging.debug("Working in read_gsi_diag()") + + filename = os.path.splitext(Path(file).stem)[0] + logging.debug(f'Filename: {filename}') + + variable = filename.split('_')[2] + logging.debug(f'Variable: {variable}') + + df_dict = {} + + column_list = ['Station_ID', 'Observation_Class', 'Observation_Type', + 'Latitude', 'Longitude', 'Pressure', 'Time', 'Prep_QC_Mark', 'Prep_Use_Flag', + 'Analysis_Use_Flag', 'Observation', 'Obs_Minus_Forecast_adjusted'] + + # Grab datetime from file + datetime = self._grab_netcdf_data(file, 'Datetime') + + if variable == 'uv': + for wtype in ['u', 'v']: + df_dict[wtype] = {} + for col in column_list: + col = f'{wtype}_' + col if col == 'Observation' else col + col = f'{wtype}_' + \ + col if col == 'Obs_Minus_Forecast_adjusted' else col + data = self._grab_netcdf_data(file, col) + df_dict[wtype][col] = data + + # Need to separate the u and v dataframes to concatenate them + u_df = pd.DataFrame(df_dict['u']) + u_df = u_df.rename({'Observation_Class': 'var_id', + 'u_Observation': 'Observation', + 'u_Obs_Minus_Forecast_adjusted': 'Obs_Minus_Forecast_adjusted'}, + axis=1) + u_df['var_id'] = self.wdqms_type_dict[self.wdqms_type]['variable_ids']['u'] + + v_df = pd.DataFrame(df_dict['v']) + v_df = v_df.rename({'Observation_Class': 'var_id', + 'v_Observation': 'Observation', + 'v_Obs_Minus_Forecast_adjusted': 'Obs_Minus_Forecast_adjusted'}, + axis=1) + v_df['var_id'] = self.wdqms_type_dict[self.wdqms_type]['variable_ids']['v'] + + df = pd.concat([u_df, v_df]) + + else: + for col in column_list: + data = self._grab_netcdf_data(file, col) + df_dict[col] = data + + df = pd.DataFrame(df_dict) + df = df.rename({'Observation_Class': 'var_id'}, axis=1) + df['var_id'] = self.wdqms_type_dict[self.wdqms_type]['variable_ids'][variable] + + # Add datetime column to dataframe + df['Datetime'] = datetime + + # Subtract longitudes > 180 by 360 to be negative + df.loc[df['Longitude'] > 180, 'Longitude'] -= 360 + + logging.debug("Exiting read_gsi_diag()") + + return df + + def _grab_netcdf_data(self, file, var): + """ + Opens and grabs data based on column name. + Args: + file : (str) netCDF GSI file + var : (str) the variable to be extracted + Returns: + data : (array) values from the specified variable + """ + logging.debug('Working in grab_netcdf_data()') + + with Dataset(file, mode='r') as f: + # Station_ID and Observation_Class variables need + # to be converted from byte string to string + if var == 'Datetime': + data = f.date_time + + elif var in ['Station_ID', 'Observation_Class']: + data = f.variables[var][:] + data = [i.tobytes(fill_value=' ', order='C') + for i in data] + data = np.array( + [''.join(i.decode('UTF-8', 'ignore').split()) + for i in data]) + + # Grab variables with only 'nobs' dimension + elif len(f.variables[var].shape) == 1: + data = f.variables[var][:] + + logging.debug("Exiting grab_netcdf_data()") + + return data + + def _df_to_csv(self, df): + """ + Produce output .csv file from dataframe. + """ + logging.info("Working in df_to_csv()") + logging.info(f'Coverting dataframe to .csv file for {self.wdqms_type} data ...') + + # Write dataframe to .csv + date = self.datetime[:-2] + cycle = self.datetime[-2:] + + hr_range = { + '00': ['21', '03'], + '06': ['03', '09'], + '12': ['09', '15'], + '18': ['15', '21'] + } + + filename = f'{self.outdir}/NCEP_{self.wdqms_type}_{date}_{cycle}.csv' + + f = open(filename, 'a') + f.write(f"# TYPE={self.wdqms_type}\n") + f.write(f"#An_Date= {date}\n") + f.write(f"#An_time= {cycle}\n") + f.write(f"#An_range=[ {hr_range[cycle][0]} to {hr_range[cycle][-1]} )\n") + f.write("#StatusFlag: 0(Used);1(Not Used);2(Rejected by DA);" + "3(Never Used by DA);4(Data Thinned);5(Rejected before DA);" + "6(Alternative Used);7(Quality Issue);8(Other Reason);9(No content)\n") + df.to_csv(f, index=False) + f.close() + + logging.info(f'{filename} file created.') + logging.info('Exiting df_to_csv()') + + return filename + + +if __name__ == "__main__": + + # Parse command line + ap = argparse.ArgumentParser() + ap.add_argument("-i", "--input_list", nargs='+', default=[], + help="List of input GSI diagnostic files") + ap.add_argument("-t", "--type", + help="WDQMS file type (SYNOP, TEMP, MARINE)") + ap.add_argument("-o", "--outdir", + help="Out directory where files will be saved") + ap.add_argument('-d', '--debug', + help="Print debugging statements to log file", + action="store_const", dest="loglevel", + const=logging.DEBUG, + default=logging.WARNING) + ap.add_argument('-v', '--verbose', + help="Print information statements about code", + action="store_const", dest="loglevel", + const=logging.INFO) + + args = ap.parse_args() + + if args.type not in ['SYNOP', 'TEMP', 'MARINE']: + raise ValueError(f"FATAL ERROR: {args.type} not a correct input. Inputs include: 'SYNOP, TEMP, MARINE'") + + WDQMS(args.input_list, args.type, args.outdir, args.loglevel) diff --git a/versions/run.ver b/versions/run.ver index 6d7339b4fe..5fe04f6514 100755 --- a/versions/run.ver +++ b/versions/run.ver @@ -1,5 +1,5 @@ -export version=v16.3.15 -export gfs_ver=v16.3.15 +export version=v16.3.16 +export gfs_ver=v16.3.16 export ukmet_ver=v2.2 export ecmwf_ver=v2.1 export nam_ver=v4.2