Merge branch 'develop' into feature/jpssrr_sea_ice

modulefiles/GDAS/hera.intel.lua

@@ -74,6 +74,7 @@ load("py-scipy/1.11.3")
 load("py-xarray/2023.7.0")
 load("py-f90nml/1.4.3")
 load("py-pip/23.1.2")
+load("py-click/8.1.7")
 
 setenv("CC","mpiicc")
 setenv("FC","mpiifort")

parm/atm/jcb-base.yaml.j2

@@ -116,6 +116,9 @@ driver_save_prior_mean: false
 driver_save_posterior_mean_increment: false
 driver_save_posterior_ensemble_increments: true
 
+# Distribution type
+distribution_type: Halo
+
 # Diagnostics
 atmosphere_ensemble_increment_prefix: "./anl/mem%{member}%/atminc."
 atmosphere_posterior_output_gaussian: "./mem%{member}%/atmanl."

parm/atm/jcb-prototype_3dvar.yaml.j2

@@ -12,7 +12,7 @@ observations:
   - ascatw.ascat_metop-b
 #  - ascatw.ascat_metop-c
   - atms_n20
-  - atms_npp
+#  - atms_npp
   - conventional_ps
   - gnssro
 #  - gpsro
@@ -23,9 +23,9 @@ observations:
 #  - ompstc_n20
   - ompstc_npp
   - satwnd.abi_goes-16
-  - satwnd.abi_goes-18
-  - satwnd.leogeo_multi
-  - satwnd.viirs_n20
-  - satwnd.viirs_npp
+#  - satwnd.abi_goes-18
+#  - satwnd.leogeo_multi
+#  - satwnd.viirs_n20
+#  - satwnd.viirs_npp
 #  - scatwind_ascat_metop-a
 #  - snowcvr

parm/atm/jcb-prototype_lgetkf_observer.yaml.j2

@@ -0,0 +1,46 @@
+# Use observations for lgetkf
+# ---------------------------
+app_path_observations: {{PARMgfs}}/gdas/jcb-gdas/observations/atmosphere-lgetkf
+app_path_observation_chronicle: {{PARMgfs}}/gdas/jcb-gdas/observation_chronicle/atmosphere
+
+# Algorithm
+# ---------
+algorithm: local_ensemble_da_observer
+
+# Model things
+# ------------
+atmosphere_background_ensemble_path: ./bkg/mem%mem%
+
+# Observation things
+# ------------------
+observations:
+#  - ADPSFC
+#  - ADPUPA
+#  - SFCSHP
+  - aircraft
+  - ascatw.ascat_metop-b
+#  - ascatw.ascat_metop-c
+  - atms_n20
+#  - atms_npp
+  - conventional_ps
+  - gnssro
+#  - gpsro
+#  - mtiasi_metop-a
+#  - mtiasi_metop-b
+#  - ompsnp_n20
+  - ompsnp_npp
+#  - ompstc_n20
+  - ompstc_npp
+  - satwnd.abi_goes-16
+#  - satwnd.abi_goes-18
+#  - satwnd.leogeo_multi
+#  - satwnd.viirs_n20
+#  - satwnd.viirs_npp
+#  - scatwind_ascat_metop-a
+#  - snowcvr
+
+# Naming conventions for observation files
+atmosphere_obsdataout_prefix: diag_lobs_
+
+# Distribution type
+distribution_type: RoundRobin

parm/atm/jcb-prototype_lgetkf_solver.yaml.j2

@@ -0,0 +1,49 @@
+# Use observations for lgetkf
+# ---------------------------
+app_path_observations: {{PARMgfs}}/gdas/jcb-gdas/observations/atmosphere-lgetkf
+app_path_observation_chronicle: {{PARMgfs}}/gdas/jcb-gdas/observation_chronicle/atmosphere
+
+# Algorithm
+# ---------
+algorithm: local_ensemble_da_solver
+
+# Model things
+# ------------
+atmosphere_background_ensemble_path: ./bkg/mem%mem%
+
+# Observation things
+# ------------------
+observations:
+#  - ADPSFC
+#  - ADPUPA
+#  - SFCSHP
+  - aircraft
+  - ascatw.ascat_metop-b
+#  - ascatw.ascat_metop-c
+  - atms_n20
+#  - atms_npp
+  - conventional_ps
+  - gnssro
+#  - gpsro
+#  - mtiasi_metop-a
+#  - mtiasi_metop-b
+#  - ompsnp_n20
+  - ompsnp_npp
+#  - ompstc_n20
+  - ompstc_npp
+  - satwnd.abi_goes-16
+#  - satwnd.abi_goes-18
+#  - satwnd.leogeo_multi
+#  - satwnd.viirs_n20
+#  - satwnd.viirs_npp
+#  - scatwind_ascat_metop-a
+#  - snowcvr
+
+# Naming conventions for observation files
+atmosphere_obsdatain_path: "{{atm_obsdataout_path}}"
+atmosphere_obsdatain_prefix: diag_lobs_
+atmosphere_obsdatain_suffix: "_{{ current_cycle | to_YMDH }}.nc"
+atmosphere_obsdataout_prefix: diag_solv_
+
+# Distribution type
+distribution_type: Halo

test/atm/global-workflow/CMakeLists.txt

@@ -31,11 +31,21 @@ add_test(NAME test_gdasapp_atm_jjob_ens_init
           ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
   WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun)
 
-add_test(NAME test_gdasapp_atm_jjob_ens_run
-  COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_run.sh
+add_test(NAME test_gdasapp_atm_jjob_ens_letkf
+  COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_letkf.sh
           ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
 	  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun)
 
+add_test(NAME test_gdasapp_atm_jjob_ens_obs
+  COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_obs.sh
+          ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun)
+
+add_test(NAME test_gdasapp_atm_jjob_ens_sol
+  COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_sol.sh
+          ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun)
+
 add_test(NAME test_gdasapp_atm_jjob_ens_inc
   COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_inc.sh
           ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}

test/atm/global-workflow/config.yaml

@@ -20,3 +20,9 @@ atmensanl:
   JCB_ALGO_YAML: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf.yaml.j2"
   LAYOUT_X_ATMENSANL: 1
   LAYOUT_Y_ATMENSANL: 1
+
+atmensanlobs:
+  JCB_ALGO_YAML: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2"
+
+atmensanlsol:
+  JCB_ALGO_YAML: "@srcdir@/test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2"

test/atm/global-workflow/jcb-prototype_lgetkf_observer.yaml.j2

@@ -0,0 +1,34 @@
+# Use observations for lgetkf
+# ---------------------------
+app_path_observations: {{PARMgfs}}/gdas/jcb-gdas/observations/atmosphere-lgetkf
+app_path_observation_chronicle: {{PARMgfs}}/gdas/jcb-gdas/observation_chronicle/atmosphere-lgetkf
+
+# Algorithm
+# ---------
+algorithm: local_ensemble_da_observer
+
+# Model things
+# ------------
+atmosphere_background_ensemble_path: ./bkg/mem%mem%
+
+# Observation things
+# ------------------
+observations:
+  - amsua_n19
+  - sondes
+
+# The observation files in the testing are appended using the yyymmddhh similar to JEDI tests
+atmosphere_obsdatain_suffix: ".{{ current_cycle | to_YMDH }}.nc"
+
+# Naming conventions for observation files
+atmosphere_obsdataout_prefix: diag_lobs_
+
+# Distribution type
+distribution_type: RoundRobin
+
+# Testing things
+# --------------
+test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/atm/global-workflow/lgetkf_observer.ref
+test_output_filename: ./lgetkf_observer.out
+test_float_relative_tolerance: 1.0e-3
+test_float_absolute_tolerance: 1.0e-5

test/atm/global-workflow/jcb-prototype_lgetkf_solver.yaml.j2

@@ -0,0 +1,34 @@
+# Use observations for lgetkf
+# ---------------------------
+app_path_observations: {{PARMgfs}}/gdas/jcb-gdas/observations/atmosphere-lgetkf
+app_path_observation_chronicle: {{PARMgfs}}/gdas/jcb-gdas/observation_chronicle/atmosphere-lgetkf
+
+# Algorithm
+# ---------
+algorithm: local_ensemble_da_solver
+
+# Model things
+# ------------
+atmosphere_background_ensemble_path: ./bkg/mem%mem%
+
+# Observation things
+# ------------------
+observations:
+  - amsua_n19
+  - sondes
+
+# Naming conventions for observation files
+atmosphere_obsdatain_path: "{{atm_obsdataout_path}}"
+atmosphere_obsdatain_prefix: diag_lobs_
+atmosphere_obsdatain_suffix: "_{{ current_cycle | to_YMDH }}.nc"
+atmosphere_obsdataout_prefix: diag_solv_
+
+# Distribution type
+distribution_type: Halo
+
+# Testing things
+# --------------
+test_reference_filename: {{ HOMEgfs }}/sorc/gdas.cd/test/atm/global-workflow/lgetkf_solver.ref
+test_output_filename: ./lgetkf_solver.out
+test_float_relative_tolerance: 1.0e-3
+test_float_absolute_tolerance: 1.0e-5

test/atm/global-workflow/jjob_ens_inc.sh

@@ -47,7 +47,7 @@ fi
 
 # Execute j-job
 if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then
-    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT --job-name=atmensanlfv3inc-%j.out
+    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT
 fi

test/atm/global-workflow/jjob_ens_init.sh

@@ -110,6 +110,10 @@ for imem in $(seq 1 $NMEM_ENS); do
     done
 done
 
+# Set lobsdiag_forenkf=.false. to run letkf as single observer and solver job
+# NOTE:  atmensanlinit creates input yaml for atmensanlletkf job
+cp $EXPDIR/config.base_lobsdiag_forenkf_false $EXPDIR/config.base
+
 # Execute j-job
 if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmensanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE

test/atm/global-workflow/jjob_ens_run.sh → test/atm/global-workflow/jjob_ens_letkf.sh

@@ -48,6 +48,9 @@ elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
 fi
 
+# Set lobsdiag_forenkf=.false. to run letkf as combined observer and solver job
+cp $EXPDIR/config.base_lobsdiag_forenkf_false $EXPDIR/config.base
+
 # Execute j-job
 if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlletkf-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF

test/atm/global-workflow/jjob_ens_obs.sh

@@ -0,0 +1,59 @@
+#! /usr/bin/env bash
+
+set -x
+bindir=$1
+srcdir=$2
+
+# Set g-w HOMEgfs
+topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P)
+export HOMEgfs=$topdir
+
+# Set variables for ctest
+export PSLOT=gdas_test
+export EXPDIR=$bindir/test/atm/global-workflow/testrun/experiments/$PSLOT
+export PDY=20210323
+export cyc=18
+export CDATE=${PDY}${cyc}
+export gPDY=20210323
+export gcyc=12
+export GDATE=${gPDY}${gcyc}
+export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS/$PSLOT
+export RUN=enkfgdas
+export CDUMP=enkfgdas
+export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIRS/$PSLOT
+export COMIN_GES=${bindir}/test/atm/bkg
+export pid=${pid:-$$}
+export jobid=$pid
+export COMROOT=$DATAROOT
+export NMEM_ENS=3
+export ACCOUNT=da-cpu
+
+# Set python path for workflow utilities and tasks
+wxflowPATH="${HOMEgfs}/ush/python"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
+export PYTHONPATH
+
+# Detemine machine from config.base
+machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
+
+# Set NETCDF and UTILROOT variables (used in config.base)
+if [[ $machine = 'HERA' ]]; then
+    NETCDF=$( which ncdump )
+    export NETCDF
+    export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
+    ncdump=$( which ncdump )
+    NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
+    export NETCDF
+    export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
+fi
+
+# Set lobsdiag_forenkf=.true. to run letkf as stand-alone observer job
+cp $EXPDIR/config.base_lobsdiag_forenkf_true $EXPDIR/config.base
+
+# Execute j-job
+if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then
+    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait --output=atmensanlobs-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS
+else
+    ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS
+fi

test/atm/global-workflow/jjob_ens_sol.sh

@@ -0,0 +1,59 @@
+#! /usr/bin/env bash
+
+set -x
+bindir=$1
+srcdir=$2
+
+# Set g-w HOMEgfs
+topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P)
+export HOMEgfs=$topdir
+
+# Set variables for ctest
+export PSLOT=gdas_test
+export EXPDIR=$bindir/test/atm/global-workflow/testrun/experiments/$PSLOT
+export PDY=20210323
+export cyc=18
+export CDATE=${PDY}${cyc}
+export gPDY=20210323
+export gcyc=12
+export GDATE=${gPDY}${gcyc}
+export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS/$PSLOT
+export RUN=enkfgdas
+export CDUMP=enkfgdas
+export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIRS/$PSLOT
+export COMIN_GES=${bindir}/test/atm/bkg
+export pid=${pid:-$$}
+export jobid=$pid
+export COMROOT=$DATAROOT
+export NMEM_ENS=3
+export ACCOUNT=da-cpu
+
+# Set python path for workflow utilities and tasks
+wxflowPATH="${HOMEgfs}/ush/python"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
+export PYTHONPATH
+
+# Detemine machine from config.base
+machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
+
+# Set NETCDF and UTILROOT variables (used in config.base)
+if [[ $machine = 'HERA' ]]; then
+    NETCDF=$( which ncdump )
+    export NETCDF
+    export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
+    ncdump=$( which ncdump )
+    NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
+    export NETCDF
+    export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
+fi
+
+# Set lobsdiag_forenkf=.true. to run letkf as stand-alone solver job
+cp $EXPDIR/config.base_lobsdiag_forenkf_true $EXPDIR/config.base
+
+# Execute j-job
+if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then
+    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait --output=atmensanlsol-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL
+else
+    ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL
+fi