From b97bce8f115b02d94dfb325012abd504ca57e280 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:22:43 -0400 Subject: [PATCH] Update ozone YAML files (#629) * Check in six updated YAMLs for ozone. --- parm/atm/obs/testing/omi_aura.yaml | 79 ++++------ parm/atm/obs/testing/omi_aura_noqc.yaml | 27 ++++ parm/atm/obs/testing/ompsnp_npp.yaml | 160 +++++++++++---------- parm/atm/obs/testing/ompsnp_npp_noqc.yaml | 32 +++++ parm/atm/obs/testing/ompstc8_npp.yaml | 61 ++++---- parm/atm/obs/testing/ompstc8_npp_noqc.yaml | 32 +++++ 6 files changed, 238 insertions(+), 153 deletions(-) create mode 100644 parm/atm/obs/testing/omi_aura_noqc.yaml create mode 100644 parm/atm/obs/testing/ompsnp_npp_noqc.yaml create mode 100644 parm/atm/obs/testing/ompstc8_npp_noqc.yaml diff --git a/parm/atm/obs/testing/omi_aura.yaml b/parm/atm/obs/testing/omi_aura.yaml index 1b786feed..a090f5945 100644 --- a/parm/atm/obs/testing/omi_aura.yaml +++ b/parm/atm/obs/testing/omi_aura.yaml @@ -19,89 +19,70 @@ obs operator: coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] -obs filters: -# range sanity check -- filter: Bounds Check +obs pre filters: +- filter: Perform Action filter variables: - name: ozoneTotal - minvalue: 0 - maxvalue: 1000 action: - name: reject + name: assign error + error parameter: 6.0 -# Toss observations for which the profile ozone error code is neither 0 nor 1 -- filter: RejectList +obs prior filters: +# GSI read routine QC +# range sanity check +- filter: Bounds Check filter variables: - name: ozoneTotal - where: - - variable: - name: MetaData/total_ozone_error_flag - minvalue: 0.1 + minvalue: 0 + maxvalue: 10000 action: name: reject -# -- filter: AcceptList - filter variables: - - name: ozoneTotal - where: - - variable: - name: MetaData/total_ozone_error_flag - minvalue: 0.9 - maxvalue: 1.1 - action: - name: accept -#exclude afbo=3 or 13 +# Do not use the data if row anomaly (bit 10)is 1 - filter: RejectList filter variables: - name: ozoneTotal where: - variable: - name: MetaData/bestOzoneAlgorithmFlag - minvalue: 2.9 - maxvalue: 3.1 - action: - name: reject + name: MetaData/totalOzoneQualityFlag + any_bit_set_of: 9 + +# Scan position check: reject scan position >= 25 - filter: RejectList filter variables: - name: ozoneTotal where: - variable: - name: MetaData/bestOzoneAlgorithmFlag - minvalue: 12.9 - maxvalue: 13.1 - action: - name: reject + name: MetaData/sensorScanPosition + minvalue: 25 -#Do not use the data if row anomaly (bit 10)is 1 -- filter: RejectList +# Accept total_ozone_error_flag values of 0 and 1, but not any others. +- filter: RejectList filter variables: - name: ozoneTotal where: - variable: - name: MetaData/row_anomaly_index - minvalue: 0.9 - maxvalue: 1.1 - action: - name: reject + name: MetaData/totalOzoneQualityCode + is_not_in: 0, 1 -#scan position check +# Use data with best ozone algorighm - filter: RejectList filter variables: - name: ozoneTotal where: - variable: - name: MetaData/sensorScanPosition - minvalue: 25 - action: - name: reject + name: MetaData/bestOzoneAlgorithmFlag + is_in: 3, 13 - # Gross check +obs post filters: +# GSI setup routine QC +# Gross check - filter: Background Check filter variables: - name: ozoneTotal - threshold: 5.0 + threshold: 10.0 + absolute threshold: 300.0 action: name: reject -passedBenchmark: 2329 \ No newline at end of file +passedBenchmark: 1182 diff --git a/parm/atm/obs/testing/omi_aura_noqc.yaml b/parm/atm/obs/testing/omi_aura_noqc.yaml new file mode 100644 index 000000000..8929c38b6 --- /dev/null +++ b/parm/atm/obs/testing/omi_aura_noqc.yaml @@ -0,0 +1,27 @@ +obs space: + name: omi_aura + obsdatain: + engine: + type: H5File + obsfile: !ENV omi_aura_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV omi_aura_diag_${CDATE}.nc4 + io pool: + max pool size: 1 + simulated variables: [ozoneTotal] +geovals: + filename: !ENV omi_aura_geoval_${CDATE}.nc4 +obs operator: + name: AtmVertInterpLay + geovals: [mole_fraction_of_ozone_in_air] + coefficients: [0.007886131] # convert from ppmv to DU + nlevels: [1] + +vector ref: GsiHofXBc +tolerance: 1.e-5 +#linear obs operator test: +## coef TL: 0.1 +## tolerance TL: 1.0e-13 +## tolerance AD: 1.0e-11 diff --git a/parm/atm/obs/testing/ompsnp_npp.yaml b/parm/atm/obs/testing/ompsnp_npp.yaml index 6d9ce0f71..f7ee9d4ec 100644 --- a/parm/atm/obs/testing/ompsnp_npp.yaml +++ b/parm/atm/obs/testing/ompsnp_npp.yaml @@ -22,99 +22,78 @@ obs operator: geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [22] - -obs filters: -# range sanity check -- filter: Bounds Check + +obs pre filters: +# Observation error assignment +- filter: Perform Action filter variables: - name: ozoneLayer - minvalue: 0 - maxvalue: 1000 action: - name: reject + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] + errors: [7.7236, 0.02, 0.02, 0.025, 0.08, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] -# Accept total_ozone_error_flag values of 0 and 2, but not any others. +obs prior filters: +# Do not assimilation where pressure is zero +# Zero pressure indicates the data is total column ozone - filter: RejectList filter variables: - name: ozoneLayer where: - variable: - name: MetaData/total_ozone_error_flag - minvalue: 0.1 - action: - name: reject -# -- filter: AcceptList - filter variables: - - name: ozoneLayer - where: - - variable: - name: MetaData/total_ozone_error_flag - minvalue: 1.9 - maxvalue: 2.1 - action: - name: accept + name: MetaData/pressure + maxvalue: 0.0001 -# -# Accept profile_ozone_error_flag values of 0, 1, and 7, but not any others. -- filter: RejectList +# Sanity check on observaton values +- filter: Bounds Check filter variables: - name: ozoneLayer - where: - - variable: - name: MetaData/profile_ozone_error_flag - minvalue: 1.1 + minvalue: 0 + maxvalue: 1000 action: name: reject -# -- filter: AcceptList + +# Total Ozone Quality Check (keeps 0, 2) +# 0 indentifies good data +# 2 identifies good data with a solar zenith angle > 84 degrees +- filter: RejectList filter variables: - name: ozoneLayer where: - variable: - name: MetaData/profile_ozone_error_flag - minvalue: 6.9 - maxvalue: 7.1 - action: - name: accept -# -# Reject all the pressure levels higher in height than pressure of 0.64hPa + name: MetaData/totalOzoneQuality + is_not_in: 0, 2 + +# Profile Ozone Quality Check (keeps 0, 1, 7) +# 0 : good data +# 1 : good data with a solar zenith angle > 84 degrees +# 7 : profile for which stray light correction applied - filter: RejectList filter variables: - name: ozoneLayer where: - variable: - name: MetaData/pressure - maxvalue: 64.0 - action: - name: reject -# -# Assign initial error value based on pressure -- filter: Perform Action - filter variables: - - name: ozoneLayer - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] - errors: [7.7236, 0.04, 0.02, 0.015, 0.03, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] + name: MetaData/profileOzoneQuality + is_not_in: 0, 1, 7 -# Gross error check with (O - B) / ObsError greater than threshold. +obs post filters: +# Gross error check - filter: Background Check filter variables: - name: ozoneLayer - absolute threshold: 100 + absolute threshold: 120 action: name: reject where: - variable: name: MetaData/pressure maxvalue: 0.001 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -126,7 +105,7 @@ obs filters: name: MetaData/pressure minvalue: 30000.0 maxvalue: 110000.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -138,7 +117,7 @@ obs filters: name: MetaData/pressure minvalue: 20000.0 maxvalue: 30000.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -150,7 +129,7 @@ obs filters: name: MetaData/pressure minvalue: 10100.0 maxvalue: 20000.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -163,7 +142,6 @@ obs filters: minvalue: 6400.0 maxvalue: 10100.0 -# - filter: Background Check filter variables: - name: ozoneLayer @@ -175,7 +153,7 @@ obs filters: name: MetaData/pressure minvalue: 4000.0 maxvalue: 6400.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -187,7 +165,7 @@ obs filters: name: MetaData/pressure minvalue: 2600.0 maxvalue: 4000.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -199,7 +177,7 @@ obs filters: name: MetaData/pressure minvalue: 1600.0 maxvalue: 2600.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -211,7 +189,7 @@ obs filters: name: MetaData/pressure minvalue: 1100.0 maxvalue: 1600.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -223,7 +201,7 @@ obs filters: name: MetaData/pressure minvalue: 700.0 maxvalue: 1100.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -235,7 +213,7 @@ obs filters: name: MetaData/pressure minvalue: 400.0 maxvalue: 700.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -247,7 +225,7 @@ obs filters: name: MetaData/pressure minvalue: 300.0 maxvalue: 400.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -259,7 +237,7 @@ obs filters: name: MetaData/pressure minvalue: 200.0 maxvalue: 300.0 -# + - filter: Background Check filter variables: - name: ozoneLayer @@ -269,6 +247,42 @@ obs filters: where: - variable: name: MetaData/pressure + maxvalue: 70.0 maxvalue: 200.0 -passedBenchmark: 3744 +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 5 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 40.0 + maxvalue: 70.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 2 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 30.0 + maxvalue: 40.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 1 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 30.0 + +passedBenchmark: 4914 diff --git a/parm/atm/obs/testing/ompsnp_npp_noqc.yaml b/parm/atm/obs/testing/ompsnp_npp_noqc.yaml new file mode 100644 index 000000000..219f9e942 --- /dev/null +++ b/parm/atm/obs/testing/ompsnp_npp_noqc.yaml @@ -0,0 +1,32 @@ +obs space: + name: ompsnp_npp + obsdatain: + engine: + type: H5File + obsfile: !ENV ompsnp_npp_obs_${CDATE}.nc4 + obsgrouping: + group variables: ["latitude"] + sort variable: "pressure" + sort order: "ascending" + obsdataout: + engine: + type: H5File + obsfile: !ENV ompsnp_npp_diag_${CDATE}.nc4 + io pool: + max pool size: 1 + simulated variables: [ozoneLayer] +geovals: + filename: !ENV ompsnp_npp_geoval_${CDATE}.nc4 +obs operator: + name: AtmVertInterpLay + geovals: [mole_fraction_of_ozone_in_air] + coefficients: [0.007886131] # convert from ppmv to DU + nlevels: [22] + +vector ref: GsiHofXBc +tolerance: 1.e-5 +#linear obs operator test: +# coef TL: 0.1 +# tolerance TL: 1.0e-13 +# tolerance AD: 1.0e-11 + diff --git a/parm/atm/obs/testing/ompstc8_npp.yaml b/parm/atm/obs/testing/ompstc8_npp.yaml index 43389dab2..aea18a5dc 100644 --- a/parm/atm/obs/testing/ompstc8_npp.yaml +++ b/parm/atm/obs/testing/ompstc8_npp.yaml @@ -23,7 +23,16 @@ obs operator: coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] -obs filters: +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +obs prior filters: +# GSI read routine QC # range sanity check - filter: Bounds Check filter variables: @@ -39,57 +48,47 @@ obs filters: - name: ozoneTotal where: - variable: - name: MetaData/total_ozone_error_flag - minvalue: 0.1 - action: - name: reject -# -- filter: AcceptList - filter variables: - - name: ozoneTotal - where: - - variable: - name: MetaData/total_ozone_error_flag - minvalue: 0.9 - maxvalue: 1.1 - action: - name: accept + name: MetaData/totalOzoneQualityCode + is_not_in: 0, 1 -#exclude afbo=3 or 13 - filter: RejectList filter variables: - name: ozoneTotal where: - variable: name: MetaData/bestOzoneAlgorithmFlag - minvalue: 2.9 - maxvalue: 3.1 - action: - name: reject + is_in: 3, 13 + +# GSI setup routine QC - filter: RejectList filter variables: - name: ozoneTotal where: - variable: - name: MetaData/bestOzoneAlgorithmFlag - minvalue: 12.9 - maxvalue: 13.1 - action: - name: reject + name: MetaData/sensorScanPosition + is_in: 1, 2, 3, 4, 35 + - variable: + name: MetaData/latitude + minvalue: 50.0 -#scan position check - filter: RejectList filter variables: - name: ozoneTotal where: - variable: name: MetaData/sensorScanPosition - minvalue: 1 - maxvalue: 4 + is_in: 1, 2, 3, 4, 35 - variable: name: MetaData/latitude - minvalue: 50.0 + maxvalue: -50.0 + +obs post filters: +- filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 10.0 + absolute threshold: 300.0 action: name: reject -passedBenchmark: 5908 +passedBenchmark: 6130 diff --git a/parm/atm/obs/testing/ompstc8_npp_noqc.yaml b/parm/atm/obs/testing/ompstc8_npp_noqc.yaml new file mode 100644 index 000000000..31027007a --- /dev/null +++ b/parm/atm/obs/testing/ompstc8_npp_noqc.yaml @@ -0,0 +1,32 @@ +obs space: + name: ompstc8_npp + obsdatain: + engine: + type: H5File + obsfile: !ENV ompstc8_npp_obs_${CDATE}.nc4 + obsgrouping: + group variables: ["latitude"] + sort variable: "pressure" + sort order: "ascending" + obsdataout: + engine: + type: H5File + obsfile: !ENV ompstc8_npp_diag_${CDATE}.nc4 + io pool: + max pool size: 1 + simulated variables: [ozoneTotal] +geovals: + filename: !ENV ompstc8_npp_geoval_${CDATE}.nc4 +obs operator: + name: AtmVertInterpLay + geovals: [mole_fraction_of_ozone_in_air] + coefficients: [0.007886131] # convert from ppmv to DU + nlevels: [1] + +vector ref: GsiHofXBc +tolerance: 1.e-5 +#linear obs operator test: +# coef TL: 0.1 +# tolerance TL: 1.0e-13 +# tolerance AD: 1.0e-11 +