Merge branch 'develop' into feature/smos2ioda

parm/atm/obs/testing/conv_ps.yaml

@@ -0,0 +1,212 @@
+obs space:
+  name: surface_ps
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV conv_ps_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV  conv_ps_diag_${CDATE}.nc4
+      overwrite: true
+  simulated variables: [stationPressure]
+geovals:
+  filename: !ENV conv_ps_geoval_${CDATE}.nc4
+
+obs operator:
+  name: SfcPCorrected
+  variables:
+  - name: stationPressure
+  da_psfc_scheme: GSI
+  station_altitude: height
+  geovar_sfc_geomz: surface_altitude
+  geovar_geomz: geopotential_height
+
+obs post filters:
+  # Observation range sanity check
+  - filter: Bounds Check
+    filter variables:
+    - name: stationPressure
+    minvalue: 37499.0
+    maxvalue: 106999.0
+    action:
+      name: reject
+
+  # Reject all ObsType 183
+  - filter: RejectList
+    where:
+    - variable:
+        name: ObsType/stationPressure
+      is_in: 183
+
+  # Reject surface pressure below 500 hPa 
+  - filter: Bounds Check
+    filter variables:
+    - name: stationPressure
+    minvalue: 50000.00
+    action:
+      name: reject
+
+  - filter: RejectList
+    where:
+    - variable:
+        name: PreQC/stationPressure
+      is_in: 4-15
+
+  # Inflate obs error based on obs type
+  - filter: Perform Action 
+    filter variables:
+    - name: stationPressure
+    where:
+    - variable: PreQC/stationPressure
+      is_in: 3, 7   
+    action:
+      name: inflate error
+      inflation factor: 1.2  
+
+  # Calculate obs error inflation factors for duplicated observations at the same location
+  - filter: Variable Assignment
+    assignments:
+    - name: ObsErrorFactorDuplicateCheck/stationPressure
+      type: float
+      function:
+        name: ObsFunction/ObsErrorFactorDuplicateCheck
+        options:
+          use_air_pressure: false 
+          variable: stationPressure 
+
+  # Reduce effective observation error based on obs type and subtype
+  # In this case: reduce effective obs error for buoy  
+  - filter: Perform Action
+    filter variables:
+    - name: stationPressure
+    where:
+    - variable:
+        name: ObsType/stationPressure
+      is_in: 180
+    - variable:
+        name: ObsSubType/stationPressure
+      is_in: 0
+    action:
+      name: inflate error
+      inflation factor: 0.7
+
+  # Reduce original observation error based on obs type and subtype
+  # In this case: reduce original obs error for buoy 
+  - filter: Variable Assignment
+    where:
+    - variable:
+        name: ObsType/stationPressure
+      is_in: 180
+    - variable:
+        name: ObsSubType/stationPressure
+      is_in: 0
+    assignments:
+    - name: ObsError/stationPressure
+      type: float
+      function:
+        name: ObsFunction/Arithmetic 
+        options: 
+          variables:
+          - name: ObsError/stationPressure
+          coefs: [0.7]
+
+  # Calculate obs error inflation factors for large discrepancies between model and observations 
+  - filter: Variable Assignment
+    assignments:
+    - name: ObsErrorFactorSfcPressure/stationPressure
+      type: float
+      function:
+        name: ObsFunction/ObsErrorFactorSfcPressure
+        options:
+          geovar_sfc_geomz: surface_altitude
+
+  # Inflate surface pressure observation based on discrepancies between
+  # model and observations due to terrian
+  - filter: Perform Action
+    filter variables:
+    - name: stationPressure
+    action:
+      name: inflate error
+      inflation variable:
+        name: ObsErrorFactorSfcPressure/stationPressure
+
+  - filter: Variable Assignment
+    assignments:
+    - name: DerivedMetaData/Innovation
+      type: float
+      function:
+        name: ObsFunction/Arithmetic
+        options:
+          variables:
+          - name: ObsValue/stationPressure
+          - name: HofX/stationPressure
+          coefs: [1, -1]
+
+  - filter: Variable Assignment
+    assignments:
+    - name: DerivedMetaData/ObsErrorBoundSfcPressure1
+      type: float
+      function:
+        name: ObsFunction/ObsErrorBoundConventional
+        options:
+          obsvar: stationPressure
+          obserr_bound_min: 100
+          obserr_bound_max: 300
+          obserr_bound_factor: 5.0 
+
+  - filter: Background Check 
+    filter variables:
+    - name: stationPressure
+    where:
+    - variable: PreQC/stationPressure
+      is_not_in: 3
+    function absolute threshold:
+    - name: DerivedMetaData/ObsErrorBoundSfcPressure1
+    action:
+      name: reject
+
+  - filter: Variable Assignment
+    assignments:
+    - name: DerivedMetaData/ObsErrorBoundSfcPressure2
+      type: float
+      function:
+        name: ObsFunction/ObsErrorBoundConventional
+        options:
+          obsvar: stationPressure
+          obserr_bound_min: 100
+          obserr_bound_max: 300
+          obserr_bound_factor: 3.5 
+
+  - filter: Background Check 
+    filter variables:
+    - name: stationPressure
+    where:
+    - variable: PreQC/stationPressure
+      is_in: 3
+    function absolute threshold:
+    - name: DerivedMetaData/ObsErrorBoundSfcPressure2
+    action:
+      name: reject
+
+  # Inflate obs error based on duplicate check
+  - filter: Perform Action
+    filter variables:
+    - name: stationPressure
+    action:
+      name: inflate error
+      inflation variable:
+        name: ObsErrorFactorDuplicateCheck/stationPressure
+
+  # Reject data based on PreUseFlag (usage in GSI) 
+  - filter: Perform Action
+    filter variables:
+    - name: stationPressure
+    where:
+    - variable: PreUseFlag/stationPressure
+      is_not_in: 0, 1
+    action:
+      name: reject 
+
+passedBenchmark: 85378 
+

parm/atm/obs/testing/conv_ps_noqc.yaml

@@ -0,0 +1,31 @@
+obs space:
+  name: surface_ps 
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV conv_ps_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV  conv_ps_diag_${CDATE}.nc4
+      overwrite: true
+  simulated variables: [stationPressure]
+geovals:
+  filename: !ENV conv_ps_geoval_${CDATE}.nc4
+
+obs operator:
+  name: SfcPCorrected
+  variables:
+  - name: stationPressure
+  da_psfc_scheme: GSI
+  station_altitude: height
+  geovar_sfc_geomz: surface_altitude
+  geovar_geomz: geopotential_height
+
+vector ref: GsiHofXBc
+tolerance: 1.e-4 
+#linear obs operator test:
+#  coef TL: 0.1
+#  tolerance TL: 1.0e-13
+#  tolerance AD: 1.0e-11
+

parm/atm/obs/testing/omi_aura.yaml

@@ -19,89 +19,70 @@ obs operator:
   coefficients: [0.007886131] # convert from ppmv to DU
   nlevels: [1]
 
-obs filters:
-# range sanity check
-- filter: Bounds Check
+obs pre filters:
+- filter: Perform Action
   filter variables:
   - name: ozoneTotal
-  minvalue: 0
-  maxvalue: 1000
   action:
-    name: reject
+    name: assign error
+    error parameter: 6.0
 
-# Toss observations for which the profile ozone error code is neither 0 nor 1
-- filter: RejectList
+obs prior filters:
+# GSI read routine QC
+# range sanity check
+- filter: Bounds Check
   filter variables:
   - name: ozoneTotal
-  where:
-  - variable:
-      name: MetaData/total_ozone_error_flag
-    minvalue: 0.1
+  minvalue: 0
+  maxvalue: 10000
   action:
     name: reject
-#
-- filter: AcceptList
-  filter variables:
-  - name: ozoneTotal
-  where:
-  - variable:
-      name: MetaData/total_ozone_error_flag
-    minvalue: 0.9
-    maxvalue: 1.1
-  action:
-    name: accept
 
-#exclude afbo=3 or 13
+# Do not use the data if row anomaly (bit 10)is 1
 - filter: RejectList
   filter variables:
   - name: ozoneTotal
   where:
   - variable:
-      name: MetaData/bestOzoneAlgorithmFlag
-    minvalue: 2.9
-    maxvalue: 3.1
-  action:
-    name: reject
+      name: MetaData/totalOzoneQualityFlag
+    any_bit_set_of: 9 
+
+# Scan position check: reject scan position >= 25
 - filter: RejectList
   filter variables:
   - name: ozoneTotal
   where:
   - variable:
-      name: MetaData/bestOzoneAlgorithmFlag
-    minvalue: 12.9
-    maxvalue: 13.1
-  action:
-    name: reject
+      name: MetaData/sensorScanPosition
+    minvalue: 25
 
-#Do not use the data if row anomaly (bit 10)is 1
-- filter: RejectList
+# Accept total_ozone_error_flag values of 0 and 1, but not any others.
+- filter: RejectList 
   filter variables:
   - name: ozoneTotal
   where:
   - variable:
-      name: MetaData/row_anomaly_index
-    minvalue: 0.9
-    maxvalue: 1.1
-  action:
-    name: reject
+      name: MetaData/totalOzoneQualityCode
+    is_not_in: 0, 1
 
-#scan position check
+# Use data with best ozone algorighm
 - filter: RejectList
   filter variables:
   - name: ozoneTotal
   where:
   - variable:
-      name: MetaData/sensorScanPosition
-    minvalue: 25
-  action:
-    name: reject
+      name: MetaData/bestOzoneAlgorithmFlag
+    is_in: 3, 13
 
- # Gross check
+obs post filters:
+# GSI setup routine QC
+# Gross check
 - filter: Background Check
   filter variables:
   - name: ozoneTotal
-  threshold: 5.0
+  threshold: 10.0
+  absolute threshold: 300.0
   action:
     name: reject
 
-passedBenchmark: 2329
+passedBenchmark: 1182 

parm/atm/obs/testing/omi_aura_noqc.yaml

@@ -0,0 +1,27 @@
+obs space:
+  name: omi_aura
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV omi_aura_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV omi_aura_diag_${CDATE}.nc4
+  io pool:
+    max pool size: 1
+  simulated variables: [ozoneTotal]
+geovals:
+  filename: !ENV omi_aura_geoval_${CDATE}.nc4
+obs operator:
+  name: AtmVertInterpLay
+  geovals: [mole_fraction_of_ozone_in_air]
+  coefficients: [0.007886131] # convert from ppmv to DU
+  nlevels: [1]
+
+vector ref: GsiHofXBc
+tolerance: 1.e-5
+#linear obs operator test:
+##  coef TL: 0.1
+##  tolerance TL: 1.0e-13
+##  tolerance AD: 1.0e-11