Concatenation of adt's into 1 ObsSpace, cp0.b QC for adt and sst

parm/soca/obs/config/adt_all.yaml

@@ -0,0 +1,50 @@
+obs space:
+  name: adt_all
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_all.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/adt_all.${PDY}${cyc}.nc4
+  simulated variables: [absoluteDynamicTopography]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: ADT
+obs error:
+  covariance model: diagonal
+obs filters:
+- filter: Domain Check
+  where:
+  - variable: {name: GeoVaLs/sea_area_fraction}
+    minvalue: 0.9
+- filter: Domain Check
+  where:
+  - variable: { name: GeoVaLs/sea_surface_temperature}
+    minvalue: 5.0
+- filter: Background Check
+  absolute threshold: 1.0
+- filter: Domain Check
+  where:
+  - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface}
+    minvalue: 500
+- filter: Perform Action
+  action:
+    name: assign error
+    error function:
+      name: ObsFunction/LinearCombination
+      options:
+        variables: [GeoVaLs/mesoscale_representation_error,
+                    ObsError/absoluteDynamicTopography]
+        coefs: [0.1,
+                0.5]
+- filter: Domain Check
+  where:
+  - variable: { name: GeoVaLs/sea_ice_area_fraction}
+    maxvalue: 0.00001
+- filter: Domain Check
+  where:
+  - variable: {name: GeoVaLs/distance_from_coast}
+    minvalue: 100e3

parm/soca/obs/config/adt_j2.yaml

@@ -0,0 +1,15 @@
+obs space:
+  name: adt_j2
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_j2.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/adt_j3.${PDY}${cyc}.nc4
+  simulated variables: [absoluteDynamicTopography]
+obs operator:
+  name: ADT
+obs error:
+  covariance model: diagonal

parm/soca/obs/config/sst_metopa_l3u_so025.yaml

@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

parm/soca/obs/config/sst_metopb_l3u_so025.yaml

@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

parm/soca/obs/config/sst_metopc_l3u_so025.yaml

@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

parm/soca/obs/config/sst_noaa18_l3u_so025.yaml

@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

parm/soca/obs/config/sst_noaa19_l3u_so025.yaml

@@ -13,6 +13,7 @@ obs space:
     max pool size: 1
 obs operator:
   name: Identity
+  observation alias file: obsop_name_map.yaml
 obs error:
   covariance model: diagonal
 obs filters:
@@ -28,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml

@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml

@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

parm/soca/obs/obs_list.yaml

@@ -1,5 +1,7 @@
 observers:
+- !INC ${OBS_YAML_DIR}/adt_all.yaml
 - !INC ${OBS_YAML_DIR}/adt_j3.yaml
+- !INC ${OBS_YAML_DIR}/adt_j2.yaml
 #- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml
 - !INC ${OBS_YAML_DIR}/sss_smap.yaml
 - !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml

scripts/exgdas_global_marine_analysis_prep.py

@@ -268,6 +268,11 @@ def find_clim_ens(input_date):
 # stage observations from R2D2 COMIN_OBS to COM_OBS
 ufsda.stage.obs(stage_cfg)
 
+# concatenate altimeters into one obs space
+# TODO (SAMG)temporary, move this into the obs procecing eventually
+adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt"
+ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True)
+
 # get the list of observations
 obs_files = []
 for ob in stage_cfg['observations']['observers']:

test/soca/create_obsdb.py

@@ -36,6 +36,7 @@ def cdl2nc(cdl_filename, nc4_filename):
     # Change the obs file name format
     obsdir = os.getenv('SOCA_TEST_OBS')
     cdl2nc(os.path.join(obsdir, 'adt.nc.cdl'), 'adt_j3_20180415.nc4')
+    cdl2nc(os.path.join(obsdir, 'adt.nc.cdl'), 'adt_j2_20180415.nc4')
     cdl2nc(os.path.join(obsdir, 'sst.nc.cdl'), 'sst_noaa19_l3u_20180415.nc4')
     cdl2nc(os.path.join(obsdir, 'sss.nc.cdl'), 'sss_smap_20180415.nc4')
     cdl2nc(os.path.join(obsdir, 'prof.nc.cdl'), 'temp_profile_fnmoc_20180415.nc4')
@@ -53,6 +54,7 @@ def cdl2nc(cdl_filename, nc4_filename):
                          'provider': 'gdasapp',
                          'experiment': 'soca',
                          'obs_types': ['adt_j3',
+                                       'adt_j2',
                                        'sst_noaa19_l3u',
                                        'sss_smap',
                                        'temp_profile_fnmoc',

ush/ufsda/soca_utils.py

@@ -53,8 +53,8 @@ def obs_specs(iodafname):
     raise Exception(f"No known obs type in {iodafname}.")
 
 
-def concatenate_ioda(iodafname):
-    flist = glob.glob(iodafname+'.*')
+def concatenate_ioda(iodafname, wildcard=".*", output_suffix="", clean=False):
+    flist = glob.glob(f"{iodafname}{wildcard}")
     flist.sort()
     nfiles = len(flist)
     if nfiles == 0:
@@ -63,7 +63,7 @@ def concatenate_ioda(iodafname):
 
     if len(flist) == 1:
         logging.info(f"Only file is {flist[0]}, rename to {iodafname}. No need to concatenate.")
-        shutil.move(flist[0], iodafname)
+        shutil.move(flist[0], iodafname+output_suffix)
         return
 
     logging.info(f"Concatenating {nfiles} files from globbing {iodafname}.*")
@@ -107,7 +107,12 @@ def concatenate_ioda(iodafname):
     globalattrs = {}
 
     # Write
-    writer = iconv.IodaWriter(iodafname, locationkeylist, dimdict)
+    writer = iconv.IodaWriter(iodafname+output_suffix, locationkeylist, dimdict)
     writer.BuildIoda(outdata, vardims, varattrs, globalattrs)
 
+    # Remove files
+    if clean:
+        for file in flist:
+            os.remove(file)
+
     return