From e40fe043976fddeb33de2f075906732b551954f1 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 1 Sep 2023 09:18:21 -0400 Subject: [PATCH] Concatenation of adt's into 1 ObsSpace, cp0.b QC for adt and sst --- parm/soca/obs/config/sst_metopa_l3u_so025.yaml | 6 +++--- parm/soca/obs/config/sst_metopb_l3u_so025.yaml | 6 +++--- parm/soca/obs/config/sst_metopc_l3u_so025.yaml | 6 +++--- parm/soca/obs/config/sst_noaa18_l3u_so025.yaml | 6 +++--- parm/soca/obs/config/sst_noaa19_l3u_so025.yaml | 7 ++++--- parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml | 6 +++--- parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml | 6 +++--- parm/soca/obs/obs_list.yaml | 2 ++ scripts/exgdas_global_marine_analysis_prep.py | 5 +++++ test/soca/create_obsdb.py | 2 ++ ush/ufsda/soca_utils.py | 13 +++++++++---- 11 files changed, 40 insertions(+), 25 deletions(-) diff --git a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml b/parm/soca/obs/config/sst_metopa_l3u_so025.yaml index c1bdcd8df..69b9d1cba 100644 --- a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopa_l3u_so025.yaml @@ -29,15 +29,15 @@ obs filters: - filter: Domain Check where: - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 0.001 + minvalue: 1.0e-8 - filter: Domain Check where: - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 0.00001 + maxvalue: 1.0e-5 - filter: Domain Check where: - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: 5.0 + minvalue: -1.0 - filter: Domain Check where: - variable: {name: GeoVaLs/distance_from_coast} diff --git a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml b/parm/soca/obs/config/sst_metopb_l3u_so025.yaml index 9826b585c..6b42880b7 100644 --- a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopb_l3u_so025.yaml @@ -29,15 +29,15 @@ obs filters: - filter: Domain Check where: - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 0.001 + minvalue: 1.0e-8 - filter: Domain Check where: - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 0.00001 + maxvalue: 1.0e-5 - filter: Domain Check where: - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: 5.0 + minvalue: -1.0 - filter: Domain Check where: - variable: {name: GeoVaLs/distance_from_coast} diff --git a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml b/parm/soca/obs/config/sst_metopc_l3u_so025.yaml index 9f1a22f0b..26d5453ec 100644 --- a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopc_l3u_so025.yaml @@ -29,15 +29,15 @@ obs filters: - filter: Domain Check where: - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 0.001 + minvalue: 1.0e-8 - filter: Domain Check where: - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 0.00001 + maxvalue: 1.0e-5 - filter: Domain Check where: - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: 5.0 + minvalue: -1.0 - filter: Domain Check where: - variable: {name: GeoVaLs/distance_from_coast} diff --git a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml b/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml index 01a960034..21bd216b7 100644 --- a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml @@ -29,15 +29,15 @@ obs filters: - filter: Domain Check where: - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 0.001 + minvalue: 1.0e-8 - filter: Domain Check where: - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 0.00001 + maxvalue: 1.0e-5 - filter: Domain Check where: - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: 5.0 + minvalue: -1.0 - filter: Domain Check where: - variable: {name: GeoVaLs/distance_from_coast} diff --git a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml b/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml index 7a51edb62..83d0db9c2 100644 --- a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml @@ -13,6 +13,7 @@ obs space: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: @@ -28,15 +29,15 @@ obs filters: - filter: Domain Check where: - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 0.001 + minvalue: 1.0e-8 - filter: Domain Check where: - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 0.00001 + maxvalue: 1.0e-5 - filter: Domain Check where: - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: 5.0 + minvalue: -1.0 - filter: Domain Check where: - variable: {name: GeoVaLs/distance_from_coast} diff --git a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml b/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml index 530e7db5a..d3154560c 100644 --- a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml @@ -29,15 +29,15 @@ obs filters: - filter: Domain Check where: - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 0.001 + minvalue: 1.0e-8 - filter: Domain Check where: - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 0.00001 + maxvalue: 1.0e-5 - filter: Domain Check where: - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: 5.0 + minvalue: -1.0 - filter: Domain Check where: - variable: {name: GeoVaLs/distance_from_coast} diff --git a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml b/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml index 2b2bbcafb..7dadb1278 100644 --- a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml @@ -29,15 +29,15 @@ obs filters: - filter: Domain Check where: - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 0.001 + minvalue: 1.0e-8 - filter: Domain Check where: - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 0.00001 + maxvalue: 1.0e-5 - filter: Domain Check where: - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: 5.0 + minvalue: -1.0 - filter: Domain Check where: - variable: {name: GeoVaLs/distance_from_coast} diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index e7c68c57a..b9d9e24a4 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -1,5 +1,7 @@ observers: +- !INC ${OBS_YAML_DIR}/adt_all.yaml - !INC ${OBS_YAML_DIR}/adt_j3.yaml +- !INC ${OBS_YAML_DIR}/adt_j2.yaml #- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml - !INC ${OBS_YAML_DIR}/sss_smap.yaml - !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py index 34eca899c..709230962 100755 --- a/scripts/exgdas_global_marine_analysis_prep.py +++ b/scripts/exgdas_global_marine_analysis_prep.py @@ -268,6 +268,11 @@ def find_clim_ens(input_date): # stage observations from R2D2 COMIN_OBS to COM_OBS ufsda.stage.obs(stage_cfg) +# concatenate altimeters into one obs space +# TODO (SAMG)temporary, move this into the obs procecing eventually +adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt" +ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True) + # get the list of observations obs_files = [] for ob in stage_cfg['observations']['observers']: diff --git a/test/soca/create_obsdb.py b/test/soca/create_obsdb.py index 6b3e753a9..99f9179c4 100755 --- a/test/soca/create_obsdb.py +++ b/test/soca/create_obsdb.py @@ -36,6 +36,7 @@ def cdl2nc(cdl_filename, nc4_filename): # Change the obs file name format obsdir = os.getenv('SOCA_TEST_OBS') cdl2nc(os.path.join(obsdir, 'adt.nc.cdl'), 'adt_j3_20180415.nc4') + cdl2nc(os.path.join(obsdir, 'adt.nc.cdl'), 'adt_j2_20180415.nc4') cdl2nc(os.path.join(obsdir, 'sst.nc.cdl'), 'sst_noaa19_l3u_20180415.nc4') cdl2nc(os.path.join(obsdir, 'sss.nc.cdl'), 'sss_smap_20180415.nc4') cdl2nc(os.path.join(obsdir, 'prof.nc.cdl'), 'temp_profile_fnmoc_20180415.nc4') @@ -53,6 +54,7 @@ def cdl2nc(cdl_filename, nc4_filename): 'provider': 'gdasapp', 'experiment': 'soca', 'obs_types': ['adt_j3', + 'adt_j2', 'sst_noaa19_l3u', 'sss_smap', 'temp_profile_fnmoc', diff --git a/ush/ufsda/soca_utils.py b/ush/ufsda/soca_utils.py index 0d3c3c0e1..83bfb9d7b 100644 --- a/ush/ufsda/soca_utils.py +++ b/ush/ufsda/soca_utils.py @@ -53,8 +53,8 @@ def obs_specs(iodafname): raise Exception(f"No known obs type in {iodafname}.") -def concatenate_ioda(iodafname): - flist = glob.glob(iodafname+'.*') +def concatenate_ioda(iodafname, wildcard=".*", output_suffix="", clean=False): + flist = glob.glob(f"{iodafname}{wildcard}") flist.sort() nfiles = len(flist) if nfiles == 0: @@ -63,7 +63,7 @@ def concatenate_ioda(iodafname): if len(flist) == 1: logging.info(f"Only file is {flist[0]}, rename to {iodafname}. No need to concatenate.") - shutil.move(flist[0], iodafname) + shutil.move(flist[0], iodafname+output_suffix) return logging.info(f"Concatenating {nfiles} files from globbing {iodafname}.*") @@ -107,7 +107,12 @@ def concatenate_ioda(iodafname): globalattrs = {} # Write - writer = iconv.IodaWriter(iodafname, locationkeylist, dimdict) + writer = iconv.IodaWriter(iodafname+output_suffix, locationkeylist, dimdict) writer.BuildIoda(outdata, vardims, varattrs, globalattrs) + # Remove files + if clean: + for file in flist: + os.remove(file) + return