From bd5cb2ee884cb8a08cb453385b9a28bb276d40bd Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:39:13 -0400 Subject: [PATCH] Adding ocean obs prep script and module (#635) * added JGLOBAL_PREP_OCEAN_OBS to ctests * first draft of marine obs prep * tweaks * new ex script, refinements to class * more obs, more problems * cleanup, obs_list with all avail yamls * linter cleanup * linter cleanup * linter corrections * rename and add info header to ex script * more obs classes * ripped out fancy OO version, now plain vanilla * fixe amsr2 ice yamls * added obs sources, cleaned up others, pep8 checked * more sst obs sources * change prints to logs * cleanup * tweaks to SST sources * removed two obs sources * added and subtracted obs yamls * more yamls * more yaml tweaks --- parm/soca/obs/config/icec_amsr2_north.yaml | 2 +- parm/soca/obs/config/icec_amsr2_south.yaml | 2 +- scripts/exglobal_prep_ocean_obs.py | 17 ++++ ush/soca/obs_list_test.yaml | 66 +++++++++++++++ ush/soca/prep_marine_obs.py | 94 ++++++++++++++++++++++ 5 files changed, 179 insertions(+), 2 deletions(-) create mode 100755 scripts/exglobal_prep_ocean_obs.py create mode 100644 ush/soca/obs_list_test.yaml create mode 100755 ush/soca/prep_marine_obs.py diff --git a/parm/soca/obs/config/icec_amsr2_north.yaml b/parm/soca/obs/config/icec_amsr2_north.yaml index 5819b770e..8554c2525 100644 --- a/parm/soca/obs/config/icec_amsr2_north.yaml +++ b/parm/soca/obs/config/icec_amsr2_north.yaml @@ -1,5 +1,5 @@ obs space: - name: icec_ssmis_f17_north + name: icec_amsr2_north obsdatain: engine: type: H5File diff --git a/parm/soca/obs/config/icec_amsr2_south.yaml b/parm/soca/obs/config/icec_amsr2_south.yaml index 89d95122b..8ebcf604e 100644 --- a/parm/soca/obs/config/icec_amsr2_south.yaml +++ b/parm/soca/obs/config/icec_amsr2_south.yaml @@ -1,5 +1,5 @@ obs space: - name: icec_ssmis_f17_south + name: icec_amsr2_south obsdatain: engine: type: H5File diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py new file mode 100755 index 000000000..da12d2b7b --- /dev/null +++ b/scripts/exglobal_prep_ocean_obs.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# exglobal_prep_ocean_obs.py +# Pepares observations for marine DA +import os +from wxflow import YAMLFile +import prep_marine_obs +import logging + +OBS_YAML = os.getenv('OBS_YAML') + +data = YAMLFile(OBS_YAML) +print(data) + +for observer in data['observers']: + obs_source_name = observer['obs space']['name'] + logging.info(f"obs_source_name: {obs_source_name}") + prep_marine_obs.obs_fetch(obs_source_name) diff --git a/ush/soca/obs_list_test.yaml b/ush/soca/obs_list_test.yaml new file mode 100644 index 000000000..0cb2f9d87 --- /dev/null +++ b/ush/soca/obs_list_test.yaml @@ -0,0 +1,66 @@ +observers: +- !INC ${OBS_YAML_DIR}/adt_3a_egm2008.yaml +- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml +- !INC ${OBS_YAML_DIR}/adt_6a_egm2008.yaml +- !INC ${OBS_YAML_DIR}/adt_all.yaml +- !INC ${OBS_YAML_DIR}/adt_c2_egm2008.yaml +- !INC ${OBS_YAML_DIR}/adt_coperl4.yaml +- !INC ${OBS_YAML_DIR}/adt_j2_egm2008.yaml +- !INC ${OBS_YAML_DIR}/adt_j3_egm2008.yaml +- !INC ${OBS_YAML_DIR}/adt_sa_egm2008.yaml +- !INC ${OBS_YAML_DIR}/bufr_sfcshipsu.yaml +- !INC ${OBS_YAML_DIR}/bufr_sfcships.yaml +- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml +- !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml +- !INC ${OBS_YAML_DIR}/icec_emc.yaml +- !INC ${OBS_YAML_DIR}/icec_nsidc_nh.yaml +- !INC ${OBS_YAML_DIR}/icec_nsidc_sh.yaml +- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_north.yaml +- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_south.yaml +- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_north.yaml +- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_south.yaml +- !INC ${OBS_YAML_DIR}/icefb_gdr.yaml +- !INC ${OBS_YAML_DIR}/salt_bufr_dbuoyprof.yaml +- !INC ${OBS_YAML_DIR}/salt_bufr_mbuoybprof.yaml +- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc_godas.yaml +- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml +- !INC ${OBS_YAML_DIR}/salt_profile_wod.yaml +- !INC ${OBS_YAML_DIR}/sss_smap_jpl.yaml +- !INC ${OBS_YAML_DIR}/sss_smap.yaml +- !INC ${OBS_YAML_DIR}/sss_smos_esa.yaml +- !INC ${OBS_YAML_DIR}/sss_smos.yaml +- !INC ${OBS_YAML_DIR}/sss_trak_fnmoc.yaml +- !INC ${OBS_YAML_DIR}/sst_amsr2_l3u.yaml +- !INC ${OBS_YAML_DIR}/sst_drifter.yaml +- !INC ${OBS_YAML_DIR}/sst_gmi_l3u.yaml +- !INC ${OBS_YAML_DIR}/sst_metopa_l3u_so025.yaml +- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml +- !INC ${OBS_YAML_DIR}/sst_metopc_l3u_so025.yaml +- !INC ${OBS_YAML_DIR}/sst_noaa18_l3u_so025.yaml +- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u_so025.yaml +- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml +- !INC ${OBS_YAML_DIR}/sst_ostia.yaml +- !INC ${OBS_YAML_DIR}/sst_ship_fnmoc.yaml +- !INC ${OBS_YAML_DIR}/sst_trak_fnmoc.yaml +- !INC ${OBS_YAML_DIR}/sst_viirs_n20_l3u_so025.yaml +- !INC ${OBS_YAML_DIR}/sst_viirs_npp_l3u_so025.yaml +- !INC ${OBS_YAML_DIR}/sst_windsat_l3u.yaml +- !INC ${OBS_YAML_DIR}/temp_bufr_dbuoyprof.yaml +- !INC ${OBS_YAML_DIR}/temp_bufr_mbuoybprof.yaml +- !INC ${OBS_YAML_DIR}/temp_profile_fnmoc_godas.yaml +- !INC ${OBS_YAML_DIR}/temp_profile_fnmoc.yaml +- !INC ${OBS_YAML_DIR}/temp_profile_wod.yaml +- !INC ${OBS_YAML_DIR}/insitu_argo_profile.yaml +- !INC ${OBS_YAML_DIR}/insitu_glider_profile.yaml +- !INC ${OBS_YAML_DIR}/insitu_bathy_profile.yaml +- !INC ${OBS_YAML_DIR}/insitu_tesac_profile.yaml +- !INC ${OBS_YAML_DIR}/insitu_xbtctd_profile.yaml +- !INC ${OBS_YAML_DIR}/insitu_dbuoy_profile.yaml, +- !INC ${OBS_YAML_DIR}/insitu_mbuoy_profile.yaml +- !INC ${OBS_YAML_DIR}/insitu_trkob_surface.yaml +- !INC ${OBS_YAML_DIR}/insitu_altkob_surface.yaml +- !INC ${OBS_YAML_DIR}/insitu_ships_surface.yaml +- !INC ${OBS_YAML_DIR}/insitu shipsu_surface.yaml +- !INC ${OBS_YAML_DIR}/insitu_marinemammal_profile.yaml +- !INC ${OBS_YAML_DIR}/insitu_saldrn_profile.yaml +- !INC ${OBS_YAML_DIR}/insitu_axbt_profile.yaml` diff --git a/ush/soca/prep_marine_obs.py b/ush/soca/prep_marine_obs.py new file mode 100755 index 000000000..a0d137d2b --- /dev/null +++ b/ush/soca/prep_marine_obs.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +from wxflow import FileHandler +import os +import fnmatch + + +DMPDIR = os.getenv('DMPDIR') +cyc = os.getenv('cyc') +PDY = os.getenv('PDY') +CDUMP = os.getenv('CDUMP') +COMIN_OBS = os.getenv('COMIN_OBS') + +cycdir = os.path.join(DMPDIR, CDUMP + '.' + str(PDY), str(cyc)) + +# TODO: this looks good for a yaml +obs_dict = { + # '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G16-ACSPO_V2.70-v02.0-fv01.0.nc' + 'ABI_G16': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G16-ACSPO_V2.70-v02.0-fv01.0.nc '), + + # '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G17-ACSPO_V2.71-v02.0-fv01.0.nc' + 'ABI_G17': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G17-ACSPO_V2.71-v02.0-fv01.0.nc '), + + # '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G18-ACSPO_V2.71-v02.0-fv01.0.nc' + 'ABI_G18': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G18-ACSPO_V2.71-v02.0-fv01.0.nc '), + + # '20210630210000-STAR-L3C_GHRSST-SSTsubskin-AHI_H09-ACSPO_V2.70-v02.0-fv01.0.nc' + 'AHI_H09': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-AHI_H09-ACSPO_V2.70-v02.0-fv01.0.nc '), + + # 'rads_adt_3b_2021182.nc' + 'adt_3b_egm2008': ('ADT', 'rads_adt_3b_???????.nc'), + + # 'rads_adt_c2_2021182.nc' + 'adt_c2_egm2008': ('ADT', 'rads_adt_c2_???????.nc'), + + # 'rads_adt_sa_2021182.nc' + 'adt_sa_egm2008': ('ADT', 'rads_adt_sa_???????.nc'), + + # 'AMSR2-SEAICE-NH_v2r2_GW1_s202107011426180_e202107011605170_c202107011642250.nc' + 'icec_amsr2_north': ('icec', 'AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc'), + + # 'AMSR2-SEAICE-SH_v2r2_GW1_s202107011426180_e202107011605170_c202107011642250.nc' + 'icec_amsr2_south': ('icec', 'AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc'), + + # 'SM_OPER_MIR_OSUDP2_20210701T093946_20210701T103256_700_001_1.nc' + 'sss_smos': ('SSS', 'SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc'), + + # 'SMAP_L2B_SSS_NRT_34268_A_20210701T153914.h5' + 'sss_smap': ('SSS', 'SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5'), + + # '20210701145000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MA-ACSPO_V2.70-v02.0-fv01.0.nc' + 'sst_metopa_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MA-ACSPO_V2.70-v02.0-fv01.0.nc'), + + # '20210701145000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc' + 'sst_metopb_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc'), + + # '20210701150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO_V2.61-v02.0-fv01.0.nc' + 'sst_viirs_n20_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO_V2.61-v02.0-fv01.0.nc'), + + # '20210701150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO_V2.61-v02.0-fv01.0.nc' + 'sst_viirs_npp_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO_V2.61-v02.0-fv01.0.nc'), + +} + + +def obs_fetch(obs_source_name): + + try: + obs_source = obs_dict[obs_source_name] + except KeyError: + print(f'WARNING: no obs source {obs_source_name} defined, skipping') + return + + subdir = obs_source[0] + filepattern = obs_source[1] + + datadir = os.path.join(cycdir, subdir) + # TODO: check the existence of this + print('datadir:', datadir) + matching_files = [] + + for root, _, files in os.walk(datadir): + for filename in fnmatch.filter(files, filepattern): + matching_files.append((root, filename)) + + obs_cpy = [] + for obs_src in matching_files: + obs_path = os.path.join(obs_src[0], obs_src[1]) + obs_dst = os.path.join(COMIN_OBS, obs_src[1]) + obs_cpy.append([obs_path, obs_dst]) + + print(obs_cpy) + + FileHandler({'copy': obs_cpy}).sync()