-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding ocean obs prep script and module (#635)
* added JGLOBAL_PREP_OCEAN_OBS to ctests * first draft of marine obs prep * tweaks * new ex script, refinements to class * more obs, more problems * cleanup, obs_list with all avail yamls * linter cleanup * linter cleanup * linter corrections * rename and add info header to ex script * more obs classes * ripped out fancy OO version, now plain vanilla * fixe amsr2 ice yamls * added obs sources, cleaned up others, pep8 checked * more sst obs sources * change prints to logs * cleanup * tweaks to SST sources * removed two obs sources * added and subtracted obs yamls * more yamls * more yaml tweaks
- Loading branch information
1 parent
945c359
commit bd5cb2e
Showing
5 changed files
with
179 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| obs space: | ||
| name: icec_ssmis_f17_north | ||
| name: icec_amsr2_north | ||
| obsdatain: | ||
| engine: | ||
| type: H5File | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| obs space: | ||
| name: icec_ssmis_f17_south | ||
| name: icec_amsr2_south | ||
| obsdatain: | ||
| engine: | ||
| type: H5File | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| #!/usr/bin/env python3 | ||
| # exglobal_prep_ocean_obs.py | ||
| # Pepares observations for marine DA | ||
| import os | ||
| from wxflow import YAMLFile | ||
| import prep_marine_obs | ||
| import logging | ||
|
|
||
| OBS_YAML = os.getenv('OBS_YAML') | ||
|
|
||
| data = YAMLFile(OBS_YAML) | ||
| print(data) | ||
|
|
||
| for observer in data['observers']: | ||
| obs_source_name = observer['obs space']['name'] | ||
| logging.info(f"obs_source_name: {obs_source_name}") | ||
| prep_marine_obs.obs_fetch(obs_source_name) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| observers: | ||
| - !INC ${OBS_YAML_DIR}/adt_3a_egm2008.yaml | ||
| - !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml | ||
| - !INC ${OBS_YAML_DIR}/adt_6a_egm2008.yaml | ||
| - !INC ${OBS_YAML_DIR}/adt_all.yaml | ||
| - !INC ${OBS_YAML_DIR}/adt_c2_egm2008.yaml | ||
| - !INC ${OBS_YAML_DIR}/adt_coperl4.yaml | ||
| - !INC ${OBS_YAML_DIR}/adt_j2_egm2008.yaml | ||
| - !INC ${OBS_YAML_DIR}/adt_j3_egm2008.yaml | ||
| - !INC ${OBS_YAML_DIR}/adt_sa_egm2008.yaml | ||
| - !INC ${OBS_YAML_DIR}/bufr_sfcshipsu.yaml | ||
| - !INC ${OBS_YAML_DIR}/bufr_sfcships.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_emc.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_nsidc_nh.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_nsidc_sh.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_ssmis_f17_north.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_ssmis_f17_south.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_ssmis_f18_north.yaml | ||
| - !INC ${OBS_YAML_DIR}/icec_ssmis_f18_south.yaml | ||
| - !INC ${OBS_YAML_DIR}/icefb_gdr.yaml | ||
| - !INC ${OBS_YAML_DIR}/salt_bufr_dbuoyprof.yaml | ||
| - !INC ${OBS_YAML_DIR}/salt_bufr_mbuoybprof.yaml | ||
| - !INC ${OBS_YAML_DIR}/salt_profile_fnmoc_godas.yaml | ||
| - !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml | ||
| - !INC ${OBS_YAML_DIR}/salt_profile_wod.yaml | ||
| - !INC ${OBS_YAML_DIR}/sss_smap_jpl.yaml | ||
| - !INC ${OBS_YAML_DIR}/sss_smap.yaml | ||
| - !INC ${OBS_YAML_DIR}/sss_smos_esa.yaml | ||
| - !INC ${OBS_YAML_DIR}/sss_smos.yaml | ||
| - !INC ${OBS_YAML_DIR}/sss_trak_fnmoc.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_amsr2_l3u.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_drifter.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_gmi_l3u.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_metopa_l3u_so025.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_metopc_l3u_so025.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_noaa18_l3u_so025.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_noaa19_l3u_so025.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_ostia.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_ship_fnmoc.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_trak_fnmoc.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_viirs_n20_l3u_so025.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_viirs_npp_l3u_so025.yaml | ||
| - !INC ${OBS_YAML_DIR}/sst_windsat_l3u.yaml | ||
| - !INC ${OBS_YAML_DIR}/temp_bufr_dbuoyprof.yaml | ||
| - !INC ${OBS_YAML_DIR}/temp_bufr_mbuoybprof.yaml | ||
| - !INC ${OBS_YAML_DIR}/temp_profile_fnmoc_godas.yaml | ||
| - !INC ${OBS_YAML_DIR}/temp_profile_fnmoc.yaml | ||
| - !INC ${OBS_YAML_DIR}/temp_profile_wod.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_argo_profile.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_glider_profile.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_bathy_profile.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_tesac_profile.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_xbtctd_profile.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_dbuoy_profile.yaml, | ||
| - !INC ${OBS_YAML_DIR}/insitu_mbuoy_profile.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_trkob_surface.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_altkob_surface.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_ships_surface.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu shipsu_surface.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_marinemammal_profile.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_saldrn_profile.yaml | ||
| - !INC ${OBS_YAML_DIR}/insitu_axbt_profile.yaml` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| from wxflow import FileHandler | ||
| import os | ||
| import fnmatch | ||
|
|
||
|
|
||
| DMPDIR = os.getenv('DMPDIR') | ||
| cyc = os.getenv('cyc') | ||
| PDY = os.getenv('PDY') | ||
| CDUMP = os.getenv('CDUMP') | ||
| COMIN_OBS = os.getenv('COMIN_OBS') | ||
|
|
||
| cycdir = os.path.join(DMPDIR, CDUMP + '.' + str(PDY), str(cyc)) | ||
|
|
||
| # TODO: this looks good for a yaml | ||
| obs_dict = { | ||
| # '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G16-ACSPO_V2.70-v02.0-fv01.0.nc' | ||
| 'ABI_G16': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G16-ACSPO_V2.70-v02.0-fv01.0.nc '), | ||
|
|
||
| # '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G17-ACSPO_V2.71-v02.0-fv01.0.nc' | ||
| 'ABI_G17': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G17-ACSPO_V2.71-v02.0-fv01.0.nc '), | ||
|
|
||
| # '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G18-ACSPO_V2.71-v02.0-fv01.0.nc' | ||
| 'ABI_G18': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G18-ACSPO_V2.71-v02.0-fv01.0.nc '), | ||
|
|
||
| # '20210630210000-STAR-L3C_GHRSST-SSTsubskin-AHI_H09-ACSPO_V2.70-v02.0-fv01.0.nc' | ||
| 'AHI_H09': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-AHI_H09-ACSPO_V2.70-v02.0-fv01.0.nc '), | ||
|
|
||
| # 'rads_adt_3b_2021182.nc' | ||
| 'adt_3b_egm2008': ('ADT', 'rads_adt_3b_???????.nc'), | ||
|
|
||
| # 'rads_adt_c2_2021182.nc' | ||
| 'adt_c2_egm2008': ('ADT', 'rads_adt_c2_???????.nc'), | ||
|
|
||
| # 'rads_adt_sa_2021182.nc' | ||
| 'adt_sa_egm2008': ('ADT', 'rads_adt_sa_???????.nc'), | ||
|
|
||
| # 'AMSR2-SEAICE-NH_v2r2_GW1_s202107011426180_e202107011605170_c202107011642250.nc' | ||
| 'icec_amsr2_north': ('icec', 'AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc'), | ||
|
|
||
| # 'AMSR2-SEAICE-SH_v2r2_GW1_s202107011426180_e202107011605170_c202107011642250.nc' | ||
| 'icec_amsr2_south': ('icec', 'AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc'), | ||
|
|
||
| # 'SM_OPER_MIR_OSUDP2_20210701T093946_20210701T103256_700_001_1.nc' | ||
| 'sss_smos': ('SSS', 'SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc'), | ||
|
|
||
| # 'SMAP_L2B_SSS_NRT_34268_A_20210701T153914.h5' | ||
| 'sss_smap': ('SSS', 'SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5'), | ||
|
|
||
| # '20210701145000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MA-ACSPO_V2.70-v02.0-fv01.0.nc' | ||
| 'sst_metopa_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MA-ACSPO_V2.70-v02.0-fv01.0.nc'), | ||
|
|
||
| # '20210701145000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc' | ||
| 'sst_metopb_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc'), | ||
|
|
||
| # '20210701150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO_V2.61-v02.0-fv01.0.nc' | ||
| 'sst_viirs_n20_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO_V2.61-v02.0-fv01.0.nc'), | ||
|
|
||
| # '20210701150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO_V2.61-v02.0-fv01.0.nc' | ||
| 'sst_viirs_npp_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO_V2.61-v02.0-fv01.0.nc'), | ||
|
|
||
| } | ||
|
|
||
|
|
||
| def obs_fetch(obs_source_name): | ||
|
|
||
| try: | ||
| obs_source = obs_dict[obs_source_name] | ||
| except KeyError: | ||
| print(f'WARNING: no obs source {obs_source_name} defined, skipping') | ||
| return | ||
|
|
||
| subdir = obs_source[0] | ||
| filepattern = obs_source[1] | ||
|
|
||
| datadir = os.path.join(cycdir, subdir) | ||
| # TODO: check the existence of this | ||
| print('datadir:', datadir) | ||
| matching_files = [] | ||
|
|
||
| for root, _, files in os.walk(datadir): | ||
| for filename in fnmatch.filter(files, filepattern): | ||
| matching_files.append((root, filename)) | ||
|
|
||
| obs_cpy = [] | ||
| for obs_src in matching_files: | ||
| obs_path = os.path.join(obs_src[0], obs_src[1]) | ||
| obs_dst = os.path.join(COMIN_OBS, obs_src[1]) | ||
| obs_cpy.append([obs_path, obs_dst]) | ||
|
|
||
| print(obs_cpy) | ||
|
|
||
| FileHandler({'copy': obs_cpy}).sync() |