Prepare for Upcoming DGEC Transition

.gitignore

@@ -3,9 +3,13 @@ _targets
 
 #All model output
 */out/*
+#Except modeling results from manuscript
+!03_model/out/Run_Manuscript_Results/*
 
-#All model inputs
-*/in/*
+#Certain model inputs
+02_munge/in/*
+03_model/in/COAWST_model_runs/raw/*
+03_model/out/
 
 #R files
 # History files
@@ -67,3 +71,4 @@ __pycache__/
 
 #dask
 dask-worker-space/
+.Rproj.user

01_fetch/params_config_fetch_usgs_nwis.yaml

@@ -8,4 +8,4 @@ s3_bucket: 'drb-estuary-salinity'
 # start and end dates for data fetch
 # should be in the format 'YYYY-MM-DD'
 start_dt: '2000-01-01'
-end_dt: '2019-12-31'
+end_dt: '2021-12-31'

01_fetch/src/fetch_coawst_model.py

@@ -8,6 +8,8 @@
 import os
 from dask.distributed import Client
 import yaml
+import sys
+sys.path.insert(0, os.path.join('01_fetch', 'src'))
 import utils
 
 #client = Client()

01_fetch/src/fetch_noaa_nerrs.py

@@ -1,9 +1,12 @@
 import os
-import utils
 import yaml
 import glob
 from zipfile import ZipFile
 import shutil
+import sys
+sys.path.insert(0, os.path.join('01_fetch', 'src'))
+import utils
+
 
 # import config
 with open("01_fetch/params_config_fetch_noaa_nerrs.yaml", 'r') as stream:

01_fetch/src/fetch_noaa_nos.py

@@ -4,8 +4,10 @@
 import datetime
 from dateutil.relativedelta import relativedelta
 import requests, json
-import utils
 import yaml
+import sys
+sys.path.insert(0, os.path.join('01_fetch', 'src'))
+import utils
 
 # import config
 with open("01_fetch/params_config_fetch_noaa_nos.yaml", 'r') as stream:

01_fetch/src/fetch_usgs_nwis.py

@@ -3,6 +3,8 @@
 import yaml
 import datetime
 import pandas as pd
+import sys
+sys.path.insert(0, os.path.join('01_fetch', 'src'))
 import utils
 
 # import config
@@ -48,11 +50,6 @@ def process_site_info_to_csv(raw_site_info_txt, site_info_outfile_csv):
         print('uploading to s3')
         s3_client.upload_file(site_info_outfile_csv, s3_bucket, local_to_s3_pathname(site_info_outfile_csv))
 
-    # get minimum date to pull for this site
-    start_dt = site_info_df_subset[['begin_date']].min().values[0]
-
-    return start_dt
-
 def fetch_params(outfile):
     '''get table of all possible USGS site parameters'''
     params_url = 'https://help.waterdata.usgs.gov/code/parameter_cd_query?fmt=rdb&group_cd=%'
@@ -101,8 +98,9 @@ def fetch_single_site_data(site_num):
     site_info_outfile_txt = os.path.join('.', '01_fetch', 'out', 'metadata', f'usgs_nwis_site_info_{site_num}.txt')
     fetch_site_info(site_num, site_info_outfile_txt)
     site_info_outfile_csv = os.path.join('.', '01_fetch', 'out', 'metadata', f'usgs_nwis_site_info_{site_num}.csv')
-    start_dt = process_site_info_to_csv(site_info_outfile_txt, site_info_outfile_csv)
-    end_dt = datetime.datetime.today().strftime("%Y-%m-%d")
+    process_site_info_to_csv(site_info_outfile_txt, site_info_outfile_csv)
+    start_dt = config['start_dt']
+    end_dt = config['end_dt']
 
     # start and end dates for data fetch
     data_outfile_txt = os.path.join('.', '01_fetch', 'out', f'usgs_nwis_{site_num}.txt')

utils.py → 01_fetch/src/utils.py

@@ -35,8 +35,8 @@ def process_to_timestep(df, cols, agg_level, prop_obs_required):
     must have datetimes in a column named 'datetime'
     '''
     # get proportion of measurements available for timestep
-    expected_measurements = df.resample(agg_level, on='datetime').count().mode()[cols].loc[0]
-    observed_measurements = df.resample(agg_level, on='datetime').count()[cols].loc[:]
+    expected_measurements = df.resample(agg_level, on='datetime').count().mode().loc[0]
+    observed_measurements = df.resample(agg_level, on='datetime').count().loc[:]
     prop_df = observed_measurements / expected_measurements
     # calculate averages for timestep
     df = df.resample(agg_level, on='datetime').mean()
@@ -84,4 +84,4 @@ def get_datafile_list(read_location, s3_client=None, s3_bucket=None):
         prefix = os.path.join('01_fetch', 'out')
         file_prefix='usgs_nwis_0'
         raw_datafiles = [os.path.join(prefix, f) for f in os.listdir(prefix) if f.startswith(file_prefix)]
-    return raw_datafiles
+    return raw_datafiles

02_munge/src/fill_discharge_prms.py

@@ -28,8 +28,8 @@ def download_unzip_sb(sb_url, prms_predictions, destination):
         os.makedirs(destination, exist_ok=True)
         sb = sciencebasepy.SbSession()
         sb.download_file(sb_url, prms_predictions+'.zip', destination)
-        with zipfile.ZipFile(os.path.join(destination,prms_predictions+'.zip'), 'r') as zip_ref:
-            zip_ref.extractall(destination)
+    with zipfile.ZipFile(os.path.join(destination,prms_predictions+'.zip'), 'r') as zip_ref:
+        zip_ref.extractall(destination)
 
 
 

02_munge/src/munge_noaa_nerrs.py

@@ -2,6 +2,8 @@
 import pandas as pd
 import numpy as np
 import yaml
+import sys
+sys.path.insert(0, os.path.join('01_fetch', 'src'))
 import utils
 
 # import config

02_munge/src/munge_noaa_nos.py

@@ -3,8 +3,10 @@
 import pandas as pd
 import boto3
 import yaml
-import utils
 from scipy import signal
+import sys
+sys.path.insert(0, os.path.join('01_fetch', 'src'))
+import utils
 
 # import config
 with open("02_munge/params_config_munge_noaa_nos.yaml", 'r') as stream:

02_munge/src/munge_usgs_nwis.py

@@ -3,6 +3,8 @@
 import pandas as pd
 import re
 import yaml
+import sys
+sys.path.insert(0, os.path.join('01_fetch', 'src'))
 import utils
 
 # import config

03b_model/hyperparameter_config.yaml → 03_model/hyperparameter_config.yaml

@@ -1,5 +1,5 @@
-out_dir: '03b_model/out/HP_Tune_02'
-run_id: 'Run_29'  
+out_dir: '03_model/out/HP_Tune_00'
+#run_id: 'Run_29'  
 inputs: ['discharge_01463500', 
                 #'specific_conductance_01463500',
                 'discharge_01474500',