9_collaborator_data.yml

target_default: 9_collaborator_data

packages:
  - sf
  - tidyverse
  - lwgeom
  - smoothr
  - geojsonio
  - geojsonsf
  - ggplot2
  - reticulate

sources:
  - 1_network/src/calc_distance_functions.R
  - 1_network/src/geo_fabric_functions.R
  - 2_observations/src/data_munge_functions.R
  - 9_collaborator_data/src/makerspace_geojson.R
  - 9_collaborator_data/src/write_functions.R
  - 9_collaborator_data/src/psu_site_selection.R
  - 9_collaborator_data/src/subset_sntemp.R
  - 9_collaborator_data/src/reservoir_inouts.R

targets:
  9_collaborator_data:
    depends:
     - 9_collaborator_data/mkrsp/delaware_sites_summary.geojson.ind
     - 9_collaborator_data/umn/network_subset.rds.ind
     - 9_collaborator_data/umn/distance_matrix.rds.ind
     - 9_collaborator_data/umn/distance_matrix.npz.ind
     - 9_collaborator_data/umn/distance_matrix_subset.npz.ind
     - 9_collaborator_data/umn/obs_temp_full.csv.ind
     - 9_collaborator_data/umn/obs_temp_subset.csv.ind
     - 9_collaborator_data/umn/obs_flow_full.csv.ind
     - 9_collaborator_data/umn/obs_flow_subset.csv.ind
     #- 9_collaborator_data/psu/highly_observed_sites.csv.ind
     #- 9_collaborator_data/psu/highly_observed_map.png
     #- 9_collaborator_data/psu/highly_obs_distance_matrix.csv.ind
     #- 9_collaborator_data/psu/highly_obs_distance_heatmap.png
     #- 9_collaborator_data/psu/sntemp_preds_aggregated.feather.ind
     #- 9_collaborator_data/psu/temp_obs_high_obs_sites.csv.ind
     #- 9_collaborator_data/psu/flow_obs_high_obs_sites.csv.ind
     - 9_collaborator_data/res/res_io_obs.feather
     - 9_collaborator_data/res/res_io_sntemp.feather


  ##### Makerspace #####
  # generate file to load for map (http://delaware-basin-test-website.s3-website-us-west-2.amazonaws.com/)

  9_collaborator_data/mkrsp/delaware_sites_summary.rds.ind:
    command: generate_site_summary(
      out_ind = target_name,
      dat_ind = '2_observations/out/all_drb_temp_obs.rds.ind',
      crosswalk_ind = '2_observations/out/crosswalk_site_reach.rds.ind')

  # convert summary to geojson

  9_collaborator_data/mkrsp/delaware_sites_summary.geojson.ind:
    command: generate_site_geojson(summary_ind = '9_collaborator_data/mkrsp/delaware_sites_summary.rds.ind', out_ind = target_name)

  ###### University of Minnesota #####
  # also should include:
    # 1_network/out/network.rds
    # 2_observations/out/basin_temp_data.rds

  # subset network segments and then distance matrix to only those reaches whose outlets match up with
  # SNTemp prediction points, because these are the reaches we'll have SNTemp driver data for, and
  # for the purpose of the neural networks, the distance matrix is the only representation of network
  # connectivity, so once we have it correct (by doing the network corrections in 1_network)
  # we can safely subset to only the sntemp segments and still have a connected network. This effectively
  # treats the reaches that were split into 2 reaches as now being 1 reach again (appropriate because
  # that's what the inputs from SNTemp into the NN will describe).
  9_collaborator_data/umn/sntemp_segs.rds:
    command: select_sntemp_subsegs(
      network_ind = '1_network/out/network.rds.ind',
      out_rds = target_name)
  9_collaborator_data/umn/distance_matrix.rds.ind:
    command: subset_dist_to_subsegs(
      subsegs_rds = '9_collaborator_data/umn/sntemp_segs.rds',
      dist_ind = '1_network/out/subseg_distance_matrix.rds.ind',
      out_ind = target_name)

  # networks and distance matrices
  # npy version of distance matrix - this target seems to rebuild a few times if given the chance. huh.
  9_collaborator_data/umn/distance_matrix.npz.ind:
    command: save_dist_matrices(
      dist_mat_ind = '9_collaborator_data/umn/distance_matrix.rds.ind',
      out_ind = target_name)

  9_collaborator_data/umn/network_subset.rds.ind:
    command: make_subnetwork(
      out_ind = target_name,
      lower_reach = I('2748_1'),
      network_ind = '1_network/out/network.rds.ind',
      distance_ind = '1_network/out/subseg_distance_matrix.rds.ind',
      summary_ind = '9_collaborator_data/mkrsp/delaware_sites_summary.rds.ind')


  9_collaborator_data/umn/network_subset_lordville.rds.ind:
    command: make_subnetwork(
      out_ind = target_name,
      lower_reach = I('139_1'),
      network_ind = '1_network/out/network.rds.ind',
      distance_ind = '1_network/out/subseg_distance_matrix.rds.ind',
      summary_ind = '9_collaborator_data/mkrsp/delaware_sites_summary.rds.ind')

  # create distance matrix of subset
  9_collaborator_data/umn/network_subset_sntemp_segs.rds:
    command: select_sntemp_subsegs(
      network_ind = '9_collaborator_data/umn/network_subset.rds.ind',
      out_rds = target_name)
  9_collaborator_data/umn/network_lordville_sntemp_segs.rds:
    command: select_sntemp_subsegs(
      network_ind = '9_collaborator_data/umn/network_subset_lordville.rds.ind',
      out_rds = target_name)

  9_collaborator_data/umn/distance_matrix_subset.rds.ind:
    command: subset_dist_to_subsegs(
      subsegs_rds = '9_collaborator_data/umn/network_subset_sntemp_segs.rds',
      dist_ind = '1_network/out/subseg_distance_matrix.rds.ind',
      out_ind = target_name)

  9_collaborator_data/umn/distance_matrix_lordville.rds.ind:
    command: subset_dist_to_subsegs(
      subsegs_rds = '9_collaborator_data/umn/network_lordville_sntemp_segs.rds',
      dist_ind = '1_network/out/subseg_distance_matrix.rds.ind',
      out_ind = target_name)


  # npy version of distance matrix_subset
  9_collaborator_data/umn/distance_matrix_subset.npz.ind:
    command: save_dist_matrices(dist_mat_ind = '9_collaborator_data/umn/distance_matrix_subset.rds.ind', out_ind = target_name)

  # temp data
  9_collaborator_data/umn/obs_temp_full.csv.ind:
    command: write_to_csv(dat_ind = '2_observations/out/obs_temp_drb.rds.ind', out_ind = target_name)

  9_collaborator_data/umn/obs_temp_subset.csv.ind:
    command: filter_subset(
      dat_ind = '2_observations/out/obs_temp_drb.rds.ind',
      subnet_ind = '9_collaborator_data/umn/network_subset.rds.ind',
      out_ind = target_name)

  9_collaborator_data/umn/obs_temp_subset_lordville.csv.ind:
    command: filter_subset(
      dat_ind = '2_observations/out/obs_temp_drb.rds.ind',
      subnet_ind = '9_collaborator_data/umn/network_subset_lordville.rds.ind',
      out_ind = target_name)

  # flow_data
  9_collaborator_data/umn/obs_flow_full.csv.ind:
    command: write_to_csv(dat_ind = '2_observations/out/obs_flow_drb.rds.ind', out_ind = target_name)

  9_collaborator_data/umn/obs_flow_subset.csv.ind:
    command: filter_subset(
      dat_ind = '2_observations/out/obs_flow_drb.rds.ind',
      subnet_ind = '9_collaborator_data/umn/network_subset.rds.ind',
      out_ind = target_name)

  9_collaborator_data/umn/obs_temp_subset.zarr.tar.ind:
    command: write_zarr_tarfile(dat_ind = '9_collaborator_data/umn/obs_temp_subset.csv.ind', out_ind = target_name)
    depends: '9_collaborator_data/src/write_functions_py.py'
    
  9_collaborator_data/umn/obs_temp_full.zarr.tar.ind:
    command: write_zarr_tarfile(dat_ind = '9_collaborator_data/umn/obs_temp_full.csv.ind', out_ind = target_name)
    depends: '9_collaborator_data/src/write_functions_py.py'

  9_collaborator_data/umn/obs_flow_subset.zarr.tar.ind:
    command: write_zarr_tarfile(dat_ind = '9_collaborator_data/umn/obs_flow_subset.csv.ind', out_ind = target_name)
    depends: '9_collaborator_data/src/write_functions_py.py'
    
  9_collaborator_data/umn/obs_flow_full.zarr.tar.ind:
    command: write_zarr_tarfile(dat_ind = '9_collaborator_data/umn/obs_flow_full.csv.ind', out_ind = target_name)
    depends: '9_collaborator_data/src/write_functions_py.py'

  9_collaborator_data/umn/uncal_sntemp_input_output.zarr.tar.ind:
    command: write_zarr_tarfile(dat_ind = '3_predictions/out/uncal_sntemp_input_output.feather.ind', out_ind = target_name)
    depends: '9_collaborator_data/src/write_functions_py.py'

  9_collaborator_data/umn/obs_flow_subset_lordville.csv.ind:
    command: filter_subset(
      dat_ind = '2_observations/out/obs_flow_drb.rds.ind',
      subnet_ind = '9_collaborator_data/umn/network_subset_lordville.rds.ind',
      out_ind = target_name)

  # reservoir metadata
  9_collaborator_data/umn/reservoir_features.csv.ind:
    command: get_res_features(dat_ind = '1_network/out/subseg_reservoir_mapping.rds.ind', out_ind = target_name)

  #### Penn State ####
  9_collaborator_data/psu/highly_observed_sites.rds:
    command: filter_sites(
      out_file = target_name,
      dat_ind = '2_observations/out/obs_temp_drb.rds.ind',
      geo_dat = I('1_network/in/GeospatialFabric_National.gdb'),
      years = I(2),
      obs_per_year = I(150),
      min_drainage = I(100),
      max_drainage = I(10000))

  9_collaborator_data/psu/highly_observed_sites.csv.ind:
    command: write_sites(dat = '9_collaborator_data/psu/highly_observed_sites.rds', out_ind = target_name)

  9_collaborator_data/psu/highly_observed_map.png:
    command: map_highly_obs(
      out_file = target_name,
      cross_ind = '2_observations/out/crosswalk_site_reach.rds.ind',
      network_ind = '1_network/out/network.rds.ind',
      dat = '9_collaborator_data/psu/highly_observed_sites.rds',
      title = I('Sites with >=2 years with >150 obs'))

  9_collaborator_data/psu/highly_obs_distance_matrix.rds.ind:
    command: subset_dist_to_subsegs(
      subsegs_rds = '9_collaborator_data/psu/highly_observed_sites.rds',
      dist_ind = '1_network/out/subseg_distance_matrix.rds.ind',
      out_ind = target_name)

  # this target builds a few times for some reason
  9_collaborator_data/psu/highly_obs_distance_matrix.csv.ind:
    command: write_distance(
      out_ind = target_name,
      dat_ind = '9_collaborator_data/psu/highly_obs_distance_matrix.rds.ind',
      dist_type = I('updown'))

  9_collaborator_data/psu/highly_obs_distance_heatmap.png:
    command: dist_heatmap2(
      out_file = target_name,
      dist_ind = '9_collaborator_data/psu/highly_obs_distance_matrix.rds.ind',
      dist_type = I('updown'),
      labels = I('subseg_id'),
      title = I('Full Network - Upstream & Downstream'))

  9_collaborator_data/psu/prms_azrh.csv:
    command: download.file(
      destfile = target_name,
      url = I('https://raw.githubusercontent.com/nhm-params-v10-usgs/nhmparamdb_v10_CONUS/434d5da5c4f5a1184d4bd07839322ab1bcb4c3c5/azrh.csv'))

  9_collaborator_data/psu/high_obs_upstream_sites.csv:
    command: get_upstream_sites(
      out_file = target_name,
      dist_ind = '1_network/out/subseg_distance_matrix.rds.ind',
      network_ind = '1_network/out/network.rds.ind',
      sites = '9_collaborator_data/psu/highly_observed_sites.rds',
      azrh_file = '9_collaborator_data/psu/prms_azrh.csv',
      geo_dat = I('1_network/in/GeospatialFabric_National.gdb'))

  9_collaborator_data/psu/sntemp_preds_high_obs_sites.feather:
    command: subset_sntemp_preds(
      out_file = target_name,
      sub_net_file = '9_collaborator_data/psu/high_obs_upstream_sites.csv',
      full_data_ind = '3_predictions/in/uncal_sntemp_input_output.feather.ind')

  9_collaborator_data/psu/sntemp_preds_aggregated.feather.ind:
    command: aggregate_sntemp_preds(
      ind_file = target_name,
      sub_net_file ='9_collaborator_data/psu/high_obs_upstream_sites.csv',
      subset_data_file = '9_collaborator_data/psu/sntemp_preds_high_obs_sites.feather')

  9_collaborator_data/psu/temp_obs_high_obs_sites.csv.ind:
    command: filter_obs(
      dat_ind = '2_observations/out/obs_temp_drb.rds.ind',
      subset = '9_collaborator_data/psu/highly_observed_sites.rds',
      out_ind = target_name)

  9_collaborator_data/psu/flow_obs_high_obs_sites.csv.ind:
    command: filter_obs(
      dat_ind = '2_observations/out/obs_flow_drb.rds.ind',
      subset = '9_collaborator_data/psu/highly_observed_sites.rds',
      out_ind = target_name)

  # data for Amy McHugh and Jon Janowicz
  9_collaborator_data/usgs/sntemp_pred_obs_subset.csv:
    command: filter_sntemp_obs(
      sntemp_ind = '3_predictions/in/uncal_sntemp_input_output.feather.ind',
      flow_ind = '2_observations/out/obs_flow_drb.rds.ind',
      temp_ind = '2_observations/out/obs_temp_drb.rds.ind',
      network_ind = '1_network/out/network.rds.ind',
      out_file = target_name)

  #### Reservoirs ####

  # Hand-code the IDs at the outflows of the reservoirs of interest.
  # The two Pepacton sites are both along the same outflow route, but
  # one site is really close to the reservoir (01417000), and the
  # other is a little downstream (01417500). They differ in data
  # availability (see 9_collaborator_data/res/pep_io_*.png, created
  # as a side effect of building 9_collaborator_data/res/res_io_obs.feather),
  # but those plots show that 01417000 is pretty good since 1980, so we
  # actually just drop 01417500 and go with 01417000 in res_io_obs.feather.
  res_outflow_ids:
    command: list(
      Cannonsville = I('01425000'),
      Pepacton = I(c('01417000', '01417500')))

  # Find sites with lots of flow and/or temp data on inflows to each reservoir.
  # This algorithm uses NLDI and hence currently only locates NWIS sites, overlooking
  # any other relevant sites in WQP
  res_inflow_ids:
    command: find_inout_obs_sites(
      res_outflow_ids = res_outflow_ids,
      flow_ind = '2_observations/in/daily_flow.rds.ind',
      temp_ind = '2_observations/out/all_drb_temp_obs.rds.ind',
      min_obs_flow = I(1000),
      min_obs_temp = I(50),
      max_dist_km = I(100))

  9_collaborator_data/res/res_io_obs.feather.ind:
    command: get_inout_obs_all(
      target_name,
      res_inflow_ids,
      res_outflow_ids,
      flow_ind = '2_observations/in/daily_flow.rds.ind',
      temp_ind = '2_observations/out/all_drb_temp_obs.rds.ind')

  9_collaborator_data/res/res_io_sntemp.feather.ind:
    command: get_inout_sntemp_all(
      target_name,
      sntemp_ind = '3_predictions/out/uncal_sntemp_input_output.feather')