Add optional parameter to download input data for reference sim

reference_simulations/README.md

@@ -26,6 +26,9 @@ Manager (SLURM).
  - download_latest_data.py
         Run this first to download the reference sim data from the google drive. The new data will be
         compared to these files. They will end up in a directory `latest_ref_data`.
+        An optional parameter "--with-inputs" will download the input data stored
+        on the Google Drive such as large beams and sky models.
+        The input data is saved into the same directory as the script.
 
  - compare_with_last.py
         Given the paths to the latest output files (uvh5), this will compare the data in those files

reference_simulations/download_latest_data.py

@@ -14,9 +14,23 @@
 parser = argparse.ArgumentParser(
     description="A script to download the latest archived reference simulation data."
 )
-parser.add_argument('generation', type=int, help="Generation of reference "
-                                                 "sims to download (1 or 2).", default=1)
-
+parser.add_argument(
+    'generation',
+    type=int,
+    nargs="?",
+    help="Generation of reference sims to download (1 or 2).",
+    default=1,
+)
+parser.add_argument(
+    "--with-inputs",
+    dest="inputs",
+    type=bool,
+    help=(
+        "Additionally download input data from the google drive for simulations whose "
+        "inputs cannot be stored on github directly."
+    ),
+    default=False
+)
 args = parser.parse_args()
 
 if args.generation not in [1, 2]:
@@ -54,3 +68,16 @@
         fname = os.path.join(target_dir, fname)
         with open(fname, 'wb') as ofile:
             ofile.write(r.content)
+
+if args.inputs:
+    with open("gdrive_input_ids.dat", "r") as dfile:
+        for line in dfile:
+            line = line.strip()
+            file_id, name = line.split()[:2]
+            req = requests.get(urlbase, params={"id": file_id})
+            if req.status_code != 200:
+                print(f"Unable to downalod {name}")
+                continue
+            print(name)
+            with open(name, "wb") as ofile:
+                ofile.write(r.content)

reference_simulations/gdrive_input_ids.dat

@@ -0,0 +1,3 @@
+17d58ixuBJDGkbbl41sREMYixAoyKNlBN   HERA_NicCST_fullfreq.uvbeam           bin    840.3 MB   2020-08-05 11:08:11
+1d3F5c29J2-XLvwP3Zr4hOA2I_Nt2Fyel   healpix_flat_EoR_spectrum_noise.hdf5  bin    773.2 MB   2020-08-05 01:00:11
+19kslTwOIQY-iQylPe-CHbX6BMCUhR4ab   healpix_gsm_shell.hdf5                bin    757.5 MB   2020-08-05 00:59:35