From 2b80e07947ea2fa4ba591d8608c3c308166a4971 Mon Sep 17 00:00:00 2001
From: twsearle <14909402+twsearle@users.noreply.github.com>
Date: Fri, 20 Mar 2020 14:02:16 +0000
Subject: [PATCH 1/5] Move _to_datetime to util.py

---
 forest/_profile.py                 |  2 +-
 forest/components/time.py          |  2 +-
 forest/db/control.py               |  2 +-
 forest/drivers/earth_networks.py   |  2 +-
 forest/drivers/eida50.py           |  2 +-
 forest/drivers/ghrsstl4.py         | 15 +----------
 forest/drivers/gridded_forecast.py | 20 +--------------
 forest/intake_loader.py            | 10 ++++----
 forest/series.py                   |  2 +-
 forest/util.py                     | 22 ++++++++++++++++
 test/test_ghrsstl4.py              | 24 ------------------
 test/test_gridded_forecast.py      | 29 ----------------------
 test/test_util.py                  | 40 ++++++++++++++++++++++++++++++
 13 files changed, 75 insertions(+), 97 deletions(-)
 create mode 100644 test/test_util.py

diff --git a/forest/_profile.py b/forest/_profile.py
index 04fec4b30..8dc4aa9a9 100644
--- a/forest/_profile.py
+++ b/forest/_profile.py
@@ -53,7 +53,7 @@
 from forest.observe import Observable
 from forest.redux import Action
 from forest.util import initial_time as _initial_time
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest.screen import SET_POSITION
 try:
     import iris
diff --git a/forest/components/time.py b/forest/components/time.py
index 954261a38..74f803ca1 100644
--- a/forest/components/time.py
+++ b/forest/components/time.py
@@ -1,7 +1,7 @@
 """Time navigation component"""
 from forest import rx
 from forest.observe import Observable
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 import forest.db.control
 import bokeh.plotting
 import numpy as np
diff --git a/forest/db/control.py b/forest/db/control.py
index 5d95c9a04..e080067a8 100644
--- a/forest/db/control.py
+++ b/forest/db/control.py
@@ -7,7 +7,7 @@
 from . import util
 from collections import namedtuple
 from forest.observe import Observable
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest.export import export
 from typing import List, Any
 
diff --git a/forest/drivers/earth_networks.py b/forest/drivers/earth_networks.py
index 8daeacd72..04ea25bce 100644
--- a/forest/drivers/earth_networks.py
+++ b/forest/drivers/earth_networks.py
@@ -4,7 +4,7 @@
 import datetime as dt
 import pandas as pd
 from forest import geo
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest.old_state import old_state, unique
 import bokeh.models
 import bokeh.palettes
diff --git a/forest/drivers/eida50.py b/forest/drivers/eida50.py
index c8b7b51b8..70a540f1b 100644
--- a/forest/drivers/eida50.py
+++ b/forest/drivers/eida50.py
@@ -9,7 +9,7 @@
 from forest.exceptions import FileNotFound, IndexNotFound
 from forest.old_state import old_state, unique
 from forest.util import coarsify
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest import (
         geo,
         locate,
diff --git a/forest/drivers/ghrsstl4.py b/forest/drivers/ghrsstl4.py
index 19be1a562..3a3da6967 100644
--- a/forest/drivers/ghrsstl4.py
+++ b/forest/drivers/ghrsstl4.py
@@ -20,6 +20,7 @@
 import glob
 from forest import geo
 from forest.view import UMView
+from forest.util import to_datetime as _to_datetime
 
 
 def empty_image():
@@ -38,20 +39,6 @@ def empty_image():
     }
 
 
-def _to_datetime(d):
-    if isinstance(d, datetime):
-        return d
-    elif isinstance(d, str):
-        try:
-            return datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
-        except ValueError:
-            return datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
-    elif isinstance(d, np.datetime64):
-        return d.astype(datetime)
-    else:
-        raise Exception("Unknown value: {}".format(d))
-
-
 def coordinates(valid_time, initial_time, pressures, pressure):
     valid = _to_datetime(valid_time)
     initial = _to_datetime(initial_time)
diff --git a/forest/drivers/gridded_forecast.py b/forest/drivers/gridded_forecast.py
index 5b396d45d..2415151e1 100644
--- a/forest/drivers/gridded_forecast.py
+++ b/forest/drivers/gridded_forecast.py
@@ -13,6 +13,7 @@
 import glob
 from forest import geo
 from forest.view import UMView
+from forest.util import to_datetime as _to_datetime
 
 
 def empty_image():
@@ -31,25 +32,6 @@ def empty_image():
     }
 
 
-def _to_datetime(d):
-
-    if isinstance(d, datetime):
-        return d
-    if isinstance(d, cftime.DatetimeNoLeap):
-        return datetime(d.year, d.month, d.day, d.hour, d.minute, d.second)
-    elif isinstance(d, cftime.DatetimeGregorian):
-        return datetime(d.year, d.month, d.day, d.hour, d.minute, d.second)
-    elif isinstance(d, str):
-        try:
-            return datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
-        except ValueError:
-            return datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
-    elif isinstance(d, np.datetime64):
-        return d.astype(datetime)
-    else:
-        raise Exception("Unknown value: {} type: {}".format(d, type(d)))
-
-
 def coordinates(valid_time, initial_time, pressures, pressure):
     valid = _to_datetime(valid_time)
     initial = _to_datetime(initial_time)
diff --git a/forest/intake_loader.py b/forest/intake_loader.py
index e68d00a3d..040e34f99 100644
--- a/forest/intake_loader.py
+++ b/forest/intake_loader.py
@@ -18,7 +18,7 @@
 except ModuleNotFoundError:
     intake = None
 
-from forest import geo
+from forest import geo, util
 from forest.drivers import gridded_forecast
 
 # Location of the Pangeo-CMIP6 intake catalogue file.
@@ -115,7 +115,7 @@ def _get_bokeh_image(cube,
     """
 
     def time_comp(select_time, time_cell):  #
-        data_time = gridded_forecast._to_datetime(time_cell.point)
+        data_time = util.to_datetime(time_cell.point)
         if abs((select_time - data_time).days) < 2:
             return True
         return False
@@ -209,7 +209,7 @@ def image(self, state):
         valid_time = state.valid_time
         pressure = state.pressure
 
-        selected_time = gridded_forecast._to_datetime(valid_time)
+        selected_time = util.to_datetime(valid_time)
 
         # the guts of creating the bokeh object has been put into a separate
         # function so that it can be cached, so if image is called multiple
@@ -314,7 +314,7 @@ def initial_times(self, pattern, variable=None):
         self._parse_pattern(pattern)
         cube = self.cube
         for cell in cube.coord('time').cells():
-            init_time = gridded_forecast._to_datetime(cell.point)
+            init_time = util.to_datetime(cell.point)
             return [init_time]
 
     def valid_times(self, pattern, variable, initial_time):
@@ -323,7 +323,7 @@ def valid_times(self, pattern, variable, initial_time):
             self._cube = None
         self._parse_pattern(pattern)
         cube = self.cube
-        valid_times = [gridded_forecast._to_datetime(cell.point) for cell in
+        valid_times = [util.to_datetime(cell.point) for cell in
                        cube.coord('time').cells()]
         return valid_times
 
diff --git a/forest/series.py b/forest/series.py
index 0bcf672de..f0feeda7a 100644
--- a/forest/series.py
+++ b/forest/series.py
@@ -39,7 +39,7 @@
 from forest.observe import Observable
 from forest.redux import Action
 from forest.util import initial_time as _initial_time
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest.screen import SET_POSITION
 
 try:
diff --git a/forest/util.py b/forest/util.py
index 991c4376f..7f936ff2d 100644
--- a/forest/util.py
+++ b/forest/util.py
@@ -1,6 +1,7 @@
 import os
 import re
 import datetime as dt
+import cftime
 from functools import partial
 import scipy.ndimage
 import numpy as np
@@ -43,3 +44,24 @@ def initial_time(path):
     groups = re.search(r"[0-9]{8}T[0-9]{4}Z", path)
     if groups:
         return dt.datetime.strptime(groups[0], "%Y%m%dT%H%MZ")
+
+
+def to_datetime(d):
+
+    if isinstance(d, dt.datetime):
+        return d
+    if isinstance(d, cftime.DatetimeNoLeap):
+        return dt.datetime(d.year, d.month, d.day, d.hour, d.minute, d.second)
+    elif isinstance(d, cftime.DatetimeGregorian):
+        return dt.datetime(d.year, d.month, d.day, d.hour, d.minute, d.second)
+    elif isinstance(d, str):
+        try:
+            return dt.datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
+        except ValueError:
+            return dt.datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
+    elif isinstance(d, np.datetime64):
+        return d.astype(dt.datetime)
+    else:
+        raise Exception("Unknown value: {} type: {}".format(d, type(d)))
+
+
diff --git a/test/test_ghrsstl4.py b/test/test_ghrsstl4.py
index 1c10bf0ef..44039fc11 100644
--- a/test/test_ghrsstl4.py
+++ b/test/test_ghrsstl4.py
@@ -18,30 +18,6 @@ def test(self):
             self.assertEqual(value, [])
 
 
-class Test_to_datetime(unittest.TestCase):
-    def test_datetime(self):
-        dt = datetime.now()
-        result = ghrsstl4._to_datetime(dt)
-        self.assertEqual(result, dt)
-
-    def test_str_with_space(self):
-        result = ghrsstl4._to_datetime('2019-10-10 01:02:34')
-        self.assertEqual(result, datetime(2019, 10, 10, 1, 2, 34))
-
-    def test_str_iso8601(self):
-        result = ghrsstl4._to_datetime('2019-10-10T01:02:34')
-        self.assertEqual(result, datetime(2019, 10, 10, 1, 2, 34))
-
-    def test_datetime64(self):
-        dt = np.datetime64('2019-10-10T11:22:33')
-        result = ghrsstl4._to_datetime(dt)
-        self.assertEqual(result, datetime(2019, 10, 10, 11, 22, 33))
-
-    def test_unsupported(self):
-        with self.assertRaisesRegex(Exception, 'Unknown value'):
-            ghrsstl4._to_datetime(12)
-
-
 @patch('forest.drivers.ghrsstl4._to_datetime')
 class Test_coordinates(unittest.TestCase):
     def test_surface_and_times(self, to_datetime):
diff --git a/test/test_gridded_forecast.py b/test/test_gridded_forecast.py
index bf452fa05..49167ecb3 100644
--- a/test/test_gridded_forecast.py
+++ b/test/test_gridded_forecast.py
@@ -20,35 +20,6 @@ def test(self):
             self.assertEqual(value, [])
 
 
-@pytest.mark.parametrize("given,expect", [
-    pytest.param('2019-10-10 01:02:34',
-                 datetime(2019, 10, 10, 1, 2, 34),
-                 id="str with space"),
-    pytest.param('2019-10-10T01:02:34',
-                 datetime(2019, 10, 10, 1, 2, 34),
-                 id="iso8601"),
-    pytest.param(np.datetime64('2019-10-10T11:22:33'),
-                 datetime(2019, 10, 10, 11, 22, 33),
-                 id="datetime64"),
-    pytest.param(cftime.DatetimeGregorian(2019, 10, 10, 11, 22, 33),
-                 datetime(2019, 10, 10, 11, 22, 33),
-                 id="cftime.DatetimeGregorian"),
-])
-def test__to_datetime(given, expect):
-    assert gridded_forecast._to_datetime(given) == expect
-
-
-class Test_to_datetime(unittest.TestCase):
-    def test_datetime(self):
-        dt = datetime.now()
-        result = gridded_forecast._to_datetime(dt)
-        self.assertEqual(result, dt)
-
-    def test_unsupported(self):
-        with self.assertRaisesRegex(Exception, 'Unknown value'):
-            gridded_forecast._to_datetime(12)
-
-
 @patch('forest.drivers.gridded_forecast._to_datetime')
 class Test_coordinates(unittest.TestCase):
     def test_surface_and_times(self, to_datetime):
diff --git a/test/test_util.py b/test/test_util.py
new file mode 100644
index 000000000..d1786da28
--- /dev/null
+++ b/test/test_util.py
@@ -0,0 +1,40 @@
+import pytest
+import cftime
+from datetime import datetime
+from unittest.mock import Mock, call, patch, sentinel
+import unittest
+
+import iris
+import numpy as np
+
+from forest import util
+
+@pytest.mark.parametrize("given,expect", [
+    pytest.param('2019-10-10 01:02:34',
+                 datetime(2019, 10, 10, 1, 2, 34),
+                 id="str with space"),
+    pytest.param('2019-10-10T01:02:34',
+                 datetime(2019, 10, 10, 1, 2, 34),
+                 id="iso8601"),
+    pytest.param(np.datetime64('2019-10-10T11:22:33'),
+                 datetime(2019, 10, 10, 11, 22, 33),
+                 id="datetime64"),
+    pytest.param(cftime.DatetimeGregorian(2019, 10, 10, 11, 22, 33),
+                 datetime(2019, 10, 10, 11, 22, 33),
+                 id="cftime.DatetimeGregorian"),
+])
+def test__to_datetime(given, expect):
+    assert util.to_datetime(given) == expect
+
+
+class Test_to_datetime(unittest.TestCase):
+    def test_datetime(self):
+        dt = datetime.now()
+        result = util.to_datetime(dt)
+        self.assertEqual(result, dt)
+
+    def test_unsupported(self):
+        with self.assertRaisesRegex(Exception, 'Unknown value'):
+            util.to_datetime(12)
+
+

From a375a26e766cde9a8ee398d19ef2c2eb8e97d433 Mon Sep 17 00:00:00 2001
From: twsearle <14909402+twsearle@users.noreply.github.com>
Date: Fri, 20 Mar 2020 14:37:01 +0000
Subject: [PATCH 2/5] Remove unused import

---
 test/test_util.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_util.py b/test/test_util.py
index d1786da28..80e68ebfb 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -1,7 +1,6 @@
 import pytest
 import cftime
 from datetime import datetime
-from unittest.mock import Mock, call, patch, sentinel
 import unittest
 
 import iris

From 1bb1161282fe7a540a35d4347ecf5ce7fd4c5489 Mon Sep 17 00:00:00 2001
From: twsearle <14909402+twsearle@users.noreply.github.com>
Date: Fri, 20 Mar 2020 15:09:50 +0000
Subject: [PATCH 3/5] Remove gridded_forecast from old interface

---
 forest/load.py         | 4 ----
 forest/navigate.py     | 4 ----
 test/test_navigator.py | 2 +-
 3 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/forest/load.py b/forest/load.py
index 87f619e89..faef6fc84 100644
--- a/forest/load.py
+++ b/forest/load.py
@@ -28,8 +28,6 @@
         intake_loader,
         nearcast)
 
-from forest.drivers import gridded_forecast
-
 
 __all__ = []
 
@@ -86,8 +84,6 @@ def file_loader(file_type, pattern, label=None):
             return rdt.Loader(pattern)
         elif file_type == 'gpm':
             return data.GPM(pattern)
-        elif file_type == 'griddedforecast':
-            return gridded_forecast.ImageLoader(label, pattern)
         elif file_type == 'intake':
             return intake_loader.IntakeLoader(pattern)
         elif file_type == 'nearcast':
diff --git a/forest/navigate.py b/forest/navigate.py
index 401a1c938..5f33e38a8 100644
--- a/forest/navigate.py
+++ b/forest/navigate.py
@@ -13,7 +13,6 @@
         rdt,
         intake_loader,
         nearcast)
-from forest.drivers import gridded_forecast
 
 
 class Navigator:
@@ -88,9 +87,6 @@ def from_file_type(cls, paths, file_type, pattern=None):
         if file_type.lower() == "rdt":
             coordinates = rdt.Coordinates()
             return cls(paths, coordinates)
-        elif file_type.lower() == 'griddedforecast':
-            # XXX This needs a "Group" object ... not "paths"
-            return gridded_forecast.Navigator(paths)
         elif file_type.lower() == 'intake':
             return intake_loader.Navigator()
         elif file_type.lower() == "nearcast":
diff --git a/test/test_navigator.py b/test/test_navigator.py
index 94049435e..45689d6bd 100644
--- a/test/test_navigator.py
+++ b/test/test_navigator.py
@@ -136,7 +136,7 @@ def test_FileSystemNavigator_from_file_type__griddedforecast(navigator_cls):
     navigator_cls.return_value = sentinel.navigator
 
     navigator = navigate.FileSystemNavigator.from_file_type(sentinel.paths,
-                                                            'grIDdeDforeCAST')
+                                                            'gridded_forecast')
 
     navigator_cls.assert_called_once_with(sentinel.paths)
     assert navigator == sentinel.navigator

From 8bba8ac7923ca8910c84324c46475cc18dafd3ab Mon Sep 17 00:00:00 2001
From: twsearle <14909402+twsearle@users.noreply.github.com>
Date: Fri, 20 Mar 2020 15:39:45 +0000
Subject: [PATCH 4/5] Fix test_navigator test for gridded_forecast

---
 test/test_drivers.py   | 2 +-
 test/test_navigator.py | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/test_drivers.py b/test/test_drivers.py
index dcd5ce8f7..cafd18589 100644
--- a/test/test_drivers.py
+++ b/test/test_drivers.py
@@ -2,7 +2,7 @@
 from forest import drivers
 
 
-@pytest.mark.parametrize("driver_name", ["earth_networks", "ghrsstl4"])
+@pytest.mark.parametrize("driver_name", ["earth_networks", "ghrsstl4", "gridded_forecast"])
 def test_singleton_dataset(driver_name):
     datasets = (
         drivers.get_dataset(driver_name),
diff --git a/test/test_navigator.py b/test/test_navigator.py
index 45689d6bd..76a799c4a 100644
--- a/test/test_navigator.py
+++ b/test/test_navigator.py
@@ -132,11 +132,13 @@ def test_FileSystemNavigator_from_file_type__rdt(coordinates_cls):
 
 
 @patch('forest.drivers.gridded_forecast.Navigator')
-def test_FileSystemNavigator_from_file_type__griddedforecast(navigator_cls):
+@patch('forest.drivers.gridded_forecast.glob.glob')
+def test_drivers__get_dataset_from__griddedforecast(glob, navigator_cls):
     navigator_cls.return_value = sentinel.navigator
+    glob.return_value = sentinel.paths
 
-    navigator = navigate.FileSystemNavigator.from_file_type(sentinel.paths,
-                                                            'gridded_forecast')
+    dataset = forest.drivers.gridded_forecast.Dataset("gridded_forecast", sentinel.settings)
+    navigator = dataset.navigator()
 
     navigator_cls.assert_called_once_with(sentinel.paths)
     assert navigator == sentinel.navigator

From 703c4f3e6dbbb26aaeddf5d3a4a5257a1677f6f6 Mon Sep 17 00:00:00 2001
From: twsearle <14909402+twsearle@users.noreply.github.com>
Date: Fri, 20 Mar 2020 16:37:11 +0000
Subject: [PATCH 5/5] Bump version to 0.13.2

---
 forest/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/forest/__init__.py b/forest/__init__.py
index df749198e..4ec1f9020 100644
--- a/forest/__init__.py
+++ b/forest/__init__.py
@@ -25,7 +25,7 @@
 .. automodule:: forest.presets
 
 """
-__version__ = '0.13.1'
+__version__ = '0.13.2'
 
 from .config import *
 from . import (