Merge pull request #312 from twsearle/move_gridded_forecast_generic_f…

…unctions

Move _to_datetime to util.py

forest/__init__.py

@@ -25,7 +25,7 @@
 .. automodule:: forest.presets
 
 """
-__version__ = '0.13.1'
+__version__ = '0.13.2'
 
 from .config import *
 from . import (

forest/_profile.py

@@ -53,7 +53,7 @@
 from forest.observe import Observable
 from forest.redux import Action
 from forest.util import initial_time as _initial_time
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest.screen import SET_POSITION
 try:
     import iris

forest/components/time.py

@@ -1,7 +1,7 @@
 """Time navigation component"""
 from forest import rx
 from forest.observe import Observable
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 import forest.db.control
 import bokeh.plotting
 import numpy as np

forest/db/control.py

@@ -7,7 +7,7 @@
 from . import util
 from collections import namedtuple
 from forest.observe import Observable
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest.export import export
 from typing import List, Any
 

forest/drivers/earth_networks.py

@@ -4,7 +4,7 @@
 import datetime as dt
 import pandas as pd
 from forest import geo
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest.old_state import old_state, unique
 import bokeh.models
 import bokeh.palettes

forest/drivers/eida50.py

@@ -9,7 +9,7 @@
 from forest.exceptions import FileNotFound, IndexNotFound
 from forest.old_state import old_state, unique
 from forest.util import coarsify
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest import (
         geo,
         locate,

forest/drivers/ghrsstl4.py

@@ -20,6 +20,7 @@
 import glob
 from forest import geo
 from forest.view import UMView
+from forest.util import to_datetime as _to_datetime
 
 
 def empty_image():
@@ -38,20 +39,6 @@ def empty_image():
     }
 
 
-def _to_datetime(d):
-    if isinstance(d, datetime):
-        return d
-    elif isinstance(d, str):
-        try:
-            return datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
-        except ValueError:
-            return datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
-    elif isinstance(d, np.datetime64):
-        return d.astype(datetime)
-    else:
-        raise Exception("Unknown value: {}".format(d))
-
-
 def coordinates(valid_time, initial_time, pressures, pressure):
     valid = _to_datetime(valid_time)
     initial = _to_datetime(initial_time)

forest/drivers/gridded_forecast.py

@@ -13,6 +13,7 @@
 import glob
 from forest import geo
 from forest.view import UMView
+from forest.util import to_datetime as _to_datetime
 
 
 def empty_image():
@@ -31,25 +32,6 @@ def empty_image():
     }
 
 
-def _to_datetime(d):
-
-    if isinstance(d, datetime):
-        return d
-    if isinstance(d, cftime.DatetimeNoLeap):
-        return datetime(d.year, d.month, d.day, d.hour, d.minute, d.second)
-    elif isinstance(d, cftime.DatetimeGregorian):
-        return datetime(d.year, d.month, d.day, d.hour, d.minute, d.second)
-    elif isinstance(d, str):
-        try:
-            return datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
-        except ValueError:
-            return datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
-    elif isinstance(d, np.datetime64):
-        return d.astype(datetime)
-    else:
-        raise Exception("Unknown value: {} type: {}".format(d, type(d)))
-
-
 def coordinates(valid_time, initial_time, pressures, pressure):
     valid = _to_datetime(valid_time)
     initial = _to_datetime(initial_time)

forest/intake_loader.py

@@ -18,7 +18,7 @@
 except ModuleNotFoundError:
     intake = None
 
-from forest import geo
+from forest import geo, util
 from forest.drivers import gridded_forecast
 
 # Location of the Pangeo-CMIP6 intake catalogue file.
@@ -115,7 +115,7 @@ def _get_bokeh_image(cube,
     """
 
     def time_comp(select_time, time_cell):  #
-        data_time = gridded_forecast._to_datetime(time_cell.point)
+        data_time = util.to_datetime(time_cell.point)
         if abs((select_time - data_time).days) < 2:
             return True
         return False
@@ -209,7 +209,7 @@ def image(self, state):
         valid_time = state.valid_time
         pressure = state.pressure
 
-        selected_time = gridded_forecast._to_datetime(valid_time)
+        selected_time = util.to_datetime(valid_time)
 
         # the guts of creating the bokeh object has been put into a separate
         # function so that it can be cached, so if image is called multiple
@@ -314,7 +314,7 @@ def initial_times(self, pattern, variable=None):
         self._parse_pattern(pattern)
         cube = self.cube
         for cell in cube.coord('time').cells():
-            init_time = gridded_forecast._to_datetime(cell.point)
+            init_time = util.to_datetime(cell.point)
             return [init_time]
 
     def valid_times(self, pattern, variable, initial_time):
@@ -323,7 +323,7 @@ def valid_times(self, pattern, variable, initial_time):
             self._cube = None
         self._parse_pattern(pattern)
         cube = self.cube
-        valid_times = [gridded_forecast._to_datetime(cell.point) for cell in
+        valid_times = [util.to_datetime(cell.point) for cell in
                        cube.coord('time').cells()]
         return valid_times
 

forest/load.py

@@ -27,8 +27,6 @@
         rdt,
         intake_loader)
 
-from forest.drivers import gridded_forecast
-
 
 __all__ = []
 
@@ -85,8 +83,6 @@ def file_loader(file_type, pattern, label=None):
             return rdt.Loader(pattern)
         elif file_type == 'gpm':
             return data.GPM(pattern)
-        elif file_type == 'griddedforecast':
-            return gridded_forecast.ImageLoader(label, pattern)
         elif file_type == 'intake':
             return intake_loader.IntakeLoader(pattern)
         else:

forest/navigate.py

@@ -12,7 +12,6 @@
         db,
         rdt,
         intake_loader)
-from forest.drivers import gridded_forecast
 
 
 class Navigator:
@@ -87,9 +86,6 @@ def from_file_type(cls, paths, file_type, pattern=None):
         if file_type.lower() == "rdt":
             coordinates = rdt.Coordinates()
             return cls(paths, coordinates)
-        elif file_type.lower() == 'griddedforecast':
-            # XXX This needs a "Group" object ... not "paths"
-            return gridded_forecast.Navigator(paths)
         elif file_type.lower() == 'intake':
             return intake_loader.Navigator()
         else:

forest/series.py

@@ -39,7 +39,7 @@
 from forest.observe import Observable
 from forest.redux import Action
 from forest.util import initial_time as _initial_time
-from forest.drivers.gridded_forecast import _to_datetime
+from forest.util import to_datetime as _to_datetime
 from forest.screen import SET_POSITION
 
 try:

forest/util.py

@@ -1,6 +1,7 @@
 import os
 import re
 import datetime as dt
+import cftime
 from functools import partial
 import scipy.ndimage
 import numpy as np
@@ -43,3 +44,24 @@ def initial_time(path):
     groups = re.search(r"[0-9]{8}T[0-9]{4}Z", path)
     if groups:
         return dt.datetime.strptime(groups[0], "%Y%m%dT%H%MZ")
+
+
+def to_datetime(d):
+
+    if isinstance(d, dt.datetime):
+        return d
+    if isinstance(d, cftime.DatetimeNoLeap):
+        return dt.datetime(d.year, d.month, d.day, d.hour, d.minute, d.second)
+    elif isinstance(d, cftime.DatetimeGregorian):
+        return dt.datetime(d.year, d.month, d.day, d.hour, d.minute, d.second)
+    elif isinstance(d, str):
+        try:
+            return dt.datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
+        except ValueError:
+            return dt.datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
+    elif isinstance(d, np.datetime64):
+        return d.astype(dt.datetime)
+    else:
+        raise Exception("Unknown value: {} type: {}".format(d, type(d)))
+
+

test/test_drivers.py

@@ -2,7 +2,7 @@
 from forest import drivers
 
 
-@pytest.mark.parametrize("driver_name", ["earth_networks", "ghrsstl4"])
+@pytest.mark.parametrize("driver_name", ["earth_networks", "ghrsstl4", "gridded_forecast"])
 def test_singleton_dataset(driver_name):
     datasets = (
         drivers.get_dataset(driver_name),

test/test_ghrsstl4.py

@@ -18,30 +18,6 @@ def test(self):
             self.assertEqual(value, [])
 
 
-class Test_to_datetime(unittest.TestCase):
-    def test_datetime(self):
-        dt = datetime.now()
-        result = ghrsstl4._to_datetime(dt)
-        self.assertEqual(result, dt)
-
-    def test_str_with_space(self):
-        result = ghrsstl4._to_datetime('2019-10-10 01:02:34')
-        self.assertEqual(result, datetime(2019, 10, 10, 1, 2, 34))
-
-    def test_str_iso8601(self):
-        result = ghrsstl4._to_datetime('2019-10-10T01:02:34')
-        self.assertEqual(result, datetime(2019, 10, 10, 1, 2, 34))
-
-    def test_datetime64(self):
-        dt = np.datetime64('2019-10-10T11:22:33')
-        result = ghrsstl4._to_datetime(dt)
-        self.assertEqual(result, datetime(2019, 10, 10, 11, 22, 33))
-
-    def test_unsupported(self):
-        with self.assertRaisesRegex(Exception, 'Unknown value'):
-            ghrsstl4._to_datetime(12)
-
-
 @patch('forest.drivers.ghrsstl4._to_datetime')
 class Test_coordinates(unittest.TestCase):
     def test_surface_and_times(self, to_datetime):

test/test_gridded_forecast.py

@@ -20,35 +20,6 @@ def test(self):
             self.assertEqual(value, [])
 
 
-@pytest.mark.parametrize("given,expect", [
-    pytest.param('2019-10-10 01:02:34',
-                 datetime(2019, 10, 10, 1, 2, 34),
-                 id="str with space"),
-    pytest.param('2019-10-10T01:02:34',
-                 datetime(2019, 10, 10, 1, 2, 34),
-                 id="iso8601"),
-    pytest.param(np.datetime64('2019-10-10T11:22:33'),
-                 datetime(2019, 10, 10, 11, 22, 33),
-                 id="datetime64"),
-    pytest.param(cftime.DatetimeGregorian(2019, 10, 10, 11, 22, 33),
-                 datetime(2019, 10, 10, 11, 22, 33),
-                 id="cftime.DatetimeGregorian"),
-])
-def test__to_datetime(given, expect):
-    assert gridded_forecast._to_datetime(given) == expect
-
-
-class Test_to_datetime(unittest.TestCase):
-    def test_datetime(self):
-        dt = datetime.now()
-        result = gridded_forecast._to_datetime(dt)
-        self.assertEqual(result, dt)
-
-    def test_unsupported(self):
-        with self.assertRaisesRegex(Exception, 'Unknown value'):
-            gridded_forecast._to_datetime(12)
-
-
 @patch('forest.drivers.gridded_forecast._to_datetime')
 class Test_coordinates(unittest.TestCase):
     def test_surface_and_times(self, to_datetime):

test/test_navigator.py

@@ -132,11 +132,13 @@ def test_FileSystemNavigator_from_file_type__rdt(coordinates_cls):
 
 
 @patch('forest.drivers.gridded_forecast.Navigator')
-def test_FileSystemNavigator_from_file_type__griddedforecast(navigator_cls):
+@patch('forest.drivers.gridded_forecast.glob.glob')
+def test_drivers__get_dataset_from__griddedforecast(glob, navigator_cls):
     navigator_cls.return_value = sentinel.navigator
+    glob.return_value = sentinel.paths
 
-    navigator = navigate.FileSystemNavigator.from_file_type(sentinel.paths,
-                                                            'grIDdeDforeCAST')
+    dataset = forest.drivers.gridded_forecast.Dataset("gridded_forecast", sentinel.settings)
+    navigator = dataset.navigator()
 
     navigator_cls.assert_called_once_with(sentinel.paths)
     assert navigator == sentinel.navigator

test/test_util.py

@@ -0,0 +1,39 @@
+import pytest
+import cftime
+from datetime import datetime
+import unittest
+
+import iris
+import numpy as np
+
+from forest import util
+
+@pytest.mark.parametrize("given,expect", [
+    pytest.param('2019-10-10 01:02:34',
+                 datetime(2019, 10, 10, 1, 2, 34),
+                 id="str with space"),
+    pytest.param('2019-10-10T01:02:34',
+                 datetime(2019, 10, 10, 1, 2, 34),
+                 id="iso8601"),
+    pytest.param(np.datetime64('2019-10-10T11:22:33'),
+                 datetime(2019, 10, 10, 11, 22, 33),
+                 id="datetime64"),
+    pytest.param(cftime.DatetimeGregorian(2019, 10, 10, 11, 22, 33),
+                 datetime(2019, 10, 10, 11, 22, 33),
+                 id="cftime.DatetimeGregorian"),
+])
+def test__to_datetime(given, expect):
+    assert util.to_datetime(given) == expect
+
+
+class Test_to_datetime(unittest.TestCase):
+    def test_datetime(self):
+        dt = datetime.now()
+        result = util.to_datetime(dt)
+        self.assertEqual(result, dt)
+
+    def test_unsupported(self):
+        with self.assertRaisesRegex(Exception, 'Unknown value'):
+            util.to_datetime(12)
+
+