diff --git a/forest/__init__.py b/forest/__init__.py index df749198e..4ec1f9020 100644 --- a/forest/__init__.py +++ b/forest/__init__.py @@ -25,7 +25,7 @@ .. automodule:: forest.presets """ -__version__ = '0.13.1' +__version__ = '0.13.2' from .config import * from . import ( diff --git a/forest/_profile.py b/forest/_profile.py index 04fec4b30..8dc4aa9a9 100644 --- a/forest/_profile.py +++ b/forest/_profile.py @@ -53,7 +53,7 @@ from forest.observe import Observable from forest.redux import Action from forest.util import initial_time as _initial_time -from forest.drivers.gridded_forecast import _to_datetime +from forest.util import to_datetime as _to_datetime from forest.screen import SET_POSITION try: import iris diff --git a/forest/components/time.py b/forest/components/time.py index 954261a38..74f803ca1 100644 --- a/forest/components/time.py +++ b/forest/components/time.py @@ -1,7 +1,7 @@ """Time navigation component""" from forest import rx from forest.observe import Observable -from forest.drivers.gridded_forecast import _to_datetime +from forest.util import to_datetime as _to_datetime import forest.db.control import bokeh.plotting import numpy as np diff --git a/forest/db/control.py b/forest/db/control.py index 5d95c9a04..e080067a8 100644 --- a/forest/db/control.py +++ b/forest/db/control.py @@ -7,7 +7,7 @@ from . import util from collections import namedtuple from forest.observe import Observable -from forest.drivers.gridded_forecast import _to_datetime +from forest.util import to_datetime as _to_datetime from forest.export import export from typing import List, Any diff --git a/forest/drivers/earth_networks.py b/forest/drivers/earth_networks.py index 8daeacd72..04ea25bce 100644 --- a/forest/drivers/earth_networks.py +++ b/forest/drivers/earth_networks.py @@ -4,7 +4,7 @@ import datetime as dt import pandas as pd from forest import geo -from forest.drivers.gridded_forecast import _to_datetime +from forest.util import to_datetime as _to_datetime from forest.old_state import old_state, unique import bokeh.models import bokeh.palettes diff --git a/forest/drivers/eida50.py b/forest/drivers/eida50.py index c8b7b51b8..70a540f1b 100644 --- a/forest/drivers/eida50.py +++ b/forest/drivers/eida50.py @@ -9,7 +9,7 @@ from forest.exceptions import FileNotFound, IndexNotFound from forest.old_state import old_state, unique from forest.util import coarsify -from forest.drivers.gridded_forecast import _to_datetime +from forest.util import to_datetime as _to_datetime from forest import ( geo, locate, diff --git a/forest/drivers/ghrsstl4.py b/forest/drivers/ghrsstl4.py index 19be1a562..3a3da6967 100644 --- a/forest/drivers/ghrsstl4.py +++ b/forest/drivers/ghrsstl4.py @@ -20,6 +20,7 @@ import glob from forest import geo from forest.view import UMView +from forest.util import to_datetime as _to_datetime def empty_image(): @@ -38,20 +39,6 @@ def empty_image(): } -def _to_datetime(d): - if isinstance(d, datetime): - return d - elif isinstance(d, str): - try: - return datetime.strptime(d, "%Y-%m-%d %H:%M:%S") - except ValueError: - return datetime.strptime(d, "%Y-%m-%dT%H:%M:%S") - elif isinstance(d, np.datetime64): - return d.astype(datetime) - else: - raise Exception("Unknown value: {}".format(d)) - - def coordinates(valid_time, initial_time, pressures, pressure): valid = _to_datetime(valid_time) initial = _to_datetime(initial_time) diff --git a/forest/drivers/gridded_forecast.py b/forest/drivers/gridded_forecast.py index 5b396d45d..2415151e1 100644 --- a/forest/drivers/gridded_forecast.py +++ b/forest/drivers/gridded_forecast.py @@ -13,6 +13,7 @@ import glob from forest import geo from forest.view import UMView +from forest.util import to_datetime as _to_datetime def empty_image(): @@ -31,25 +32,6 @@ def empty_image(): } -def _to_datetime(d): - - if isinstance(d, datetime): - return d - if isinstance(d, cftime.DatetimeNoLeap): - return datetime(d.year, d.month, d.day, d.hour, d.minute, d.second) - elif isinstance(d, cftime.DatetimeGregorian): - return datetime(d.year, d.month, d.day, d.hour, d.minute, d.second) - elif isinstance(d, str): - try: - return datetime.strptime(d, "%Y-%m-%d %H:%M:%S") - except ValueError: - return datetime.strptime(d, "%Y-%m-%dT%H:%M:%S") - elif isinstance(d, np.datetime64): - return d.astype(datetime) - else: - raise Exception("Unknown value: {} type: {}".format(d, type(d))) - - def coordinates(valid_time, initial_time, pressures, pressure): valid = _to_datetime(valid_time) initial = _to_datetime(initial_time) diff --git a/forest/intake_loader.py b/forest/intake_loader.py index e68d00a3d..040e34f99 100644 --- a/forest/intake_loader.py +++ b/forest/intake_loader.py @@ -18,7 +18,7 @@ except ModuleNotFoundError: intake = None -from forest import geo +from forest import geo, util from forest.drivers import gridded_forecast # Location of the Pangeo-CMIP6 intake catalogue file. @@ -115,7 +115,7 @@ def _get_bokeh_image(cube, """ def time_comp(select_time, time_cell): # - data_time = gridded_forecast._to_datetime(time_cell.point) + data_time = util.to_datetime(time_cell.point) if abs((select_time - data_time).days) < 2: return True return False @@ -209,7 +209,7 @@ def image(self, state): valid_time = state.valid_time pressure = state.pressure - selected_time = gridded_forecast._to_datetime(valid_time) + selected_time = util.to_datetime(valid_time) # the guts of creating the bokeh object has been put into a separate # function so that it can be cached, so if image is called multiple @@ -314,7 +314,7 @@ def initial_times(self, pattern, variable=None): self._parse_pattern(pattern) cube = self.cube for cell in cube.coord('time').cells(): - init_time = gridded_forecast._to_datetime(cell.point) + init_time = util.to_datetime(cell.point) return [init_time] def valid_times(self, pattern, variable, initial_time): @@ -323,7 +323,7 @@ def valid_times(self, pattern, variable, initial_time): self._cube = None self._parse_pattern(pattern) cube = self.cube - valid_times = [gridded_forecast._to_datetime(cell.point) for cell in + valid_times = [util.to_datetime(cell.point) for cell in cube.coord('time').cells()] return valid_times diff --git a/forest/load.py b/forest/load.py index b14566623..249152d0a 100644 --- a/forest/load.py +++ b/forest/load.py @@ -27,8 +27,6 @@ rdt, intake_loader) -from forest.drivers import gridded_forecast - __all__ = [] @@ -85,8 +83,6 @@ def file_loader(file_type, pattern, label=None): return rdt.Loader(pattern) elif file_type == 'gpm': return data.GPM(pattern) - elif file_type == 'griddedforecast': - return gridded_forecast.ImageLoader(label, pattern) elif file_type == 'intake': return intake_loader.IntakeLoader(pattern) else: diff --git a/forest/navigate.py b/forest/navigate.py index f68bf911c..0dbea7095 100644 --- a/forest/navigate.py +++ b/forest/navigate.py @@ -12,7 +12,6 @@ db, rdt, intake_loader) -from forest.drivers import gridded_forecast class Navigator: @@ -87,9 +86,6 @@ def from_file_type(cls, paths, file_type, pattern=None): if file_type.lower() == "rdt": coordinates = rdt.Coordinates() return cls(paths, coordinates) - elif file_type.lower() == 'griddedforecast': - # XXX This needs a "Group" object ... not "paths" - return gridded_forecast.Navigator(paths) elif file_type.lower() == 'intake': return intake_loader.Navigator() else: diff --git a/forest/series.py b/forest/series.py index 0bcf672de..f0feeda7a 100644 --- a/forest/series.py +++ b/forest/series.py @@ -39,7 +39,7 @@ from forest.observe import Observable from forest.redux import Action from forest.util import initial_time as _initial_time -from forest.drivers.gridded_forecast import _to_datetime +from forest.util import to_datetime as _to_datetime from forest.screen import SET_POSITION try: diff --git a/forest/util.py b/forest/util.py index 991c4376f..7f936ff2d 100644 --- a/forest/util.py +++ b/forest/util.py @@ -1,6 +1,7 @@ import os import re import datetime as dt +import cftime from functools import partial import scipy.ndimage import numpy as np @@ -43,3 +44,24 @@ def initial_time(path): groups = re.search(r"[0-9]{8}T[0-9]{4}Z", path) if groups: return dt.datetime.strptime(groups[0], "%Y%m%dT%H%MZ") + + +def to_datetime(d): + + if isinstance(d, dt.datetime): + return d + if isinstance(d, cftime.DatetimeNoLeap): + return dt.datetime(d.year, d.month, d.day, d.hour, d.minute, d.second) + elif isinstance(d, cftime.DatetimeGregorian): + return dt.datetime(d.year, d.month, d.day, d.hour, d.minute, d.second) + elif isinstance(d, str): + try: + return dt.datetime.strptime(d, "%Y-%m-%d %H:%M:%S") + except ValueError: + return dt.datetime.strptime(d, "%Y-%m-%dT%H:%M:%S") + elif isinstance(d, np.datetime64): + return d.astype(dt.datetime) + else: + raise Exception("Unknown value: {} type: {}".format(d, type(d))) + + diff --git a/test/test_drivers.py b/test/test_drivers.py index dcd5ce8f7..cafd18589 100644 --- a/test/test_drivers.py +++ b/test/test_drivers.py @@ -2,7 +2,7 @@ from forest import drivers -@pytest.mark.parametrize("driver_name", ["earth_networks", "ghrsstl4"]) +@pytest.mark.parametrize("driver_name", ["earth_networks", "ghrsstl4", "gridded_forecast"]) def test_singleton_dataset(driver_name): datasets = ( drivers.get_dataset(driver_name), diff --git a/test/test_ghrsstl4.py b/test/test_ghrsstl4.py index 1c10bf0ef..44039fc11 100644 --- a/test/test_ghrsstl4.py +++ b/test/test_ghrsstl4.py @@ -18,30 +18,6 @@ def test(self): self.assertEqual(value, []) -class Test_to_datetime(unittest.TestCase): - def test_datetime(self): - dt = datetime.now() - result = ghrsstl4._to_datetime(dt) - self.assertEqual(result, dt) - - def test_str_with_space(self): - result = ghrsstl4._to_datetime('2019-10-10 01:02:34') - self.assertEqual(result, datetime(2019, 10, 10, 1, 2, 34)) - - def test_str_iso8601(self): - result = ghrsstl4._to_datetime('2019-10-10T01:02:34') - self.assertEqual(result, datetime(2019, 10, 10, 1, 2, 34)) - - def test_datetime64(self): - dt = np.datetime64('2019-10-10T11:22:33') - result = ghrsstl4._to_datetime(dt) - self.assertEqual(result, datetime(2019, 10, 10, 11, 22, 33)) - - def test_unsupported(self): - with self.assertRaisesRegex(Exception, 'Unknown value'): - ghrsstl4._to_datetime(12) - - @patch('forest.drivers.ghrsstl4._to_datetime') class Test_coordinates(unittest.TestCase): def test_surface_and_times(self, to_datetime): diff --git a/test/test_gridded_forecast.py b/test/test_gridded_forecast.py index bf452fa05..49167ecb3 100644 --- a/test/test_gridded_forecast.py +++ b/test/test_gridded_forecast.py @@ -20,35 +20,6 @@ def test(self): self.assertEqual(value, []) -@pytest.mark.parametrize("given,expect", [ - pytest.param('2019-10-10 01:02:34', - datetime(2019, 10, 10, 1, 2, 34), - id="str with space"), - pytest.param('2019-10-10T01:02:34', - datetime(2019, 10, 10, 1, 2, 34), - id="iso8601"), - pytest.param(np.datetime64('2019-10-10T11:22:33'), - datetime(2019, 10, 10, 11, 22, 33), - id="datetime64"), - pytest.param(cftime.DatetimeGregorian(2019, 10, 10, 11, 22, 33), - datetime(2019, 10, 10, 11, 22, 33), - id="cftime.DatetimeGregorian"), -]) -def test__to_datetime(given, expect): - assert gridded_forecast._to_datetime(given) == expect - - -class Test_to_datetime(unittest.TestCase): - def test_datetime(self): - dt = datetime.now() - result = gridded_forecast._to_datetime(dt) - self.assertEqual(result, dt) - - def test_unsupported(self): - with self.assertRaisesRegex(Exception, 'Unknown value'): - gridded_forecast._to_datetime(12) - - @patch('forest.drivers.gridded_forecast._to_datetime') class Test_coordinates(unittest.TestCase): def test_surface_and_times(self, to_datetime): diff --git a/test/test_navigator.py b/test/test_navigator.py index 94049435e..76a799c4a 100644 --- a/test/test_navigator.py +++ b/test/test_navigator.py @@ -132,11 +132,13 @@ def test_FileSystemNavigator_from_file_type__rdt(coordinates_cls): @patch('forest.drivers.gridded_forecast.Navigator') -def test_FileSystemNavigator_from_file_type__griddedforecast(navigator_cls): +@patch('forest.drivers.gridded_forecast.glob.glob') +def test_drivers__get_dataset_from__griddedforecast(glob, navigator_cls): navigator_cls.return_value = sentinel.navigator + glob.return_value = sentinel.paths - navigator = navigate.FileSystemNavigator.from_file_type(sentinel.paths, - 'grIDdeDforeCAST') + dataset = forest.drivers.gridded_forecast.Dataset("gridded_forecast", sentinel.settings) + navigator = dataset.navigator() navigator_cls.assert_called_once_with(sentinel.paths) assert navigator == sentinel.navigator diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 000000000..80e68ebfb --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,39 @@ +import pytest +import cftime +from datetime import datetime +import unittest + +import iris +import numpy as np + +from forest import util + +@pytest.mark.parametrize("given,expect", [ + pytest.param('2019-10-10 01:02:34', + datetime(2019, 10, 10, 1, 2, 34), + id="str with space"), + pytest.param('2019-10-10T01:02:34', + datetime(2019, 10, 10, 1, 2, 34), + id="iso8601"), + pytest.param(np.datetime64('2019-10-10T11:22:33'), + datetime(2019, 10, 10, 11, 22, 33), + id="datetime64"), + pytest.param(cftime.DatetimeGregorian(2019, 10, 10, 11, 22, 33), + datetime(2019, 10, 10, 11, 22, 33), + id="cftime.DatetimeGregorian"), +]) +def test__to_datetime(given, expect): + assert util.to_datetime(given) == expect + + +class Test_to_datetime(unittest.TestCase): + def test_datetime(self): + dt = datetime.now() + result = util.to_datetime(dt) + self.assertEqual(result, dt) + + def test_unsupported(self): + with self.assertRaisesRegex(Exception, 'Unknown value'): + util.to_datetime(12) + +