From 9c6c4a568d9a24e1f5f0c68251ff05ec08761494 Mon Sep 17 00:00:00 2001 From: andrewgryan Date: Tue, 10 Mar 2020 13:36:05 +0000 Subject: [PATCH 1/8] refactor to dataset/driver pattern --- forest/data.py | 7 +------ forest/drivers/__init__.py | 12 ++++++++++++ forest/{ => drivers}/earth_networks.py | 13 +++++++++++++ forest/exceptions.py | 4 ++++ forest/load.py | 14 +++++++++++--- forest/main.py | 2 +- forest/navigate.py | 18 ++++++++++++++---- test/test_earth_networks.py | 18 +++++++++++++++++- test/test_load.py | 3 ++- 9 files changed, 75 insertions(+), 16 deletions(-) rename forest/{ => drivers}/earth_networks.py (92%) diff --git a/forest/data.py b/forest/data.py index 820aa5f69..96fdbca23 100644 --- a/forest/data.py +++ b/forest/data.py @@ -18,13 +18,8 @@ pass from forest import ( gridded_forecast, - saf, - satellite, - rdt, - earth_networks, geo, - disk, - nearcast) + disk) import bokeh.models from collections import OrderedDict from functools import partial diff --git a/forest/drivers/__init__.py b/forest/drivers/__init__.py index 8b1378917..4630b73f3 100644 --- a/forest/drivers/__init__.py +++ b/forest/drivers/__init__.py @@ -1 +1,13 @@ +from importlib import import_module +from forest.exceptions import DriverNotFound + +def get_dataset(driver_name, settings=None): + """Find Dataset related to file type""" + if settings is None: + settings = {} + try: + module = import_module(f"forest.drivers.{driver_name}") + except ModuleNotFoundError: + raise DriverNotFound(driver_name) + return module.Dataset(**settings) diff --git a/forest/earth_networks.py b/forest/drivers/earth_networks.py similarity index 92% rename from forest/earth_networks.py rename to forest/drivers/earth_networks.py index 93f2b6d42..f00f5aa2c 100644 --- a/forest/earth_networks.py +++ b/forest/drivers/earth_networks.py @@ -11,6 +11,19 @@ import numpy as np +class Dataset: + """High-level class to relate navigators, loaders and views""" + def __init__(self, pattern=None): + self.pattern = pattern + + def navigator(self): + """Construct appropriate navigator""" + return Navigator([]) + + def loader(self): + """Construct appropriate loader""" + return Loader([]) + class View(object): def __init__(self, loader): diff --git a/forest/exceptions.py b/forest/exceptions.py index 15b9cb6f1..edac8005f 100644 --- a/forest/exceptions.py +++ b/forest/exceptions.py @@ -1,3 +1,7 @@ +class DriverNotFound(Exception): + pass + + class FileNotFound(Exception): pass diff --git a/forest/load.py b/forest/load.py index ef2c69e5b..1414ee003 100644 --- a/forest/load.py +++ b/forest/load.py @@ -22,9 +22,10 @@ import os from forest.export import export from forest import ( + exceptions, + drivers, data, db, - earth_networks, gridded_forecast, unified_model, rdt, @@ -129,13 +130,20 @@ def from_pattern(cls, @staticmethod def file_loader(file_type, pattern, label=None, locator=None): + try: + settings = { + "pattern": pattern + } + dataset = drivers.get_dataset(file_type, settings) + return dataset.loader() + except exceptions.DriverNotFound: + pass + file_type = file_type.lower().replace("_", "") if file_type == 'rdt': return rdt.Loader(pattern) elif file_type == 'gpm': return data.GPM(pattern) - elif file_type == 'earthnetworks': - return earth_networks.Loader.pattern(pattern) elif file_type == 'eida50': return satellite.EIDA50(pattern) elif file_type == 'griddedforecast': diff --git a/forest/main.py b/forest/main.py index 62c01b371..a75b8a7eb 100644 --- a/forest/main.py +++ b/forest/main.py @@ -6,6 +6,7 @@ import os import glob from forest import _profile as profile +from forest.drivers import earth_networks from forest import ( satellite, screen, @@ -14,7 +15,6 @@ data, load, view, - earth_networks, rdt, nearcast, geo, diff --git a/forest/navigate.py b/forest/navigate.py index 6d8b45d52..4ae0b8480 100644 --- a/forest/navigate.py +++ b/forest/navigate.py @@ -7,7 +7,8 @@ ValidTimesNotFound, PressuresNotFound) from forest import ( - earth_networks, + exceptions, + drivers, db, gridded_forecast, unified_model, @@ -17,7 +18,8 @@ saf, nearcast) -from forest.drivers import ghrsstl4 +from forest.drivers import ( + ghrsstl4) class Navigator: @@ -80,6 +82,16 @@ def __init__(self, paths, coordinates=None): @classmethod def from_file_type(cls, paths, file_type, pattern=None): + try: + settings = { + "paths": paths, + "pattern": pattern} + dataset = drivers.get_dataset(file_type, settings) + return dataset.navigator() + except exceptions.DriverNotFound: + # TODO: Migrate all file types to forest.drivers + pass + if file_type.lower() == "rdt": coordinates = rdt.Coordinates() elif file_type.lower() == "eida50": @@ -95,8 +107,6 @@ def from_file_type(cls, paths, file_type, pattern=None): coordinates = unified_model.Coordinates() elif file_type.lower() == "saf": coordinates = saf.Coordinates() - elif file_type.lower() == "earth_networks": - return earth_networks.Navigator(paths) elif file_type.lower() == "nearcast": return nearcast.Navigator(pattern) else: diff --git a/test/test_earth_networks.py b/test/test_earth_networks.py index cc5b61f2c..b1525f2ca 100644 --- a/test/test_earth_networks.py +++ b/test/test_earth_networks.py @@ -1,7 +1,8 @@ import datetime as dt import numpy as np import glob -from forest import earth_networks +import forest.drivers +from forest.drivers import earth_networks LINES = [ @@ -28,3 +29,18 @@ def test_earth_networks(tmpdir): assert result["flash_type"] == "IC" assert abs(result["latitude"] - 2.75144) < atol assert abs(result["longitude"] - 31.92064) < atol + + +def test_dataset(): + dataset = forest.drivers.get_dataset("earth_networks") + assert isinstance(dataset, forest.drivers.earth_networks.Dataset) + + +def test_dataset_navigator(): + settings = { + "pattern": "*.txt" + } + dataset = forest.drivers.get_dataset("earth_networks", settings) + navigator = dataset.navigator() + assert isinstance(navigator, + forest.drivers.earth_networks.Navigator) diff --git a/test/test_load.py b/test/test_load.py index cabe278b4..8d4fe62e3 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -1,11 +1,12 @@ import yaml import forest from forest import main +from forest.drivers import earth_networks def test_earth_networks_loader_given_pattern(): loader = forest.Loader.from_pattern("Label", "EarthNetworks*.txt", "earth_networks") - assert isinstance(loader, forest.earth_networks.Loader) + assert isinstance(loader, earth_networks.Loader) def test_build_loader_given_files(): From dffc06ab549e1147baeb3467909a8048a6e5fc03 Mon Sep 17 00:00:00 2001 From: andrewgryan Date: Tue, 10 Mar 2020 14:13:37 +0000 Subject: [PATCH 2/8] implement more dataset methods --- forest/drivers/earth_networks.py | 11 +++++-- forest/exceptions.py | 4 +++ forest/load.py | 12 +------ forest/main.py | 56 +++++++++++++++++++------------- forest/navigate.py | 1 - 5 files changed, 48 insertions(+), 36 deletions(-) diff --git a/forest/drivers/earth_networks.py b/forest/drivers/earth_networks.py index f00f5aa2c..f7ff0ba86 100644 --- a/forest/drivers/earth_networks.py +++ b/forest/drivers/earth_networks.py @@ -17,13 +17,17 @@ def __init__(self, pattern=None): self.pattern = pattern def navigator(self): - """Construct appropriate navigator""" + """Construct navigator""" return Navigator([]) def loader(self): - """Construct appropriate loader""" + """Construct loader""" return Loader([]) + def map_view(self): + """Construct view""" + return View(self.loader()) + class View(object): def __init__(self, loader): @@ -43,6 +47,9 @@ def __init__(self, loader): @old_state @unique def render(self, state): + if state.valid_time is None: + return + valid_time = _to_datetime(state.valid_time) frame = self.loader.load_date(valid_time) x, y = geo.web_mercator( diff --git a/forest/exceptions.py b/forest/exceptions.py index edac8005f..21ea132a6 100644 --- a/forest/exceptions.py +++ b/forest/exceptions.py @@ -2,6 +2,10 @@ class DriverNotFound(Exception): pass +class UnknownFileType(Exception): + pass + + class FileNotFound(Exception): pass diff --git a/forest/load.py b/forest/load.py index 1414ee003..c96fb34d7 100644 --- a/forest/load.py +++ b/forest/load.py @@ -23,7 +23,6 @@ from forest.export import export from forest import ( exceptions, - drivers, data, db, gridded_forecast, @@ -130,15 +129,6 @@ def from_pattern(cls, @staticmethod def file_loader(file_type, pattern, label=None, locator=None): - try: - settings = { - "pattern": pattern - } - dataset = drivers.get_dataset(file_type, settings) - return dataset.loader() - except exceptions.DriverNotFound: - pass - file_type = file_type.lower().replace("_", "") if file_type == 'rdt': return rdt.Loader(pattern) @@ -159,4 +149,4 @@ def file_loader(file_type, pattern, label=None, locator=None): elif file_type == 'nearcast': return nearcast.NearCast(pattern) else: - raise Exception("unrecognised file_type: {}".format(file_type)) + raise exceptions.UnknownFileType("unrecognised file_type: {}".format(file_type)) diff --git a/forest/main.py b/forest/main.py index a75b8a7eb..f9f22b16a 100644 --- a/forest/main.py +++ b/forest/main.py @@ -6,8 +6,9 @@ import os import glob from forest import _profile as profile -from forest.drivers import earth_networks from forest import ( + drivers, + exceptions, satellite, screen, tools, @@ -95,32 +96,43 @@ def main(argv=None): database = None if group.locator == "database": database = db.get_database(group.database_path) - loader = load.Loader.group_args( - group, args, database=database) + try: + loader = load.Loader.group_args( + group, args, database=database) + except exceptions.UnknownFileType: + # TODO: Deprecate load.Loader.group_args() + continue data.add_loader(group.label, loader) renderers = {} viewers = {} - for name, loader in data.LOADERS.items(): - if isinstance(loader, rdt.Loader): - viewer = rdt.View(loader) - elif isinstance(loader, earth_networks.Loader): - viewer = earth_networks.View(loader) - elif isinstance(loader, data.GPM): - viewer = view.GPMView(loader, color_mapper) - elif isinstance(loader, satellite.EIDA50): - viewer = view.EIDA50(loader, color_mapper) - elif isinstance(loader, nearcast.NearCast): - viewer = view.NearCast(loader, color_mapper) - viewer.set_hover_properties(nearcast.NEARCAST_TOOLTIPS) - elif isinstance(loader, intake_loader.IntakeLoader): - viewer = view.UMView(loader, color_mapper) - viewer.set_hover_properties(intake_loader.INTAKE_TOOLTIPS, - intake_loader.INTAKE_FORMATTERS) + for group in config.file_groups: + if group.label in data.LOADERS: + loader = data.LOADERS[group.label] + if isinstance(loader, rdt.Loader): + viewer = rdt.View(loader) + elif isinstance(loader, data.GPM): + viewer = view.GPMView(loader, color_mapper) + elif isinstance(loader, satellite.EIDA50): + viewer = view.EIDA50(loader, color_mapper) + elif isinstance(loader, nearcast.NearCast): + viewer = view.NearCast(loader, color_mapper) + viewer.set_hover_properties(nearcast.NEARCAST_TOOLTIPS) + elif isinstance(loader, intake_loader.IntakeLoader): + viewer = view.UMView(loader, color_mapper) + viewer.set_hover_properties(intake_loader.INTAKE_TOOLTIPS, + intake_loader.INTAKE_FORMATTERS) + else: + viewer = view.UMView(loader, color_mapper) else: - viewer = view.UMView(loader, color_mapper) - viewers[name] = viewer - renderers[name] = [ + # Use dataset interface + settings = { + "pattern": group.pattern + } + dataset = drivers.get_dataset(group.file_type, settings) + viewer = dataset.map_view() + viewers[group.label] = viewer + renderers[group.label] = [ viewer.add_figure(f) for f in figures] diff --git a/forest/navigate.py b/forest/navigate.py index 4ae0b8480..498af7759 100644 --- a/forest/navigate.py +++ b/forest/navigate.py @@ -84,7 +84,6 @@ def __init__(self, paths, coordinates=None): def from_file_type(cls, paths, file_type, pattern=None): try: settings = { - "paths": paths, "pattern": pattern} dataset = drivers.get_dataset(file_type, settings) return dataset.navigator() From f1193bf9d15f139c1dc8ff728174ec73c7b5d8d5 Mon Sep 17 00:00:00 2001 From: andrewgryan Date: Tue, 10 Mar 2020 14:22:15 +0000 Subject: [PATCH 3/8] minimal interface for dataset --- forest/drivers/earth_networks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/forest/drivers/earth_networks.py b/forest/drivers/earth_networks.py index f7ff0ba86..a1e28787d 100644 --- a/forest/drivers/earth_networks.py +++ b/forest/drivers/earth_networks.py @@ -15,18 +15,18 @@ class Dataset: """High-level class to relate navigators, loaders and views""" def __init__(self, pattern=None): self.pattern = pattern + if pattern is not None: + self._paths = glob.glob(pattern) + else: + self._paths = [] def navigator(self): """Construct navigator""" - return Navigator([]) - - def loader(self): - """Construct loader""" - return Loader([]) + return Navigator(self._paths) def map_view(self): """Construct view""" - return View(self.loader()) + return View(Loader(self._paths)) class View(object): From 1315fa88477cfa02d237beff0c9e4a00a9e14863 Mon Sep 17 00:00:00 2001 From: andrewgryan Date: Tue, 10 Mar 2020 14:24:47 +0000 Subject: [PATCH 4/8] fix test by using rdt instead of earth_networks --- test/test_load.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/test_load.py b/test/test_load.py index 8d4fe62e3..e6d2a1b31 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -1,12 +1,11 @@ import yaml import forest -from forest import main -from forest.drivers import earth_networks +from forest import main, rdt -def test_earth_networks_loader_given_pattern(): - loader = forest.Loader.from_pattern("Label", "EarthNetworks*.txt", "earth_networks") - assert isinstance(loader, earth_networks.Loader) +def test_rdt_loader_given_pattern(): + loader = forest.Loader.from_pattern("Label", "RDT*.json", "rdt") + assert isinstance(loader, rdt.Loader) def test_build_loader_given_files(): From aebba8b9861efe7b8f3b1153148d0f56473341a8 Mon Sep 17 00:00:00 2001 From: andrewgryan Date: Tue, 10 Mar 2020 16:23:31 +0000 Subject: [PATCH 5/8] add per-server dataset caching given driver name and settings --- forest/drivers/__init__.py | 21 +++++++++++++++++++++ test/test_drivers.py | 9 +++++++++ 2 files changed, 30 insertions(+) create mode 100644 test/test_drivers.py diff --git a/forest/drivers/__init__.py b/forest/drivers/__init__.py index 4630b73f3..c747317fe 100644 --- a/forest/drivers/__init__.py +++ b/forest/drivers/__init__.py @@ -1,7 +1,28 @@ from importlib import import_module from forest.exceptions import DriverNotFound +from functools import wraps +_CACHE = {} + + +def _cache(f): + # Ensure per-server dataset instances + def wrapped(driver_name, settings=None): + uid = _uid(driver_name, settings) + if uid not in _CACHE: + _CACHE[uid] = f(driver_name, settings) + return _CACHE[uid] + return wrapped + + +def _uid(driver_name, settings): + if settings is None: + return (driver_name,) + return (driver_name,) + tuple(settings[k] for k in sorted(settings.keys())) + + +@_cache def get_dataset(driver_name, settings=None): """Find Dataset related to file type""" if settings is None: diff --git a/test/test_drivers.py b/test/test_drivers.py new file mode 100644 index 000000000..b18665786 --- /dev/null +++ b/test/test_drivers.py @@ -0,0 +1,9 @@ +from forest import drivers + + +def test_singleton_dataset(): + driver_name = "earth_networks" + datasets = ( + drivers.get_dataset(driver_name), + drivers.get_dataset(driver_name)) + assert id(datasets[0]) == id(datasets[1]) From 29ab5421271a0cfade9381345126eaae0c8d945f Mon Sep 17 00:00:00 2001 From: andrewgryan Date: Tue, 10 Mar 2020 16:35:14 +0000 Subject: [PATCH 6/8] pin bokeh version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d3622e8e2..eb2f43212 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -bokeh +bokeh=1.4.0 # Port to 2.0.0 in future datashader iris intake From b0144fb9a116fcd629b7628425ba632e8a6b73f6 Mon Sep 17 00:00:00 2001 From: andrewgryan Date: Wed, 11 Mar 2020 10:50:15 +0000 Subject: [PATCH 7/8] fix middleware to emit valid_times, fix unique decorator, fix empty image earth networks view --- forest/db/control.py | 16 +++++++++++++--- forest/drivers/earth_networks.py | 7 +++++-- forest/old_state.py | 6 ++++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/forest/db/control.py b/forest/db/control.py index 02be9d4c7..6b2d865df 100644 --- a/forest/db/control.py +++ b/forest/db/control.py @@ -301,14 +301,24 @@ def _pressure(self, store, action): yield set_value(key, value) def _pattern(self, store, action): - value = action["payload"]["value"] - variables = self.navigator.variables(pattern=value) - initial_times = self.navigator.initial_times(pattern=value) + pattern = action["payload"]["value"] + variables = self.navigator.variables(pattern=pattern) + initial_times = self.navigator.initial_times(pattern=pattern) initial_times = list(reversed(initial_times)) yield action yield set_value("variables", variables) yield set_value("initial_times", initial_times) + # Set valid_times if pattern, variable and initial_time present + kwargs = { + "pattern": pattern, + "variable": store.state.get("variable"), + "initial_time": store.state.get("initial_time"), + } + if all(kwargs[k] is not None for k in ["variable", "initial_time"]): + valid_times = self.navigator.valid_times(**kwargs) + yield set_value("valid_times", valid_times) + def _variable(self, store, action): for attr in ["pattern", "initial_time"]: if attr not in store.state: diff --git a/forest/drivers/earth_networks.py b/forest/drivers/earth_networks.py index a1e28787d..d0ae5309a 100644 --- a/forest/drivers/earth_networks.py +++ b/forest/drivers/earth_networks.py @@ -34,7 +34,7 @@ def __init__(self, loader): self.loader = loader palette = bokeh.palettes.all_palettes['Spectral'][11][::-1] self.color_mapper = bokeh.models.LinearColorMapper(low=-1000, high=0, palette=palette) - self.source = bokeh.models.ColumnDataSource({ + self.empty_image = { "x": [], "y": [], "date": [], @@ -42,7 +42,8 @@ def __init__(self, loader): "latitude": [], "flash_type": [], "time_since_flash": [] - }) + } + self.source = bokeh.models.ColumnDataSource(self.empty_image) @old_state @unique @@ -52,6 +53,8 @@ def render(self, state): valid_time = _to_datetime(state.valid_time) frame = self.loader.load_date(valid_time) + if len(frame) == 0: + return self.empty_image x, y = geo.web_mercator( frame.longitude, frame.latitude) diff --git a/forest/old_state.py b/forest/old_state.py index a2b48a35b..8ac6f9e5c 100644 --- a/forest/old_state.py +++ b/forest/old_state.py @@ -31,12 +31,14 @@ def wrapper(*args): if len(args) == 2: self, value = args + key = (id(self), value) # Distinguish wrapped methods else: value, = args + key = value - if (not called) or (value != previous): + if (not called) or (key != previous): called = True - previous = value + previous = key if len(args) == 2: result = f(self, value) else: From dce4b6637e674c14c9620dffda828ce94994660a Mon Sep 17 00:00:00 2001 From: andrewgryan Date: Wed, 11 Mar 2020 10:58:31 +0000 Subject: [PATCH 8/8] bump version to 0.12.7 --- forest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forest/__init__.py b/forest/__init__.py index b40648c62..0b7b71727 100644 --- a/forest/__init__.py +++ b/forest/__init__.py @@ -25,7 +25,7 @@ .. automodule:: forest.presets """ -__version__ = '0.12.6' +__version__ = '0.12.7' from .config import * from . import (