From 7ade3487c50af2258ad43cff07c8758b0bf9ccc0 Mon Sep 17 00:00:00 2001 From: Andrew Kiss <31054815+aekiss@users.noreply.github.com> Date: Fri, 20 Jan 2023 11:24:54 +1100 Subject: [PATCH 1/8] Update README.md (#315) add ACCESS-OM2 conditions of use --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 21e1d49..37e69c6 100644 --- a/README.md +++ b/README.md @@ -46,5 +46,13 @@ If you like the cookbook, you may like to interact more closely with us: * If you find a problem, or have a suggestion for improvement, please log an issue. * All code submitted as part of the `cosima-cookbook` itself must be formatted with [black](https://github.com/psf/black) +## Conditions of use for ACCESS-OM2 data + +We request that users of ACCESS-OM2 model [code](https://github.com/COSIMA/access-om2) or output data: +1. consider citing Kiss et al. (2020) ([http://doi.org/10.5194/gmd-13-401-2020](http://doi.org/10.5194/gmd-13-401-2020)) +2. include an acknowledgement such as the following: + + *The authors thank the Consortium for Ocean-Sea Ice Modelling in Australia (COSIMA; [http://www.cosima.org.au](http://www.cosima.org.au)) for making the ACCESS-OM2 suite of models available at [https://github.com/COSIMA/access-om2](https://github.com/COSIMA/access-om2).* +3. let us know of any publications which use these models or data so we can add them to [our list](https://scholar.google.com/citations?hl=en&user=inVqu_4AAAAJ). [![Documentation Status](https://readthedocs.org/projects/cosima-cookbook/badge/?version=latest)](https://cosima-cookbook.readthedocs.org/en/latest) From a0edaea6fad8b26e2d61d054901b24bd8b8b2c85 Mon Sep 17 00:00:00 2001 From: Angus Gibson Date: Fri, 20 Jan 2023 13:36:56 +1100 Subject: [PATCH 2/8] Fix attribute indexing test This was relying on output from sqlalchemy's inpspect, which changed. I've added a utility function to only compare the expected keys of a dictionary, rather than the entire thing. --- test/test_indexing.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test/test_indexing.py b/test/test_indexing.py index 9f5a862..d8a9432 100644 --- a/test/test_indexing.py +++ b/test/test_indexing.py @@ -21,6 +21,14 @@ def rm_tree(pth): pth.rmdir() +def assert_dictionaries_same(expected, actual): + for key in expected.keys(): + if key not in actual or expected[key] != actual[key]: + return False + + return True + + @pytest.fixture def unreadable_dir(tmp_path): expt_path = tmp_path / "expt_dir" @@ -351,11 +359,14 @@ def test_index_attributes(session_db): database.build_index("test/data/querying", session) inspector = inspect(session.get_bind()) - assert inspector.get_indexes("ncattributes")[0] == { - "name": "ix_ncattributes_ncvar_id", - "column_names": ["ncvar_id"], - "unique": 0, - } + assert assert_dictionaries_same( + { + "name": "ix_ncattributes_ncvar_id", + "column_names": ["ncvar_id"], + "unique": 0, + }, + inspector.get_indexes("ncattributes")[0], + ) ncfile = "output000/ocean.nc" From 71a2e24127d84ae59aea6fb9e8fc145f779eca18 Mon Sep 17 00:00:00 2001 From: Angus Gibson Date: Fri, 20 Jan 2023 13:38:43 +1100 Subject: [PATCH 3/8] Remove distributed indexing test This relies on the now-deprecated dask-backed distributed indexing. --- test/test_indexing.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/test_indexing.py b/test/test_indexing.py index d8a9432..79bfab9 100644 --- a/test/test_indexing.py +++ b/test/test_indexing.py @@ -399,16 +399,6 @@ def test_index_attributes(session_db): assert attr in v.attrs and v.attrs[attr] == attr_val -def test_distributed(client, session_db): - session, db = session_db - database.build_index("test/data/indexing/broken_file", session, client) - - assert db.exists() - q = session.query(database.NCExperiment) - r = q.all() - assert len(r) == 1 - - def test_prune_broken(session_db): session, db = session_db database.build_index("test/data/indexing/broken_file", session) From 97d7b4a925ce14a3bc28a56f10534e3529d070fd Mon Sep 17 00:00:00 2001 From: Angus Gibson Date: Tue, 24 Jan 2023 11:30:34 +1100 Subject: [PATCH 4/8] Correct dictionary syntax in QueryWarning about ambiguous attributes --- cosima_cookbook/querying.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosima_cookbook/querying.py b/cosima_cookbook/querying.py index 8ff2e47..f338013 100644 --- a/cosima_cookbook/querying.py +++ b/cosima_cookbook/querying.py @@ -520,7 +520,7 @@ def _ncfiles_for_variable( warnings.warn( f"Your query returns variables from files with different {attr}: {unique_attributes}. " "This could lead to unexpected behaviour! Disambiguate by passing " - f"attrs={{'{attr}'=''}} to getvar, specifying the desired attribute value.", + f"attrs={{'{attr}':''}} to getvar, specifying the desired attribute value.", QueryWarning, ) From d07e64144a5d68ade54aea70b4f86c7b119d8c27 Mon Sep 17 00:00:00 2001 From: Angus Gibson Date: Wed, 7 Jun 2023 10:42:35 +1000 Subject: [PATCH 5/8] Fix examples link in README Closes #323 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 37e69c6..beab5c9 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Alternatively, you might prefer to download `vdi_jupyter` or the `gadi_jupyter` The COSIMA Cookbook relies on several components: 1. There needs to be a database of simulations -- on the NCI system, model output that is stored in the COSIMA space on the `/g/data/ik11/` directory. - 2. Once you have access to data, the best place to start is the [`cosima-recipes`](https://github.com/COSIMA/cosima-recipes) repository which includes a series of jupyter notebooks containing examples that guide you through to use the cookbook to load model output and then proceed doing simple (or elaborate) computations. The best starting point of exploring the [`cosima-recipes`](https://github.com/COSIMA/cosima-recipes) is the [Documented Examples](https://cosima-recipes.readthedocs.io/en/latest/documented_examples/index.html). A collection of useful examples leveraging the `cosima-cookbook` is also found [here](https://github.com/COSIMA/ACCESS-OM2-1-025-010deg-report/tree/master/figures). + 2. Once you have access to data, the best place to start is the [`cosima-recipes`](https://github.com/COSIMA/cosima-recipes) repository which includes a series of jupyter notebooks containing examples that guide you through to use the cookbook to load model output and then proceed doing simple (or elaborate) computations. The best starting point of exploring the [`cosima-recipes`](https://github.com/COSIMA/cosima-recipes) is the [Documented Examples](https://cosima-recipes.readthedocs.io/en/latest/documented_examples.html). A collection of useful examples leveraging the `cosima-cookbook` is also found [here](https://github.com/COSIMA/ACCESS-OM2-1-025-010deg-report/tree/master/figures). ## Contributing to the Cookbook From da30dd69817312d9244a1207862ceca4e91d3086 Mon Sep 17 00:00:00 2001 From: Angus Gibson Date: Wed, 7 Jun 2023 10:52:40 +1000 Subject: [PATCH 6/8] Apply new formatting rules --- cosima_cookbook/database.py | 1 - cosima_cookbook/diagnostics/overturning.py | 1 - cosima_cookbook/diagnostics/simple.py | 1 - cosima_cookbook/explore.py | 7 ------- cosima_cookbook/netcdf_index.py | 1 - cosima_cookbook/plots/lineplots.py | 1 - cosima_cookbook/plots/overturning.py | 2 -- cosima_cookbook/querying.py | 1 - test/test_dates.py | 5 ----- test/test_explore.py | 4 ---- test/test_indexing.py | 1 - test/test_metadata.py | 1 - test/test_querying.py | 6 ------ test/test_update.py | 1 - 14 files changed, 33 deletions(-) diff --git a/cosima_cookbook/database.py b/cosima_cookbook/database.py index 1bccab3..55999ce 100644 --- a/cosima_cookbook/database.py +++ b/cosima_cookbook/database.py @@ -844,7 +844,6 @@ def build_index( indexed = 0 for directory in [Path(d) for d in directories]: - expt = find_experiment(session, directory) if expt is None: expt = NCExperiment( diff --git a/cosima_cookbook/diagnostics/overturning.py b/cosima_cookbook/diagnostics/overturning.py index 7f53fbf..5b365d6 100644 --- a/cosima_cookbook/diagnostics/overturning.py +++ b/cosima_cookbook/diagnostics/overturning.py @@ -225,7 +225,6 @@ def op(p): @memory.cache def zonal_mean(expt, variable, n=10, resolution=1): - zonal_var = get_nc_variable( expt, "ocean.nc", diff --git a/cosima_cookbook/diagnostics/simple.py b/cosima_cookbook/diagnostics/simple.py index 306e4be..350f807 100644 --- a/cosima_cookbook/diagnostics/simple.py +++ b/cosima_cookbook/diagnostics/simple.py @@ -21,7 +21,6 @@ def annual_scalar(expt, variables): annual_average = darray.resample(time="A").mean("time") for v in annual_average.data_vars: - avar = annual_average.variables[v] dvar = darray.variables[v] avar.attrs["long_name"] = dvar.attrs["long_name"] + " (annual average)" diff --git a/cosima_cookbook/explore.py b/cosima_cookbook/explore.py index e8e0603..6be42ea 100644 --- a/cosima_cookbook/explore.py +++ b/cosima_cookbook/explore.py @@ -25,7 +25,6 @@ def return_value_or_empty(value): class DatabaseExtension: - # DEPRECATED def __init__(self, session, experiments=None): @@ -147,7 +146,6 @@ def _update_selector(self, variables): for vals in variables.sort_values(["name"])[ ["name", "long_name", "units"] ].values: - var, name, units = map(str, vals) if firstvar is None: @@ -329,7 +327,6 @@ class VariableSelectorInfo(VariableSelector): def __init__( self, parent, variables, daterange, frequency, cellmethods, rows=10, **kwargs ): - # The cellmethods widget needs access to the session and experiment self.session = parent.session self.experiment = parent.experiment_name @@ -586,7 +583,6 @@ class DatabaseExplorer(VBox): variables = None def __init__(self, session=None, de=None): - if session is None: session = database.create_session() self.session = session @@ -611,7 +607,6 @@ def __init__(self, session=None, de=None): self._set_handlers() def _make_widgets(self): - style = "" # Gui header @@ -850,7 +845,6 @@ def data(self): class ExperimentExplorer(VBox): - session = None _loaded_data = None experiment_name = None @@ -858,7 +852,6 @@ class ExperimentExplorer(VBox): experiments = None def __init__(self, session=None, experiment=None): - if session is None: session = database.create_session() self.session = session diff --git a/cosima_cookbook/netcdf_index.py b/cosima_cookbook/netcdf_index.py index 172c0a7..4f87343 100755 --- a/cosima_cookbook/netcdf_index.py +++ b/cosima_cookbook/netcdf_index.py @@ -201,7 +201,6 @@ def build_index(use_bag=False, careful=False, expt_dir_list=None): ) def index_variables(ncfile): - matched = find_output.match(ncfile) if matched is None: return [] diff --git a/cosima_cookbook/plots/lineplots.py b/cosima_cookbook/plots/lineplots.py index 714fd91..4a1c319 100644 --- a/cosima_cookbook/plots/lineplots.py +++ b/cosima_cookbook/plots/lineplots.py @@ -69,7 +69,6 @@ def annual_scalar(expts=[], variables=[]): # plotting each variable in a separate plot for variable in variables: - plt.figure(figsize=(12, 6)) for result in results: diff --git a/cosima_cookbook/plots/overturning.py b/cosima_cookbook/plots/overturning.py index 2151211..b528114 100644 --- a/cosima_cookbook/plots/overturning.py +++ b/cosima_cookbook/plots/overturning.py @@ -7,7 +7,6 @@ def psi_avg(expts, n=10, clev=np.arange(-20, 20, 2)): - if not isinstance(expts, list): expts = [expts] @@ -66,7 +65,6 @@ def psi_avg(expts, n=10, clev=np.arange(-20, 20, 2)): def zonal_mean(expts, variable, n=10, resolution=1): - if not isinstance(expts, list): expts = [expts] diff --git a/cosima_cookbook/querying.py b/cosima_cookbook/querying.py index f338013..835d0c9 100644 --- a/cosima_cookbook/querying.py +++ b/cosima_cookbook/querying.py @@ -160,7 +160,6 @@ def get_variables( ] if experiment: - # Create aliases so as to able to join to the NCAttribute table # twice, for the name and value ncas1 = aliased(NCAttributeString) diff --git a/test/test_dates.py b/test/test_dates.py index a3430f5..85650d3 100644 --- a/test/test_dates.py +++ b/test/test_dates.py @@ -71,7 +71,6 @@ def teardown_module(module): def test_format_parse_datetime(): - dates = [ cftime.num2date(t, units="days since 01-01-01", calendar="noleap") for t in times @@ -94,7 +93,6 @@ def test_format_parse_datetime(): def test_rebase_times(): - # Should be a 10 year offset between original times and rebased times assert not np.any( (times + 365 * 10) @@ -113,7 +111,6 @@ def test_rebase_times(): def test_rebase_variable(): - timesvar = xr.DataArray( times, attrs={"units": "days since 1980-01-01", "calendar": "noleap"} ) @@ -154,7 +151,6 @@ def test_rebase_variable(): def test_matching_time_units(): - testfile = "test/data/ocean_sealevel.nc" ds = xr.open_dataset(testfile, decode_times=False) @@ -213,7 +209,6 @@ def test_matching_time_units(): def test_chunking(): - # An offset is required as the target units are ahead of the data in time target_units = "days since 2000-01-01" diff --git a/test/test_explore.py b/test/test_explore.py index 2b0a8a2..086215f 100644 --- a/test/test_explore.py +++ b/test/test_explore.py @@ -68,7 +68,6 @@ def session(tmp_path_factory): def test_database_explorer(session): - dbx = cc.explore.DatabaseExplorer(session=session) assert dbx.session is session @@ -129,7 +128,6 @@ def test_database_explorer(session): def test_experiment_explorer(session): - ee1 = cc.explore.ExperimentExplorer(session=session) # Experiment selector @@ -169,7 +167,6 @@ def test_experiment_explorer(session): def test_get_data(session): - ee = cc.explore.ExperimentExplorer(session=session) assert ee.data is None @@ -190,7 +187,6 @@ def test_get_data(session): def test_model_property(session): - # Grab all variables and ensure the SQL classification matches the python version # May be some holes, as not ensured all cases covered for expt in cc.querying.get_experiments(session, all=True).experiment: diff --git a/test/test_indexing.py b/test/test_indexing.py index 79bfab9..c7b2275 100644 --- a/test/test_indexing.py +++ b/test/test_indexing.py @@ -44,7 +44,6 @@ def unreadable_dir(tmp_path): def test_find_files(): - files = database.find_files("test/data/indexing/") assert len(files) == 17 diff --git a/test/test_metadata.py b/test/test_metadata.py index 23e7b46..8ba803d 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -305,7 +305,6 @@ def test_get_experiments_with_keywords(session_db): def test_getvar_with_metadata(session_db): - session, db = session_db database.build_index("test/data/indexing/metadata", session) diff --git a/test/test_querying.py b/test/test_querying.py index 019dd90..6253b7c 100644 --- a/test/test_querying.py +++ b/test/test_querying.py @@ -56,7 +56,6 @@ def test_invalid_query(session): def test_warning_on_ambiguous_attr(session): - with pytest.warns(QueryWarning) as record: cc.querying._ncfiles_for_variable( "querying_disambiguation", @@ -118,7 +117,6 @@ def test_warning_on_ambiguous_attr(session): def test_disambiguation_on_default_attr(session): - files = cc.querying._ncfiles_for_variable( "querying_disambiguation", "v", @@ -432,7 +430,6 @@ def test_get_variables(session): def test_model_property(session): - filename_map = { "ocean": ( "output/ocean/ice.nc", @@ -480,7 +477,6 @@ def test_model_property(session): def test_is_restart_property(session): - filename_map = { True: ( "output/restart/ice.nc", @@ -515,7 +511,6 @@ def test_is_restart_property(session): def test_is_coordinate_property(session): - units_map = { True: ( "degrees_", @@ -552,7 +547,6 @@ def test_get_frequencies(session): def test_disambiguation_by_frequency(session): - with pytest.warns(UserWarning) as record: assert len(cc.querying._ncfiles_for_variable("querying", "time", session)) == 3 diff --git a/test/test_update.py b/test/test_update.py index b51fa64..a48dee3 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -3,7 +3,6 @@ def test_database_update(tmp_path): - args = shlex.split( "-db {db} test/data/update/experiment_a test/data/update/experiment_b".format( db=tmp_path / "test.db" From 5fb8ae12df5aad9a0dc7c9ad711be271ce101903 Mon Sep 17 00:00:00 2001 From: Angus Gibson Date: Tue, 1 Aug 2023 14:47:25 +1000 Subject: [PATCH 7/8] Fix links in getting started guide Closes #327 --- docs/source/getting_started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 2c948d5..5bc7fe6 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -64,6 +64,6 @@ in the COSIMA recipes repository which demonstrates the available tools. Tutorials and examples ====================== -COSIMA recipes provides `tutorials `_ -and `documented examples `_ +COSIMA recipes provides `tutorials `_ +and `documented examples `_ which can be used to learn how to use the Cookbook and for ideas and inspiration for your own analysis. From fe298bdfc79b207c644723abb08f4808a94f4fa4 Mon Sep 17 00:00:00 2001 From: dougiesquire Date: Thu, 12 Oct 2023 14:56:55 +1100 Subject: [PATCH 8/8] pin sqlalchemy<2.0 --- conda/meta.yaml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index 0ccdfd9..b23ef82 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -26,7 +26,7 @@ requirements: - netcdf4 - joblib - tqdm - - sqlalchemy + - sqlalchemy<2.0 - ipywidgets - cftime>1.2.1 - lxml diff --git a/setup.py b/setup.py index cc3585e..c1b4c7a 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ 'bokeh', 'netcdf4', 'tqdm', - 'sqlalchemy', + 'sqlalchemy<2.0', 'cftime', 'f90nml', 'joblib',