From 9d328cebe3e9902ccae6d9f30b1b2c97fb511e2c Mon Sep 17 00:00:00 2001 From: Romain Beucher Date: Fri, 4 Oct 2024 10:52:49 +1000 Subject: [PATCH] Add test coverage for BarpaTranslator --- tests/data/esm_datastore/barpa-py18.csv | 6 ++ tests/data/esm_datastore/barpa-py18.json | 92 ++++++++++++++++++++++++ tests/test_translators.py | 21 ++++++ 3 files changed, 119 insertions(+) create mode 100644 tests/data/esm_datastore/barpa-py18.csv create mode 100644 tests/data/esm_datastore/barpa-py18.json diff --git a/tests/data/esm_datastore/barpa-py18.csv b/tests/data/esm_datastore/barpa-py18.csv new file mode 100644 index 0000000..b76fd4d --- /dev/null +++ b/tests/data/esm_datastore/barpa-py18.csv @@ -0,0 +1,6 @@ +path,file_type,project_id,MIP_era,activity_id,domain_id,RCM_institution_id,driving_source_id,driving_experiment_id,driving_variant_label,source_id,version_realisation,version,variable_id,freq,start_time,end_time,time_range +/g/data/py18/BARPA/output/CMIP6/DD/AUS-15/BOM/ERA5/evaluation/r1i1p1f1/BARPA-R/v1-r1/1hr/evspsblpot/v20231001/evspsblpot_AUS-15_ERA5_evaluation_r1i1p1f1_BOM_BARPA-R_v1-r1_1hr_199201-199212.nc,f,output,CMIP6,DD,AUS-15,BOM,ERA5,evaluation,r1i1p1f1,BARPA-R,v1-r1,v20231001,evspsblpot,1hr,199201,199212,199201-199212 +/g/data/py18/BARPA/output/CMIP6/DD/AUS-15/BOM/ERA5/evaluation/r1i1p1f1/BARPA-R/v1-r1/1hr/evspsblpot/v20231001/evspsblpot_AUS-15_ERA5_evaluation_r1i1p1f1_BOM_BARPA-R_v1-r1_1hr_198601-198612.nc,f,output,CMIP6,DD,AUS-15,BOM,ERA5,evaluation,r1i1p1f1,BARPA-R,v1-r1,v20231001,evspsblpot,1hr,198601,198612,198601-198612 +/g/data/py18/BARPA/output/CMIP6/DD/AUS-15/BOM/ERA5/evaluation/r1i1p1f1/BARPA-R/v1-r1/1hr/vasmax/v20231001/vasmax_AUS-15_ERA5_evaluation_r1i1p1f1_BOM_BARPA-R_v1-r1_1hr_199201-199212.nc,f,output,CMIP6,DD,AUS-15,BOM,ERA5,evaluation,r1i1p1f1,BARPA-R,v1-r1,v20231001,vasmax,1hr,199201,199212,199201-199212 +/g/data/py18/BARPA/output/CMIP6/DD/AUS-15/BOM/ERA5/evaluation/r1i1p1f1/BARPA-R/v1-r1/1hr/zmla/v20231001/zmla_AUS-15_ERA5_evaluation_r1i1p1f1_BOM_BARPA-R_v1-r1_1hr_198701-198712.nc,f,output,CMIP6,DD,AUS-15,BOM,ERA5,evaluation,r1i1p1f1,BARPA-R,v1-r1,v20231001,zmla,1hr,198701,198712,198701-198712 +/g/data/py18/BARPA/output/CMIP6/DD/AUS-15/BOM/ERA5/evaluation/r1i1p1f1/BARPA-R/v1-r1/1hr/ta200m/v20231001/ta200m_AUS-15_ERA5_evaluation_r1i1p1f1_BOM_BARPA-R_v1-r1_1hr_199601-199612.nc,f,output,CMIP6,DD,AUS-15,BOM,ERA5,evaluation,r1i1p1f1,BARPA-R,v1-r1,v20231001,ta200m,1hr,199601,199612,199601-199612 diff --git a/tests/data/esm_datastore/barpa-py18.json b/tests/data/esm_datastore/barpa-py18.json new file mode 100644 index 0000000..3d03240 --- /dev/null +++ b/tests/data/esm_datastore/barpa-py18.json @@ -0,0 +1,92 @@ +{ + "id": "barpa-py18", + "title": "barpa-py18", + "description": "Datasets on Gadi, both publised and replicated. All file versions present are in the listing\nMaintained By: NCI\nContact: help@nci.org.au", + "assets": { + "column_name": "path", + "format": "netcdf" + }, + "aggregation_control": { + "variable_column_name": "variable", + "groupby_attrs": [ + "file_type", + "project_id", + "MIP_era", + "activity_id", + "domain_id", + "RCM_institution_id", + "driving_source_id", + "driving_experiment_id", + "driving_variant_label", + "source_id", + "version_realisation", + "version", + "variable_id", + "freq" + ], + "aggregations": [ + { + "type": "join_existing", + "attribute_name": "time_range", + "options": { + "dim": "time" + } + } + ] + }, + "esmcat_version": "0.1.0", + "catalog_file": "barpa-py18.csv.gz", + "attributes": [ + { + "column_name": "file_type" + }, + { + "column_name": "project_id" + }, + { + "column_name": "MIP_era" + }, + { + "column_name": "activity_id" + }, + { + "column_name": "domain_id" + }, + { + "column_name": "RCM_institution_id" + }, + { + "column_name": "driving_source_id" + }, + { + "column_name": "driving_experiment_id" + }, + { + "column_name": "driving_variant_label" + }, + { + "column_name": "source_id" + }, + { + "column_name": "version_realisation" + }, + { + "column_name": "version" + }, + { + "column_name": "variable_id" + }, + { + "column_name": "freq" + }, + { + "column_name": "start_time" + }, + { + "column_name": "end_time" + }, + { + "column_name": "time_range" + } + ] +} \ No newline at end of file diff --git a/tests/test_translators.py b/tests/test_translators.py index 78ec2ef..a70ddc5 100644 --- a/tests/test_translators.py +++ b/tests/test_translators.py @@ -7,6 +7,7 @@ from access_nri_intake.catalog import CORE_COLUMNS, TRANSLATOR_GROUPBY_COLUMNS from access_nri_intake.catalog.translators import ( + BarpaTranslator, Cmip5Translator, Cmip6Translator, DefaultTranslator, @@ -244,3 +245,23 @@ def test_EraiTranslator(test_data, groupby, n_entries): df = EraiTranslator(esmds, CORE_COLUMNS).translate(groupby) assert all(df["model"] == model) assert len(df) == n_entries + + +@pytest.mark.parametrize( + "groupby, n_entries", + [ + (None, 5), + (TRANSLATOR_GROUPBY_COLUMNS, 5), + (["variable"], 4), + (["realm"], 2), + (["frequency"], 2), + (["description"], 1), + ], +) +def test_BarpaTranslator(test_data, groupby, n_entries): + "Test BARPA datastore translator" "" + esmds = intake.open_esm_datastore(test_data / "esm_datastore/barpa-py18.json") + esmds.name = "name" + esmds.description = "description" + df = BarpaTranslator(esmds, CORE_COLUMNS).translate(groupby) + assert len(df) == n_entries