From 270162dcc49a457481cc445f87c9789ecd65e5ad Mon Sep 17 00:00:00 2001 From: Sean Bryan Date: Fri, 1 Dec 2023 11:06:56 +1100 Subject: [PATCH] Add payu test suite for spatial configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spatial tests use the [payu framework][payu]. The payu framework was chosen so that we: - Encourage uptake of payu amongst users of CABLE - Have the foundations in place for running coupled models (atmosphere + land) with payu - Can easily test longer running simulations (payu makes it easy to run a model multiple times and have state persist in the model via restart files) The design of the spatial tests assumes each payu experiment is tailored to running CABLE with a specific meteorological forcing. This has the benefit that all the required inputs are already defined in the payu configuration file. An alternative would be to build up the spatial namelist configurations from scratch. This would be problematic as it is unclear if CABLE requires 'forcing specific' namelist options to be enabled to run with a particular met forcing. That is, CABLE does not allow for easy plug and play with different met forcings via the namelist file. The run directory structure is organised as follows: runs/ ├── spatial │ └── tasks │ ├── (a payu control / experiment directory) │ └── ... ├── payu-laboratory │ └── ... └── fluxsite └── ... Note we have a separate payu-laboratory directory. This is so we keep all CABLE outputs produced by benchcab under the bench_example work directory. This change includes the following additional features: - Add the ability to build the CABLE executable with MPI at runtime so that we run the spatial configurations with MPI. - Add the --mpi flag to benchcab build command so that the user can run the MPI build step independently. - Add subcommands to run each step of the spatial workflow in isolation. - Add payu key in the benchcab config file so that users can easily configure payu experiments and add optional command line arguments to the payu run command. - Add met_forcings key to specify different met forcings and their respective payu experiment. Fixes #5 [payu]: https://github.com/payu-org/payu [cable_example]: https://github.com/CABLE-LSM/cable_example --- benchcab/benchcab.py | 128 +++++++++++----- benchcab/cli.py | 43 ++++-- benchcab/data/config-schema.yml | 24 ++- benchcab/fluxsite.py | 78 ++-------- benchcab/internal.py | 32 +++- benchcab/model.py | 39 +++-- benchcab/spatial.py | 175 ++++++++++++++++++++++ benchcab/utils/dict.py | 41 +++++ benchcab/utils/namelist.py | 36 +++++ benchcab/workdir.py | 11 ++ docs/user_guide/config_options.md | 67 ++++++++- docs/user_guide/expected_output.md | 31 ++-- docs/user_guide/index.md | 96 ++++++++---- tests/conftest.py | 14 ++ tests/test_cli.py | 19 ++- tests/test_fluxsite.py | 120 +++------------ tests/test_model.py | 121 ++++++++++----- tests/test_namelist.py | 87 +++++++++++ tests/test_spatial.py | 233 +++++++++++++++++++++++++++++ tests/test_workdir.py | 20 +++ 20 files changed, 1098 insertions(+), 317 deletions(-) create mode 100644 benchcab/spatial.py create mode 100644 benchcab/utils/dict.py create mode 100644 benchcab/utils/namelist.py create mode 100644 tests/test_namelist.py create mode 100644 tests/test_spatial.py diff --git a/benchcab/benchcab.py b/benchcab/benchcab.py index fab16f4c..3b96eda6 100644 --- a/benchcab/benchcab.py +++ b/benchcab/benchcab.py @@ -10,24 +10,20 @@ from subprocess import CalledProcessError from typing import Optional -from benchcab import internal +from benchcab import fluxsite, internal, spatial from benchcab.comparison import run_comparisons, run_comparisons_in_parallel from benchcab.config import read_config from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface -from benchcab.fluxsite import ( - Task, - get_fluxsite_comparisons, - get_fluxsite_tasks, - run_tasks, - run_tasks_in_parallel, -) from benchcab.internal import get_met_forcing_file_names from benchcab.model import Model from benchcab.utils.fs import mkdir, next_path from benchcab.utils.pbs import render_job_script from benchcab.utils.repo import SVNRepo, create_repo from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface -from benchcab.workdir import setup_fluxsite_directory_tree +from benchcab.workdir import ( + setup_fluxsite_directory_tree, + setup_spatial_directory_tree, +) class Benchcab: @@ -47,7 +43,8 @@ def __init__( self._config: Optional[dict] = None self._models: list[Model] = [] - self.tasks: list[Task] = [] # initialise fluxsite tasks lazily + self._fluxsite_tasks: list[fluxsite.FluxsiteTask] = [] + self._spatial_tasks: list[spatial.SpatialTask] = [] def _validate_environment(self, project: str, modules: list): """Performs checks on current user environment.""" @@ -114,20 +111,34 @@ def _get_models(self, config: dict) -> list[Model]: self._models.append(Model(repo=repo, model_id=id, **sub_config)) return self._models - def _initialise_tasks(self, config: dict) -> list[Task]: - """A helper method that initialises and returns the `tasks` attribute.""" - self.tasks = get_fluxsite_tasks( - models=self._get_models(config), - science_configurations=config.get( - "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS - ), - fluxsite_forcing_file_names=get_met_forcing_file_names( - config.get("fluxsite", {}).get( - "experiment", internal.FLUXSITE_DEFAULT_EXPERIMENT - ) - ), - ) - return self.tasks + def _get_fluxsite_tasks(self, config: dict) -> list[fluxsite.FluxsiteTask]: + if not self._fluxsite_tasks: + self._fluxsite_tasks = fluxsite.get_fluxsite_tasks( + models=self._get_models(config), + science_configurations=config.get( + "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS + ), + fluxsite_forcing_file_names=get_met_forcing_file_names( + config.get("fluxsite", {}).get( + "experiment", internal.FLUXSITE_DEFAULT_EXPERIMENT + ) + ), + ) + return self._fluxsite_tasks + + def _get_spatial_tasks(self, config) -> list[spatial.SpatialTask]: + if not self._spatial_tasks: + self._spatial_tasks = spatial.get_spatial_tasks( + models=self._get_models(config), + met_forcings=config.get("spatial", {}).get( + "met_forcings", internal.SPATIAL_DEFAULT_MET_FORCINGS + ), + science_configurations=config.get( + "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS + ), + payu_args=config.get("spatial", {}).get("payu", {}).get("args"), + ) + return self._spatial_tasks def validate_config(self, config_path: str, verbose: bool): """Endpoint for `benchcab validate_config`.""" @@ -180,6 +191,7 @@ def fluxsite_submit_job( "The NetCDF output for each task is written to " f"{internal.FLUXSITE_DIRS['OUTPUT']}/_out.nc" ) + print("") def checkout(self, config_path: str, verbose: bool): """Endpoint for `benchcab checkout`.""" @@ -213,7 +225,7 @@ def checkout(self, config_path: str, verbose: bool): print("") - def build(self, config_path: str, verbose: bool): + def build(self, config_path: str, verbose: bool, mpi=False): """Endpoint for `benchcab build`.""" config = self._get_config(config_path) self._validate_environment(project=config["project"], modules=config["modules"]) @@ -226,11 +238,11 @@ def build(self, config_path: str, verbose: bool): ) repo.custom_build(modules=config["modules"], verbose=verbose) else: - build_mode = "with MPI" if internal.MPI else "serially" + build_mode = "with MPI" if mpi else "serially" print(f"Compiling CABLE {build_mode} for realisation {repo.name}...") - repo.pre_build(verbose=verbose) - repo.run_build(modules=config["modules"], verbose=verbose) - repo.post_build(verbose=verbose) + repo.pre_build(verbose=verbose, mpi=mpi) + repo.run_build(modules=config["modules"], verbose=verbose, mpi=mpi) + repo.post_build(verbose=verbose, mpi=mpi) print(f"Successfully compiled CABLE for realisation {repo.name}") print("") @@ -239,11 +251,10 @@ def fluxsite_setup_work_directory(self, config_path: str, verbose: bool): config = self._get_config(config_path) self._validate_environment(project=config["project"], modules=config["modules"]) - tasks = self.tasks if self.tasks else self._initialise_tasks(config) print("Setting up run directory tree for fluxsite tests...") setup_fluxsite_directory_tree(verbose=verbose) print("Setting up tasks...") - for task in tasks: + for task in self._get_fluxsite_tasks(config): task.setup_task(verbose=verbose) print("Successfully setup fluxsite tasks") print("") @@ -252,8 +263,8 @@ def fluxsite_run_tasks(self, config_path: str, verbose: bool): """Endpoint for `benchcab fluxsite-run-tasks`.""" config = self._get_config(config_path) self._validate_environment(project=config["project"], modules=config["modules"]) + tasks = self._get_fluxsite_tasks(config) - tasks = self.tasks if self.tasks else self._initialise_tasks(config) print("Running fluxsite tasks...") try: multiprocess = config["fluxsite"]["multiprocess"] @@ -263,9 +274,9 @@ def fluxsite_run_tasks(self, config_path: str, verbose: bool): ncpus = config.get("pbs", {}).get( "ncpus", internal.FLUXSITE_DEFAULT_PBS["ncpus"] ) - run_tasks_in_parallel(tasks, n_processes=ncpus, verbose=verbose) + fluxsite.run_tasks_in_parallel(tasks, n_processes=ncpus, verbose=verbose) else: - run_tasks(tasks, verbose=verbose) + fluxsite.run_tasks(tasks, verbose=verbose) print("Successfully ran fluxsite tasks") print("") @@ -279,8 +290,9 @@ def fluxsite_bitwise_cmp(self, config_path: str, verbose: bool): "nccmp/1.8.5.0" ) # use `nccmp -df` for bitwise comparisons - tasks = self.tasks if self.tasks else self._initialise_tasks(config) - comparisons = get_fluxsite_comparisons(tasks) + comparisons = fluxsite.get_fluxsite_comparisons( + self._get_fluxsite_tasks(config) + ) print("Running comparison tasks...") try: @@ -311,10 +323,46 @@ def fluxsite( else: self.fluxsite_submit_job(config_path, verbose, skip) - def spatial(self, config_path: str, verbose: bool): + def spatial_setup_work_directory(self, config_path: str, verbose: bool): + """Endpoint for `benchcab spatial-setup-work-dir`.""" + config = self._get_config(config_path) + self._validate_environment(project=config["project"], modules=config["modules"]) + + print("Setting up run directory tree for spatial tests...") + setup_spatial_directory_tree() + print("Setting up tasks...") + try: + payu_config = config["spatial"]["payu"]["config"] + except KeyError: + payu_config = None + for task in self._get_spatial_tasks(config): + task.setup_task(payu_config=payu_config, verbose=verbose) + print("Successfully setup spatial tasks") + print("") + + def spatial_run_tasks(self, config_path: str, verbose: bool): + """Endpoint for `benchcab spatial-run-tasks`.""" + config = self._get_config(config_path) + self._validate_environment(project=config["project"], modules=config["modules"]) + + print("Running spatial tasks...") + spatial.run_tasks(tasks=self._get_spatial_tasks(config), verbose=verbose) + print("Successfully dispatched payu jobs") + print("") + + def spatial(self, config_path: str, verbose: bool, skip: list): """Endpoint for `benchcab spatial`.""" + self.checkout(config_path, verbose) + self.build(config_path, verbose, mpi=True) + self.spatial_setup_work_directory(config_path, verbose) + self.spatial_run_tasks(config_path, verbose) - def run(self, config_path: str, no_submit: bool, verbose: bool, skip: list[str]): + def run(self, config_path: str, verbose: bool, skip: list[str]): """Endpoint for `benchcab run`.""" - self.fluxsite(config_path, no_submit, verbose, skip) - self.spatial(config_path, verbose) + self.checkout(config_path, verbose) + self.build(config_path, verbose) + self.build(config_path, verbose, mpi=True) + self.fluxsite_setup_work_directory(config_path, verbose) + self.spatial_setup_work_directory(config_path, verbose) + self.fluxsite_submit_job(config_path, verbose, skip) + self.spatial_run_tasks(config_path, verbose) diff --git a/benchcab/cli.py b/benchcab/cli.py index 4e19d268..fee7086f 100644 --- a/benchcab/cli.py +++ b/benchcab/cli.py @@ -38,9 +38,9 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: action="store_true", ) - # parent parser that contains arguments common to all run specific subcommands - args_run_subcommand = argparse.ArgumentParser(add_help=False) - args_run_subcommand.add_argument( + # parent parser that contains the argument for --no-submit + args_no_submit = argparse.ArgumentParser(add_help=False) + args_no_submit.add_argument( "--no-submit", action="store_true", help="Force benchcab to execute tasks on the current compute node.", @@ -80,7 +80,6 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: parents=[ args_help, args_subcommand, - args_run_subcommand, args_composite_subcommand, ], help="Run all test suites for CABLE.", @@ -109,7 +108,7 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: parents=[ args_help, args_subcommand, - args_run_subcommand, + args_no_submit, args_composite_subcommand, ], help="Run the fluxsite test suite for CABLE.", @@ -140,6 +139,11 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: config file.""", add_help=False, ) + parser_build.add_argument( + "--mpi", + action="store_true", + help="Enable MPI build.", + ) parser_build.set_defaults(func=app.build) # subcommand: 'benchcab fluxsite-setup-work-dir' @@ -168,9 +172,9 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: "fluxsite-run-tasks", parents=[args_help, args_subcommand], help="Run the fluxsite tasks of the main fluxsite command.", - description="""Runs the fluxsite tasks for the fluxsite test suite. Note, this command should - ideally be run inside a PBS job. This command is invoked by the PBS job script generated by - `benchcab run`.""", + description="""Runs the fluxsite tasks for the fluxsite test suite. + Note, this command should ideally be run inside a PBS job. This command + is invoked by the PBS job script generated by `benchcab run`.""", add_help=False, ) parser_fluxsite_run_tasks.set_defaults(func=app.fluxsite_run_tasks) @@ -192,11 +196,32 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: # subcommand: 'benchcab spatial' parser_spatial = subparsers.add_parser( "spatial", - parents=[args_help, args_subcommand], + parents=[args_help, args_subcommand, args_composite_subcommand], help="Run the spatial tests only.", description="""Runs the default spatial test suite for CABLE.""", add_help=False, ) parser_spatial.set_defaults(func=app.spatial) + # subcommand: 'benchcab spatial-setup-work-dir' + parser_spatial_setup_work_dir = subparsers.add_parser( + "spatial-setup-work-dir", + parents=[args_help, args_subcommand], + help="Run the work directory setup step of the spatial command.", + description="""Generates the spatial run directory tree in the current working + directory so that spatial tasks can be run.""", + add_help=False, + ) + parser_spatial_setup_work_dir.set_defaults(func=app.spatial_setup_work_directory) + + # subcommand 'benchcab spatial-run-tasks' + parser_spatial_run_tasks = subparsers.add_parser( + "spatial-run-tasks", + parents=[args_help, args_subcommand], + help="Run the spatial tasks of the main spatial command.", + description="Runs the spatial tasks for the spatial test suite.", + add_help=False, + ) + parser_spatial_run_tasks.set_defaults(func=app.spatial_run_tasks) + return main_parser diff --git a/benchcab/data/config-schema.yml b/benchcab/data/config-schema.yml index 892d3abf..7826ac99 100644 --- a/benchcab/data/config-schema.yml +++ b/benchcab/data/config-schema.yml @@ -95,4 +95,26 @@ fluxsite: schema: type: "string" required: false - \ No newline at end of file + +spatial: + type: "dict" + required: false + schema: + met_forcings: + type: "dict" + required: false + minlength: 1 + keysrules: + type: "string" + valuesrules: + type: "string" + payu: + type: "dict" + required: false + schema: + config: + type: "dict" + required: false + args: + type: "string" + required: false \ No newline at end of file diff --git a/benchcab/fluxsite.py b/benchcab/fluxsite.py index 3797c1cd..41cd8035 100644 --- a/benchcab/fluxsite.py +++ b/benchcab/fluxsite.py @@ -9,7 +9,6 @@ import sys from pathlib import Path from subprocess import CalledProcessError -from typing import Any, Dict, TypeVar import f90nml import flatdict @@ -19,70 +18,9 @@ from benchcab.comparison import ComparisonTask from benchcab.model import Model from benchcab.utils.fs import chdir, mkdir +from benchcab.utils.namelist import patch_namelist, patch_remove_namelist from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface -# fmt: off -# ====================================================== -# Copyright (c) 2017 - 2022 Samuel Colvin and other contributors -# from https://github.com/pydantic/pydantic/blob/fd2991fe6a73819b48c906e3c3274e8e47d0f761/pydantic/utils.py#L200 - -KeyType = TypeVar('KeyType') - - -def deep_update(mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any]) -> Dict[KeyType, Any]: - updated_mapping = mapping.copy() - for updating_mapping in updating_mappings: - for k, v in updating_mapping.items(): - if k in updated_mapping and isinstance(updated_mapping[k], dict) and isinstance(v, dict): - updated_mapping[k] = deep_update(updated_mapping[k], v) - else: - updated_mapping[k] = v - return updated_mapping - -# ====================================================== -# fmt: on - - -def deep_del( - mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any] -) -> Dict[KeyType, Any]: - """Deletes all key-value 'leaf nodes' in `mapping` specified by `updating_mappings`.""" - updated_mapping = mapping.copy() - for updating_mapping in updating_mappings: - for key, value in updating_mapping.items(): - if isinstance(updated_mapping[key], dict) and isinstance(value, dict): - updated_mapping[key] = deep_del(updated_mapping[key], value) - else: - del updated_mapping[key] - return updated_mapping - - -def patch_namelist(nml_path: Path, patch: dict): - """Writes a namelist patch specified by `patch` to `nml_path`. - - The `patch` dictionary must comply with the `f90nml` api. - """ - if not nml_path.exists(): - f90nml.write(patch, nml_path) - return - - nml = f90nml.read(nml_path) - f90nml.write(deep_update(nml, patch), nml_path, force=True) - - -def patch_remove_namelist(nml_path: Path, patch_remove: dict): - """Removes a subset of namelist parameters specified by `patch_remove` from `nml_path`. - - The `patch_remove` dictionary must comply with the `f90nml` api. - """ - nml = f90nml.read(nml_path) - try: - f90nml.write(deep_del(nml, patch_remove), nml_path, force=True) - except KeyError as exc: - msg = f"Namelist parameters specified in `patch_remove` do not exist in {nml_path.name}." - raise KeyError(msg) from exc - - f90_logical_repr = {True: ".true.", False: ".false."} @@ -90,7 +28,7 @@ class CableError(Exception): """Custom exception class for CABLE errors.""" -class Task: +class FluxsiteTask: """A class used to represent a single fluxsite task.""" root_dir: Path = internal.CWD @@ -358,10 +296,10 @@ def get_fluxsite_tasks( models: list[Model], science_configurations: list[dict], fluxsite_forcing_file_names: list[str], -) -> list[Task]: +) -> list[FluxsiteTask]: """Returns a list of fluxsite tasks to run.""" tasks = [ - Task( + FluxsiteTask( model=model, met_forcing_file=file_name, sci_conf_id=sci_conf_id, @@ -374,14 +312,16 @@ def get_fluxsite_tasks( return tasks -def run_tasks(tasks: list[Task], verbose=False): +def run_tasks(tasks: list[FluxsiteTask], verbose=False): """Runs tasks in `tasks` serially.""" for task in tasks: task.run(verbose=verbose) def run_tasks_in_parallel( - tasks: list[Task], n_processes=internal.FLUXSITE_DEFAULT_PBS["ncpus"], verbose=False + tasks: list[FluxsiteTask], + n_processes=internal.FLUXSITE_DEFAULT_PBS["ncpus"], + verbose=False, ): """Runs tasks in `tasks` in parallel across multiple processes.""" run_task = operator.methodcaller("run", verbose=verbose) @@ -390,7 +330,7 @@ def run_tasks_in_parallel( def get_fluxsite_comparisons( - tasks: list[Task], root_dir=internal.CWD + tasks: list[FluxsiteTask], root_dir=internal.CWD ) -> list[ComparisonTask]: """Returns a list of `ComparisonTask` objects to run comparisons with. diff --git a/benchcab/internal.py b/benchcab/internal.py index bf6b6799..0871ca6c 100644 --- a/benchcab/internal.py +++ b/benchcab/internal.py @@ -19,7 +19,6 @@ "walltime": "6:00:00", "storage": [], } -MPI = False FLUXSITE_DEFAULT_MULTIPROCESS = True # DIRECTORY PATHS/STRUCTURE: @@ -62,6 +61,7 @@ # Fluxsite directory tree FLUXSITE_DIRS = {} + # Relative path to root directory for CABLE fluxsite runs FLUXSITE_DIRS["RUN"] = RUN_DIR / "fluxsite" @@ -80,17 +80,43 @@ # Relative path to directory that stores bitwise comparison results FLUXSITE_DIRS["BITWISE_CMP"] = FLUXSITE_DIRS["ANALYSIS"] / "bitwise-comparisons" -# Path to met files: +# Relative path to root directory for CABLE spatial runs +SPATIAL_RUN_DIR = RUN_DIR / "spatial" + +# Relative path to tasks directory (contains payu control directories configured +# for each spatial task) +SPATIAL_TASKS_DIR = SPATIAL_RUN_DIR / "tasks" + +# A custom payu laboratory directory for payu runs +PAYU_LABORATORY_DIR = RUN_DIR / "payu-laboratory" + +# Path to PLUMBER2 site forcing data directory (doi: 10.25914/5fdb0902607e1): MET_DIR = Path("/g/data/ks32/CLEX_Data/PLUMBER2/v1-0/Met/") +# Default met forcings to use in the spatial test suite. Each met +# forcing has a corresponding payu experiment that is configured to run CABLE +# with that forcing. +SPATIAL_DEFAULT_MET_FORCINGS = { + "crujra_access": "https://github.com/CABLE-LSM/cable_example.git", +} + # CABLE SVN root url: CABLE_SVN_ROOT = "https://trac.nci.org.au/svn/cable" +# Relative path to temporary build directory (serial) +TMP_BUILD_DIR = Path("offline", ".tmp") + +# Relative path to temporary build directory (MPI) +TMP_BUILD_DIR_MPI = Path("offline", ".mpitmp") + # CABLE GitHub URL: CABLE_GIT_URL = "https://github.com/CABLE-LSM/CABLE.git" # CABLE executable file name: -CABLE_EXE = "cable-mpi" if MPI else "cable" +CABLE_EXE = "cable" + +# CABLE MPI executable file name: +CABLE_MPI_EXE = "cable-mpi" # CABLE namelist file name: CABLE_NML = "cable.nml" diff --git a/benchcab/model.py b/benchcab/model.py index 269f5b70..97eee0ba 100644 --- a/benchcab/model.py +++ b/benchcab/model.py @@ -58,7 +58,7 @@ def model_id(self) -> int: def model_id(self, value: int): self._model_id = value - def get_exe_path(self) -> Path: + def get_exe_path(self, mpi=False) -> Path: """Return the path to the built executable.""" return ( self.root_dir @@ -66,7 +66,7 @@ def get_exe_path(self) -> Path: / self.name / self.src_dir / "offline" - / internal.CABLE_EXE + / (internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE) ) def custom_build(self, modules: list[str], verbose=False): @@ -108,10 +108,14 @@ def custom_build(self, modules: list[str], verbose=False): verbose=verbose, ) - def pre_build(self, verbose=False): + def pre_build(self, mpi=False, verbose=False): """Runs CABLE pre-build steps.""" path_to_repo = self.root_dir / internal.SRC_DIR / self.name - tmp_dir = path_to_repo / self.src_dir / "offline" / ".tmp" + tmp_dir = ( + path_to_repo + / self.src_dir + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) if not tmp_dir.exists(): if verbose: print(f"mkdir {tmp_dir.relative_to(self.root_dir)}") @@ -151,10 +155,14 @@ def pre_build(self, verbose=False): verbose=verbose, ) - def run_build(self, modules: list[str], verbose=False): + def run_build(self, modules: list[str], mpi=False, verbose=False): """Runs CABLE build scripts.""" path_to_repo = self.root_dir / internal.SRC_DIR / self.name - tmp_dir = path_to_repo / self.src_dir / "offline" / ".tmp" + tmp_dir = ( + path_to_repo + / self.src_dir + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) with chdir(tmp_dir), self.modules_handler.load(modules, verbose=verbose): env = os.environ.copy() @@ -163,28 +171,31 @@ def run_build(self, modules: list[str], verbose=False): env["CFLAGS"] = "-O2 -fp-model precise" env["LDFLAGS"] = f"-L{env['NETCDF_ROOT']}/lib/Intel -O0" env["LD"] = "-lnetcdf -lnetcdff" - env["FC"] = "mpif90" if internal.MPI else "ifort" + env["FC"] = "mpif90" if mpi else "ifort" self.subprocess_handler.run_cmd( "make -f Makefile", env=env, verbose=verbose ) self.subprocess_handler.run_cmd( - f"./{'parallel_cable' if internal.MPI else 'serial_cable'} \"{env['FC']}\" " + f"./{'parallel_cable' if mpi else 'serial_cable'} \"{env['FC']}\" " f"\"{env['CFLAGS']}\" \"{env['LDFLAGS']}\" \"{env['LD']}\" \"{env['NCMOD']}\"", env=env, verbose=verbose, ) - def post_build(self, verbose=False): + def post_build(self, mpi=False, verbose=False): """Runs CABLE post-build steps.""" path_to_repo = self.root_dir / internal.SRC_DIR / self.name - tmp_dir = path_to_repo / self.src_dir / "offline" / ".tmp" + tmp_dir = ( + path_to_repo + / self.src_dir + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) + exe = internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE rename( - (tmp_dir / internal.CABLE_EXE).relative_to(self.root_dir), - (path_to_repo / self.src_dir / "offline" / internal.CABLE_EXE).relative_to( - self.root_dir - ), + (tmp_dir / exe).relative_to(self.root_dir), + (path_to_repo / self.src_dir / "offline" / exe).relative_to(self.root_dir), verbose=verbose, ) diff --git a/benchcab/spatial.py b/benchcab/spatial.py new file mode 100644 index 00000000..9205aad8 --- /dev/null +++ b/benchcab/spatial.py @@ -0,0 +1,175 @@ +# Copyright 2022 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. +# SPDX-License-Identifier: Apache-2.0 + +"""A module containing functions and data structures for running spatial tasks.""" + +from pathlib import Path +from typing import Optional + +import git +import yaml + +from benchcab import internal +from benchcab.model import Model +from benchcab.utils.dict import deep_update +from benchcab.utils.fs import chdir +from benchcab.utils.namelist import patch_namelist, patch_remove_namelist +from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface + + +class SpatialTask: + """A class used to represent a single spatial task.""" + + root_dir: Path = internal.CWD + subprocess_handler: SubprocessWrapperInterface = SubprocessWrapper() + + def __init__( + self, + model: Model, + met_forcing_name: str, + met_forcing_payu_experiment: str, + sci_conf_id: int, + sci_config: dict, + payu_args: Optional[str] = None, + ) -> None: + self.model = model + self.met_forcing_name = met_forcing_name + self.met_forcing_payu_experiment = met_forcing_payu_experiment + self.sci_conf_id = sci_conf_id + self.sci_config = sci_config + self.payu_args = payu_args + + def get_task_name(self) -> str: + """Returns the file name convention used for this task.""" + return f"{self.met_forcing_name}_R{self.model.model_id}_S{self.sci_conf_id}" + + def setup_task(self, payu_config: Optional[dict] = None, verbose=False): + """Does all file manipulations to run cable with payu for this task.""" + + if verbose: + print(f"Setting up task: {self.get_task_name()}") + + self.clone_experiment(verbose=verbose) + self.configure_experiment(payu_config, verbose=verbose) + self.update_namelist(verbose=verbose) + + def clone_experiment(self, verbose=False): + """Clone the payu experiment from GitHub.""" + url = self.met_forcing_payu_experiment + path = internal.SPATIAL_TASKS_DIR / self.get_task_name() + if verbose: + print(f"git clone {url} {path}") + _ = git.Repo.clone_from(url, path) + + def configure_experiment(self, payu_config: Optional[dict] = None, verbose=False): + """Configure the payu experiment for this task.""" + task_dir = self.root_dir / internal.SPATIAL_TASKS_DIR / self.get_task_name() + exp_config_path = task_dir / "config.yaml" + with exp_config_path.open("r", encoding="utf-8") as file: + config = yaml.safe_load(file) + if config is None: + config = {} + + if verbose: + print( + " Updating experiment config parameters in", + task_dir.relative_to(self.root_dir) / "config.yaml", + ) + + if payu_config: + config = deep_update(config, payu_config) + + config["exe"] = str(self.model.get_exe_path(mpi=True)) + + # Here we prepend inputs to the `input` list so that payu knows to use + # our inputs over the pre-existing inputs in the config file: + config["input"] = [ + # Note: only necessary for CABLE v2 + str( + self.root_dir + / internal.CABLE_AUX_DIR + / "core" + / "biogeophys" + / "def_veg_params_zr_clitt_albedo_fix.txt" + ), + # Note: only necessary for CABLE v2 + str( + self.root_dir + / internal.CABLE_AUX_DIR + / "core" + / "biogeophys" + / "def_soil_params.txt" + ), + *config.get("input", []), + ] + + config["laboratory"] = str(self.root_dir / internal.PAYU_LABORATORY_DIR) + + with exp_config_path.open("w", encoding="utf-8") as file: + yaml.dump(config, file) + + def update_namelist(self, verbose=False): + """Update the namelist file for this task.""" + nml_path = ( + self.root_dir + / internal.SPATIAL_TASKS_DIR + / self.get_task_name() + / internal.CABLE_NML + ) + if verbose: + print(f" Adding science configurations to CABLE namelist file {nml_path}") + patch_namelist(nml_path, self.sci_config) + + if self.model.patch: + if verbose: + print( + f" Adding branch specific configurations to CABLE namelist file {nml_path}" + ) + patch_namelist(nml_path, self.model.patch) + + if self.model.patch_remove: + if verbose: + print( + f" Removing branch specific configurations from CABLE namelist file {nml_path}" + ) + patch_remove_namelist(nml_path, self.model.patch_remove) + + def run(self, verbose=False) -> None: + """Runs a single spatial task.""" + + task_dir = self.root_dir / internal.SPATIAL_TASKS_DIR / self.get_task_name() + with chdir(task_dir): + self.subprocess_handler.run_cmd( + f"payu run {self.payu_args}" if self.payu_args else "payu run", + verbose=verbose, + ) + + +def run_tasks(tasks: list[SpatialTask], verbose=False): + """Runs tasks in `tasks` sequentially.""" + + for task in tasks: + task.run(verbose=verbose) + + +def get_spatial_tasks( + models: list[Model], + met_forcings: dict[str, str], + science_configurations: list[dict], + payu_args: Optional[str] = None, +): + """Returns a list of spatial tasks to run.""" + tasks = [ + SpatialTask( + model=model, + met_forcing_name=met_forcing_name, + met_forcing_payu_experiment=met_forcing_payu_experiment, + sci_conf_id=sci_conf_id, + sci_config=sci_config, + payu_args=payu_args, + ) + for model in models + for met_forcing_name, met_forcing_payu_experiment in met_forcings.items() + for sci_conf_id, sci_config in enumerate(science_configurations) + ] + return tasks diff --git a/benchcab/utils/dict.py b/benchcab/utils/dict.py new file mode 100644 index 00000000..c76e6c78 --- /dev/null +++ b/benchcab/utils/dict.py @@ -0,0 +1,41 @@ +# Copyright 2022 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. +# SPDX-License-Identifier: Apache-2.0 + +"""Utility functions for manipulating nested dictionaries.""" + +from typing import Any, Dict, TypeVar + +# fmt: off +# ====================================================== +# Copyright (c) 2017 - 2022 Samuel Colvin and other contributors +# from https://github.com/pydantic/pydantic/blob/fd2991fe6a73819b48c906e3c3274e8e47d0f761/pydantic/utils.py#L200 + +KeyType = TypeVar('KeyType') + + +def deep_update(mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any]) -> Dict[KeyType, Any]: # noqa + updated_mapping = mapping.copy() + for updating_mapping in updating_mappings: + for k, v in updating_mapping.items(): + if k in updated_mapping and isinstance(updated_mapping[k], dict) and isinstance(v, dict): + updated_mapping[k] = deep_update(updated_mapping[k], v) + else: + updated_mapping[k] = v + return updated_mapping + +# ====================================================== +# fmt: on + + +def deep_del( + mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any] +) -> Dict[KeyType, Any]: + """Deletes all key-value 'leaf nodes' in `mapping` specified by `updating_mappings`.""" + updated_mapping = mapping.copy() + for updating_mapping in updating_mappings: + for key, value in updating_mapping.items(): + if isinstance(updated_mapping[key], dict) and isinstance(value, dict): + updated_mapping[key] = deep_del(updated_mapping[key], value) + else: + del updated_mapping[key] + return updated_mapping diff --git a/benchcab/utils/namelist.py b/benchcab/utils/namelist.py new file mode 100644 index 00000000..d6a6de48 --- /dev/null +++ b/benchcab/utils/namelist.py @@ -0,0 +1,36 @@ +# Copyright 2022 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. +# SPDX-License-Identifier: Apache-2.0 + +"""Contains utility functions for manipulating Fortran namelist files.""" + +from pathlib import Path + +import f90nml + +from benchcab.utils.dict import deep_del, deep_update + + +def patch_namelist(nml_path: Path, patch: dict): + """Writes a namelist patch specified by `patch` to `nml_path`. + + The `patch` dictionary must comply with the `f90nml` api. + """ + if not nml_path.exists(): + f90nml.write(patch, nml_path) + return + + nml = f90nml.read(nml_path) + f90nml.write(deep_update(nml, patch), nml_path, force=True) + + +def patch_remove_namelist(nml_path: Path, patch_remove: dict): + """Removes a subset of namelist parameters specified by `patch_remove` from `nml_path`. + + The `patch_remove` dictionary must comply with the `f90nml` api. + """ + nml = f90nml.read(nml_path) + try: + f90nml.write(deep_del(nml, patch_remove), nml_path, force=True) + except KeyError as exc: + msg = f"Namelist parameters specified in `patch_remove` do not exist in {nml_path.name}." + raise KeyError(msg) from exc diff --git a/benchcab/workdir.py b/benchcab/workdir.py index d695337b..f2b32a25 100644 --- a/benchcab/workdir.py +++ b/benchcab/workdir.py @@ -28,3 +28,14 @@ def setup_fluxsite_directory_tree(verbose=False): """ for path in internal.FLUXSITE_DIRS.values(): mkdir(path, verbose=verbose, parents=True, exist_ok=True) + + +def setup_spatial_directory_tree(verbose=False): + """Generate the directory structure for running spatial tests.""" + + for path in [ + internal.SPATIAL_RUN_DIR, + internal.SPATIAL_TASKS_DIR, + internal.PAYU_LABORATORY_DIR, + ]: + mkdir(path, verbose=verbose, parents=True, exist_ok=True) diff --git a/docs/user_guide/config_options.md b/docs/user_guide/config_options.md index 1d22c314..3a60a5cc 100644 --- a/docs/user_guide/config_options.md +++ b/docs/user_guide/config_options.md @@ -163,6 +163,69 @@ fluxsites: ``` +## spatial + +Contains settings specific to spatial tests. + +This key is _optional_. **Default** settings for the spatial tests will be used if it is not present. + +```yaml +spatial: + met_forcings: + crujra_access: https://github.com/CABLE-LSM/cable_example.git + payu: + config: + walltime: 1:00:00 + args: -n 2 +``` + +### [met_forcings](#met_forcings) + +Specify one or more spatial met forcings to use in the spatial test suite. Each entry is a key-value pair where the key is the name of the met forcing and the value is a URL to a payu experiment that is configured to run CABLE with that forcing. + +This key is _optional_. **Default** values for the `met_forcings` key is as follows: + +```yaml +spatial: + met_forcings: + crujra_access: https://github.com/CABLE-LSM/cable_example.git +``` + +### [payu](#payu) + +Contains settings specific to the payu workflow manager. + +This key is _optional_. **Default** values for the payu settings will apply if not specified. + +```yaml +spatial: + payu: + config: + walltime: 1:00:00 + args: -n 2 +``` + +[`config`](#+payu.config){ #+payu.config } + +: **Default:** unset, _optional key_. :octicons-dash-24: Specify global configuration options for running payu. Settings specified here are passed into to the payu configuration file for each experiment. + +```yaml +spatial: + payu: + config: + walltime: 1:00:00 +``` + +[`args`](#+payu.args){ #+payu.args } + +: **Default:** unset, _optional key_. :octicons-dash-24: Specify command line arguments to the `payu run` command in the form of a string. Arguments are used for all spatial payu runs. + +```yaml +spatial: + payu: + args: -n 2 +``` + ## realisations Entries for each CABLE branch to use. Each entry is a key-value pair and are listed as follows: @@ -337,7 +400,7 @@ realisations: ### [patch_remove](#patch_remove) -: **Default:** unset, no effect, _optional key. :octicons-dash-24: Specifies branch-specific namelist settings to be removed from the `cable.nml` namelist settings. When the `patch_remove` key is specified, the specified namelists are removed from all namelist files for this branch for all science configurations run by `benchcab`. When specifying a namelist parameter in `patch_remove`, the value of the namelist parameter is ignored. +: **Default:** unset, _optional key. :octicons-dash-24: Specifies branch-specific namelist settings to be removed from the `cable.nml` namelist settings. When the `patch_remove` key is specified, the specified namelists are removed from all namelist files for this branch for all science configurations run by `benchcab`. When specifying a namelist parameter in `patch_remove`, the value of the namelist parameter is ignored. : The `patch_remove` key must be a dictionary-like data structure that is compliant with the [`f90nml`][f90nml-github] python package. ```yaml @@ -356,7 +419,7 @@ realisations: ## science_configurations -: **Default:** unset, no impact, _optional key_. :octicons-dash-24: User defined science configurations. Science configurations that are specified here will replace [the default science configurations](default_science_configurations.md). In the output filenames, each configuration is identified with S where N is an integer starting from 0 for the first listed configuration and increasing by 1 for each subsequent configuration. +: **Default:** unset, _optional key_. :octicons-dash-24: User defined science configurations. Science configurations that are specified here will replace [the default science configurations](default_science_configurations.md). In the output filenames, each configuration is identified with S where N is an integer starting from 0 for the first listed configuration and increasing by 1 for each subsequent configuration. ```yaml science_configurations: [ diff --git a/docs/user_guide/expected_output.md b/docs/user_guide/expected_output.md index a126fb69..66963a4d 100644 --- a/docs/user_guide/expected_output.md +++ b/docs/user_guide/expected_output.md @@ -6,11 +6,11 @@ Other sub-commands should print out part of this output. ``` $ benchcab run -Creating src directory: /scratch/tm70/sb8430/bench_example/src +Creating src directory Checking out repositories... -Successfully checked out trunk at revision 9550 -Successfully checked out test-branch at revision 9550 -Successfully checked out CABLE-AUX at revision 9550 +Successfully checked out trunk at revision 9672 +Successfully checked out test-branch at revision 9672 +Successfully checked out CABLE-AUX at revision 9672 Writing revision number info to rev_number-1.log Compiling CABLE serially for realisation trunk... @@ -18,24 +18,31 @@ Successfully compiled CABLE for realisation trunk Compiling CABLE serially for realisation test-branch... Successfully compiled CABLE for realisation test-branch +Compiling CABLE with MPI for realisation trunk... +Successfully compiled CABLE for realisation trunk +Compiling CABLE with MPI for realisation test-branch... +Successfully compiled CABLE for realisation test-branch + Setting up run directory tree for fluxsite tests... -Creating runs/fluxsite/logs directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/logs -Creating runs/fluxsite/outputs directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/outputs -Creating runs/fluxsite/tasks directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/tasks -Creating runs/fluxsite/analysis directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/analysis -Creating runs/fluxsite/analysis/bitwise-comparisons directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/analysis/bitwise-comparisons -Creating task directories... Setting up tasks... Successfully setup fluxsite tasks +Setting up run directory tree for spatial tests... +Setting up tasks... +Successfully setup spatial tasks + Creating PBS job script to run fluxsite tasks on compute nodes: benchmark_cable_qsub.sh -PBS job submitted: 82479088.gadi-pbs +PBS job submitted: 100563227.gadi-pbs The CABLE log file for each task is written to runs/fluxsite/logs/_log.txt The CABLE standard output for each task is written to runs/fluxsite/tasks//out.txt The NetCDF output for each task is written to runs/fluxsite/outputs/_out.nc + +Running spatial tasks... +Successfully dispatched payu jobs + ``` -The PBS schedule job should print out the following to the job log file: +The benchmark_cable_qsub.sh PBS job should print out the following to the job log file: ``` Running fluxsite tasks... Successfully ran fluxsite tasks diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index e45e4f2c..3d93b9fd 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -8,12 +8,6 @@ In this guide, we will describe: `benchcab` has been designed to work on NCI machine exclusively. It might be extended later on to other systems. -!!! warning "Limitations" - Currently, - - * `benchcab` can only run simulations at flux sites. - * spin-up for CASA simulations are not supported. - ## Pre-requisites To use `benchcab`, you need to join the following projects at NCI: @@ -44,15 +38,24 @@ You need to load the module on each new session at NCI on login or compute nodes - *New feature:* running two versions of CABLE with the same standard set of science configurations except one version is patched to use a new feature. - *Ensemble run:* running any number of versions of CABLE with the same set of customised science configurations. -The regression and new feature run modes should be used as necessary when evaluating new development in CABLE. +The regression and new feature run modes should be used as necessary when evaluating new developments in CABLE. -The code will: (i) check out and (ii) build the code branches. Then it will run each executable across N standard science configurations for a given number of sites. It is possible to produce some plots locally from the output produced. But [the modelevaluation website][meorg] can be used for further benchmarking and evaluation. +The `benchcab` tool: + +- checks out the model versions specified by the user +- builds the required executables +- runs each model version across N standard science configurations for a variety of meteorological forcings +- performs bitwise comparison checks on model outputs across model versions + +The user can then pipe the model outputs into a benchmark analysis via [modelevaluation.org][meorg] to assess model performance. ### Create a work directory #### Choose a location -You can run the benchmark from any directory you want under `/scratch` or `/g/data`. `/scratch` is preferred as the data in the run directory does not need to be preserved for a long time. The code will create sub-directories as needed. Please ensure you have enough space to store the CABLE outputs in your directory, at least temporary, until you upload them to [modelevaluation.org][meorg]. You will need about 16GB for the outputs for the `forty-two-site` experiment (with 4 different science configurations). +You can run the benchmark from any directory you want under `/scratch` or `/g/data`. `/scratch` is preferred as the data in the run directory does not need to be preserved for a long time. The code will create sub-directories as needed. Please ensure you have enough space to store the CABLE outputs in your directory, at least temporarily until you upload them to [modelevaluation.org][meorg]. + +The full test suite will require about 22GB of storage space. !!! Warning "The HOME directory is unsuitable" @@ -83,7 +86,7 @@ cd bench_example !!! warning `benchcab` will stop if it is not run within a work directory with the proper structure. -Currently, `benchcab` can only run CABLE for flux sites. **To run the whole workflow**, run +Currently, `benchcab` can only run CABLE for flux site and offline spatial configurations. **To run the whole workflow**, run ```bash benchcab run @@ -93,7 +96,8 @@ The tool will follow the steps: 1. Checkout the code branches. The codes will be stored under `src/` directory in your work directory. The sub-directories are created automatically. 2. Compile the source code from all branches -3. Setup and launch a PBS job to run the simulations in parallel. When `benchcab` launches the PBS job, it will print out the job ID to the terminal. You can check the status of the job with `qstat`. `benchcab` will not warn you when the simulations are over. +3. Setup and launch a PBS job to run the flux site simulations in parallel. When `benchcab` launches the PBS job, it will print out the job ID to the terminal. You can check the status of the job with `qstat`. `benchcab` will not warn you when the simulations are over. +4. Setup and run an ensemble of offline spatial runs using the [`payu`][payu-github] framework. !!! tip "Expected output" @@ -119,22 +123,27 @@ The following files and directories are created when `benchcab run` executes suc ├── benchmark_cable_qsub.sh.o ├── rev_number-1.log ├── runs -│   └── fluxsite -│   ├── logs -│ │ ├── _log.txt -│ │ └── ... -│   ├── outputs -│ │ ├── _out.nc -│ │ └── ... -│   ├── analysis -│ │ └── bitwise-comparisons -│   └── tasks -│ ├── -│ │ ├── cable (executable) -│ │ ├── cable.nml -│ │ ├── cable_soilparm.nml -│ │ └── pft_params.nml -│ └── ... +│   ├── fluxsite +│   │ ├── logs +│ │ │ ├── _log.txt +│ │ │ └── ... +│   │ ├── outputs +│ │ │ ├── _out.nc +│ │ │ └── ... +│   │ ├── analysis +│ │ │ └── bitwise-comparisons +│   │ └── tasks +│ │ ├── +│ │ │ ├── cable (executable) +│ │ │ ├── cable.nml +│ │ │ ├── cable_soilparm.nml +│ │ │ └── pft_params.nml +│ │ └── ... +│   ├── spatial +│   │ └── tasks +│ │ ├── (a payu control / experiment directory) +│ │ └── ... +│   └── payu-laboratory └── src ├── CABLE-AUX ├── @@ -155,11 +164,11 @@ The following files and directories are created when `benchcab run` executes suc `runs/fluxsite/` -: directory that contains the log files, output files, and tasks for running CABLE. +: directory that contains the log files, output files, and tasks for running CABLE in the fluxsite configuration. `runs/fluxsite/tasks` -: directory that contains task directories. A task consists of a CABLE run for a branch (realisation), a meteorological forcing, and a science configuration. In the above directory structure, `` uses the following naming convention: +: directory that contains fluxsite task directories. A task consists of a CABLE run for a branch (realisation), a meteorological forcing, and a science configuration. In the above directory structure, `` uses the following naming convention: ``` _R_S @@ -183,6 +192,29 @@ The following files and directories are created when `benchcab run` executes suc : directory that contains the standard output produced by the bitwise comparison command: `benchcab fluxsite-bitwise-cmp`. Standard output is only saved when the netcdf files being compared differ from each other +`runs/spatial/` + +: directory that contains task directories for running CABLE in the offline spatial configuration. + +`runs/spatial/tasks` + +: directory that contains payu control directories (or experiments) configured for each spatial task. A task consists of a CABLE run for a branch (realisation), a meteorological forcing, and a science configuration. In the above directory structure, `` uses the following naming convention: + +``` +_R_S +``` + +: where `met_forcing_name` is the name of the spatial met forcing, `realisation_key` is the branch key specified in the config file, and `science_config_key` identifies the science configuration used. See the [`met_forcings`](config_options.md#met_forcings) option for more information on how to configure the met forcings used. + + +`runs/spatial/tasks//` + +: a payu control directory (or experiment). See [Configuring your experiment](https://payu.readthedocs.io/en/latest/config.html) for more information on payu experiments. + +`runs/payu-laboratory/` + +: a custom payu laboratory directory. See [Laboratory Structure](https://payu.readthedocs.io/en/latest/design.html#laboratory-structure) for more information on the payu laboratory directory. + !!! warning "Re-running `benchcab` multiple times in the same working directory" We recommend the user to manually delete the generated files when re-running `benchcab`. Re-running `benchcab` multiple times in the same working directory is currently not yet supported (see issue [CABLE-LSM/benchcab#20](https://github.com/CABLE-LSM/benchcab/issues/20)). To clean the current working directory, run the following command in the working directory @@ -192,10 +224,11 @@ The following files and directories are created when `benchcab run` executes suc ## Analyse the output with [modelevaluation.org][meorg] - - Once the benchmarking has finished running all the simulations, you need to upload the output files to [modelevaluation.org][meorg] via the web interface. To do this: +!!! warning "Limitations" + Model evaluation for offline spatial outputs is not yet available (see issue [CABLE-LSM/benchcab#193](https://github.com/CABLE-LSM/benchcab/issues/193)). + 1. Go to [modelevaluation.org][meorg] and login or create a new account. 2. Navigate to the `benchcab-evaluation` workspace. To do this, click the **Current Workspace** button at the top of the page, and select `benchcab-evaluation` under "Workspaces Shared With Me".
@@ -272,3 +305,4 @@ Alternatively, you can also access the ACCESS-NRI User support via [the ACCESS-H [benchmark_5]: https://modelevaluation.org/modelOutput/display/diLdf49PfpEwZemTz [benchmark_42]: https://modelevaluation.org/modelOutput/display/pvkuY5gpR2n4FKZw3 [run_CABLE_v2]: running_CABLE_v2.md +[payu-github]: https://github.com/payu-org/payu diff --git a/tests/conftest.py b/tests/conftest.py index 807e0203..1cd5db91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,6 +90,20 @@ def config(): }, "multiprocessing": True, }, + "spatial": { + "met_forcings": { + "crujra_access": "https://github.com/CABLE-LSM/cable_example.git", + "gswp": "foo", + }, + "payu": { + "config": { + "ncpus": 16, + "walltime": "1:00:00", + "mem": "64GB", + }, + "args": "-n 2", + }, + }, } diff --git a/tests/test_cli.py b/tests/test_cli.py index f4c4e038..c190d25c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,7 +15,6 @@ def test_cli_parser(): res = vars(parser.parse_args(["run"])) assert res == { "config_path": "config.yaml", - "no_submit": False, "verbose": False, "skip": [], "func": app.run, @@ -42,6 +41,7 @@ def test_cli_parser(): assert res == { "config_path": "config.yaml", "verbose": False, + "mpi": False, "func": app.build, } @@ -93,9 +93,26 @@ def test_cli_parser(): assert res == { "config_path": "config.yaml", "verbose": False, + "skip": [], "func": app.spatial, } + # Success case: default spatial-setup-work-dir command + res = vars(parser.parse_args(["spatial-setup-work-dir"])) + assert res == { + "config_path": "config.yaml", + "verbose": False, + "func": app.spatial_setup_work_directory, + } + + # Success case: default spatial-run-tasks command + res = vars(parser.parse_args(["spatial-run-tasks"])) + assert res == { + "config_path": "config.yaml", + "verbose": False, + "func": app.spatial_run_tasks, + } + # Failure case: pass --no-submit to a non 'run' command with pytest.raises(SystemExit): parser.parse_args(["fluxsite-setup-work-dir", "--no-submit"]) diff --git a/tests/test_fluxsite.py b/tests/test_fluxsite.py index 8f97f0c4..e76d9b6d 100644 --- a/tests/test_fluxsite.py +++ b/tests/test_fluxsite.py @@ -8,7 +8,6 @@ import contextlib import io import math -from pathlib import Path import f90nml import netCDF4 @@ -17,12 +16,10 @@ from benchcab import __version__, internal from benchcab.fluxsite import ( CableError, - Task, + FluxsiteTask, get_comparison_name, get_fluxsite_comparisons, get_fluxsite_tasks, - patch_namelist, - patch_remove_namelist, ) from benchcab.model import Model from benchcab.utils.repo import Repo @@ -62,8 +59,8 @@ def model(mock_cwd, mock_subprocess_handler, mock_repo): @pytest.fixture() def task(model, mock_cwd, mock_subprocess_handler): - """Returns a mock `Task` instance.""" - _task = Task( + """Returns a mock `FluxsiteTask` instance.""" + _task = FluxsiteTask( model=model, met_forcing_file="forcing-file.nc", sci_conf_id=0, @@ -75,7 +72,7 @@ def task(model, mock_cwd, mock_subprocess_handler): class TestGetTaskName: - """tests for `Task.get_task_name()`.""" + """tests for `FluxsiteTask.get_task_name()`.""" def test_task_name_convention(self, task): """Success case: check task name convention.""" @@ -83,7 +80,7 @@ def test_task_name_convention(self, task): class TestGetLogFilename: - """Tests for `Task.get_log_filename()`.""" + """Tests for `FluxsiteTask.get_log_filename()`.""" def test_log_filename_convention(self, task): """Success case: check log file name convention.""" @@ -91,7 +88,7 @@ def test_log_filename_convention(self, task): class TestGetOutputFilename: - """Tests for `Task.get_output_filename()`.""" + """Tests for `FluxsiteTask.get_output_filename()`.""" def test_output_filename_convention(self, task): """Success case: check output file name convention.""" @@ -99,11 +96,11 @@ def test_output_filename_convention(self, task): class TestFetchFiles: - """Tests for `Task.fetch_files()`.""" + """Tests for `FluxsiteTask.fetch_files()`.""" @pytest.fixture(autouse=True) def _setup(self, task): - """Setup precondition for `Task.fetch_files()`.""" + """Setup precondition for `FluxsiteTask.fetch_files()`.""" internal.NAMELIST_DIR.mkdir() (internal.NAMELIST_DIR / internal.CABLE_NML).touch() (internal.NAMELIST_DIR / internal.CABLE_SOIL_NML).touch() @@ -129,11 +126,11 @@ def test_required_files_are_copied_to_task_dir(self, task): class TestCleanTask: - """Tests for `Task.clean_task()`.""" + """Tests for `FluxsiteTask.clean_task()`.""" @pytest.fixture(autouse=True) def _setup(self, task): - """Setup precondition for `Task.clean_task()`.""" + """Setup precondition for `FluxsiteTask.clean_task()`.""" task_dir = internal.FLUXSITE_DIRS["TASKS"] / task.get_task_name() task_dir.mkdir(parents=True) (task_dir / internal.CABLE_NML).touch() @@ -161,91 +158,12 @@ def test_clean_files(self, task): assert not (internal.FLUXSITE_DIRS["LOG"] / task.get_log_filename()).exists() -class TestPatchNamelist: - """Tests for `patch_namelist()`.""" - - @pytest.fixture() - def nml_path(self): - """Return a path to a namelist file used for testing.""" - return Path("test.nml") - - def test_patch_on_non_existing_namelist_file(self, nml_path): - """Success case: patch non-existing namelist file.""" - patch = {"cable": {"file": "/path/to/file", "bar": 123}} - patch_namelist(nml_path, patch) - assert f90nml.read(nml_path) == patch - - def test_patch_on_non_empty_namelist_file(self, nml_path): - """Success case: patch non-empty namelist file.""" - f90nml.write({"cable": {"file": "/path/to/file", "bar": 123}}, nml_path) - patch_namelist(nml_path, {"cable": {"some": {"parameter": True}, "bar": 456}}) - assert f90nml.read(nml_path) == { - "cable": { - "file": "/path/to/file", - "bar": 456, - "some": {"parameter": True}, - } - } - - def test_empty_patch_does_nothing(self, nml_path): - """Success case: empty patch does nothing.""" - f90nml.write({"cable": {"file": "/path/to/file", "bar": 123}}, nml_path) - prev = f90nml.read(nml_path) - patch_namelist(nml_path, {}) - assert f90nml.read(nml_path) == prev - - -class TestPatchRemoveNamelist: - """Tests for `patch_remove_namelist()`.""" - - @pytest.fixture() - def nml(self): - """Return a namelist dictionary used for testing.""" - return { - "cable": { - "cable_user": { - "some_parameter": True, - "new_feature": True, - }, - }, - } - - @pytest.fixture() - def nml_path(self, nml): - """Create a namelist file and return its path.""" - _nml_path = Path("test.nml") - f90nml.write(nml, _nml_path) - return _nml_path - - def test_remove_namelist_parameter_from_derived_type(self, nml_path): - """Success case: remove a namelist parameter from derrived type.""" - patch_remove_namelist( - nml_path, {"cable": {"cable_user": {"new_feature": True}}} - ) - assert f90nml.read(nml_path) == { - "cable": {"cable_user": {"some_parameter": True}} - } - - def test_empty_patch_remove_does_nothing(self, nml_path, nml): - """Success case: empty patch_remove does nothing.""" - patch_remove_namelist(nml_path, {}) - assert f90nml.read(nml_path) == nml - - def test_key_error_raised_for_non_existent_namelist_parameter(self, nml_path): - """Failure case: test patch_remove KeyError exeption.""" - with pytest.raises( - KeyError, - match=f"Namelist parameters specified in `patch_remove` do not exist in {nml_path.name}.", - ): - patch_remove_namelist(nml_path, {"cable": {"foo": {"bar": True}}}) - - class TestSetupTask: - """Tests for `Task.setup_task()`.""" + """Tests for `FluxsiteTask.setup_task()`.""" @pytest.fixture(autouse=True) def _setup(self, task): - """Setup precondition for `Task.setup_task()`.""" + """Setup precondition for `FluxsiteTask.setup_task()`.""" (internal.NAMELIST_DIR).mkdir() (internal.NAMELIST_DIR / internal.CABLE_NML).touch() (internal.NAMELIST_DIR / internal.CABLE_SOIL_NML).touch() @@ -328,11 +246,11 @@ def test_standard_output(self, task, verbosity, expected): class TestRunCable: - """Tests for `Task.run_cable()`.""" + """Tests for `FluxsiteTask.run_cable()`.""" @pytest.fixture(autouse=True) def _setup(self, task): - """Setup precondition for `Task.run_cable()`.""" + """Setup precondition for `FluxsiteTask.run_cable()`.""" task_dir = internal.FLUXSITE_DIRS["TASKS"] / task.get_task_name() task_dir.mkdir(parents=True) @@ -361,7 +279,7 @@ def test_cable_error_exception(self, task, mock_subprocess_handler): class TestAddProvenanceInfo: - """Tests for `Task.add_provenance_info()`.""" + """Tests for `FluxsiteTask.add_provenance_info()`.""" @pytest.fixture() def nml(self): @@ -385,7 +303,7 @@ def nc_output_path(self, task): @pytest.fixture(autouse=True) def _setup(self, task, nml): - """Setup precondition for `Task.add_provenance_info()`.""" + """Setup precondition for `FluxsiteTask.add_provenance_info()`.""" task_dir = internal.FLUXSITE_DIRS["TASKS"] / task.get_task_name() task_dir.mkdir(parents=True) fluxsite_output_dir = internal.FLUXSITE_DIRS["OUTPUT"] @@ -437,7 +355,7 @@ class TestGetFluxsiteTasks: @pytest.fixture() def models(self, mock_repo): - """Return a list of `CableRepository` instances used for testing.""" + """Return a list of `Model` instances used for testing.""" return [Model(repo=mock_repo, model_id=id) for id in range(2)] @pytest.fixture() @@ -479,7 +397,7 @@ class TestGetFluxsiteComparisons: def test_comparisons_for_two_branches_with_two_tasks(self, mock_cwd, mock_repo): """Success case: comparisons for two branches with two tasks.""" tasks = [ - Task( + FluxsiteTask( model=Model(repo=mock_repo, model_id=model_id), met_forcing_file="foo.nc", sci_config={"foo": "bar"}, @@ -506,7 +424,7 @@ def test_comparisons_for_two_branches_with_two_tasks(self, mock_cwd, mock_repo): def test_comparisons_for_three_branches_with_three_tasks(self, mock_cwd, mock_repo): """Success case: comparisons for three branches with three tasks.""" tasks = [ - Task( + FluxsiteTask( model=Model(repo=mock_repo, model_id=model_id), met_forcing_file="foo.nc", sci_config={"foo": "bar"}, diff --git a/tests/test_model.py b/tests/test_model.py index b35ca3c4..bd86aeea 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -29,18 +29,24 @@ class MockRepo(Repo): def __init__(self) -> None: self.handle = "trunk" - def checkout(self, verbose=False): + def checkout(self, path: Path, verbose=False): pass def get_branch_name(self) -> str: return self.handle - def get_revision(self) -> str: + def get_revision(self, path: Path) -> str: pass return MockRepo() +@pytest.fixture(params=[False, True]) +def mpi(request): + """Return a parametrized mpi flag for testing.""" + return request.param + + @pytest.fixture() def model( mock_repo, mock_cwd, mock_subprocess_handler, mock_environment_modules_handler @@ -74,11 +80,15 @@ def test_undefined_model_id(self, model): class TestGetExePath: """Tests for `Model.get_exe_path()`.""" - def test_serial_exe_path(self, model, mock_cwd): - """Success case: get path to serial executable.""" + @pytest.mark.parametrize( + ("mpi", "expected_exe"), + [(False, internal.CABLE_EXE), (True, internal.CABLE_MPI_EXE)], + ) + def test_get_exe_path(self, model, mock_cwd, mpi, expected_exe): + """Success case: get path to executable.""" assert ( - model.get_exe_path() - == mock_cwd / internal.SRC_DIR / model.name / "offline" / internal.CABLE_EXE + model.get_exe_path(mpi=mpi) + == mock_cwd / internal.SRC_DIR / model.name / "offline" / expected_exe ) @@ -164,10 +174,18 @@ def _setup(self, model): (internal.SRC_DIR / model.name / "offline" / "serial_cable").touch() (internal.SRC_DIR / model.name / "offline" / "foo.f90").touch() - def test_source_files_and_scripts_are_copied_to_tmp_dir(self, model): + @pytest.fixture() + def tmp_dir(self, model, mpi): + """Return the relative path to the temporary build directory.""" + return ( + internal.SRC_DIR + / model.name + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) + + def test_source_files_and_scripts_are_copied_to_tmp_dir(self, model, mpi, tmp_dir): """Success case: test source files and scripts are copied to .tmp.""" - model.pre_build() - tmp_dir = internal.SRC_DIR / model.name / "offline" / ".tmp" + model.pre_build(mpi=mpi) assert (tmp_dir / "Makefile").exists() assert (tmp_dir / "parallel_cable").exists() assert (tmp_dir / "serial_cable").exists() @@ -211,37 +229,52 @@ def modules(self): return ["foo", "bar"] @pytest.fixture() - def env(self, netcdf_root): - """Return a dictionary containing the required environment variables.""" + def expected_env(self, netcdf_root, mpi): + """Return a dictionary of expected environment variables to be defined.""" return { "NCDIR": f"{netcdf_root}/lib/Intel", "NCMOD": f"{netcdf_root}/include/Intel", "CFLAGS": "-O2 -fp-model precise", "LDFLAGS": f"-L{netcdf_root}/lib/Intel -O0", "LD": "-lnetcdf -lnetcdff", - "FC": "ifort", + "FC": "mpif90" if mpi else "ifort", } @pytest.fixture(autouse=True) def _setup(self, model, netcdf_root): """Setup precondition for `Model.run_build()`.""" - (internal.SRC_DIR / model.name / "offline" / ".tmp").mkdir(parents=True) + (internal.SRC_DIR / model.name / internal.TMP_BUILD_DIR).mkdir(parents=True) + (internal.SRC_DIR / model.name / internal.TMP_BUILD_DIR_MPI).mkdir(parents=True) # This is required so that we can use the NETCDF_ROOT environment variable # when running `make`, and `serial_cable` and `parallel_cable` scripts: os.environ["NETCDF_ROOT"] = netcdf_root + @pytest.fixture() + def expected_commands(self, netcdf_root, mpi): + """Return a list of expected commands to be executed.""" + return ( + [ + "make -f Makefile", + './parallel_cable "mpif90" "-O2 -fp-model precise"' + f' "-L{netcdf_root}/lib/Intel -O0" "-lnetcdf -lnetcdff" ' + f'"{netcdf_root}/include/Intel"', + ] + if mpi + else [ + "make -f Makefile", + './serial_cable "ifort" "-O2 -fp-model precise"' + f' "-L{netcdf_root}/lib/Intel -O0" "-lnetcdf -lnetcdff" ' + f'"{netcdf_root}/include/Intel"', + ] + ) + def test_build_command_execution( - self, model, mock_subprocess_handler, modules, netcdf_root + self, model, mock_subprocess_handler, modules, mpi, expected_commands ): """Success case: test build commands are run.""" - model.run_build(modules) - assert mock_subprocess_handler.commands == [ - "make -f Makefile", - './serial_cable "ifort" "-O2 -fp-model precise"' - f' "-L{netcdf_root}/lib/Intel -O0" "-lnetcdf -lnetcdff" ' - f'"{netcdf_root}/include/Intel"', - ] + model.run_build(modules, mpi=mpi) + assert mock_subprocess_handler.commands == expected_commands def test_modules_loaded_at_runtime( self, model, mock_environment_modules_handler, modules @@ -256,11 +289,11 @@ def test_modules_loaded_at_runtime( ) in mock_environment_modules_handler.commands def test_commands_are_run_with_environment_variables( - self, model, mock_subprocess_handler, modules, env + self, model, mock_subprocess_handler, modules, mpi, expected_env ): """Success case: test commands are run with the correct environment variables.""" - model.run_build(modules) - for kv in env.items(): + model.run_build(modules, mpi=mpi) + for kv in expected_env.items(): assert kv in mock_subprocess_handler.env.items() @pytest.mark.parametrize( @@ -283,18 +316,38 @@ class TestPostBuild: @pytest.fixture(autouse=True) def _setup(self, model): """Setup precondition for `Model.post_build()`.""" - (internal.SRC_DIR / model.name / "offline" / ".tmp").mkdir(parents=True) - ( - internal.SRC_DIR / model.name / "offline" / ".tmp" / internal.CABLE_EXE - ).touch() + tmp_build_dir = internal.SRC_DIR / model.name / internal.TMP_BUILD_DIR + tmp_build_dir.mkdir(parents=True) + (tmp_build_dir / internal.CABLE_EXE).touch() + + tmp_build_dir_mpi = internal.SRC_DIR / model.name / internal.TMP_BUILD_DIR_MPI + tmp_build_dir_mpi.mkdir(parents=True) + (tmp_build_dir_mpi / internal.CABLE_MPI_EXE).touch() + + @pytest.fixture() + def tmp_dir(self, model, mpi): + """Return the relative path to the temporary build directory.""" + return ( + internal.SRC_DIR + / model.name + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) + + @pytest.fixture() + def exe(self, mpi): + """Return the name of the CABLE executable.""" + return internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE + + @pytest.fixture() + def offline_dir(self, model): + """Return the relative path to the offline source directory.""" + return internal.SRC_DIR / model.name / "offline" - def test_exe_moved_to_offline_dir(self, model): + def test_exe_moved_to_offline_dir(self, model, mpi, tmp_dir, exe, offline_dir): """Success case: test executable is moved to offline directory.""" - model.post_build() - tmp_dir = internal.SRC_DIR / model.name / "offline" / ".tmp" - assert not (tmp_dir / internal.CABLE_EXE).exists() - offline_dir = internal.SRC_DIR / model.name / "offline" - assert (offline_dir / internal.CABLE_EXE).exists() + model.post_build(mpi=mpi) + assert not (tmp_dir / exe).exists() + assert (offline_dir / exe).exists() @pytest.mark.parametrize( ("verbosity", "expected"), diff --git a/tests/test_namelist.py b/tests/test_namelist.py new file mode 100644 index 00000000..87d629cf --- /dev/null +++ b/tests/test_namelist.py @@ -0,0 +1,87 @@ +"""`pytest` tests for namelist.py.""" + +from pathlib import Path + +import f90nml +import pytest + +from benchcab.utils.namelist import patch_namelist, patch_remove_namelist + + +class TestPatchNamelist: + """Tests for `patch_namelist()`.""" + + @pytest.fixture() + def nml_path(self): + """Return a path to a namelist file used for testing.""" + return Path("test.nml") + + def test_patch_on_non_existing_namelist_file(self, nml_path): + """Success case: patch non-existing namelist file.""" + patch = {"cable": {"file": "/path/to/file", "bar": 123}} + patch_namelist(nml_path, patch) + assert f90nml.read(nml_path) == patch + + def test_patch_on_non_empty_namelist_file(self, nml_path): + """Success case: patch non-empty namelist file.""" + f90nml.write({"cable": {"file": "/path/to/file", "bar": 123}}, nml_path) + patch_namelist(nml_path, {"cable": {"some": {"parameter": True}, "bar": 456}}) + assert f90nml.read(nml_path) == { + "cable": { + "file": "/path/to/file", + "bar": 456, + "some": {"parameter": True}, + } + } + + def test_empty_patch_does_nothing(self, nml_path): + """Success case: empty patch does nothing.""" + f90nml.write({"cable": {"file": "/path/to/file", "bar": 123}}, nml_path) + prev = f90nml.read(nml_path) + patch_namelist(nml_path, {}) + assert f90nml.read(nml_path) == prev + + +class TestPatchRemoveNamelist: + """Tests for `patch_remove_namelist()`.""" + + @pytest.fixture() + def nml(self): + """Return a namelist dictionary used for testing.""" + return { + "cable": { + "cable_user": { + "some_parameter": True, + "new_feature": True, + }, + }, + } + + @pytest.fixture() + def nml_path(self, nml): + """Create a namelist file and return its path.""" + _nml_path = Path("test.nml") + f90nml.write(nml, _nml_path) + return _nml_path + + def test_remove_namelist_parameter_from_derived_type(self, nml_path): + """Success case: remove a namelist parameter from derrived type.""" + patch_remove_namelist( + nml_path, {"cable": {"cable_user": {"new_feature": True}}} + ) + assert f90nml.read(nml_path) == { + "cable": {"cable_user": {"some_parameter": True}} + } + + def test_empty_patch_remove_does_nothing(self, nml_path, nml): + """Success case: empty patch_remove does nothing.""" + patch_remove_namelist(nml_path, {}) + assert f90nml.read(nml_path) == nml + + def test_key_error_raised_for_non_existent_namelist_parameter(self, nml_path): + """Failure case: test patch_remove KeyError exeption.""" + with pytest.raises( + KeyError, + match=f"Namelist parameters specified in `patch_remove` do not exist in {nml_path.name}.", + ): + patch_remove_namelist(nml_path, {"cable": {"foo": {"bar": True}}}) diff --git a/tests/test_spatial.py b/tests/test_spatial.py new file mode 100644 index 00000000..6d8a1286 --- /dev/null +++ b/tests/test_spatial.py @@ -0,0 +1,233 @@ +"""`pytest` tests for spatial.py. + +Note: explicit teardown for generated files and directories are not required as +the working directory used for testing is cleaned up in the `_run_around_tests` +pytest autouse fixture. +""" + +import contextlib +import io + +import f90nml +import pytest +import yaml + +from benchcab import internal +from benchcab.model import Model +from benchcab.spatial import SpatialTask, get_spatial_tasks +from benchcab.utils.repo import Repo + + +@pytest.fixture() +def mock_repo(): + class MockRepo(Repo): + def __init__(self) -> None: + self.branch = "test-branch" + self.revision = "1234" + + def checkout(self, verbose=False): + pass + + def get_branch_name(self) -> str: + return self.branch + + def get_revision(self) -> str: + return self.revision + + return MockRepo() + + +@pytest.fixture() +def model(mock_cwd, mock_subprocess_handler, mock_repo): + """Returns a `Model` instance.""" + _model = Model( + model_id=1, + repo=mock_repo, + patch={"cable": {"some_branch_specific_setting": True}}, + ) + _model.subprocess_handler = mock_subprocess_handler + _model.root_dir = mock_cwd + return _model + + +@pytest.fixture() +def task(model, mock_cwd, mock_subprocess_handler): + """Returns a mock `SpatialTask` instance.""" + _task = SpatialTask( + model=model, + met_forcing_name="crujra_access", + met_forcing_payu_experiment="https://github.com/CABLE-LSM/cable_example.git", + sci_conf_id=0, + sci_config={"cable": {"some_setting": True}}, + ) + _task.subprocess_handler = mock_subprocess_handler + _task.root_dir = mock_cwd + return _task + + +class TestGetTaskName: + """Tests for `SpatialTask.get_task_name()`.""" + + def test_task_name_convention(self, task): + """Success case: check task name convention.""" + assert task.get_task_name() == "crujra_access_R1_S0" + + +class TestConfigureExperiment: + """Tests for `SpatialTask.configure_experiment()`.""" + + @pytest.fixture(autouse=True) + def _create_task_dir(self): + task_dir = internal.SPATIAL_TASKS_DIR / "crujra_access_R1_S0" + task_dir.mkdir(parents=True) + (task_dir / "config.yaml").touch() + (task_dir / "cable.nml").touch() + + def test_payu_config_parameters(self, task, mock_cwd): + """Success case: check config.yaml parameters.""" + task.configure_experiment(payu_config={"some_parameter": "foo"}) + config_path = internal.SPATIAL_TASKS_DIR / task.get_task_name() / "config.yaml" + with config_path.open("r", encoding="utf-8") as file: + config = yaml.safe_load(file) + assert ( + config["exe"] + == f"{mock_cwd}/src/test-branch/offline/{internal.CABLE_MPI_EXE}" + ) + assert config["input"] == [ + f"{mock_cwd}/src/CABLE-AUX/core/biogeophys/def_veg_params_zr_clitt_albedo_fix.txt", + f"{mock_cwd}/src/CABLE-AUX/core/biogeophys/def_soil_params.txt", + ] + assert config["laboratory"] == str(mock_cwd / internal.PAYU_LABORATORY_DIR) + assert config["some_parameter"] == "foo" + + # TODO(Sean) fix for issue https://github.com/CABLE-LSM/benchcab/issues/162 + @pytest.mark.skip( + reason="""This will always fail since `parametrize()` parameters are + dependent on the `mock_cwd` fixture.""" + ) + @pytest.mark.parametrize( + ("verbosity", "expected"), + [ + (False, ""), + ( + True, + " Updating experiment config parameters in " + "runs/spatial/tasks/crujra_access_R1_S0/config.yaml\n", + ), + ], + ) + def test_standard_output(self, task, verbosity, expected): + """Success case: test standard output.""" + with contextlib.redirect_stdout(io.StringIO()) as buf: + task.configure_experiment(verbose=verbosity) + assert buf.getvalue() == expected + + +class TestUpdateNamelist: + """Tests for `SpatialTask.update_namelist()`.""" + + @pytest.fixture(autouse=True) + def _create_task_dir(self): + task_dir = internal.SPATIAL_TASKS_DIR / "crujra_access_R1_S0" + task_dir.mkdir(parents=True) + (task_dir / "config.yaml").touch() + (task_dir / "cable.nml").touch() + + def test_namelist_parameters_are_patched(self, task): + """Success case: test namelist parameters are patched.""" + task.update_namelist() + res_nml = f90nml.read( + str(internal.SPATIAL_TASKS_DIR / task.get_task_name() / internal.CABLE_NML) + ) + assert res_nml["cable"] == { + "some_setting": True, + "some_branch_specific_setting": True, + } + + # TODO(Sean) fix for issue https://github.com/CABLE-LSM/benchcab/issues/162 + @pytest.mark.skip( + reason="""This will always fail since `parametrize()` parameters are + dependent on the `mock_cwd` fixture.""" + ) + @pytest.mark.parametrize( + ("verbosity", "expected"), + [ + (False, ""), + ( + True, + " Updating experiment config parameters in " + "config.yaml\n" + " Adding science configurations to CABLE namelist file " + "runs/spatial/tasks/crujra_access_R1_S0/cable.nml\n" + " Adding branch specific configurations to CABLE namelist file " + "runs/spatial/tasks/crujra_access_R1_S0/cable.nml\n", + ), + ], + ) + def test_standard_output(self, task, verbosity, expected): + """Success case: test standard output.""" + with contextlib.redirect_stdout(io.StringIO()) as buf: + task.update_namelist(verbose=verbosity) + assert buf.getvalue() == expected + + +class TestRun: + """Tests for `SpatialTask.run()`.""" + + @pytest.fixture(autouse=True) + def _setup(self, task): + task_dir = internal.SPATIAL_TASKS_DIR / task.get_task_name() + task_dir.mkdir(parents=True, exist_ok=True) + + def test_payu_run_command(self, task, mock_subprocess_handler): + """Success case: test payu run command.""" + task.run() + assert "payu run" in mock_subprocess_handler.commands + + def test_payu_run_with_optional_arguments(self, task, mock_subprocess_handler): + """Success case: test payu run command with optional arguments.""" + task.payu_args = "--some-flag" + task.run() + assert "payu run --some-flag" in mock_subprocess_handler.commands + + +class TestGetSpatialTasks: + """Tests for `get_spatial_tasks()`.""" + + @pytest.fixture() + def models(self, mock_repo): + """Return a list of `Model` instances used for testing.""" + return [Model(repo=mock_repo, model_id=id) for id in range(2)] + + @pytest.fixture() + def met_forcings(self, config): + """Return a list of spatial met forcing specifications.""" + return config["spatial"]["met_forcings"] + + @pytest.fixture() + def science_configurations(self, config): + """Return a list of science configurations used for testing.""" + return config["science_configurations"] + + def test_task_product_across_branches_forcings_and_configurations( + self, models, met_forcings, science_configurations + ): + """Success case: test task product across branches, forcings and configurations.""" + tasks = get_spatial_tasks( + models=models, + met_forcings=met_forcings, + science_configurations=science_configurations, + ) + met_forcing_names = list(met_forcings.keys()) + assert [ + (task.model, task.met_forcing_name, task.sci_config) for task in tasks + ] == [ + (models[0], met_forcing_names[0], science_configurations[0]), + (models[0], met_forcing_names[0], science_configurations[1]), + (models[0], met_forcing_names[1], science_configurations[0]), + (models[0], met_forcing_names[1], science_configurations[1]), + (models[1], met_forcing_names[0], science_configurations[0]), + (models[1], met_forcing_names[0], science_configurations[1]), + (models[1], met_forcing_names[1], science_configurations[0]), + (models[1], met_forcing_names[1], science_configurations[1]), + ] diff --git a/tests/test_workdir.py b/tests/test_workdir.py index 62dd710c..ae67676e 100644 --- a/tests/test_workdir.py +++ b/tests/test_workdir.py @@ -12,6 +12,7 @@ from benchcab.workdir import ( clean_directory_tree, setup_fluxsite_directory_tree, + setup_spatial_directory_tree, ) @@ -37,6 +38,25 @@ def test_directory_structure_generated(self, fluxsite_directory_list): assert path.exists() +class TestSetupSpatialDirectoryTree: + """Tests for `setup_spatial_directory_tree()`.""" + + @pytest.fixture() + def spatial_directory_list(self): + """Return the list of work directories we want benchcab to create.""" + return [ + Path("runs", "spatial"), + Path("runs", "spatial", "tasks"), + Path("runs", "payu-laboratory"), + ] + + def test_directory_structure_generated(self, spatial_directory_list): + """Success case: generate spatial directory structure.""" + setup_spatial_directory_tree() + for path in spatial_directory_list: + assert path.exists() + + class TestCleanDirectoryTree: """Tests for `clean_directory_tree()`."""