Add payu test suite for spatial configuration

The spatial test suite runs CABLE with CRUJRA forcing at ACCESS
resolution (see [here][cable_example] for more details) over all science
configurations and model versions.

Spatial tests use the [payu framework][payu]. The payu framework was
chosen so that we:
- Encourage uptake of payu amongst users of CABLE
- Have the foundations in place for running coupled models (atmosphere +
  land) with payu
- Can easily test longer running simulations (payu makes it easy to run
  a model multiple times and have state persist in the model via restart
  files)

The run directory structure is organised as follows:

runs/
├── spatial
│   └── tasks
│	├── <spatial-task-name> (a payu control / experiment directory)
│	└── ...
├── payu-laboratory
│   └── ...
└── fluxsite
    └── ...

Note we have a separate payu-laboratory directory. This is so we keep
all CABLE outputs produced by benchcab under the bench_example work
directory.

Add the ability to build the CABLE executable with MPI at runtime so
that we run the spatial configurations with MPI.

Add the --mpi flag to benchcab build command so that the user can
run the MPI build step independently.

Add utility functions for git API requests and manipulating namelist
files.

Add subcommands to run each step of the spatial workflow in isolation.

Add payu key in the benchcab config file so that users can easily
configure payu experiments and pass in command line arguments to the
`payu run` command.

Fixes #5

[payu]: https://github.com/payu-org/payu
[cable_example]: https://github.com/CABLE-LSM/cable_example

.conda/benchcab-dev.yaml

@@ -9,3 +9,4 @@ dependencies:
   - pytest-cov
   - pyyaml
   - flatdict
+  - gitpython

benchcab/benchcab.py

@@ -1,5 +1,6 @@
 """Contains the main program entry point for `benchcab`."""
 
+import functools
 import grp
 import os
 import shutil
@@ -8,24 +9,20 @@
 from subprocess import CalledProcessError
 from typing import Optional
 
-from benchcab import internal
+from benchcab import fluxsite, internal, spatial
 from benchcab.cli import generate_parser
 from benchcab.comparison import run_comparisons, run_comparisons_in_parallel
 from benchcab.config import read_config
 from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface
-from benchcab.fluxsite import (
-    Task,
-    get_fluxsite_comparisons,
-    get_fluxsite_tasks,
-    run_tasks,
-    run_tasks_in_parallel,
-)
 from benchcab.internal import get_met_forcing_file_names
 from benchcab.repository import CableRepository
 from benchcab.utils.fs import mkdir, next_path
 from benchcab.utils.pbs import render_job_script
 from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface
-from benchcab.workdir import setup_fluxsite_directory_tree
+from benchcab.workdir import (
+    setup_fluxsite_directory_tree,
+    setup_spatial_directory_tree,
+)
 
 
 class Benchcab:
@@ -48,7 +45,6 @@ def __init__(
             CableRepository(**config, repo_id=id)
             for id, config in enumerate(self.config["realisations"])
         ]
-        self.tasks: list[Task] = []  # initialise fluxsite tasks lazily
         self.benchcab_exe_path = benchcab_exe_path
 
         if validate_env:
@@ -102,9 +98,10 @@ def _validate_environment(self, project: str, modules: list):
                 )
                 sys.exit(1)
 
-    def _initialise_tasks(self) -> list[Task]:
-        """A helper method that initialises and returns the `tasks` attribute."""
-        self.tasks = get_fluxsite_tasks(
+    @functools.cache
+    def _fluxsite_tasks(self) -> list[fluxsite.FluxsiteTask]:
+        """Generate the list of fluxsite tasks and cache the result."""
+        return fluxsite.get_fluxsite_tasks(
             repos=self.repos,
             science_configurations=self.config.get(
                 "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS
@@ -113,7 +110,18 @@ def _initialise_tasks(self) -> list[Task]:
                 self.config["experiment"]
             ),
         )
-        return self.tasks
+
+    @functools.cache
+    def _spatial_tasks(self) -> list[spatial.SpatialTask]:
+        """Generate the list of spatial tasks and cache the result."""
+        return spatial.get_spatial_tasks(
+            repos=self.repos,
+            met_forcings=internal.SPATIAL_DEFAULT_FORCINGS,
+            science_configurations=self.config.get(
+                "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS
+            ),
+            payu_args=self.config.get("spatial", {}).get("payu", {}).get("args"),
+        )
 
     def fluxsite_submit_job(self) -> None:
         """Submits the PBS job script step in the fluxsite test workflow."""
@@ -187,7 +195,7 @@ def checkout(self):
 
         print("")
 
-    def build(self):
+    def build(self, mpi=False):
         """Endpoint for `benchcab build`."""
         for repo in self.repos:
             if repo.build_script:
@@ -199,30 +207,28 @@ def build(self):
                     modules=self.config["modules"], verbose=self.args.verbose
                 )
             else:
-                build_mode = "with MPI" if internal.MPI else "serially"
+                build_mode = "with MPI" if mpi else "serially"
                 print(f"Compiling CABLE {build_mode} for realisation {repo.name}...")
-                repo.pre_build(verbose=self.args.verbose)
+                repo.pre_build(mpi=mpi, verbose=self.args.verbose)
                 repo.run_build(
-                    modules=self.config["modules"], verbose=self.args.verbose
+                    modules=self.config["modules"], mpi=mpi, verbose=self.args.verbose
                 )
-                repo.post_build(verbose=self.args.verbose)
+                repo.post_build(mpi=mpi, verbose=self.args.verbose)
             print(f"Successfully compiled CABLE for realisation {repo.name}")
         print("")
 
     def fluxsite_setup_work_directory(self):
         """Endpoint for `benchcab fluxsite-setup-work-dir`."""
-        tasks = self.tasks if self.tasks else self._initialise_tasks()
         print("Setting up run directory tree for fluxsite tests...")
         setup_fluxsite_directory_tree(verbose=self.args.verbose)
         print("Setting up tasks...")
-        for task in tasks:
+        for task in self._fluxsite_tasks():
             task.setup_task(verbose=self.args.verbose)
         print("Successfully setup fluxsite tasks")
         print("")
 
     def fluxsite_run_tasks(self):
         """Endpoint for `benchcab fluxsite-run-tasks`."""
-        tasks = self.tasks if self.tasks else self._initialise_tasks()
         print("Running fluxsite tasks...")
         try:
             multiprocess = self.config["fluxsite"]["multiprocess"]
@@ -232,9 +238,11 @@ def fluxsite_run_tasks(self):
             ncpus = self.config.get("pbs", {}).get(
                 "ncpus", internal.FLUXSITE_DEFAULT_PBS["ncpus"]
             )
-            run_tasks_in_parallel(tasks, n_processes=ncpus, verbose=self.args.verbose)
+            fluxsite.run_tasks_in_parallel(
+                self._fluxsite_tasks(), n_processes=ncpus, verbose=self.args.verbose
+            )
         else:
-            run_tasks(tasks, verbose=self.args.verbose)
+            fluxsite.run_tasks(self._fluxsite_tasks(), verbose=self.args.verbose)
         print("Successfully ran fluxsite tasks")
         print("")
 
@@ -245,8 +253,7 @@ def fluxsite_bitwise_cmp(self):
                 "nccmp/1.8.5.0"
             )  # use `nccmp -df` for bitwise comparisons
 
-        tasks = self.tasks if self.tasks else self._initialise_tasks()
-        comparisons = get_fluxsite_comparisons(tasks)
+        comparisons = fluxsite.get_fluxsite_comparisons(self._fluxsite_tasks())
 
         print("Running comparison tasks...")
         try:
@@ -277,13 +284,42 @@ def fluxsite(self):
         else:
             self.fluxsite_submit_job()
 
+    def spatial_setup_work_directory(self):
+        """Endpoint for `benchcab spatial-setup-work-dir`."""
+        print("Setting up run directory tree for spatial tests...")
+        setup_spatial_directory_tree()
+        print("Setting up tasks...")
+        try:
+            payu_config = self.config["spatial"]["payu"]["config"]
+        except KeyError:
+            payu_config = None
+        for task in self._spatial_tasks():
+            task.setup_task(payu_config=payu_config, verbose=self.args.verbose)
+        print("Successfully setup spatial tasks")
+        print("")
+
+    def spatial_run_tasks(self):
+        """Endpoint for `benchcab spatial-run-tasks`."""
+        print("Running spatial tasks...")
+        spatial.run_tasks(tasks=self._spatial_tasks(), verbose=self.args.verbose)
+        print("")
+
     def spatial(self):
         """Endpoint for `benchcab spatial`."""
+        self.checkout()
+        self.build(mpi=True)
+        self.spatial_setup_work_directory()
+        self.spatial_run_tasks()
 
     def run(self):
         """Endpoint for `benchcab run`."""
-        self.fluxsite()
-        self.spatial()
+        self.checkout()
+        self.build()
+        self.build(mpi=True)
+        self.fluxsite_setup_work_directory()
+        self.spatial_setup_work_directory()
+        self.fluxsite_submit_job()
+        self.spatial_run_tasks()
 
     def main(self):
         """Main function for `benchcab`."""
@@ -294,7 +330,7 @@ def main(self):
             self.checkout()
 
         if self.args.subcommand == "build":
-            self.build()
+            self.build(mpi=self.args.mpi)
 
         if self.args.subcommand == "fluxsite":
             self.fluxsite()
@@ -314,6 +350,12 @@ def main(self):
         if self.args.subcommand == "spatial":
             self.spatial()
 
+        if self.args.subcommand == "spatial-setup-work-dir":
+            self.spatial_setup_work_directory()
+
+        if self.args.subcommand == "spatial-run-tasks":
+            self.spatial_run_tasks()
+
 
 def main():
     """Main program entry point for `benchcab`.

benchcab/cli.py

@@ -30,9 +30,9 @@ def generate_parser() -> argparse.ArgumentParser:
         action="store_true",
     )
 
-    # parent parser that contains arguments common to all run specific subcommands
-    args_run_subcommand = argparse.ArgumentParser(add_help=False)
-    args_run_subcommand.add_argument(
+    # parent parser that contains the argument for --no-submit
+    args_no_submit = argparse.ArgumentParser(add_help=False)
+    args_no_submit.add_argument(
         "--no-submit",
         action="store_true",
         help="Force benchcab to execute tasks on the current compute node.",
@@ -72,7 +72,6 @@ def generate_parser() -> argparse.ArgumentParser:
         parents=[
             args_help,
             args_subcommand,
-            args_run_subcommand,
             args_composite_subcommand,
         ],
         help="Run all test suites for CABLE.",
@@ -87,7 +86,7 @@ def generate_parser() -> argparse.ArgumentParser:
         parents=[
             args_help,
             args_subcommand,
-            args_run_subcommand,
+            args_no_submit,
             args_composite_subcommand,
         ],
         help="Run the fluxsite test suite for CABLE.",
@@ -108,14 +107,19 @@ def generate_parser() -> argparse.ArgumentParser:
     )
 
     # subcommand: 'benchcab build'
-    subparsers.add_parser(
+    build_parser = subparsers.add_parser(
         "build",
         parents=[args_help, args_subcommand],
         help="Run the build step in the benchmarking workflow.",
         description="""Build the CABLE offline executable for each repository specified in the
         config file.""",
         add_help=False,
     )
+    build_parser.add_argument(
+        "--mpi",
+        action="store_true",
+        help="Enable MPI build.",
+    )
 
     # subcommand: 'benchcab fluxsite-setup-work-dir'
     subparsers.add_parser(
@@ -141,9 +145,9 @@ def generate_parser() -> argparse.ArgumentParser:
         "fluxsite-run-tasks",
         parents=[args_help, args_subcommand],
         help="Run the fluxsite tasks of the main fluxsite command.",
-        description="""Runs the fluxsite tasks for the fluxsite test suite. Note, this command should
-        ideally be run inside a PBS job. This command is invoked by the PBS job script generated by
-        `benchcab run`.""",
+        description="""Runs the fluxsite tasks for the fluxsite test suite.
+        Note, this command should ideally be run inside a PBS job. This command
+        is invoked by the PBS job script generated by `benchcab run`.""",
         add_help=False,
     )
 
@@ -163,10 +167,29 @@ def generate_parser() -> argparse.ArgumentParser:
     # subcommand: 'benchcab spatial'
     subparsers.add_parser(
         "spatial",
-        parents=[args_help, args_subcommand],
+        parents=[args_help, args_subcommand, args_composite_subcommand],
         help="Run the spatial tests only.",
         description="""Runs the default spatial test suite for CABLE.""",
         add_help=False,
     )
 
+    # subcommand: 'benchcab spatial-setup-work-dir'
+    subparsers.add_parser(
+        "spatial-setup-work-dir",
+        parents=[args_help, args_subcommand],
+        help="Run the work directory setup step of the spatial command.",
+        description="""Generates the spatial run directory tree in the current working
+        directory so that spatial tasks can be run.""",
+        add_help=False,
+    )
+
+    # subcommand 'benchcab spatial-run-tasks'
+    subparsers.add_parser(
+        "spatial-run-tasks",
+        parents=[args_help, args_subcommand],
+        help="Run the spatial tasks of the main spatial command.",
+        description="Runs the spatial tasks for the spatial test suite.",
+        add_help=False,
+    )
+
     return main_parser

benchcab/config.py

@@ -92,6 +92,34 @@ def check_config(config: dict):
             msg = "The 'multiprocessing' key must be a boolean."
             raise TypeError(msg)
 
+    # the "spatial" key is optional
+    if "spatial" in config:
+        if not isinstance(config["spatial"], dict):
+            msg = "The 'spatial' key must be a dictionary."
+            raise TypeError(msg)
+        # the "payu" key is optional
+        if "payu" in config["spatial"]:
+            if not isinstance(config["spatial"]["payu"], dict):
+                msg = "The 'payu' key must be a dictionary."
+                raise TypeError(msg)
+            # the "config" key is optional
+            if "config" in config["spatial"]["payu"]:
+                if not isinstance(config["spatial"]["payu"]["config"], dict):
+                    msg = "The 'config' key must be a dictionary."
+                    raise TypeError(msg)
+            # the "args" key is optional
+            if "args" in config["spatial"]["payu"]:
+                if not isinstance(config["spatial"]["payu"]["args"], str):
+                    msg = "The 'args' key must be a string."
+                    raise TypeError(msg)
+        # the "met_forcings" key is optional
+        if "met_forcings" in config["spatial"]:
+            if not isinstance(config["spatial"]["met_forcings"], list) or any(
+                not isinstance(val, str) for val in config["spatial"]["met_forcings"]
+            ):
+                msg = "The 'met_forcings' key must be a list of strings."
+                raise TypeError(msg)
+
     valid_experiments = (
         list(internal.MEORG_EXPERIMENTS) + internal.MEORG_EXPERIMENTS["five-site-test"]
     )