Add payu test suite for spatial configuration

The spatial test suite runs CABLE with CRUJRA forcing at ACCESS
resolution (see [here][cable_example] for more details) over all science
configurations and model versions.

Spatial tests use the [payu framework][payu]. The payu framework was
chosen so that we:
- Encourage uptake of payu amongst users of CABLE
- Have the foundations in place for running coupled models (atmosphere +
  land) with payu.
- Can easily test longer running simulations (payu makes it easy to run
  a model multiple times and have state persist in the model via restart
  files).

The run directory structure is organised as follows:

runs/
├── spatial
│   └── tasks
│	├── <spatial-task-name> (a payu control / experiment directory)
│	└── ...
├── payu-laboratory
│   └── ...
└── fluxsite
    └── ...

Note we have a separate payu-laboratory directory. This is so we keep
all CABLE outputs produced by benchcab under the bench_example work
directory.

Add the ability to build the CABLE executable with MPI at runtime so
that we run the spatial configurations with MPI.

Add the --mpi flag to benchcab build command so that the user can
run the MPI build step independently.

Add utility functions for git API requests and manipulating namelist
files.

Fixes #5

[payu]: https://github.com/payu-org/payu
[cable_example]: [email protected]:CABLE-LSM/cable_example.git

.conda/benchcab-dev.yaml

@@ -10,3 +10,4 @@ dependencies:
   - pytest-cov
   - pyyaml
   - flatdict
+  - gitpython

benchcab/benchcab.py

@@ -11,15 +11,14 @@
 from benchcab import internal
 from benchcab.internal import get_met_forcing_file_names
 from benchcab.config import read_config
-from benchcab.workdir import setup_fluxsite_directory_tree, setup_src_dir
-from benchcab.repository import CableRepository
-from benchcab.fluxsite import (
-    get_fluxsite_tasks,
-    get_fluxsite_comparisons,
-    run_tasks,
-    run_tasks_in_parallel,
-    Task,
+from benchcab.workdir import (
+    setup_src_dir,
+    setup_fluxsite_directory_tree,
+    setup_spatial_directory_tree,
 )
+from benchcab.repository import CableRepository
+from benchcab import fluxsite
+from benchcab import spatial
 from benchcab.comparison import run_comparisons, run_comparisons_in_parallel
 from benchcab.cli import generate_parser
 from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface
@@ -48,7 +47,15 @@ def __init__(
             CableRepository(**config, repo_id=id)
             for id, config in enumerate(self.config["realisations"])
         ]
-        self.tasks: list[Task] = []  # initialise fluxsite tasks lazily
+        self.science_configurations = self.config.get(
+            "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS
+        )
+        self.fluxsite_tasks: list[
+            fluxsite.FluxsiteTask
+        ] = []  # initialise fluxsite tasks lazily
+        self.spatial_tasks = spatial.get_spatial_tasks(
+            repos=self.repos, science_configurations=self.science_configurations
+        )
         self.benchcab_exe_path = benchcab_exe_path
 
         if validate_env:
@@ -103,18 +110,16 @@ def _validate_environment(self, project: str, modules: list):
                 )
                 sys.exit(1)
 
-    def _initialise_tasks(self) -> list[Task]:
+    def _initialise_tasks(self) -> list[fluxsite.FluxsiteTask]:
         """A helper method that initialises and returns the `tasks` attribute."""
-        self.tasks = get_fluxsite_tasks(
+        self.fluxsite_tasks = fluxsite.get_fluxsite_tasks(
             repos=self.repos,
-            science_configurations=self.config.get(
-                "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS
-            ),
+            science_configurations=self.science_configurations,
             fluxsite_forcing_file_names=get_met_forcing_file_names(
                 self.config["experiment"]
             ),
         )
-        return self.tasks
+        return self.fluxsite_tasks
 
     def fluxsite_submit_job(self) -> None:
         """Submits the PBS job script step in the fluxsite test workflow."""
@@ -189,7 +194,7 @@ def checkout(self):
 
         print("")
 
-    def build(self):
+    def build(self, mpi=False):
         """Endpoint for `benchcab build`."""
         for repo in self.repos:
             if repo.build_script:
@@ -201,30 +206,38 @@ def build(self):
                     modules=self.config["modules"], verbose=self.args.verbose
                 )
             else:
-                build_mode = "with MPI" if internal.MPI else "serially"
+                build_mode = "with MPI" if mpi else "serially"
                 print(f"Compiling CABLE {build_mode} for realisation {repo.name}...")
-                repo.pre_build(verbose=self.args.verbose)
+                repo.pre_build(mpi=mpi, verbose=self.args.verbose)
                 repo.run_build(
-                    modules=self.config["modules"], verbose=self.args.verbose
+                    modules=self.config["modules"], mpi=mpi, verbose=self.args.verbose
                 )
-                repo.post_build(verbose=self.args.verbose)
+                repo.post_build(mpi=mpi, verbose=self.args.verbose)
             print(f"Successfully compiled CABLE for realisation {repo.name}")
         print("")
 
     def fluxsite_setup_work_directory(self):
         """Endpoint for `benchcab fluxsite-setup-work-dir`."""
-        tasks = self.tasks if self.tasks else self._initialise_tasks()
+
+        if not self.fluxsite_tasks:
+            self._initialise_tasks()
+
         print("Setting up run directory tree for fluxsite tests...")
-        setup_fluxsite_directory_tree(fluxsite_tasks=tasks, verbose=self.args.verbose)
+        setup_fluxsite_directory_tree(
+            fluxsite_tasks=self.fluxsite_tasks, verbose=self.args.verbose
+        )
         print("Setting up tasks...")
-        for task in tasks:
+        for task in self.fluxsite_tasks:
             task.setup_task(verbose=self.args.verbose)
         print("Successfully setup fluxsite tasks")
         print("")
 
     def fluxsite_run_tasks(self):
         """Endpoint for `benchcab fluxsite-run-tasks`."""
-        tasks = self.tasks if self.tasks else self._initialise_tasks()
+
+        if not self.fluxsite_tasks:
+            self._initialise_tasks()
+
         print("Running fluxsite tasks...")
         try:
             multiprocess = self.config["fluxsite"]["multiprocess"]
@@ -234,9 +247,11 @@ def fluxsite_run_tasks(self):
             ncpus = self.config.get("pbs", {}).get(
                 "ncpus", internal.FLUXSITE_DEFAULT_PBS["ncpus"]
             )
-            run_tasks_in_parallel(tasks, n_processes=ncpus, verbose=self.args.verbose)
+            fluxsite.run_tasks_in_parallel(
+                self.fluxsite_tasks, n_processes=ncpus, verbose=self.args.verbose
+            )
         else:
-            run_tasks(tasks, verbose=self.args.verbose)
+            fluxsite.run_tasks(self.fluxsite_tasks, verbose=self.args.verbose)
         print("Successfully ran fluxsite tasks")
         print("")
 
@@ -248,8 +263,10 @@ def fluxsite_bitwise_cmp(self):
                 "nccmp/1.8.5.0"
             )  # use `nccmp -df` for bitwise comparisons
 
-        tasks = self.tasks if self.tasks else self._initialise_tasks()
-        comparisons = get_fluxsite_comparisons(tasks)
+        if not self.fluxsite_tasks:
+            self._initialise_tasks()
+
+        comparisons = fluxsite.get_fluxsite_comparisons(self.fluxsite_tasks)
 
         print("Running comparison tasks...")
         try:
@@ -280,13 +297,38 @@ def fluxsite(self):
         else:
             self.fluxsite_submit_job()
 
+    def spatial_setup_work_directory(self):
+        """Endpoint for `benchcab spatial-setup-work-dir`."""
+        print("Setting up run directory tree for spatial tests...")
+        setup_spatial_directory_tree()
+        print("Setting up tasks...")
+        for task in self.spatial_tasks:
+            task.setup_task(verbose=self.args.verbose)
+        print("Successfully setup spatial tasks")
+        print("")
+
+    def spatial_run_tasks(self):
+        """Endpoint for `benchcab spatial-run-tasks`."""
+        print("Running spatial tasks...")
+        spatial.run_tasks(tasks=self.spatial_tasks, verbose=self.args.verbose)
+        print("")
+
     def spatial(self):
         """Endpoint for `benchcab spatial`."""
+        self.checkout()
+        self.build(mpi=True)
+        self.spatial_setup_work_directory()
+        self.spatial_run_tasks()
 
     def run(self):
         """Endpoint for `benchcab run`."""
-        self.fluxsite()
-        self.spatial()
+        self.checkout()
+        self.build()
+        self.build(mpi=True)
+        self.fluxsite_setup_work_directory()
+        self.spatial_setup_work_directory()
+        self.fluxsite_submit_job()
+        self.spatial_run_tasks()
 
     def main(self):
         """Main function for `benchcab`."""
@@ -298,7 +340,7 @@ def main(self):
             self.checkout()
 
         if self.args.subcommand == "build":
-            self.build()
+            self.build(mpi=self.args.mpi)
 
         if self.args.subcommand == "fluxsite":
             self.fluxsite()

benchcab/cli.py

@@ -32,9 +32,9 @@ def generate_parser() -> argparse.ArgumentParser:
         action="store_true",
     )
 
-    # parent parser that contains arguments common to all run specific subcommands
-    args_run_subcommand = argparse.ArgumentParser(add_help=False)
-    args_run_subcommand.add_argument(
+    # parent parser that contains the argument for --no-submit
+    args_no_submit = argparse.ArgumentParser(add_help=False)
+    args_no_submit.add_argument(
         "--no-submit",
         action="store_true",
         help="Force benchcab to execute tasks on the current compute node.",
@@ -74,7 +74,6 @@ def generate_parser() -> argparse.ArgumentParser:
         parents=[
             args_help,
             args_subcommand,
-            args_run_subcommand,
             args_composite_subcommand,
         ],
         help="Run all test suites for CABLE.",
@@ -89,7 +88,7 @@ def generate_parser() -> argparse.ArgumentParser:
         parents=[
             args_help,
             args_subcommand,
-            args_run_subcommand,
+            args_no_submit,
             args_composite_subcommand,
         ],
         help="Run the fluxsite test suite for CABLE.",
@@ -110,14 +109,19 @@ def generate_parser() -> argparse.ArgumentParser:
     )
 
     # subcommand: 'benchcab build'
-    subparsers.add_parser(
+    build_parser = subparsers.add_parser(
         "build",
         parents=[args_help, args_subcommand],
         help="Run the build step in the benchmarking workflow.",
         description="""Build the CABLE offline executable for each repository specified in the
         config file.""",
         add_help=False,
     )
+    build_parser.add_argument(
+        "--mpi",
+        action="store_true",
+        help="Enable MPI build.",
+    )
 
     # subcommand: 'benchcab fluxsite-setup-work-dir'
     subparsers.add_parser(
@@ -143,9 +147,9 @@ def generate_parser() -> argparse.ArgumentParser:
         "fluxsite-run-tasks",
         parents=[args_help, args_subcommand],
         help="Run the fluxsite tasks of the main fluxsite command.",
-        description="""Runs the fluxsite tasks for the fluxsite test suite. Note, this command should
-        ideally be run inside a PBS job. This command is invoked by the PBS job script generated by
-        `benchcab run`.""",
+        description="""Runs the fluxsite tasks for the fluxsite test suite.
+        Note, this command should ideally be run inside a PBS job. This command
+        is invoked by the PBS job script generated by `benchcab run`.""",
         add_help=False,
     )
 
@@ -165,7 +169,7 @@ def generate_parser() -> argparse.ArgumentParser:
     # subcommand: 'benchcab spatial'
     subparsers.add_parser(
         "spatial",
-        parents=[args_help, args_subcommand],
+        parents=[args_help, args_subcommand, args_composite_subcommand],
         help="Run the spatial tests only.",
         description="""Runs the default spatial test suite for CABLE.""",
         add_help=False,