Update to CMake build (#278)

The CABLE build system was transitioned from Makefile to CMake: see
CABLE-LSM/CABLE#216,
CABLE-LSM/CABLE#200.

This change adds support for building CABLE via CMake in benchcab so
that CABLE versions which branch from the HEAD of main can be tested.

Closes #258

docs/user_guide/running_CABLE_v2.md

@@ -41,6 +41,7 @@ There are two differences one might need to deal with to run with an older versi
 
 - ensure your build script is using exactly the same modules as listed in the [`config.yaml` file](config_options.md#modules) of the `benchcab` work directory. For this you can either modify the modules in your build script or in the `config.yaml` file. Make sure to commit the build script with the correct modules to your branch before running `benchcab`.
 - give the path to the build script for that older branch using the [`build_script` option](config_options.md#build_script) in the `config.yaml` file.
+- give the path to the install directory of built executables using the [`install_dir` option](config_options.md#install_dir) in the `config.yaml`.
 
 ## FLUXNET data
 

src/benchcab/benchcab.py

@@ -255,13 +255,11 @@ def build(self, config_path: str, mpi=False):
                 repo.custom_build(modules=config["modules"])
 
             else:
-                build_mode = "with MPI" if mpi else "serially"
+                build_mode = "serial and MPI" if mpi else "serial"
                 self.logger.info(
                     f"Compiling CABLE {build_mode} for realisation {repo.name}..."
                 )
-                repo.pre_build(mpi=mpi)
-                repo.run_build(modules=config["modules"], mpi=mpi)
-                repo.post_build(mpi=mpi)
+                repo.build(modules=config["modules"], mpi=mpi)
             self.logger.info(f"Successfully compiled CABLE for realisation {repo.name}")
 
     def fluxsite_setup_work_directory(self, config_path: str):
@@ -366,7 +364,6 @@ def spatial(self, config_path: str, skip: list):
     def run(self, config_path: str, skip: list[str]):
         """Endpoint for `benchcab run`."""
         self.checkout(config_path)
-        self.build(config_path)
         self.build(config_path, mpi=True)
         self.fluxsite_setup_work_directory(config_path)
         self.spatial_setup_work_directory(config_path)

src/benchcab/data/test/integration.sh

@@ -18,8 +18,6 @@ mkdir -p $TEST_DIR
 # Clone local checkout for CABLE
 git clone $CABLE_REPO $CABLE_DIR
 cd $CABLE_DIR
-# Note: This is temporary, to be removed once #258 is fixed
-git reset --hard 67a52dc5721f0da78ee7d61798c0e8a804dcaaeb
 
 # Clone the example repo
 git clone $EXAMPLE_REPO $TEST_DIR
@@ -36,7 +34,6 @@ realisations:
   - repo:
       git:
         branch: main
-        commit: 67a52dc5721f0da78ee7d61798c0e8a804dcaaeb  # Note: This is temporary, to be removed once #258 is fixed
 modules: [
   intel-compiler/2021.1.1,
   netcdf/4.7.4,
@@ -50,4 +47,4 @@ fluxsite:
       - scratch/$PROJECT
 EOL
 
-benchcab run -v
+benchcab run -v

src/benchcab/internal.py

@@ -12,6 +12,12 @@
 
 CONFIG_REQUIRED_KEYS = ["realisations", "modules"]
 
+# CMake module used for compilation:
+CMAKE_MODULE = "cmake/3.24.2"
+
+# Number of parallel jobs used when compiling with CMake:
+CMAKE_BUILD_PARALLEL_LEVEL = 4
+
 # Parameters for job script:
 QSUB_FNAME = "benchmark_cable_qsub.sh"
 FLUXSITE_DEFAULT_PBS: PBSConfig = {

src/benchcab/model.py

@@ -13,7 +13,7 @@
 from benchcab import internal
 from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface
 from benchcab.utils import get_logger
-from benchcab.utils.fs import chdir, copy2, rename
+from benchcab.utils.fs import chdir, prepend_path
 from benchcab.utils.repo import GitRepo, LocalRepo, Repo
 from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface
 
@@ -87,7 +87,7 @@ def get_exe_path(self, mpi=False) -> Path:
         exe = internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE
         if self.install_dir:
             return internal.SRC_DIR / self.name / self.install_dir / exe
-        return internal.SRC_DIR / self.name / self.src_dir / "offline" / exe
+        return internal.SRC_DIR / self.name / "bin" / exe
 
     def custom_build(self, modules: list[str]):
         """Build CABLE using a custom build script."""
@@ -118,60 +118,37 @@ def custom_build(self, modules: list[str]):
         with chdir(build_script_path.parent), self.modules_handler.load(modules):
             self.subprocess_handler.run_cmd(f"./{tmp_script_path.name}")
 
-    def pre_build(self, mpi=False):
-        """Runs CABLE pre-build steps."""
+    def build(self, modules: list[str], mpi=False):
+        """Build CABLE with CMake."""
         path_to_repo = internal.SRC_DIR / self.name
-        tmp_dir = (
-            path_to_repo
-            / self.src_dir
-            / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR)
-        )
-        if not tmp_dir.exists():
-            self.logger.debug(f"mkdir {tmp_dir}")
-            tmp_dir.mkdir()
-
-        for pattern in internal.OFFLINE_SOURCE_FILES:
-            for path in (path_to_repo / self.src_dir).glob(pattern):
-                if not path.is_file():
-                    continue
-                copy2(path, tmp_dir)
-
-        copy2(path_to_repo / self.src_dir / "offline" / "Makefile", tmp_dir)
-
-    def run_build(self, modules: list[str], mpi=False):
-        """Runs CABLE build scripts."""
-        path_to_repo = internal.SRC_DIR / self.name
-        tmp_dir = (
-            path_to_repo
-            / self.src_dir
-            / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR)
-        )
-
-        with chdir(tmp_dir), self.modules_handler.load(modules):
+        cmake_args = [
+            "-DCMAKE_BUILD_TYPE=Release",
+            "-DCABLE_MPI=" + ("ON" if mpi else "OFF"),
+        ]
+        with chdir(path_to_repo), self.modules_handler.load(
+            [internal.CMAKE_MODULE, *modules]
+        ):
             env = os.environ.copy()
-            env["NCDIR"] = f"{env['NETCDF_ROOT']}/lib/Intel"
-            env["NCMOD"] = f"{env['NETCDF_ROOT']}/include/Intel"
-            env["CFLAGS"] = "-O2 -fp-model precise"
-            env["LDFLAGS"] = f"-L{env['NETCDF_ROOT']}/lib/Intel -O0"
-            env["LD"] = "-lnetcdf -lnetcdff"
-            env["FC"] = "mpif90" if mpi else "ifort"
+            # This is required so that the netcdf-fortran library is discoverable by
+            # pkg-config:
+            prepend_path(
+                "PKG_CONFIG_PATH", f"{env['NETCDF_BASE']}/lib/Intel/pkgconfig", env=env
+            )
 
-            self.subprocess_handler.run_cmd("make mpi" if mpi else "make", env=env)
+            if self.modules_handler.module_is_loaded("openmpi"):
+                # This is required so that the openmpi MPI libraries are discoverable
+                # via CMake's `find_package` mechanism:
+                prepend_path(
+                    "CMAKE_PREFIX_PATH", f"{env['OPENMPI_BASE']}/include/Intel", env=env
+                )
 
-    def post_build(self, mpi=False):
-        """Runs CABLE post-build steps."""
-        path_to_repo = internal.SRC_DIR / self.name
-        tmp_dir = (
-            path_to_repo
-            / self.src_dir
-            / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR)
-        )
-        exe = internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE
+            env["CMAKE_BUILD_PARALLEL_LEVEL"] = str(internal.CMAKE_BUILD_PARALLEL_LEVEL)
 
-        rename(
-            tmp_dir / exe,
-            path_to_repo / self.src_dir / "offline" / exe,
-        )
+            self.subprocess_handler.run_cmd(
+                "cmake -S . -B build " + " ".join(cmake_args), env=env
+            )
+            self.subprocess_handler.run_cmd("cmake --build build ", env=env)
+            self.subprocess_handler.run_cmd("cmake --install build --prefix .", env=env)
 
 
 def remove_module_lines(file_path: Path) -> None:

src/benchcab/utils/fs.py

@@ -72,3 +72,19 @@ def mkdir(new_path: Path, **kwargs):
     """
     get_logger().debug(f"Creating {new_path} directory")
     new_path.mkdir(**kwargs)
+
+
+def prepend_path(var: str, path: str, env=os.environ):
+    """Prepend path to environment variable.
+
+    Parameters
+    ----------
+    var : str
+        Name of environment variable.
+    path : str
+        Path to prepend.
+    env : dict
+        Environment dictionary.
+
+    """
+    env[var] = os.pathsep.join(filter(None, [path, env.get(var)]))

tests/test_fluxsite.py

@@ -10,7 +10,6 @@
 import f90nml
 import netCDF4
 import pytest
-
 from benchcab import __version__, internal
 from benchcab.fluxsite import (
     CableError,
@@ -107,7 +106,7 @@ def _setup(self, task):
         (internal.FLUXSITE_DIRS["OUTPUT"]).mkdir(parents=True)
         (internal.FLUXSITE_DIRS["LOG"]).mkdir(parents=True)
 
-        exe_build_dir = internal.SRC_DIR / "test-branch" / "offline"
+        exe_build_dir = internal.SRC_DIR / "test-branch" / "bin"
         exe_build_dir.mkdir(parents=True)
         (exe_build_dir / internal.CABLE_EXE).touch()
 
@@ -170,7 +169,7 @@ def _setup(self, task):
         (internal.FLUXSITE_DIRS["OUTPUT"]).mkdir(parents=True)
         (internal.FLUXSITE_DIRS["LOG"]).mkdir(parents=True)
 
-        exe_build_dir = internal.SRC_DIR / "test-branch" / "offline"
+        exe_build_dir = internal.SRC_DIR / "test-branch" / "bin"
         exe_build_dir.mkdir(parents=True)
         (exe_build_dir / internal.CABLE_EXE).touch()
 

tests/test_fs.py

@@ -6,11 +6,11 @@
 """
 
 import logging
+import os
 from pathlib import Path
 
 import pytest
-
-from benchcab.utils.fs import chdir, mkdir, next_path
+from benchcab.utils.fs import chdir, mkdir, next_path, prepend_path
 
 
 class TestNextPath:
@@ -68,3 +68,25 @@ def test_mkdir(self, test_path, kwargs):
         mkdir(test_path, **kwargs)
         assert test_path.exists()
         test_path.rmdir()
+
+
+class TestPrependPath:
+    """Tests for `prepend_path()`."""
+
+    var = "PATHS"
+    new_path = "/path/to/bar"
+
+    @pytest.mark.parametrize(
+        ("env", "expected"),
+        [
+            ({}, new_path),
+            (
+                {var: "/path/to/foo"},
+                f"{new_path}{os.pathsep}/path/to/foo",
+            ),
+        ],
+    )
+    def test_prepend_path(self, env, expected):
+        """Success case: test prepend_path for unset and existing variables."""
+        prepend_path(self.var, self.new_path, env=env)
+        assert env[self.var] == expected