Use cable local checkout

benchcab/benchcab.py

@@ -22,6 +22,8 @@
 from benchcab.utils.repo import create_repo
 from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface
 from benchcab.workdir import (
+    clean_realisation_files,
+    clean_submission_files,
     setup_fluxsite_directory_tree,
     setup_spatial_directory_tree,
 )
@@ -74,7 +76,7 @@
             self.logger.error("benchcab is currently implemented only on Gadi")
             sys.exit(1)
 
-        namelist_dir = Path(internal.CWD / internal.NAMELIST_DIR)
+        namelist_dir = Path(internal.NAMELIST_DIR)
         if not namelist_dir.exists():
             self.logger.error(
                 "Cannot find 'namelists' directory in current working directory"
@@ -225,7 +227,14 @@
         rev_number_log = ""
 
         for model in self._get_models(config):
-            model.repo.checkout()
+            try:
+                model.repo.checkout()
+            except Exception:
+                msg = "Try using `benchcab clean realisations` first"
+                self.logger.error(
+                    "Model checkout failed, probably due to existing realisation name"
+                )
+                raise FileExistsError(msg)
             rev_number_log += f"{model.name}: {model.repo.get_revision()}\n"
 
         rev_number_log_path = next_path("rev_number-*.log")
@@ -315,6 +324,13 @@
         else:
             self.fluxsite_submit_job(config_path, skip)
 
+    def clean(self, config_path: str, clean_option: str):
+        """Endpoint for `benchcab clean`."""
+        if clean_option in ["all", "realisations"]:
+            clean_realisation_files()
+        if clean_option in ["all", "submissions"]:
+            clean_submission_files()
+
     def spatial_setup_work_directory(self, config_path: str):
         """Endpoint for `benchcab spatial-setup-work-dir`."""
         config = self._get_config(config_path)

benchcab/cli.py

@@ -224,4 +224,26 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser:
     )
     parser_spatial_run_tasks.set_defaults(func=app.spatial_run_tasks)
 
+    # subcommand: 'benchcab clean'
+    parser_clean = subparsers.add_parser(
+        "clean",
+        parents=[args_help, args_subcommand],
+        help="Cleanup files created by running benchcab.",
+        description="""Removes src/ and runs/ directories, along with log files in the 
+        project root directory. The user has to specify which stage of files to remove 
+        via \{all, realisations, submissions\} subcommand.""",
+        add_help=False,
+    )
+
+    parser_clean.add_argument(
+        "clean_option",
+        choices=["all", "realisations", "submissions"],
+        help="""Can be one of three options:
+
+        realisations: deletes src/
+        submissions: deletes runs/ and benchmark submission files
+        all: deletes in both stages of submissions and realisations""",
+    )
+
+    parser_clean.set_defaults(func=app.clean)
     return main_parser

benchcab/data/config-schema.yml

@@ -24,7 +24,7 @@ realisations:
         schema:
           git:
             type: "dict"
-            excludes: "svn"
+            excludes: ["svn", "local"]
             schema:
               branch:
                 type: "string"
@@ -37,13 +37,20 @@ realisations:
                 required: false
           svn:
             type: "dict"
-            excludes: "git"
+            excludes: ["git", "local"]
             schema:
               branch_path:
                 type: "string"
                 required: true
               revision:
                 type: "integer"
+          local:
+            type: "dict"
+            excludes: ["git", "svn"]
+            schema:
+              path:
+                type: "string"
+                required: true
       name:
         nullable: true
         type: "string"

benchcab/data/test/integration.sh

@@ -2,13 +2,25 @@
 
 set -ex
 
+CABLE_REPO="[email protected]:CABLE-LSM/CABLE.git"
+CABLE_DIR=/scratch/$PROJECT/$USER/benchcab/CABLE
+
 TEST_DIR=/scratch/$PROJECT/$USER/benchcab/integration
 EXAMPLE_REPO="[email protected]:CABLE-LSM/bench_example.git"
 
-# Remove the test work space, then recreate
+# Remove CABLE and test work space, then recreate
+rm -rf $CABLE_DIR
+mkdir -p $CABLE_DIR
+
 rm -rf $TEST_DIR
 mkdir -p $TEST_DIR
 
+# Clone local checkout for CABLE
+git clone $CABLE_REPO $CABLE_DIR
+cd $CABLE_DIR
+# Note: This is temporary, to be removed once #258 is fixed
+git reset --hard 67a52dc5721f0da78ee7d61798c0e8a804dcaaeb
+
 # Clone the example repo
 git clone $EXAMPLE_REPO $TEST_DIR
 cd $TEST_DIR
@@ -18,10 +30,13 @@ cat > config.yaml << EOL
 project: $PROJECT
 
 realisations:
+  - repo:
+      local:
+        path: $CABLE_DIR
   - repo:
       git:
         branch: main
-
+        commit: 67a52dc5721f0da78ee7d61798c0e8a804dcaaeb  # Note: This is temporary, to be removed once #258 is fixed
 modules: [
   intel-compiler/2021.1.1,
   netcdf/4.7.4,

benchcab/internal.py

@@ -24,9 +24,6 @@
 
 # DIRECTORY PATHS/STRUCTURE:
 
-# Path to the user's current working directory
-CWD = Path.cwd()
-
 # Default system paths in Unix
 SYSTEM_PATHS = ["/bin", "/usr/bin", "/usr/local/bin"]
 

benchcab/model.py

@@ -14,7 +14,7 @@
 from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface
 from benchcab.utils import get_logger
 from benchcab.utils.fs import chdir, copy2, rename
-from benchcab.utils.repo import GitRepo, Repo
+from benchcab.utils.repo import GitRepo, LocalRepo, Repo
 from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface
 
 
@@ -62,7 +62,7 @@ def __init__(
         # TODO(Sean) we should not have to know whether `repo` is a `GitRepo` or
         # `SVNRepo`, we should only be working with the `Repo` interface.
         # See issue https://github.com/CABLE-LSM/benchcab/issues/210
-        if isinstance(repo, GitRepo):
+        if isinstance(repo, (GitRepo, LocalRepo)):
             self.src_dir = Path("src")
 
     @property

benchcab/utils/repo.py

@@ -46,6 +46,59 @@
         """
 
 
+class LocalRepo(Repo):
+    """Concrete implementation of the `Repo` class using local path backend."""
+
+    def __init__(self, path: str, realisation_path: str) -> None:
+        """_summary_.
+
+        Parameters
+        ----------
+        realisation_path : str
+            Path for local checkout of CABLE
+        path : str
+            Directory where CABLE is symlinked from, assigned as `local_path`
+
+        """
+        self.name = Path(path).name
+        self.local_path = path
+        self.realisation_path = (
+            realisation_path / self.name
+            if realisation_path.is_dir()
+            else realisation_path
+        )
+        self.logger = get_logger()
+
+    def checkout(self):
+        """Checkout the source code."""
+        self.realisation_path.symlink_to(self.local_path)
+        self.logger.info(
+            f"Created symlink from to {self.realisation_path} named {self.name}"
+        )
+
+    def get_revision(self) -> str:
+        """Return the latest revision of the source code.
+
+        Returns
+        -------
+        str
+            Human readable string describing the latest revision.
+
+        """
+        return f"Local CABLE build: {Path(self.local_path).absolute().as_posix}"
+
+    def get_branch_name(self) -> str:
+        """Return the branch name of the source code.
+
+        Returns
+        -------
+        str
+            Branch name of the source code.
+
+        """
+        return Path(self.realisation_path).absolute().as_posix()
+
+
 class GitRepo(Repo):
     """A concrete implementation of the `Repo` class using a Git backend.
 
@@ -59,7 +112,11 @@
     subprocess_handler = SubprocessWrapper()
 
     def __init__(
-        self, url: str, branch: str, path: Path, commit: Optional[str] = None
+        self,
+        url: str,
+        branch: str,
+        realisation_path: Path,
+        commit: Optional[str] = None,
     ) -> None:
         """Return a `GitRepo` instance.
 
@@ -69,7 +126,7 @@
             URL pointing to the GitHub repository.
         branch: str
             Name of a branch on the GitHub repository.
-        path: Path
+        realisation_path: Path
             Path to a directory in which the repository is cloned into. If
             `path` points to an existing directory, the repository will be
             cloned into `path / branch`.
@@ -80,7 +137,9 @@
         """
         self.url = url
         self.branch = branch
-        self.path = path / branch if path.is_dir() else path
+        self.realisation_path = (
+            realisation_path / branch if realisation_path.is_dir() else realisation_path
+        )
         self.commit = commit
         self.logger = get_logger()
 
@@ -90,11 +149,11 @@
         # remote progress. See
         # https://gitpython.readthedocs.io/en/stable/reference.html#git.remote.RemoteProgress
         self.subprocess_handler.run_cmd(
-            f"git clone --branch {self.branch} -- {self.url} {self.path}"
+            f"git clone --branch {self.branch} -- {self.url} {self.realisation_path}"
         )
         if self.commit:
             self.logger.debug(f"Reset to commit {self.commit} (hard reset)")
-            repo = git.Repo(self.path)
+            repo = git.Repo(self.realisation_path)
             repo.head.reset(self.commit, working_tree=True)
         self.logger.info(
             f"Successfully checked out {self.branch} - {self.get_revision()}"
@@ -109,7 +168,7 @@
             Human readable string describing the latest revision.
 
         """
-        repo = git.Repo(self.path)
+        repo = git.Repo(self.realisation_path)
         return f"commit {repo.head.commit.hexsha}"
 
     def get_branch_name(self) -> str:
@@ -140,7 +199,7 @@
         self,
         svn_root: str,
         branch_path: str,
-        path: Path,
+        realisation_path: Path,
         revision: Optional[int] = None,
     ) -> None:
         """Return an `SVNRepo` instance.
@@ -151,7 +210,7 @@
             URL pointing to the root of the SVN repository.
         branch_path: str
             Path to a branch relative to `svn_root`.
-        path: Path
+        realisation_path: Path
             Path to a directory in which the branch is checked out into. If
             `path` points to an existing directory, the repository will be
             cloned into `path / <branch_name>` where `<branch_name>` is the
@@ -164,7 +223,11 @@
         self.svn_root = svn_root
         self.branch_path = branch_path
         self.revision = revision
-        self.path = path / Path(branch_path).name if path.is_dir() else path
+        self.realisation_path = (
+            realisation_path / Path(branch_path).name
+            if realisation_path.is_dir()
+            else realisation_path
+        )
         self.logger = get_logger()
 
     def checkout(self):
@@ -174,12 +237,12 @@
         if self.revision:
             cmd += f" -r {self.revision}"
 
-        cmd += f" {internal.CABLE_SVN_ROOT}/{self.branch_path} {self.path}"
+        cmd += f" {internal.CABLE_SVN_ROOT}/{self.branch_path} {self.realisation_path}"
 
         self.subprocess_handler.run_cmd(cmd)
 
         self.logger.info(
-            f"Successfully checked out {self.path.name} - {self.get_revision()}"
+            f"Successfully checked out {self.realisation_path.name} - {self.get_revision()}"
         )
 
     def get_revision(self) -> str:
@@ -192,7 +255,7 @@
 
         """
         proc = self.subprocess_handler.run_cmd(
-            f"svn info --show-item last-changed-revision {self.path}",
+            f"svn info --show-item last-changed-revision {self.realisation_path}",
             capture_output=True,
         )
         return f"last-changed-revision {proc.stdout.strip()}"
@@ -233,7 +296,11 @@
     if "git" in spec:
         if "url" not in spec["git"]:
             spec["git"]["url"] = internal.CABLE_GIT_URL
-        return GitRepo(path=path, **spec["git"])
+        return GitRepo(realisation_path=path, **spec["git"])
     if "svn" in spec:
-        return SVNRepo(svn_root=internal.CABLE_SVN_ROOT, path=path, **spec["svn"])
+        return SVNRepo(
+            svn_root=internal.CABLE_SVN_ROOT, realisation_path=path, **spec["svn"]
+        )
+    if "local" in spec:
+        return LocalRepo(realisation_path=path, **spec["local"])
     raise RepoSpecError

benchcab/workdir.py

@@ -4,19 +4,29 @@
 """Functions for generating the directory structure used for `benchcab`."""
 
 import shutil
+from pathlib import Path
 
 from benchcab import internal
 from benchcab.utils.fs import mkdir
 
 
-def clean_directory_tree():
-    """Remove pre-existing directories in current working directory."""
+def clean_realisation_files():
+    """Remove files/directories related to CABLE realisation source codes."""
     if internal.SRC_DIR.exists():
+        for realisation in internal.SRC_DIR.iterdir():
+            if realisation.is_symlink():
+                realisation.unlink()
         shutil.rmtree(internal.SRC_DIR)
 
+
+def clean_submission_files():
+    """Remove files/directories related to PBS jobs."""
     if internal.RUN_DIR.exists():
         shutil.rmtree(internal.RUN_DIR)
 
+    for pbs_job_file in Path.cwd().glob(f"{internal.QSUB_FNAME}*"):
+        pbs_job_file.unlink()
+
 
 def setup_fluxsite_directory_tree():
     """Generate the directory structure used by `benchcab`."""