Use cable local checkout

benchcab/benchcab.py

@@ -22,6 +22,8 @@
 from benchcab.utils.repo import create_repo
 from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface
 from benchcab.workdir import (
+    clean_realisation_files,
+    clean_submission_files,
     setup_fluxsite_directory_tree,
     setup_spatial_directory_tree,
 )
@@ -225,7 +227,14 @@
         rev_number_log = ""
 
         for model in self._get_models(config):
-            model.repo.checkout()
+            try:
+                model.repo.checkout()
+            except Exception:
+                msg = "Try using `benchcab clean realisations` first"
+                self.logger.error(
+                    "Model checkout failed, probably due to existing realisation name"
+                )
+                raise FileExistsError(msg)
             rev_number_log += f"{model.name}: {model.repo.get_revision()}\n"
 
         rev_number_log_path = next_path("rev_number-*.log")
@@ -315,6 +324,13 @@
         else:
             self.fluxsite_submit_job(config_path, skip)
 
+    def clean(self, config_path: str, clean_option: str):
+        """Endpoint for `benchcab clean`."""
+        if clean_option in ["all", "realisations"]:
+            clean_realisation_files()
+        if clean_option in ["all", "submissions"]:
+            clean_submission_files()
+
     def spatial_setup_work_directory(self, config_path: str):
         """Endpoint for `benchcab spatial-setup-work-dir`."""
         config = self._get_config(config_path)

benchcab/cli.py

@@ -224,4 +224,26 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser:
     )
     parser_spatial_run_tasks.set_defaults(func=app.spatial_run_tasks)
 
+    # subcommand: 'benchcab clean'
+    parser_clean = subparsers.add_parser(
+        "clean",
+        parents=[args_help, args_subcommand],
+        help="Cleanup files created by running benchcab.",
+        description="""Removes src/ and runs/ directories, along with log files in the 
+        project root directory. The user has to specify which stage of files to remove 
+        via \{all, realisations, submissions\} subcommand.""",
+        add_help=False,
+    )
+
+    parser_clean.add_argument(
+        "clean_option",
+        choices=["all", "realisations", "submissions"],
+        help="""Can be one of three options:
+
+        submissions: deletes src/ and revision log files
+        realisations: deletes runs/ and benchmark submission files
+        all: deletes in both stages of submissions and realisations""",
+    )
+
+    parser_clean.set_defaults(func=app.clean)
     return main_parser

benchcab/data/config-schema.yml

@@ -24,7 +24,7 @@ realisations:
         schema:
           git:
             type: "dict"
-            excludes: "svn"
+            excludes: ["svn", "local"]
             schema:
               branch:
                 type: "string"
@@ -37,13 +37,20 @@ realisations:
                 required: false
           svn:
             type: "dict"
-            excludes: "git"
+            excludes: ["git", "local"]
             schema:
               branch_path:
                 type: "string"
                 required: true
               revision:
                 type: "integer"
+          local:
+            type: "dict"
+            excludes: ["git", "svn"]
+            schema:
+              path:
+                type: "string"
+                required: true
       name:
         nullable: true
         type: "string"

benchcab/data/test/integration.sh

@@ -2,13 +2,25 @@
 
 set -ex
 
+CABLE_REPO="[email protected]:CABLE-LSM/CABLE.git"
+CABLE_DIR=/scratch/$PROJECT/$USER/benchcab/CABLE
+
 TEST_DIR=/scratch/$PROJECT/$USER/benchcab/integration
 EXAMPLE_REPO="[email protected]:CABLE-LSM/bench_example.git"
 
-# Remove the test work space, then recreate
+# Remove CABLE and test work space, then recreate
+rm -rf $CABLE_DIR
+mkdir -p $CABLE_DIR
+
 rm -rf $TEST_DIR
 mkdir -p $TEST_DIR
 
+# Clone local checkout for CABLE
+git clone $CABLE_REPO $CABLE_DIR
+cd $CABLE_DIR
+# Note: This is temporary, to be removed once #258 is fixed
+git reset --hard 67a52dc5721f0da78ee7d61798c0e8a804dcaaeb
+
 # Clone the example repo
 git clone $EXAMPLE_REPO $TEST_DIR
 cd $TEST_DIR
@@ -18,10 +30,13 @@ cat > config.yaml << EOL
 project: $PROJECT
 
 realisations:
+  - repo:
+      local:
+        path: $CABLE_DIR
   - repo:
       git:
         branch: main
-
+        commit: 67a52dc5721f0da78ee7d61798c0e8a804dcaaeb  # Note: This is temporary, to be removed once #258 is fixed
 modules: [
   intel-compiler/2021.1.1,
   netcdf/4.7.4,

benchcab/model.py

@@ -14,7 +14,7 @@
 from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface
 from benchcab.utils import get_logger
 from benchcab.utils.fs import chdir, copy2, rename
-from benchcab.utils.repo import GitRepo, Repo
+from benchcab.utils.repo import GitRepo, LocalRepo, Repo
 from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface
 
 
@@ -62,7 +62,7 @@ def __init__(
         # TODO(Sean) we should not have to know whether `repo` is a `GitRepo` or
         # `SVNRepo`, we should only be working with the `Repo` interface.
         # See issue https://github.com/CABLE-LSM/benchcab/issues/210
-        if isinstance(repo, GitRepo):
+        if isinstance(repo, (GitRepo, LocalRepo)):
             self.src_dir = Path("src")
 
     @property

benchcab/utils/repo.py

@@ -46,6 +46,59 @@
         """
 
 
+class LocalRepo(Repo):
+    """Concrete implementation of the `Repo` class using local path backend."""
+
+    def __init__(self, path: str, realisation_path: str) -> None:
+        """_summary_.
+
+        Parameters
+        ----------
+        realisation_path : str
+            Path for local checkout of CABLE
+        path : str
+            Directory where CABLE is symlinked from
+
+        """
+        self.name = Path(path).name
+        self.local_path = path
+        self.realisation_path = (
+            realisation_path / self.name
+            if realisation_path.is_dir()
+            else realisation_path
+        )
+        self.logger = get_logger()
+
+    def checkout(self):
+        """Checkout the source code."""
+        self.realisation_path.symlink_to(self.local_path)
+        self.logger.info(
+            f"Created symlink from to {self.realisation_path} named {self.name}"
+        )
+
+    def get_revision(self) -> str:
+        """Return the latest revision of the source code.
+
+        Returns
+        -------
+        str
+            Human readable string describing the latest revision.
+
+        """
+        return f"Local CABLE build: {Path(self.local_path).absolute().as_posix}"
+
+    def get_branch_name(self) -> str:
+        """Return the branch name of the source code.
+
+        Returns
+        -------
+        str
+            Branch name of the source code.
+
+        """
+        return Path(self.realisation_path).absolute().as_posix()
+
+
 class GitRepo(Repo):
     """A concrete implementation of the `Repo` class using a Git backend.
 
@@ -59,7 +112,11 @@
     subprocess_handler = SubprocessWrapper()
 
     def __init__(
-        self, url: str, branch: str, path: Path, commit: Optional[str] = None
+        self,
+        url: str,
+        branch: str,
+        realisation_path: Path,
+        commit: Optional[str] = None,
     ) -> None:
         """Return a `GitRepo` instance.
 
@@ -69,7 +126,7 @@
             URL pointing to the GitHub repository.
         branch: str
             Name of a branch on the GitHub repository.
-        path: Path
+        realisation_path: Path
             Path to a directory in which the repository is cloned into. If
             `path` points to an existing directory, the repository will be
             cloned into `path / branch`.
@@ -80,7 +137,9 @@
         """
         self.url = url
         self.branch = branch
-        self.path = path / branch if path.is_dir() else path
+        self.path = (
+            realisation_path / branch if realisation_path.is_dir() else realisation_path
+        )
         self.commit = commit
         self.logger = get_logger()
 
@@ -140,7 +199,7 @@
         self,
         svn_root: str,
         branch_path: str,
-        path: Path,
+        realisation_path: Path,
         revision: Optional[int] = None,
     ) -> None:
         """Return an `SVNRepo` instance.
@@ -151,7 +210,7 @@
             URL pointing to the root of the SVN repository.
         branch_path: str
             Path to a branch relative to `svn_root`.
-        path: Path
+        realisation_path: Path
             Path to a directory in which the branch is checked out into. If
             `path` points to an existing directory, the repository will be
             cloned into `path / <branch_name>` where `<branch_name>` is the
@@ -164,7 +223,11 @@
         self.svn_root = svn_root
         self.branch_path = branch_path
         self.revision = revision
-        self.path = path / Path(branch_path).name if path.is_dir() else path
+        self.path = (
+            realisation_path / Path(branch_path).name
+            if realisation_path.is_dir()
+            else realisation_path
+        )
         self.logger = get_logger()
 
     def checkout(self):
@@ -233,7 +296,11 @@
     if "git" in spec:
         if "url" not in spec["git"]:
             spec["git"]["url"] = internal.CABLE_GIT_URL
-        return GitRepo(path=path, **spec["git"])
+        return GitRepo(realisation_path=path, **spec["git"])
     if "svn" in spec:
-        return SVNRepo(svn_root=internal.CABLE_SVN_ROOT, path=path, **spec["svn"])
+        return SVNRepo(
+            svn_root=internal.CABLE_SVN_ROOT, realisation_path=path, **spec["svn"]
+        )
+    if "local" in spec:
+        return LocalRepo(realisation_path=path, **spec["local"])
     raise RepoSpecError

benchcab/workdir.py

@@ -9,14 +9,26 @@
 from benchcab.utils.fs import mkdir
 
 
-def clean_directory_tree():
-    """Remove pre-existing directories in current working directory."""
+def clean_realisation_files():
+    """Remove files/directories related to CABLE realisation source codes."""
     if internal.SRC_DIR.exists():
+        for realisation in internal.SRC_DIR.iterdir():
+            if realisation.is_symlink():
+                realisation.unlink()
         shutil.rmtree(internal.SRC_DIR)
 
+    for rev_log_file in internal.CWD.glob("rev_number-*.log"):
+        rev_log_file.unlink()
+
+
+def clean_submission_files():
+    """Remove files/directories related to PBS jobs."""
     if internal.RUN_DIR.exists():
         shutil.rmtree(internal.RUN_DIR)
 
+    for pbs_job_file in internal.CWD.glob(f"{internal.QSUB_FNAME}*"):
+        pbs_job_file.unlink()
+
 
 def setup_fluxsite_directory_tree():
     """Generate the directory structure used by `benchcab`."""

docs/user_guide/config_options.md

@@ -352,6 +352,23 @@ realisations:
 
 : **Default:** URL of the [CABLE GitHub repository][cable-github], _optional key_. :octicons-dash-24: Specify the GitHub repository url to clone from when checking out the branch.
 
+#### [`local`](#+repo.local){ #+repo.local}
+
+Contains settings to specify CABLE checkouts on a local repository.
+
+This key is _optional_. No default.
+
+```yaml
+realisations:
+  - repo:
+      local:
+        path: /scratch/tm70/ab1234/CABLE
+```
+
+[`path`](#+repo.local.path){ #+repo.local.path}
+
+: **Default:** _required key, no default_. :octicons-dash-24: Specify the local checkout path of CABLE branch.
+
 ### [name](#name)
 
 : **Default:** base name of [branch_path](#+repo.svn.branch_path) if an SVN repository is given, for Git repositories the default is the branch name, _optional key_. :octicons-dash-24: An alias name used internally by `benchcab` for the branch. The `name` key also specifies the directory name of the source code when retrieving from SVN or GitHub.

docs/user_guide/index.md

@@ -91,6 +91,12 @@ The tool will follow the steps:
 3. Setup and launch a PBS job to run the flux site simulations in parallel. When `benchcab` launches the PBS job, it will print out the job ID to the terminal. You can check the status of the job with `qstat`. `benchcab` will not warn you when the simulations are over.
 4. Setup and run an ensemble of offline spatial runs using the [`payu`][payu-github] framework.
 
+!!! info
+    In case the code branches are already checked out before running Step (1) - `benchcab` will fail. This could happen on re-runs of `benchcab`. In that case, run `benchcab clean realisations` before the `checkout` step.
+
+!!! warning
+    It is dangerous to delete `src/` via `rm -rf`, since `src/` may contain symlinks to local directories that could also be affected. Use `benchcab clean realisations` instead, which would also delete unecessary log files like `rev_number-*.log`. 
+
 !!! tip "Expected output"
 
     You can see [an example of the expected output](expected_output.md) printed out to the screen by `benchcab run` to check if the tool has worked as expected.