diff --git a/docs/user_guide/config_options.md b/docs/user_guide/config_options.md index 9e3c204..cf154c7 100644 --- a/docs/user_guide/config_options.md +++ b/docs/user_guide/config_options.md @@ -471,6 +471,15 @@ science_configurations: [ ] ``` +## codecov + +: **Default:** False, _optional key. :octicons-dash-24: Specifies whether to build `benchcab` with code-coverage flags, which can then be used in post-run analysis (`benchcab gen_codecov`). + +```yaml +codecov: + true +``` + [meorg]: https://modelevaluation.org/ [forty-two-me]: https://modelevaluation.org/experiment/display/s6k22L3WajmiS9uGv [five-me]: https://modelevaluation.org/experiment/display/Nb37QxkAz3FczWDd7 diff --git a/src/benchcab/benchcab.py b/src/benchcab/benchcab.py index ac6ef08..648ae10 100644 --- a/src/benchcab/benchcab.py +++ b/src/benchcab/benchcab.py @@ -14,6 +14,11 @@ from benchcab import fluxsite, internal, spatial from benchcab.comparison import run_comparisons, run_comparisons_in_parallel from benchcab.config import read_config +from benchcab.coverage import ( + get_coverage_tasks_default, + run_coverage_tasks, + run_coverages_in_parallel, +) from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface from benchcab.internal import get_met_forcing_file_names from benchcab.model import Model @@ -203,6 +208,7 @@ def fluxsite_submit_job(self, config_path: str, skip: list[str]) -> None: modules=config["modules"], pbs_config=config["fluxsite"]["pbs"], skip_bitwise_cmp="fluxsite-bitwise-cmp" in skip, + skip_codecov="gen_codecov" in skip or not config["codecov"], verbose=is_verbose(), benchcab_path=str(self.benchcab_exe_path), ) @@ -226,6 +232,29 @@ def fluxsite_submit_job(self, config_path: str, skip: list[str]) -> None: logger.info("The NetCDF output for each task is written to:") logger.info(f"{internal.FLUXSITE_DIRS['OUTPUT']}/_out.nc") + def gen_codecov(self, config_path: str): + """Endpoint for `benchcab codecov`.""" + logger = self._get_logger() + config = self._get_config(config_path) + self._validate_environment(project=config["project"], modules=config["modules"]) + + coverage_tasks = get_coverage_tasks_default( + models=self._get_models(config=config) + ) + + if not config["codecov"]: + msg = """`config.yaml` should have set `codecov: true` before building and + running `gen_codecov`.""" + raise ValueError(msg) + + logger.info("Running coverage tasks...") + if config["fluxsite"]["multiprocess"]: + ncpus = config["fluxsite"]["pbs"]["ncpus"] + run_coverages_in_parallel(coverage_tasks, n_processes=ncpus) + else: + run_coverage_tasks(coverage_tasks) + logger.info("Successfully ran coverage tasks") + def checkout(self, config_path: str): """Endpoint for `benchcab checkout`.""" logger = self._get_logger() @@ -271,7 +300,11 @@ def build(self, config_path: str, mpi=False): logger.info( f"Compiling CABLE {build_mode} for realisation {repo.name}..." ) - repo.build(modules=config["modules"], mpi=mpi) + repo.build( + modules=config["modules"], + mpi=mpi, + coverage=config["codecov"], + ) logger.info(f"Successfully compiled CABLE for realisation {repo.name}") def fluxsite_setup_work_directory(self, config_path: str): @@ -334,6 +367,8 @@ def fluxsite(self, config_path: str, no_submit: bool, skip: list[str]): self.fluxsite_run_tasks(config_path) if "fluxsite-bitwise-cmp" not in skip: self.fluxsite_bitwise_cmp(config_path) + if "codecov" not in skip: + self.gen_codecov(config_path) else: self.fluxsite_submit_job(config_path, skip) diff --git a/src/benchcab/cli.py b/src/benchcab/cli.py index a86c689..e029811 100644 --- a/src/benchcab/cli.py +++ b/src/benchcab/cli.py @@ -244,6 +244,17 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: submissions: deletes runs/ and benchmark submission files all: deletes in both stages of submissions and realisations""", ) - parser_clean.set_defaults(func=app.clean) + + # subcommand: 'benchcab gen_codecov" + parser_codecov = subparsers.add_parser( + "gen_codecov", + parents=[args_help, args_subcommand], + help="Runs code coverage tasks when runs are finised.", + description="""Uses profmerge and codecov utilties to do code coverage + analysis. Note: All sources must be built using Intel compiler. + """, + add_help=False, + ) + parser_codecov.set_defaults(func=app.gen_codecov) return main_parser diff --git a/src/benchcab/config.py b/src/benchcab/config.py index a144450..102a9bb 100644 --- a/src/benchcab/config.py +++ b/src/benchcab/config.py @@ -120,6 +120,8 @@ def read_optional_key(config: dict): "pbs", {} ) + config["codecov"] = config.get("codecov", False) + def read_config_file(config_path: str) -> dict: """Load the config file in a dict. diff --git a/src/benchcab/coverage.py b/src/benchcab/coverage.py new file mode 100644 index 0000000..e324b46 --- /dev/null +++ b/src/benchcab/coverage.py @@ -0,0 +1,95 @@ +# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. +# SPDX-License-Identifier: Apache-2.0 + +"""A module containing functions and data structures for running coverage tasks.""" + +import multiprocessing +import operator +from contextlib import nullcontext +from pathlib import Path +from typing import Optional + +from benchcab import internal +from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface +from benchcab.model import Model +from benchcab.utils import get_logger +from benchcab.utils.fs import chdir +from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface + + +class CoverageTask: + """A class used to represent a single coverage report generation task.""" + + subprocess_handler: SubprocessWrapperInterface = SubprocessWrapper() + modules_handler: EnvironmentModulesInterface = EnvironmentModules() + + def __init__( + self, + coverage_dir: str, + project_name: Optional[str] = "CABLE", + dpi_file: Optional[str] = "pgopti.dpi", + spi_file: Optional[str] = "pgopti.spi", + ) -> None: + """Constructor. + + Parameters + ---------- + coverage_dir: + Name of directory where coverage analysis is to be done + + project_name: + Name of project on which codecov is run + dpi_file: + name of DPI file created after merging .dyn files created after all runs + spi_file: + Static profile information on compilation + + """ + self.logger = get_logger() + self.coverage_dir = coverage_dir + self.project_name = project_name + self.dpi_file = dpi_file + self.spi_file = spi_file + + def run(self) -> None: + """Executes `profmerge` and `codecov` to run codecov analysis for a given realisation.""" + if not Path(self.coverage_dir).is_dir(): + msg = f"""The coverage directory: {self.coverage_dir} + does not exist. Did you run the jobs and/or set `coverage: true` in `config.yaml` + before the building stage""" + raise OSError(msg) + + self.logger.info(f"Generating coverage report in {self.coverage_dir}") + + # Load intel-compiler in case we run from CLI, otherwise assuming + # PBS jobscript loads + with chdir(self.coverage_dir), ( + nullcontext() + if self.modules_handler.module_is_loaded("intel-compiler") + else self.modules_handler.load([internal.DEFAULT_MODULES["intel-compiler"]]) + ): + self.subprocess_handler.run_cmd(f"profmerge -prof-dpi {self.dpi_file}") + self.subprocess_handler.run_cmd( + f"codecov -prj {self.project_name} -dpi {self.dpi_file} -spi {self.spi_file}" + ) + + +def run_coverage_tasks(coverage_tasks: list[CoverageTask]) -> None: + """Runs coverage tasks serially.""" + for task in coverage_tasks: + task.run() + + +def get_coverage_tasks_default(models: list[Model]) -> list[CoverageTask]: + """Returns list of Coveragee Tasks setting default values for optional parameters.""" + return [CoverageTask(model.get_coverage_dir()) for model in models] + + +def run_coverages_in_parallel( + coverage_tasks: list[CoverageTask], + n_processes=internal.FLUXSITE_DEFAULT_PBS["ncpus"], +) -> None: + """Runs coverage tasks in parallel across multiple processes.""" + run_task = operator.methodcaller("run") + with multiprocessing.Pool(n_processes) as pool: + pool.map(run_task, coverage_tasks, chunksize=1) diff --git a/src/benchcab/data/config-schema.yml b/src/benchcab/data/config-schema.yml index b28bfeb..07a55f4 100644 --- a/src/benchcab/data/config-schema.yml +++ b/src/benchcab/data/config-schema.yml @@ -130,4 +130,8 @@ spatial: args: nullable: true type: "string" - required: false \ No newline at end of file + required: false + +codecov: + type: "boolean" + required: false \ No newline at end of file diff --git a/src/benchcab/data/pbs_jobscript.j2 b/src/benchcab/data/pbs_jobscript.j2 index ea125bd..8812cd0 100644 --- a/src/benchcab/data/pbs_jobscript.j2 +++ b/src/benchcab/data/pbs_jobscript.j2 @@ -16,4 +16,9 @@ module load {{module}} set -ev {{benchcab_path}} fluxsite-run-tasks --config={{config_path}}{{verbose_flag}} -{% if skip_bitwise_cmp == False %}{{benchcab_path}} fluxsite-bitwise-cmp --config={{config_path}}{{verbose_flag}}{% endif %} \ No newline at end of file +{%- if skip_bitwise_cmp == False %} +{{benchcab_path}} fluxsite-bitwise-cmp --config={{config_path}}{{verbose_flag}} +{%- endif %} +{%- if skip_codecov == False %} +{{benchcab_path}} gen_codecov --config={{config_path}}{{verbose_flag}} +{%- endif %} \ No newline at end of file diff --git a/src/benchcab/data/test/config-optional.yml b/src/benchcab/data/test/config-optional.yml index 6ba4c33..f4605e8 100644 --- a/src/benchcab/data/test/config-optional.yml +++ b/src/benchcab/data/test/config-optional.yml @@ -35,6 +35,8 @@ realisations: branch_path: branches/Users/ccc561/v3.0-YP-changes name: git_branch +codecov: + true modules: [ intel-compiler/2021.1.1, diff --git a/src/benchcab/data/test/pbs_jobscript_no_skip_codecov.sh b/src/benchcab/data/test/pbs_jobscript_no_skip_codecov.sh new file mode 100644 index 0000000..ee64bda --- /dev/null +++ b/src/benchcab/data/test/pbs_jobscript_no_skip_codecov.sh @@ -0,0 +1,21 @@ +#!/bin/bash +#PBS -l wd +#PBS -l ncpus=18 +#PBS -l mem=30GB +#PBS -l walltime=6:00:00 +#PBS -q normal +#PBS -P tm70 +#PBS -j oe +#PBS -m e +#PBS -l storage=gdata/ks32+gdata/hh5+gdata/wd9 + +module purge +module load foo +module load bar +module load baz + +set -ev + +/absolute/path/to/benchcab fluxsite-run-tasks --config=/path/to/config.yaml +/absolute/path/to/benchcab fluxsite-bitwise-cmp --config=/path/to/config.yaml +/absolute/path/to/benchcab gen_codecov --config=/path/to/config.yaml \ No newline at end of file diff --git a/src/benchcab/data/test/pbs_jobscript_skip_bitwise.sh b/src/benchcab/data/test/pbs_jobscript_skip_bitwise.sh index d2dae5e..b5bf882 100644 --- a/src/benchcab/data/test/pbs_jobscript_skip_bitwise.sh +++ b/src/benchcab/data/test/pbs_jobscript_skip_bitwise.sh @@ -16,4 +16,4 @@ module load baz set -ev -/absolute/path/to/benchcab fluxsite-run-tasks --config=/path/to/config.yaml +/absolute/path/to/benchcab fluxsite-run-tasks --config=/path/to/config.yaml \ No newline at end of file diff --git a/src/benchcab/internal.py b/src/benchcab/internal.py index 08deb07..d095171 100644 --- a/src/benchcab/internal.py +++ b/src/benchcab/internal.py @@ -13,7 +13,10 @@ CONFIG_REQUIRED_KEYS = ["realisations", "modules"] # CMake module used for compilation: -CMAKE_MODULE = "cmake/3.24.2" +DEFAULT_MODULES = { + "cmake": "cmake/3.24.2", + "intel-compiler": "intel-compiler/2021.10.0", +} # Number of parallel jobs used when compiling with CMake: CMAKE_BUILD_PARALLEL_LEVEL = 4 @@ -51,6 +54,9 @@ # Path CABLE grid info file GRID_FILE = CABLE_AUX_DIR / "offline" / "gridinfo_CSIRO_1x1.nc" +# Relative path to directory that stores codecov files +CODECOV_DIR = RUN_DIR / "coverage" + # Fluxsite directory tree FLUXSITE_DIRS: dict[str, Path] = {} @@ -243,7 +249,7 @@ FLUXSITE_DEFAULT_EXPERIMENT = "forty-two-site-test" -OPTIONAL_COMMANDS = ["fluxsite-bitwise-cmp"] +OPTIONAL_COMMANDS = ["fluxsite-bitwise-cmp", "gen_codecov"] def get_met_forcing_file_names(experiment: str) -> list[str]: diff --git a/src/benchcab/model.py b/src/benchcab/model.py index 9bc6485..6e6246b 100644 --- a/src/benchcab/model.py +++ b/src/benchcab/model.py @@ -13,7 +13,7 @@ from benchcab import internal from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface from benchcab.utils import get_logger -from benchcab.utils.fs import chdir, prepend_path +from benchcab.utils.fs import chdir, mkdir, prepend_path from benchcab.utils.repo import GitRepo, LocalRepo, Repo from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface @@ -82,6 +82,40 @@ def model_id(self) -> int: def model_id(self, value: int): self._model_id = value + def get_coverage_dir(self) -> Path: + """Get absolute path for code coverage analysis.""" + return (internal.CODECOV_DIR / f"R{self.model_id}").absolute() + + def _get_build_flags(self, mpi: bool, coverage: bool, compiler_id: str) -> dict: + """Get flags for CMake build.""" + # Supported compilers for code coverage + codecov_compilers = ["ifort", "ifx"] + + build_flags = {} + + build_flags["build_type"] = "Debug" if coverage else "Release" + build_flags["mpi"] = "ON" if mpi else "OFF" + + build_flags["flags_init"] = "" + + if coverage: + if compiler_id not in codecov_compilers: + msg = f"""For code coverage, the only supported compilers are {codecov_compilers} + User has {compiler_id} in their environment""" + raise RuntimeError(msg) + + codecov_dir = self.get_coverage_dir() + + self.logger.info("Building with Intel code coverage") + + # `ifort` checks for pre-existing profile directories before compilation + mkdir(codecov_dir, parents=True, exist_ok=True) + + self.logger.debug(f"Analysis directory set as {codecov_dir}") + build_flags["flags_init"] += f'"-prof-gen=srcpos -prof-dir={codecov_dir}"' + + return build_flags + def get_exe_path(self, mpi=False) -> Path: """Return the path to the built executable.""" exe = internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE @@ -118,46 +152,67 @@ def custom_build(self, modules: list[str]): with chdir(build_script_path.parent), self.modules_handler.load(modules): self.subprocess_handler.run_cmd(f"./{tmp_script_path.name}") - def build(self, modules: list[str], mpi=False): + def build(self, modules: list[str], mpi: bool, coverage: bool): """Build CABLE with CMake.""" path_to_repo = internal.SRC_DIR / self.name - cmake_args = [ - "-DCMAKE_BUILD_TYPE=Release", - "-DCABLE_MPI=" + ("ON" if mpi else "OFF"), - ] - with chdir(path_to_repo), self.modules_handler.load( - [internal.CMAKE_MODULE, *modules] - ): - env = os.environ.copy() - - # This is required to prevent CMake from finding the conda - # installation of netcdf-fortran (#279): - env.pop("LDFLAGS", None) - - # This is required to prevent CMake from finding MPI libraries in - # the conda environment (#279): - env.pop("CMAKE_PREFIX_PATH", None) - - # This is required so that the netcdf-fortran library is discoverable by - # pkg-config: - prepend_path( - "PKG_CONFIG_PATH", f"{env['NETCDF_BASE']}/lib/Intel/pkgconfig", env=env + + with self.modules_handler.load([internal.DEFAULT_MODULES["cmake"], *modules]): + + # $FC is loaded after compiler module is loaded, + # but we need runs/ dir relative to project rootdir + env_fc = os.environ.get("FC", "") + self.logger.debug( + f"Getting environment variable for compiler $FC = {env_fc}" ) + build_flags = self._get_build_flags(mpi, coverage, env_fc) + env_fc = None + + with chdir(path_to_repo): + env = os.environ.copy() + + cmake_args = [ + f"-DCABLE_MPI={build_flags['mpi']}", + f"-DCMAKE_BUILD_TYPE={build_flags['build_type']}", + f"-DCMAKE_Fortran_FLAGS_INIT={build_flags['flags_init']}", + "-DCMAKE_VERBOSE_MAKEFILE=ON", + ] + + # This is required to prevent CMake from finding the conda + # installation of netcdf-fortran (#279): + env.pop("LDFLAGS", None) - if self.modules_handler.module_is_loaded("openmpi"): - # This is required so that the openmpi MPI libraries are discoverable - # via CMake's `find_package` mechanism: + # This is required to prevent CMake from finding MPI libraries in + # the conda environment (#279): + env.pop("CMAKE_PREFIX_PATH", None) + + # This is required so that the netcdf-fortran library is discoverable by + # pkg-config: prepend_path( - "CMAKE_PREFIX_PATH", f"{env['OPENMPI_BASE']}/include/Intel", env=env + "PKG_CONFIG_PATH", + f"{env['NETCDF_BASE']}/lib/Intel/pkgconfig", + env=env, ) - env["CMAKE_BUILD_PARALLEL_LEVEL"] = str(internal.CMAKE_BUILD_PARALLEL_LEVEL) + if self.modules_handler.module_is_loaded("openmpi"): + # This is required so that the openmpi MPI libraries are discoverable + # via CMake's `find_package` mechanism: + prepend_path( + "CMAKE_PREFIX_PATH", + f"{env['OPENMPI_BASE']}/include/Intel", + env=env, + ) + + env["CMAKE_BUILD_PARALLEL_LEVEL"] = str( + internal.CMAKE_BUILD_PARALLEL_LEVEL + ) - self.subprocess_handler.run_cmd( - "cmake -S . -B build " + " ".join(cmake_args), env=env - ) - self.subprocess_handler.run_cmd("cmake --build build ", env=env) - self.subprocess_handler.run_cmd("cmake --install build --prefix .", env=env) + self.subprocess_handler.run_cmd( + "cmake -S . -B build " + " ".join(cmake_args), env=env + ) + self.subprocess_handler.run_cmd("cmake --build build ", env=env) + self.subprocess_handler.run_cmd( + "cmake --install build --prefix .", env=env + ) def remove_module_lines(file_path: Path) -> None: diff --git a/src/benchcab/utils/pbs.py b/src/benchcab/utils/pbs.py index 2ae8a4f..01486d0 100644 --- a/src/benchcab/utils/pbs.py +++ b/src/benchcab/utils/pbs.py @@ -3,7 +3,7 @@ """Contains helper functions for manipulating PBS job scripts.""" -from typing import TypedDict +from typing import Optional, TypedDict from benchcab.utils import interpolate_file_template @@ -23,8 +23,9 @@ def render_job_script( modules: list, benchcab_path: str, pbs_config: PBSConfig, - verbose=False, - skip_bitwise_cmp=False, + verbose: Optional[bool] = False, + skip_bitwise_cmp: Optional[bool] = False, + skip_codecov: Optional[bool] = True, ) -> str: """Returns the text for a PBS job script that executes all computationally expensive commands. @@ -45,6 +46,7 @@ def render_job_script( benchcab_path=benchcab_path, config_path=config_path, skip_bitwise_cmp=skip_bitwise_cmp, + skip_codecov=skip_codecov, ) return interpolate_file_template("pbs_jobscript.j2", **context) diff --git a/tests/test_cli.py b/tests/test_cli.py index c190d25..eb15630 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -113,6 +113,14 @@ def test_cli_parser(): "func": app.spatial_run_tasks, } + # Success case: default gen_codecov command + res = vars(parser.parse_args(["gen_codecov"])) + assert res == { + "config_path": "config.yaml", + "verbose": False, + "func": app.gen_codecov, + } + # Failure case: pass --no-submit to a non 'run' command with pytest.raises(SystemExit): parser.parse_args(["fluxsite-setup-work-dir", "--no-submit"]) diff --git a/tests/test_config.py b/tests/test_config.py index de4cb14..5f741be 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -81,6 +81,7 @@ def all_optional_default_config(no_optional_config) -> dict: "payu": {"config": {}, "args": None}, "met_forcings": internal.SPATIAL_DEFAULT_MET_FORCINGS, }, + "codecov": False, } for c_r in config["realisations"]: c_r["name"] = None @@ -119,6 +120,7 @@ def all_optional_custom_config(no_optional_config) -> dict: "crujra_access": "https://github.com/CABLE-LSM/cable_example.git" }, }, + "codecov": True, } branch_names = ["svn_trunk", "git_branch"] diff --git a/tests/test_coverage.py b/tests/test_coverage.py new file mode 100644 index 0000000..51014ce --- /dev/null +++ b/tests/test_coverage.py @@ -0,0 +1,58 @@ +"""`pytest` tests for `coverage.py`. + +Note: explicit teardown for generated files and directories are not required as +the working directory used for testing is cleaned up in the `_run_around_tests` +pytest autouse fixture. +""" + +from pathlib import Path + +import pytest + +from benchcab import internal +from benchcab.coverage import CoverageTask + +COVERAGE_REALISATION = "R0" +PROJECT_NAME = "BENCHCAB_TEST" +DPI_FILE = "test.dpi" +SPI_FILE = "test.spi" + + +@pytest.fixture() +def coverage_task( + coverage_dir, mock_subprocess_handler, mock_environment_modules_handler +): + """Returns a mock `CoverageTask` instance for testing against.""" + _coverage_task = CoverageTask( + coverage_dir=coverage_dir, + project_name=PROJECT_NAME, + dpi_file=DPI_FILE, + spi_file=SPI_FILE, + ) + _coverage_task.subprocess_handler = mock_subprocess_handler + _coverage_task.modules_handler = mock_environment_modules_handler + return _coverage_task + + +class TestRun: + """Tests for `CoverageTask.run()`.""" + + @pytest.fixture(autouse=True) + def coverage_dir(self) -> Path: + """Create and return the fluxsite bitwise coverage directory.""" + coverage_path = internal.CODECOV_DIR / COVERAGE_REALISATION + coverage_path.mkdir(parents=True) + return coverage_path + + def test_profmerge_execution(self, coverage_task, mock_subprocess_handler): + """Success case: test profmerge is executed.""" + coverage_task.run() + assert f"profmerge -prof-dpi {DPI_FILE}" in mock_subprocess_handler.commands + + def test_codecov_execution(self, coverage_task, mock_subprocess_handler): + """Success case: test codecov is executed.""" + coverage_task.run() + assert ( + f"codecov -prj {PROJECT_NAME} -dpi {DPI_FILE} -spi {SPI_FILE}" + in mock_subprocess_handler.commands + ) diff --git a/tests/test_model.py b/tests/test_model.py index a8f6b6d..cdf6a3e 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -5,13 +5,17 @@ pytest autouse fixture. """ +import re from pathlib import Path import pytest + from benchcab import internal from benchcab.model import Model, remove_module_lines from benchcab.utils.repo import Repo +TEST_MODEL_ID = 0 + @pytest.fixture() def mock_repo(): @@ -44,7 +48,7 @@ def mpi(request): @pytest.fixture() def model(mock_repo, mock_subprocess_handler, mock_environment_modules_handler): """Return a mock `Model` instance for testing against.""" - _model = Model(repo=mock_repo) + _model = Model(repo=mock_repo, model_id=TEST_MODEL_ID) _model.subprocess_handler = mock_subprocess_handler _model.modules_handler = mock_environment_modules_handler return _model @@ -83,6 +87,62 @@ def test_get_exe_path(self, model, mpi, expected_exe): ) +class TestGetBuildFlags: + """Tests for `Model.get_build_flags()`.""" + + @pytest.fixture(params=[False, True]) + def codecov(self, request) -> bool: + """Return a parametrized codecov option for testing.""" + return request.param + + @pytest.fixture(params=["gfortran", "ifort"]) + def compiler_id(self, request): + """Return a parametrized compiler_id flag for testing.""" + return request.param + + @pytest.fixture() + def cmake_flags(self, codecov, mpi, compiler_id): + """Generate build flags used in CMake build.""" + codecov_build_type = { + False: "Release", + True: "Debug", + } + + mpi_args = {True: "ON", False: "OFF"} + + codecov_init_args = { + False: "", + True: ( + f"\"-prof-gen=srcpos -prof-dir={(internal.CODECOV_DIR / 'R0').absolute()}\"" + if compiler_id in ["ifort", "ifx"] + else "" + ), + } + + return { + "build_type": codecov_build_type[codecov], + "mpi": mpi_args[mpi], + "flags_init": codecov_init_args[codecov], + } + + def test_get_build_flags(self, model, mpi, codecov, compiler_id, cmake_flags): + """Failure case: If coverage flags are passed to non-Intel compiler.""" + codecov_compilers = ["ifort", "ifx"] + if compiler_id not in codecov_compilers and codecov: + with pytest.raises( + RuntimeError, + match=re.escape( + f"""For code coverage, the only supported compilers are {codecov_compilers} + User has {compiler_id} in their environment""" + ), + ): + model._get_build_flags(mpi, codecov, compiler_id) + return + + # Success case: get expected build flags to pass to CMake. + assert model._get_build_flags(mpi, codecov, compiler_id) == cmake_flags + + # TODO(Sean) remove for issue https://github.com/CABLE-LSM/benchcab/issues/211 @pytest.mark.skip( reason="""Skip tests for `checkout` until tests for repo.py diff --git a/tests/test_pbs.py b/tests/test_pbs.py index fb84edd..bfccd68 100644 --- a/tests/test_pbs.py +++ b/tests/test_pbs.py @@ -39,3 +39,14 @@ def test_skip_bitwise_comparison_step(self): skip_bitwise_cmp=True, benchcab_path="/absolute/path/to/benchcab", ) == load_package_data("test/pbs_jobscript_skip_bitwise.sh") + + def test_no_skip_coverage_step(self): + """Success case: not skip coverage step.""" + assert render_job_script( + project="tm70", + config_path="/path/to/config.yaml", + modules=["foo", "bar", "baz"], + pbs_config=internal.FLUXSITE_DEFAULT_PBS, + skip_codecov=False, + benchcab_path="/absolute/path/to/benchcab", + ) == load_package_data("test/pbs_jobscript_no_skip_codecov.sh")