From adf42e393797e96620a81ff079d9f0fa773cf0c3 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Fri, 4 Aug 2023 23:24:54 -0400 Subject: [PATCH] ci(benchmarks): benchmark all platforms, refactor dist scripts --- .build_rtd_docs/conf.py | 16 ++-- .build_rtd_docs/index.rst | 2 +- .github/workflows/docs.yml | 131 ++++++++++++++++++++++---------- .gitignore | 3 +- distribution/benchmark.py | 57 ++++++++------ distribution/build_dist.py | 7 +- distribution/build_docs.py | 92 +++++++++++----------- distribution/build_makefiles.py | 2 +- distribution/check_dist.py | 1 - distribution/update_version.py | 14 ++-- distribution/utils.py | 2 +- 11 files changed, 191 insertions(+), 136 deletions(-) diff --git a/.build_rtd_docs/conf.py b/.build_rtd_docs/conf.py index bfd38623e25..c2e7f9b78c8 100644 --- a/.build_rtd_docs/conf.py +++ b/.build_rtd_docs/conf.py @@ -7,6 +7,7 @@ # -- Path setup -------------------------------------------------------------- import os +from pathlib import Path import shutil # If extensions (or modules to document with autodoc) are in another directory, @@ -42,19 +43,20 @@ # -- import version from doc/version.py ------------------------------------- from version import __version__ -# -- copy run-time comparison markdown -------------------------------------- -print("Copy the run-time comparison table") +# -- copy run-time comparison tables ---------------------------------------- +print("Copy run-time comparison tables") dstdir = "_mf6run" -fpth = "run-time-comparison.md" -src = os.path.join("..", "distribution", fpth) -dst = os.path.join(dstdir, fpth) +src = Path(os.path.join("..", "distribution", ".benchmarks")) +srcs = list(src.glob("run-time-comparison*")) # clean up an existing _mf6run directory if os.path.isdir(dstdir): shutil.rmtree(dstdir) # make the _mf6run directory os.makedirs(dstdir) -# copy the file -shutil.copy(src, dst) +# copy the files +for f in srcs: + dst = os.path.join(dstdir, f.name) + shutil.copy(f, dst) # -- build the mf6io markdown files ----------------------------------------- print("Build the mf6io markdown files") diff --git a/.build_rtd_docs/index.rst b/.build_rtd_docs/index.rst index e5b8ee8cae1..6b41a7ea7d4 100644 --- a/.build_rtd_docs/index.rst +++ b/.build_rtd_docs/index.rst @@ -13,5 +13,5 @@ Contents: MODFLOW 6 Source Code Documentation mf6io - _mf6run/run-time-comparison.md + _mf6run/run-time-comparison*.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0a1a47e9bc5..3d046714614 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,16 +14,25 @@ on: paths-ignore: - '.github/workflows/release.yml' jobs: - rtd_build: - name: Build ReadTheDocs - runs-on: ubuntu-22.04 + benchmark: + runs-on: ${{ matrix.os }} defaults: run: shell: bash -l {0} - env: - GCC_V: 12 - + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-22.04, compiler_toolchain: gcc, compiler_version: 13} + - {os: macos-12, compiler_toolchain: gcc, compiler_version: 13} + - {os: windows-2022, compiler_toolchain: gcc, compiler_version: 12} + - {os: ubuntu-22.04, compiler_toolchain: intel, compiler_version: 2022.2} + - {os: windows-2022, compiler_toolchain: intel, compiler_version: 2022.2} + - {os: ubuntu-22.04, compiler_toolchain: intel-classic, compiler_version: 2021.7.0} + - {os: macos-12, compiler_toolchain: intel-classic, compiler_version: 2021.7.0} + - {os: windows-2022, compiler_toolchain: intel-classic, compiler_version: 2021.7.0} steps: + - name: Checkout modflow6 uses: actions/checkout@v3 with: @@ -35,12 +44,6 @@ jobs: repository: MODFLOW-USGS/modflow6-examples path: modflow6-examples - - name: Checkout usgslatex - uses: actions/checkout@v3 - with: - repository: MODFLOW-USGS/usgslatex - path: usgslatex - - name: Install Conda environment from environment.yml uses: mamba-org/setup-micromamba@v1 with: @@ -48,31 +51,14 @@ jobs: cache-environment: true cache-downloads: true - - name: Install additional packages for Sphinx using pip - working-directory: modflow6/.build_rtd_docs - run: pip install -r requirements.rtd.txt - - name: Print python package versions run: pip list - - name: Install TeX Live - run: | - sudo apt-get update - sudo apt install texlive-latex-extra texlive-science texlive-font-utils - - - name: Install USGS LaTeX style files and Univers font - working-directory: usgslatex/usgsLaTeX - run: sudo ./install.sh --all-users - - - name: Test building files from dfn's for LaTeX - working-directory: modflow6/autotest - run: pytest -v build_mfio_tex.py - - - name: Setup GNU Fortran ${{ env.GCC_V }} + - name: Setup ${{ matrix.compiler_toolchain }} ${{ matrix.compiler_version }} uses: awvwgk/setup-fortran@main with: - compiler: gcc - version: ${{ env.GCC_V }} + compiler: ${{ matrix.compiler_toolchain }} + version: ${{ matrix.compiler_version }} - name: Cache modflow6 examples id: cache-examples @@ -105,22 +91,87 @@ jobs: run: python benchmark.py env: GITHUB_TOKEN: ${{ github.token }} + + - name: Rename/show benchmarks + id: rename-benchmarks + working-directory: modflow6/distribution/.benchmarks + run: | + orig="run-time-comparison.md" + name="run-time-comparison-${{ matrix.os }}-${{ matrix.compiler_toolchain }}-${{ matrix.compiler_version }}.md" + cat $orig + mv $orig $name + echo "name=$name" >> $GITHUB_OUTPUT + + - name: Upload benchmarks + uses: actions/upload-artifact@v3 + with: + name: run-time-comparison + path: modflow6/distribution/.benchmarks/${{ steps.rename-benchmarks.outputs.name }} - - name: Run sphinx + rtd_build: + name: Benchmark & build RTD + needs: benchmark + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash -l {0} + steps: + + - name: Checkout modflow6 + uses: actions/checkout@v3 + with: + path: modflow6 + + - name: Checkout usgslatex + uses: actions/checkout@v3 + with: + repository: MODFLOW-USGS/usgslatex + path: usgslatex + + - name: Install Conda environment from environment.yml + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: modflow6/environment.yml + cache-environment: true + cache-downloads: true + + - name: Install additional packages for Sphinx using pip working-directory: modflow6/.build_rtd_docs - run: make html + run: pip install -r requirements.rtd.txt - - name: Show benchmarks - working-directory: modflow6/distribution - run: cat run-time-comparison.md + - name: Print python package versions + run: pip list - - name: Upload benchmarks - uses: actions/upload-artifact@v3 + - name: Install TeX Live + run: | + sudo apt-get update + sudo apt install texlive-latex-extra texlive-science texlive-font-utils + + - name: Install USGS LaTeX style files and Univers font + working-directory: usgslatex/usgsLaTeX + run: sudo ./install.sh --all-users + + - name: Test building files from dfn's for LaTeX + working-directory: modflow6/autotest + run: pytest -v build_mfio_tex.py + + - name: Download benchmarks + uses: actions/download-artifact@v3 with: name: run-time-comparison - path: modflow6/distribution/run-time-comparison.md + path: modflow6/distribution + + - name: Show benchmarks + working-directory: modflow6/distribution + run: cat run-time-comparison-*.md + + - name: Run sphinx + if: runner.os == 'Linux' + working-directory: modflow6/.build_rtd_docs + run: make html - name: Upload results + if: runner.os == 'Linux' uses: actions/upload-artifact@v3 with: name: rtd-files-for-${{ github.sha }} diff --git a/.gitignore b/.gitignore index 56b28c72ec8..1430503e279 100644 --- a/.gitignore +++ b/.gitignore @@ -116,7 +116,8 @@ CMakeFiles/ latex/ html/ xml/ -distribution/run-time-comparison.md +distribution/run-time-comparison*.md +distribution/.benchmarks/run-time-comparison*.md doc/ReleaseNotes/run-time-comparison.tex # Intel Fortran Visual Studio intermediate files diff --git a/distribution/benchmark.py b/distribution/benchmark.py index b92d533fec3..c240f2cd133 100644 --- a/distribution/benchmark.py +++ b/distribution/benchmark.py @@ -5,35 +5,38 @@ import sys import textwrap from multiprocessing import Pool -from os import PathLike +from os import PathLike, environ from pathlib import Path +from platform import system from typing import List, Tuple import flopy import pytest from modflow_devtools.build import meson_build from modflow_devtools.download import download_and_unzip, get_latest_version -from modflow_devtools.misc import get_model_paths +from modflow_devtools.misc import get_model_paths, set_env +from modflow_devtools.ostags import get_github_ostag, get_modflow_ostag from utils import get_project_root_path _verify = False +_github_ostag = get_github_ostag() +_modflow_ostag = get_modflow_ostag() +_is_windows = _github_ostag == "Windows" +_app_ext = ".exe" if _is_windows else "" +_soext = ".dll" if _is_windows else ".so" +_github_repo = "MODFLOW-USGS/modflow6" +_markdown_file_name = "run-time-comparison.md" _project_root_path = get_project_root_path() -_examples_repo_path = _project_root_path.parent / "modflow6-examples" _build_path = _project_root_path / "builddir" _bin_path = _project_root_path / "bin" -_github_repo = "MODFLOW-USGS/modflow6" -_markdown_file_name = "run-time-comparison.md" -_is_windows = sys.platform.lower() == "win32" -_app_ext = ".exe" if _is_windows else "" -_soext = ".dll" if _is_windows else ".so" -_ostag = "win64" if _is_windows else "linux" if sys.platform.lower().startswith("linux") else "mac" +_examples_repo_path = _project_root_path.parent / "modflow6-examples" def download_previous_version(output_path: PathLike) -> Tuple[str, Path]: output_path = Path(output_path).expanduser().absolute() version = get_latest_version(_github_repo) - distname = f"mf{version}_{_ostag}" + distname = f"mf{version}_{_modflow_ostag}" url = ( f"https://github.com/{_github_repo}" + f"/releases/download/{version}/{distname}.zip" @@ -331,7 +334,7 @@ def run_benchmarks( output_path: PathLike, excluded: List[str] = [], ): - """Benchmark current development version against previous release with example models.""" + """Benchmark current development version against previous release on example models.""" build_path = Path(build_path).expanduser().absolute() current_bin_path = Path(current_bin_path).expanduser().absolute() @@ -339,14 +342,11 @@ def run_benchmarks( examples_path = Path(examples_path).expanduser().absolute() output_path = Path(output_path).expanduser().absolute() + # make sure example models exist example_dirs = get_model_paths(examples_path, excluded=excluded) assert any(example_dirs), f"No example model paths found, have models been built?" - # results_path = output_path / _markdown_file_name - # if results_path.is_file(): - # print(f"Benchmark results already exist: {results_path}") - # return - + # make sure binaries exist exe_name = f"mf6{_app_ext}" current_exe = current_bin_path / exe_name previous_exe = previous_bin_path / exe_name @@ -362,11 +362,22 @@ def run_benchmarks( if not previous_exe.is_file(): version, download_path = download_previous_version(output_path) print(f"Rebuilding latest MODFLOW 6 release {version} in development mode") - meson_build( - project_path=download_path, - build_path=build_path, - bin_path=previous_bin_path, - ) + + def rebuild_prev_release(): + meson_build( + project_path=download_path, + build_path=build_path, + bin_path=previous_bin_path, + ) + + # temp workaround until next release, + # ifx fails to build 6.4.2 on Windows + # most likely due to backspace issues + if _github_ostag == "Windows" and environ.get("FC") == "ifx": + with set_env(FC="ifort", CC="icl"): + rebuild_prev_release() + else: + rebuild_prev_release() print(f"Benchmarking MODFLOW 6 versions:") print(f" current: {current_exe}") @@ -445,8 +456,8 @@ def test_run_benchmarks(tmp_path): "-o", "--output-path", required=False, - default=str(_project_root_path / "distribution" / ""), - help="Location to create the zip archive", + default=str(_project_root_path / "distribution" / ".benchmarks"), + help="Location to create benchmark result files", ) parser.add_argument( "-e", diff --git a/distribution/build_dist.py b/distribution/build_dist.py index 68c454ec650..6964f93b02f 100644 --- a/distribution/build_dist.py +++ b/distribution/build_dist.py @@ -10,14 +10,14 @@ from shutil import copytree import pytest +from build_docs import build_documentation +from build_makefiles import (build_mf5to6_makefile, build_mf6_makefile, + build_zbud6_makefile) from modflow_devtools.build import meson_build from modflow_devtools.download import download_and_unzip, get_release from modflow_devtools.markers import requires_exe from modflow_devtools.misc import get_model_paths -from build_docs import build_documentation -from build_makefiles import (build_mf5to6_makefile, build_mf6_makefile, - build_zbud6_makefile) from utils import get_project_root_path, run_command # default paths @@ -323,7 +323,6 @@ def build_distribution( bin_path=output_path / "bin", output_path=output_path / "doc", examples_repo_path=examples_repo_path, - # benchmarks_path=_benchmarks_path / "run-time-comparison.md", full=full, overwrite=overwrite, ) diff --git a/distribution/build_docs.py b/distribution/build_docs.py index 353ba713a31..5755e5047bc 100644 --- a/distribution/build_docs.py +++ b/distribution/build_docs.py @@ -25,7 +25,6 @@ from modflow_devtools.markers import requires_exe, requires_github from modflow_devtools.misc import is_in_ci, run_cmd, set_dir -from benchmark import run_benchmarks from utils import convert_line_endings, get_project_root_path # paths @@ -49,6 +48,7 @@ ] # OS-specific extensions +_github_ostags = ["Linux", "macOS", "Windows"] _system = platform.system() _eext = ".exe" if _system == "Windows" else "" _soext = ".dll" if _system == "Windows" else ".so" if _system == "Linux" else ".dylib" @@ -105,8 +105,8 @@ def clean_tex_files(): def download_benchmarks( - output_path: PathLike, verbose: bool = False, repo_owner: str = "MODFLOW-USGS" -) -> Optional[Path]: + output_path: PathLike, repo_owner: str = "MODFLOW-USGS", verbose: bool = False, +) -> List[Optional[Path]]: output_path = Path(output_path).expanduser().absolute() name = "run-time-comparison" # todo make configurable repo = f"{repo_owner}/modflow6" # todo make configurable, add pytest/cli args @@ -117,20 +117,26 @@ def download_benchmarks( reverse=True, ) artifacts = [ - a for a in artifacts if a["workflow_run"]["head_branch"] == "develop" # todo make configurable + a + for a in artifacts + if a["workflow_run"]["head_branch"] == "develop" # todo make configurable ] most_recent = next(iter(artifacts), None) - print(f"Found most recent benchmarks (artifact {most_recent['id']})") if most_recent: - print(f"Downloading benchmarks (artifact {most_recent['id']})") + if verbose: + print(f"Downloading benchmarks (artifact {most_recent['id']}) to {output_path}") + download_artifact(repo, id=most_recent["id"], path=output_path, verbose=verbose) - print(f"Downloaded benchmarks to {output_path}") - path = output_path / f"{name}.md" - assert path.is_file() - return path + + if verbose: + print(f"Found benchmark results:") + paths = list(output_path.glob(f"{name}*.md")) + + assert any(paths) + return paths else: print(f"No benchmarks found") - return None + return [] @pytest.fixture @@ -141,60 +147,50 @@ def github_user() -> Optional[str]: @flaky @requires_github def test_download_benchmarks(tmp_path, github_user): - path = download_benchmarks( + paths = download_benchmarks( tmp_path, - verbose=True, repo_owner=github_user if github_user else "MODFLOW-USGS", + verbose=True, ) - if path: - assert path.name == "run-time-comparison.md" + assert any(paths) -def build_benchmark_tex( - output_path: PathLike, overwrite: bool = False, repo_owner: str = "MODFLOW-USGS" -): +def build_benchmark_tex(output_path: PathLike, repo_owner: str = "MODFLOW-USGS", verbose: bool = False): _benchmarks_path.mkdir(parents=True, exist_ok=True) - benchmarks_path = _benchmarks_path / "run-time-comparison.md" - - # download benchmark artifacts if any exist on GitHub - if not benchmarks_path.is_file(): - benchmarks_path = download_benchmarks(_benchmarks_path, repo_owner=repo_owner) - - # run benchmarks again if no benchmarks found on GitHub or overwrite requested - if overwrite or not benchmarks_path.is_file(): - run_benchmarks( - build_path=_project_root_path / "builddir", - current_bin_path=_project_root_path / "bin", - previous_bin_path=_project_root_path / "bin" / "rebuilt", - examples_path=_examples_repo_path / "examples", - output_path=output_path, + benchmarks_paths = list(_benchmarks_path.glob("run-time-comparison*.md")) + + # download benchmark artifacts if we don't already have one or more + if not any(benchmarks_paths): + benchmarks_paths = download_benchmarks(_benchmarks_path, repo_owner=repo_owner, verbose=verbose) + + # convert Linux markdown benchmark results to LaTeX + linux_benchmark_md = benchmarks_paths[0] if len(benchmarks_paths) == 1 else next( + iter([p for p in benchmarks_paths if "linux" in str(p)]), None + ) + if not linux_benchmark_md: + raise ValueError( + "No Linux benchmark results found" ) - # convert markdown benchmark results to LaTeX with set_dir(_release_notes_path): tex_path = Path("run-time-comparison.tex") tex_path.unlink(missing_ok=True) out, err, ret = run_cmd( - sys.executable, "mk_runtimecomp.py", benchmarks_path, verbose=True + sys.executable, + "mk_runtimecomp.py", + linux_benchmark_md, + verbose=True, ) assert not ret, out + err assert tex_path.is_file() - if (_distribution_path / f"{benchmarks_path.stem}.md").is_file(): - assert (_docs_path / "ReleaseNotes" / f"{benchmarks_path.stem}.tex").is_file() + assert (_docs_path / "ReleaseNotes" / f"run-time-comparison.tex").is_file() @flaky @requires_github -def test_build_benchmark_tex(tmp_path): - benchmarks_path = _benchmarks_path / "run-time-comparison.md" - tex_path = _distribution_path / f"{benchmarks_path.stem}.tex" - - try: - build_benchmark_tex(tmp_path) - assert benchmarks_path.is_file() - finally: - tex_path.unlink(missing_ok=True) +def test_build_benchmark_tex(tmp_path, github_user): + build_benchmark_tex(tmp_path, repo_owner=github_user, verbose=True) def build_mf6io_tex_from_dfn(overwrite: bool = False): @@ -475,10 +471,8 @@ def build_documentation( # convert LaTeX to PDF build_pdfs_from_tex(tex_paths=_dev_dist_tex_paths, output_path=output_path) else: - # convert benchmarks to LaTex, running them first if necessary - build_benchmark_tex( - output_path=output_path, overwrite=overwrite, repo_owner=repo_owner - ) + # convert benchmarks to LaTex + build_benchmark_tex(output_path=output_path, repo_owner=repo_owner) # download example docs latest = get_release(f"{repo_owner}/modflow6-examples", "latest") diff --git a/distribution/build_makefiles.py b/distribution/build_makefiles.py index e9fb09645db..1384fcd93a0 100644 --- a/distribution/build_makefiles.py +++ b/distribution/build_makefiles.py @@ -3,12 +3,12 @@ from os import environ from pathlib import Path -import pymake import pytest from flaky import flaky from modflow_devtools.markers import requires_exe from modflow_devtools.misc import set_dir +import pymake from utils import get_modified_time, get_project_root_path _project_root_path = get_project_root_path() diff --git a/distribution/check_dist.py b/distribution/check_dist.py index 6248204c306..3f2fb50d2b5 100644 --- a/distribution/check_dist.py +++ b/distribution/check_dist.py @@ -6,7 +6,6 @@ import pytest - # OS-specific extensions _system = platform.system() _eext = ".exe" if _system == "Windows" else "" diff --git a/distribution/update_version.py b/distribution/update_version.py index e09e0c69aef..41b02db6818 100755 --- a/distribution/update_version.py +++ b/distribution/update_version.py @@ -34,13 +34,13 @@ import textwrap from collections import OrderedDict from datetime import datetime -from packaging.version import Version from pathlib import Path from typing import Optional import pytest -from filelock import FileLock import yaml +from filelock import FileLock +from packaging.version import Version from utils import get_modified_time @@ -292,11 +292,7 @@ def update_version( try: lock = FileLock(lock_path) previous = Version(version_file_path.read_text().strip()) - version = ( - version - if version - else previous - ) + version = version if version else previous with lock: update_version_txt_and_py(version, timestamp) @@ -320,7 +316,9 @@ def update_version( [ None, _initial_version, - Version(f"{_initial_version.major}.{_initial_version.minor}.dev{_initial_version.micro}"), + Version( + f"{_initial_version.major}.{_initial_version.minor}.dev{_initial_version.micro}" + ), ], ) @pytest.mark.parametrize("approved", [True, False]) diff --git a/distribution/utils.py b/distribution/utils.py index a301987e51c..6878c3c661a 100644 --- a/distribution/utils.py +++ b/distribution/utils.py @@ -9,8 +9,8 @@ from pathlib import Path from warnings import warn -from modflow_devtools.markers import requires_exe import pytest +from modflow_devtools.markers import requires_exe _project_root_path = Path(__file__).resolve().parent.parent