ci(benchmarks): benchmark all platforms, refactor dist scripts

.build_rtd_docs/conf.py

@@ -7,6 +7,7 @@
 # -- Path setup --------------------------------------------------------------
 
 import os
+from pathlib import Path
 import shutil
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -42,19 +43,20 @@
 # -- import version from doc/version.py -------------------------------------
 from version import __version__
 
-# -- copy run-time comparison markdown --------------------------------------
-print("Copy the run-time comparison table")
+# -- copy run-time comparison tables ----------------------------------------
+print("Copy run-time comparison tables")
 dstdir = "_mf6run"
-fpth = "run-time-comparison.md"
-src = os.path.join("..", "distribution", fpth)
-dst = os.path.join(dstdir, fpth)
+src = Path(os.path.join("..", "distribution", ".benchmarks")) 
+srcs = list(src.glob("run-time-comparison*"))
 # clean up an existing _mf6run directory
 if os.path.isdir(dstdir):
     shutil.rmtree(dstdir)
 # make the _mf6run directory
 os.makedirs(dstdir)
-# copy the file
-shutil.copy(src, dst)
+# copy the files
+for f in srcs:
+    dst = os.path.join(dstdir, f.name)
+    shutil.copy(f, dst)
 
 # -- build the mf6io markdown files -----------------------------------------
 print("Build the mf6io markdown files")

.build_rtd_docs/index.rst

@@ -13,5 +13,5 @@ Contents:
 
    MODFLOW 6 Source Code Documentation <https://modflow-usgs.github.io/modflow6/>
    mf6io
-   _mf6run/run-time-comparison.md
+   _mf6run/run-time-comparison*.md
 

.github/workflows/docs.yml

@@ -14,16 +14,25 @@ on:
     paths-ignore:
       - '.github/workflows/release.yml'
 jobs:
-  rtd_build:
-    name: Build ReadTheDocs
-    runs-on: ubuntu-22.04
+  benchmark:
+    runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash -l {0}
-    env:
-      GCC_V: 12
-
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {os: ubuntu-22.04, compiler_toolchain: gcc, compiler_version: 13}
+          - {os: macos-12, compiler_toolchain: gcc, compiler_version: 13}
+          - {os: windows-2022, compiler_toolchain: gcc, compiler_version: 12}
+          - {os: ubuntu-22.04, compiler_toolchain: intel, compiler_version: 2022.2}
+          - {os: windows-2022, compiler_toolchain: intel, compiler_version: 2022.2}
+          - {os: ubuntu-22.04, compiler_toolchain: intel-classic, compiler_version: 2021.7.0}
+          - {os: macos-12, compiler_toolchain: intel-classic, compiler_version: 2021.7.0}
+          - {os: windows-2022, compiler_toolchain: intel-classic, compiler_version: 2021.7.0}
     steps:
+
       - name: Checkout modflow6
         uses: actions/checkout@v3
         with:
@@ -35,44 +44,21 @@ jobs:
           repository: MODFLOW-USGS/modflow6-examples
           path: modflow6-examples
 
-      - name: Checkout usgslatex
-        uses: actions/checkout@v3
-        with:
-          repository: MODFLOW-USGS/usgslatex
-          path: usgslatex
-
       - name: Install Conda environment from environment.yml
         uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: modflow6/environment.yml
           cache-environment: true
           cache-downloads: true
 
-      - name: Install additional packages for Sphinx using pip
-        working-directory: modflow6/.build_rtd_docs
-        run: pip install -r requirements.rtd.txt
-
       - name: Print python package versions
         run: pip list
 
-      - name: Install TeX Live
-        run: |
-          sudo apt-get update
-          sudo apt install texlive-latex-extra texlive-science texlive-font-utils
-
-      - name: Install USGS LaTeX style files and Univers font
-        working-directory: usgslatex/usgsLaTeX
-        run: sudo ./install.sh --all-users
-
-      - name: Test building files from dfn's for LaTeX
-        working-directory: modflow6/autotest
-        run: pytest -v build_mfio_tex.py
-
-      - name: Setup GNU Fortran ${{ env.GCC_V }}
+      - name: Setup ${{ matrix.compiler_toolchain }} ${{ matrix.compiler_version }}
         uses: awvwgk/setup-fortran@main
         with:
-          compiler: gcc
-          version: ${{ env.GCC_V }}
+          compiler: ${{ matrix.compiler_toolchain }}
+          version: ${{ matrix.compiler_version }}
 
       - name: Cache modflow6 examples
         id: cache-examples
@@ -105,22 +91,87 @@ jobs:
         run: python benchmark.py
         env:
           GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Rename/show benchmarks
+        id: rename-benchmarks
+        working-directory: modflow6/distribution/.benchmarks
+        run: |
+          orig="run-time-comparison.md"
+          name="run-time-comparison-${{ matrix.os }}-${{ matrix.compiler_toolchain }}-${{ matrix.compiler_version }}.md"
+          cat $orig
+          mv $orig $name
+          echo "name=$name" >> $GITHUB_OUTPUT
+      
+      - name: Upload benchmarks
+        uses: actions/upload-artifact@v3
+        with:
+          name: run-time-comparison
+          path: modflow6/distribution/.benchmarks/${{ steps.rename-benchmarks.outputs.name }}
 
-      - name: Run sphinx
+  rtd_build:
+    name: Benchmark & build RTD
+    needs: benchmark
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+
+      - name: Checkout modflow6
+        uses: actions/checkout@v3
+        with:
+          path: modflow6
+
+      - name: Checkout usgslatex
+        uses: actions/checkout@v3
+        with:
+          repository: MODFLOW-USGS/usgslatex
+          path: usgslatex
+
+      - name: Install Conda environment from environment.yml
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: modflow6/environment.yml
+          cache-environment: true
+          cache-downloads: true
+
+      - name: Install additional packages for Sphinx using pip
         working-directory: modflow6/.build_rtd_docs
-        run: make html
+        run: pip install -r requirements.rtd.txt
 
-      - name: Show benchmarks
-        working-directory: modflow6/distribution
-        run: cat run-time-comparison.md
+      - name: Print python package versions
+        run: pip list
 
-      - name: Upload benchmarks
-        uses: actions/upload-artifact@v3
+      - name: Install TeX Live
+        run: |
+          sudo apt-get update
+          sudo apt install texlive-latex-extra texlive-science texlive-font-utils
+
+      - name: Install USGS LaTeX style files and Univers font
+        working-directory: usgslatex/usgsLaTeX
+        run: sudo ./install.sh --all-users
+
+      - name: Test building files from dfn's for LaTeX
+        working-directory: modflow6/autotest
+        run: pytest -v build_mfio_tex.py
+
+      - name: Download benchmarks
+        uses: actions/download-artifact@v3
         with:
           name: run-time-comparison
-          path: modflow6/distribution/run-time-comparison.md
+          path: modflow6/distribution
+
+      - name: Show benchmarks
+        working-directory: modflow6/distribution
+        run: cat run-time-comparison-*.md
+
+      - name: Run sphinx
+        if: runner.os == 'Linux'
+        working-directory: modflow6/.build_rtd_docs
+        run: make html
 
       - name: Upload results
+        if: runner.os == 'Linux'
         uses: actions/upload-artifact@v3
         with:
           name: rtd-files-for-${{ github.sha }}

.gitignore

@@ -116,7 +116,8 @@ CMakeFiles/
 latex/
 html/
 xml/
-distribution/run-time-comparison.md
+distribution/run-time-comparison*.md
+distribution/.benchmarks/run-time-comparison*.md
 doc/ReleaseNotes/run-time-comparison.tex
 
 # Intel Fortran Visual Studio intermediate files

distribution/benchmark.py

@@ -5,35 +5,38 @@
 import sys
 import textwrap
 from multiprocessing import Pool
-from os import PathLike
+from os import PathLike, environ
 from pathlib import Path
+from platform import system
 from typing import List, Tuple
 
 import flopy
 import pytest
 from modflow_devtools.build import meson_build
 from modflow_devtools.download import download_and_unzip, get_latest_version
-from modflow_devtools.misc import get_model_paths
+from modflow_devtools.misc import get_model_paths, set_env
+from modflow_devtools.ostags import get_github_ostag, get_modflow_ostag
 
 from utils import get_project_root_path
 
 _verify = False
+_github_ostag = get_github_ostag()
+_modflow_ostag = get_modflow_ostag()
+_is_windows = _github_ostag == "Windows"
+_app_ext = ".exe" if _is_windows else ""
+_soext = ".dll" if _is_windows else ".so"
+_github_repo = "MODFLOW-USGS/modflow6"
+_markdown_file_name = "run-time-comparison.md"
 _project_root_path = get_project_root_path()
-_examples_repo_path = _project_root_path.parent / "modflow6-examples"
 _build_path = _project_root_path / "builddir"
 _bin_path = _project_root_path / "bin"
-_github_repo = "MODFLOW-USGS/modflow6"
-_markdown_file_name = "run-time-comparison.md"
-_is_windows = sys.platform.lower() == "win32"
-_app_ext = ".exe" if _is_windows else ""
-_soext = ".dll" if _is_windows else ".so"
-_ostag = "win64" if _is_windows else "linux" if sys.platform.lower().startswith("linux") else "mac"
+_examples_repo_path = _project_root_path.parent / "modflow6-examples"
 
 
 def download_previous_version(output_path: PathLike) -> Tuple[str, Path]:
     output_path = Path(output_path).expanduser().absolute()
     version = get_latest_version(_github_repo)
-    distname = f"mf{version}_{_ostag}"
+    distname = f"mf{version}_{_modflow_ostag}"
     url = (
         f"https://github.com/{_github_repo}"
         + f"/releases/download/{version}/{distname}.zip"
@@ -331,22 +334,19 @@ def run_benchmarks(
     output_path: PathLike,
     excluded: List[str] = [],
 ):
-    """Benchmark current development version against previous release with example models."""
+    """Benchmark current development version against previous release on example models."""
 
     build_path = Path(build_path).expanduser().absolute()
     current_bin_path = Path(current_bin_path).expanduser().absolute()
     previous_bin_path = Path(previous_bin_path).expanduser().absolute()
     examples_path = Path(examples_path).expanduser().absolute()
     output_path = Path(output_path).expanduser().absolute()
 
+    # make sure example models exist
     example_dirs = get_model_paths(examples_path, excluded=excluded)
     assert any(example_dirs), f"No example model paths found, have models been built?"
 
-    # results_path = output_path / _markdown_file_name
-    # if results_path.is_file():
-    #     print(f"Benchmark results already exist: {results_path}")
-    #     return
-
+    # make sure binaries exist
     exe_name = f"mf6{_app_ext}"
     current_exe = current_bin_path / exe_name
     previous_exe = previous_bin_path / exe_name
@@ -362,11 +362,22 @@ def run_benchmarks(
     if not previous_exe.is_file():
         version, download_path = download_previous_version(output_path)
         print(f"Rebuilding latest MODFLOW 6 release {version} in development mode")
-        meson_build(
-            project_path=download_path,
-            build_path=build_path,
-            bin_path=previous_bin_path,
-        )
+
+        def rebuild_prev_release():
+            meson_build(
+                project_path=download_path,
+                build_path=build_path,
+                bin_path=previous_bin_path,
+            )
+
+        # temp workaround until next release,
+        # ifx fails to build 6.4.2 on Windows
+        # most likely due to backspace issues
+        if _github_ostag == "Windows" and environ.get("FC") == "ifx":
+            with set_env(FC="ifort", CC="icl"):
+                rebuild_prev_release()
+        else:
+            rebuild_prev_release()
 
     print(f"Benchmarking MODFLOW 6 versions:")
     print(f"    current: {current_exe}")
@@ -445,8 +456,8 @@ def test_run_benchmarks(tmp_path):
         "-o",
         "--output-path",
         required=False,
-        default=str(_project_root_path / "distribution" / ""),
-        help="Location to create the zip archive",
+        default=str(_project_root_path / "distribution" / ".benchmarks"),
+        help="Location to create benchmark result files",
     )
     parser.add_argument(
         "-e",

distribution/build_dist.py

@@ -10,14 +10,14 @@
 from shutil import copytree
 
 import pytest
+from build_docs import build_documentation
+from build_makefiles import (build_mf5to6_makefile, build_mf6_makefile,
+                             build_zbud6_makefile)
 from modflow_devtools.build import meson_build
 from modflow_devtools.download import download_and_unzip, get_release
 from modflow_devtools.markers import requires_exe
 from modflow_devtools.misc import get_model_paths
 
-from build_docs import build_documentation
-from build_makefiles import (build_mf5to6_makefile, build_mf6_makefile,
-                             build_zbud6_makefile)
 from utils import get_project_root_path, run_command
 
 # default paths
@@ -323,7 +323,6 @@ def build_distribution(
             bin_path=output_path / "bin",
             output_path=output_path / "doc",
             examples_repo_path=examples_repo_path,
-            # benchmarks_path=_benchmarks_path / "run-time-comparison.md",
             full=full,
             overwrite=overwrite,
         )