From 5b804218913c2c57e100a970b31b3e3691eee194 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 20 Jun 2024 11:13:13 -0400 Subject: [PATCH] feat(macos): add distribution for apple silicon (#29) Introduce a statically linked distribution built with GCC 12 for ARM macs. Also, update integration.yml: * add a macos-14 test job building with GCC 12 and requisite workarounds * bump linux test job runner image ubuntu-20.04 -> ubuntu-22.04 --- .github/workflows/integration.yml | 57 ++++++++++++-- .github/workflows/release.yml | 85 +++++++++++++++++---- scripts/build_programs.py | 123 ------------------------------ 3 files changed, 118 insertions(+), 147 deletions(-) delete mode 100755 scripts/build_programs.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bc3d83c..1005de0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu-20.04, macos-latest, windows-2019 ] + os: [ ubuntu-22.04, macos-13, macos-14, windows-2019 ] defaults: run: shell: bash -l {0} @@ -33,17 +33,39 @@ jobs: cache-downloads: true init-shell: bash - - name: Setup Intel fortran + - name: Setup ${{ contains(fromJSON('["macos-14"]'), matrix.os) && 'gcc' || 'intel-classic' }} ${{ contains(fromJSON('["macos-14"]'), matrix.os) && 12 || 2021.7 }} uses: fortran-lang/setup-fortran@v1 with: - compiler: intel-classic - version: 2021.7 + compiler: ${{ contains(fromJSON('["macos-14"]'), matrix.os) && 'gcc' || 'intel-classic' }} + version: ${{ contains(fromJSON('["macos-14"]'), matrix.os) && 12 || 2021.7 }} + - name: Set LDFLAGS (macOS) + if: matrix.os == 'macos-14' + run: | + os_ver=$(sw_vers -productVersion | cut -d'.' -f1) + if (( "$os_ver" > 12 )); then + ldflags="$LDFLAGS -Wl,-ld_classic" + echo "LDFLAGS=$ldflags" >> $GITHUB_ENV + fi + + - name: Hide dylibs (macOS) + if: matrix.os == 'macos-14' + run: | + version="12" + libpath="/opt/homebrew/opt/gcc@$version/lib/gcc/$version" + mv $libpath/libgfortran.5.dylib $libpath/libgfortran.5.dylib.bak + mv $libpath/libquadmath.0.dylib $libpath/libquadmath.0.dylib.bak + mv $libpath/libstdc++.6.dylib $libpath/libstdc++.6.dylib.bak + # only necessary because we need mf5to6 for mf6 autotests - name: Build modflow6 working-directory: modflow6 run: | - meson setup builddir --prefix=$(pwd) --libdir=bin -Ddebug=false + setupargs="--prefix=$(pwd) --libdir=bin -Ddebug=false" + if [[ "${{ matrix.os }}" == "macos-14" ]]; then + setupargs="$setupargs -Doptimization=1" + fi + meson setup builddir $setupargs meson install -C builddir - name: Get OS tag @@ -53,11 +75,24 @@ jobs: echo "ostag=$ostag" >> $GITHUB_OUTPUT - name: Build programs + uses: nick-fields/retry@v3 + with: + shell: bash + timeout_minutes: 40 + command: | + ostag="${{ steps.ostag.outputs.ostag }}" + mkdir $ostag + make-program : --appdir $ostag --zip $ostag.zip --verbose + make-program mf2005,mflgr,mfnwt,mfusg --appdir $ostag --double --keep --zip $ostag.zip --verbose + if [[ "${{ matrix.os }}" == "macos-14" ]]; then + make-program mf6 --appdir $ostag --keep --zip $ostag.zip --verbose --fflags='-O1' + fi + make-code-json --appdir $ostag --zip $ostag.zip --verbose + + - name: Move programs run: | # build programs ostag="${{ steps.ostag.outputs.ostag }}" - mkdir $ostag - python executables/scripts/build_programs.py -p $ostag # move programs where mf6 autotests expect them mkdir modflow6/bin/downloaded @@ -83,6 +118,12 @@ jobs: sudo chmod +x modflow6/bin/* sudo chmod +x modflow6/bin/downloaded/* fi + + - name: Check linked libs (macOS) + if: matrix.os == 'macos-14' + run: | + ostag="${{ steps.ostag.outputs.ostag }}" + find $ostag -perm +111 -type f | xargs -I{} sh -c "otool -L {}" - name: Upload programs uses: actions/upload-artifact@v3 @@ -94,7 +135,7 @@ jobs: if: runner.os == 'Linux' uses: actions/upload-artifact@v3 with: - name: metadata + name: ${{ steps.ostag.outputs.ostag }} path: | ./code.json ./code.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28ea8c6..cddd328 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,8 +7,6 @@ on: branches: - master workflow_dispatch: -env: - DIST: dist jobs: build: name: Build distribution @@ -16,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-latest, windows-2019] + os: [ubuntu-22.04, macos-13, macos-14, windows-2019] defaults: run: shell: bash @@ -25,11 +23,35 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - - name: Setup Intel fortran + - name: Setup ${{ contains(fromJSON('["macos-14"]'), matrix.os) && 'gcc' || 'intel-classic' }} ${{ contains(fromJSON('["macos-14"]'), matrix.os) && 12 || 2021.7 }} uses: fortran-lang/setup-fortran@v1 with: - compiler: intel-classic - version: 2021.7 + compiler: ${{ contains(fromJSON('["macos-14"]'), matrix.os) && 'gcc' || 'intel-classic' }} + version: ${{ contains(fromJSON('["macos-14"]'), matrix.os) && 12 || 2021.7 }} + + - name: Set LDFLAGS (ARM mac) + if: matrix.os == 'macos-14' + run: | + os_ver=$(sw_vers -productVersion | cut -d'.' -f1) + if (( "$os_ver" > 12 )); then + ldflags="$LDFLAGS -Wl,-ld_classic" + echo "LDFLAGS=$ldflags" >> $GITHUB_ENV + fi + + - name: Hide dylibs (ARM mac) + if: matrix.os == 'macos-14' + run: | + version="12" + libpath="/opt/homebrew/opt/gcc@$version/lib/gcc/$version" + mv $libpath/libgfortran.5.dylib $libpath/libgfortran.5.dylib.bak + mv $libpath/libquadmath.0.dylib $libpath/libquadmath.0.dylib.bak + mv $libpath/libstdc++.6.dylib $libpath/libstdc++.6.dylib.bak + + - name: Setup Xcode CLT (mac) + uses: maxim-lobanov/setup-xcode@v1 + if: runner.os == 'macOS' + with: + xcode-version: "14.3.1" - uses: oprypin/find-latest-tag@v1 id: tag @@ -53,20 +75,44 @@ jobs: pip install -r requirements.txt pip list + - name: Get OS tag + id: ostag + run: | + ostag=$(python -c "from modflow_devtools.ostags import get_ostag; print(get_ostag())") + echo "ostag=$ostag" >> $GITHUB_OUTPUT + - name: Build programs - run: python scripts/build_programs.py + uses: nick-fields/retry@v3 + with: + shell: bash + timeout_minutes: 40 + command: | + ostag="${{ steps.ostag.outputs.ostag }}" + mkdir $ostag + make-program : --appdir $ostag --zip $ostag.zip --verbose + make-program mf2005,mflgr,mfnwt,mfusg --appdir $ostag --double --keep --zip $ostag.zip --verbose + if [[ "${{ matrix.os }}" == "macos-14" ]]; then + make-program mf6 --appdir $ostag --keep --zip $ostag.zip --verbose --fflags='-O1' + fi + make-code-json --appdir $ostag --zip $ostag.zip --verbose + + - name: Check linked libs (ARM mac) + if: matrix.os == 'macos-14' + run: | + ostag="${{ steps.ostag.outputs.ostag }}" + find $ostag -perm +111 -type f | xargs -I{} sh -c "otool -L {}" - name: Upload distribution archive uses: actions/upload-artifact@v3 with: - name: ${{ env.DIST }} + name: ${{ steps.ostag.outputs.ostag }} path: ./*.zip - name: Upload distribution metadata if: runner.os == 'Linux' uses: actions/upload-artifact@v3 with: - name: ${{ env.DIST }} + name: ${{ steps.ostag.outputs.ostag }} path: | ./code.json ./code.md @@ -93,6 +139,7 @@ jobs: run: | python -m pip install --upgrade pip pip install https://github.com/modflowpy/pymake/zipball/master + pip install https://github.com/MODFLOW-USGS/modflow-devtools/zipball/develop - name: Get last release tag id: last-tag @@ -112,14 +159,20 @@ jobs: echo "$repo current version is $current" echo "$repo next version is $next" + - name: Get OS tag + id: ostag + run: | + ostag=$(python -c "from modflow_devtools.ostags import get_ostag; print(get_ostag())") + echo "ostag=$ostag" >> $GITHUB_OUTPUT + - name: Download distribution uses: actions/download-artifact@v3 with: - name: ${{ env.DIST }} - path: ${{ env.DIST }} + name: ${{ steps.ostag.outputs.ostag }} + path: ${{ steps.ostag.outputs.ostag }} - name: List distribution files - run: ls -l ${{ env.DIST }} + run: ls -l ${{ steps.ostag.outputs.ostag }} - name: Create release body header shell: python @@ -134,7 +187,7 @@ jobs: - name: Build release body run: | - cat Header.md ${{ env.DIST }}/code.md > BodyFile.md + cat Header.md ${{ steps.ostag.outputs.ostag }}/code.md > BodyFile.md cat BodyFile.md # interactive debugging @@ -147,8 +200,8 @@ jobs: id: update-readme run: | # update readme from metadata - cp ${{ env.DIST }}/code.md code.md - cp ${{ env.DIST }}/code.json code.json + cp ${{ steps.ostag.outputs.ostag }}/code.md code.md + cp ${{ steps.ostag.outputs.ostag }}/code.json code.json python scripts/update_readme.py # determine whether changes need to be committed @@ -195,7 +248,7 @@ jobs: tag: ${{ steps.next-tag.outputs.tag }} name: "MODFLOW and related programs binary executables" bodyFile: "./BodyFile.md" - artifacts: "${{ env.DIST }}/*" + artifacts: "${{ steps.ostag.outputs.ostag }}/*" draft: false allowUpdates: true token: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/build_programs.py b/scripts/build_programs.py deleted file mode 100755 index 276231a..0000000 --- a/scripts/build_programs.py +++ /dev/null @@ -1,123 +0,0 @@ -import argparse -import subprocess -import sys -import textwrap -from pathlib import Path - -from modflow_devtools.ostags import get_modflow_ostag - -DEFAULT_RETRIES = 3 -DPRECISION_EXES = ["mf2005", "mflgr", "mfnwt", "mfusg"] - - -def get_fc() -> str: - """Determine Intel Fortran compiler to use based on the current platform.""" - return "ifort" - - -def get_cc() -> str: - """Determine Intel C compiler to use based on the current platform.""" - if sys.platform.startswith("linux"): - return "icc" - elif sys.platform.startswith("win"): - return "icl" - elif sys.platform.startswith("darwin"): - return "icc" - raise ValueError(f"platform {sys.platform!r} not supported") - - -def run_cmd(args) -> bool: - success = False - for idx in range(DEFAULT_RETRIES): - p = subprocess.run(args) - if p.returncode == 0: - success = True - break - print(f"{args[0]} run {idx + 1}/{DEFAULT_RETRIES} failed...rerunning") - return success - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog=f"Build MODFLOW-related binaries and metadata files", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent( - """\ - Build MODFLOW-related executables, shared libraries and metadata files with pymake. - """ - ), - ) - parser.add_argument( - "-k", - "--keep", - action=argparse.BooleanOptionalAction, - help="Whether to keep (not recreate) existing binaries", - ) - parser.add_argument( - "-p", - "--path", - required=False, - default=get_modflow_ostag(), - help="Path to create built binaries and metadata files", - ) - parser.add_argument( - "-r", - "--retries", - type=int, - required=False, - default=DEFAULT_RETRIES, - help="Number of times to retry a failed build", - ) - args = parser.parse_args() - keep = bool(args.keep) - path = Path(args.path) - path.mkdir(parents=True, exist_ok=True) - zip_path = Path(path).with_suffix(".zip") - dp_exes = ",".join(DPRECISION_EXES) - retries = args.retries - fc = get_fc() - cc = get_cc() - - assert run_cmd( - [ - "make-program", - ":", - "--appdir", - path, - "-fc", - fc, - "-cc", - cc, - "--zip", - f"{path}.zip", - ] - + (["--keep"] if keep else []) - ), "could not build default precision binaries" - assert run_cmd( - [ - "make-program", - dp_exes, - "--appdir", - path, - "--double", - "--keep", - "-fc", - fc, - "-cc", - cc, - "--zip", - str(zip_path), - ] - ), f"could not build double precision binaries: {dp_exes}" - assert run_cmd( - [ - "make-code-json", - "-ad", - str(path), - "--verbose", - "--zip", - str(zip_path) - ] - ), "could not make code.json" - assert zip_path.is_file(), "could not build distribution zipfile" - print(f"created distribution zipfile: {zip_path}")