From 98e4c46276200b8083fee7b1224ff9d1051dc12b Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 21 Nov 2023 16:17:30 -0500 Subject: [PATCH] ci: multiple updates/simplifications (#22) * add --keep argument to build script * remove branch filters from push trigger * remove unneeded windows compatibility steps * bump actions versions, various refactoring * use fortran-lang/setup-fortran action * add requirements.txt for python deps --- .github/workflows/integration.yml | 83 ++++++----------------- .github/workflows/release.yml | 107 ++++++++++++++---------------- requirements.txt | 2 + scripts/build_executables.py | 17 ++++- 4 files changed, 87 insertions(+), 122 deletions(-) create mode 100644 requirements.txt diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b0a1f1d..589efab 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,10 +1,6 @@ name: Integration testing on: push: - branches: - - master - - develop* - - ci-diagnose* paths-ignore: - '**.md' pull_request: @@ -28,13 +24,13 @@ jobs: shell: bash -l {0} steps: - - name: Checkout action - uses: actions/checkout@v3 + - name: Checkout repo + uses: actions/checkout@v4 with: path: executables - name: Checkout modflow6 - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: MODFLOW-USGS/modflow6 path: modflow6 @@ -42,83 +38,46 @@ jobs: - name: Setup Micromamba uses: mamba-org/setup-micromamba@v1 with: - micromamba-root-path: ${{ runner.temp }}/micromamba-root environment-file: modflow6/environment.yml cache-environment: true cache-downloads: true - init-shell: >- - bash - powershell - - - name: Setup ifort - uses: modflowpy/install-intelfortran-action@v1 + init-shell: bash - - name: Fix Micromamba path (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: | - # https://github.com/modflowpy/install-intelfortran-action#conda-scripts - $mamba_bin = "${{ runner.temp }}\micromamba-root\envs\modflow6\Scripts" - echo $mamba_bin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: Setup Intel fortran + uses: fortran-lang/setup-fortran@v1 + with: + compiler: intel-classic + version: 2021.7 - - name: Build modflow6 (Linux & Mac) + - name: Build modflow6 working-directory: modflow6 run: | - meson setup builddir -Ddebug=false --prefix=$(pwd) --libdir=bin - meson compile -v -C builddir + meson setup builddir --prefix=$(pwd) --libdir=bin -Ddebug=false meson install -C builddir - # - name: Show meson build log - # run: cat modflow6/builddir/meson-logs/meson-log.txt + - name: Show meson build log + if: failure() + working-directory: modflow6 + run: cat builddir/meson-logs/meson-log.txt + + - name: Unit test modflow6 + working-directory: modflow6 + run: meson test --verbose --no-rebuild -C builddir - name: Update FloPy working-directory: modflow6/autotest run: python update_flopy.py - name: Get executables - if: runner.os != 'Windows' - working-directory: modflow6/autotest - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - pytest -v -s get_exes.py - sudo rm -rf ../bin/downloaded/* - - - name: Get executables - if: runner.os == 'Windows' - shell: pwsh working-directory: modflow6/autotest env: GITHUB_TOKEN: ${{ github.token }} - run: | - pytest -v -s get_exes.py - rm -Force ../bin/downloaded/* - - - name: Show pymake version - run: | - pip show mfpymake - python -c "import pymake; print(pymake.__version__)" - - - name: Build executables - if: runner.os != 'Windows' - working-directory: executables/scripts - run: python build_executables.py -p ../../modflow6/bin/downloaded + run: pytest -v -s get_exes.py - name: Build executables - if: runner.os == 'Windows' - shell: cmd working-directory: executables/scripts run: python build_executables.py -p ../../modflow6/bin/downloaded - - # - name: Build executables - # uses: nick-fields/retry@v2 - # with: - # timeout_minutes: 40 - # max_attempts: 5 - # command: | - # cd modflow6/bin/downloaded - # python ../../../scripts/build_executables.py - + - name: Set executable permission if: runner.os != 'Windows' working-directory: modflow6/bin/downloaded diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c19e17..e64967d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,9 +7,11 @@ on: - master - develop workflow_dispatch: +env: + DIST: dist jobs: - executables-intel: - name: pymake CI intel on different OSs + build: + name: Build distribution runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -21,21 +23,24 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Setup Intel OneAPI Compilers - uses: modflowpy/install-intelfortran-action@v1 + - name: Setup Intel fortran + uses: fortran-lang/setup-fortran@v1 + with: + compiler: intel-classic + version: 2021.7 - uses: oprypin/find-latest-tag@v1 + id: tag with: - repository: MODFLOW-USGS/executables # The repository to scan. - releases-only: true # We know that all relevant tags have a GitHub release for them. - id: executables # The step ID to refer to later. - - - name: Setup persistent environment variables + repository: ${{ github.repository }} + releases-only: true + + - name: Set environment variables run: | - echo "MODFLOW-USGS/executables is at version ${{ steps.executables.outputs.tag }}" - echo "RELEASE_VERSION=${{ steps.executables.outputs.tag }}" >> $GITHUB_ENV + echo "${{ github.repository }} version ${{ steps.tag.outputs.tag }}" + echo "RELEASE_VERSION=${{ steps.tag.outputs.tag }}" >> $GITHUB_ENV - name: Setup Python uses: actions/setup-python@v4 @@ -45,38 +50,31 @@ jobs: - name: Install Python dependencies run: | python -m pip install --upgrade pip - pip install wheel - pip install https://github.com/modflowpy/pymake/zipball/master + pip install -r requirements.txt pip list - - name: build executables on Linux and macOS - # if: runner.os != 'Windows' + - name: Build executables run: python scripts/build_executables.py - # - name: build executables on Windows - # if: runner.os == 'Windows' - # shell: cmd - # run: python scripts/build_executables.py - - - name: Upload a Build Artifact + - name: Upload distribution archive uses: actions/upload-artifact@v3 with: - name: release_build + name: ${{ env.DIST }} path: ./*.zip - - name: Upload additional Build Artifacts + - name: Upload distribution metadata if: runner.os == 'Linux' uses: actions/upload-artifact@v3 with: - name: release_build + name: ${{ env.DIST }} path: | ./code.json ./code.md # make the release if previous job was successful release: - name: Make a release - needs: executables-intel + name: Make release + needs: build runs-on: ubuntu-latest defaults: run: @@ -96,33 +94,32 @@ jobs: python -m pip install --upgrade pip pip install https://github.com/modflowpy/pymake/zipball/master - - name: Get the latest release tag + - name: Get last release tag + id: last-tag uses: oprypin/find-latest-tag@v1 with: - repository: MODFLOW-USGS/executables # The repository to scan. - releases-only: true # We know that all relevant tags have a GitHub release for them. - id: executables # The step ID to refer to later. + repository: ${{ github.repository }} + releases-only: true - - name: Get latest release tag + - name: Get next release tag + id: next-tag run: | - current="${{ steps.executables.outputs.tag }}" - # next="${current%.*}.$((${current##*.}+1))" - next=$(echo "${{ steps.executables.outputs.tag }} + 1.0" | bc) + current="${{ steps.last-tag.outputs.tag }}" + next=$(echo "${{ steps.last-tag.outputs.tag }} + 1.0" | bc) echo "RELEASE_VERSION=$current" >> $GITHUB_ENV - echo "NEXT_VERSION=$next" >> $GITHUB_ENV - + echo "tag=$next" >> $GITHUB_OUTPUT repo="${{ github.repository }}" echo "$repo current version is $current" echo "$repo next version is $next" - - name: Download release artifact + - name: Download distribution uses: actions/download-artifact@v3 with: - name: release_build - path: ./release_build/ + name: ${{ env.DIST }} + path: ${{ env.DIST }} - - name: List artifact files - run: ls -l ./release_build/ + - name: List distribution files + run: ls -l ${{ env.DIST }} - name: Create release body header shell: python @@ -137,9 +134,10 @@ jobs: - name: Build release body run: | - cat Header.md ./release_build/code.md > BodyFile.md + cat Header.md ${{ env.DIST }}/code.md > BodyFile.md cat BodyFile.md + # interactive debugging # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 # with: @@ -149,8 +147,8 @@ jobs: id: update-readme run: | # update readme from metadata - cp release_build/code.md code.md - cp release_build/code.json code.json + cp ${{ env.DIST }}/code.md code.md + cp ${{ env.DIST }}/code.json code.json python scripts/update_readme.py # determine whether changes need to be committed @@ -164,14 +162,14 @@ jobs: fi echo "changes=$changes" >> $GITHUB_OUTPUT + # open PR if manual trigger - name: Draft pull request - # only open PR on manual trigger if: github.event_name == 'workflow_dispatch' && steps.update-readme.outputs.changes == 'true' env: GH_TOKEN: ${{ github.token }} run: | # commit and push - branch="update-readme-${{ env.NEXT_VERSION }}" + branch="update-readme-${{ steps.next-tag.outputs.tag }}" git config core.sharedRepository true git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" @@ -183,26 +181,21 @@ jobs: # create PR body=' - # MODFLOW executables release '${{ env.NEXT_VERSION }}' + # MODFLOW executables release '${{ steps.next-tag.outputs.tag }}' This PR updates `README.md` with the latest release information. ' - gh pr create -B "master" -H "$branch" --title "Release ${{ env.NEXT_VERSION }}" --draft --body "$body" + gh pr create -B "master" -H "$branch" --title "Release ${{ steps.next-tag.outputs.tag }}" --draft --body "$body" + # create new release if manual trigger - name: Create release - # only create new release on manual trigger if: github.event_name == 'workflow_dispatch' uses: ncipollo/release-action@v1 with: - tag: ${{ env.NEXT_VERSION }} + tag: ${{ steps.next-tag.outputs.tag }} name: "MODFLOW and related programs binary executables" bodyFile: "./BodyFile.md" - artifacts: "./release_build/*" + artifacts: "${{ env.DIST }}/*" draft: false allowUpdates: true token: ${{ secrets.GITHUB_TOKEN }} - -# - name: Delete Artifact -# uses: GeekyEggo/delete-artifact@v1.0.0 -# with: -# name: release_build diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..069b1f5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +wheel +https://github.com/modflowpy/pymake/zipball/master \ No newline at end of file diff --git a/scripts/build_executables.py b/scripts/build_executables.py index b2f8a02..96ebb93 100755 --- a/scripts/build_executables.py +++ b/scripts/build_executables.py @@ -50,6 +50,12 @@ def run_cmd(args) -> bool: """ ), ) + parser.add_argument( + "-k", + "--keep", + action=argparse.BooleanOptionalAction, + help="Whether to keep (not recreate) existing binaries", + ) parser.add_argument( "-p", "--path", @@ -68,6 +74,9 @@ def run_cmd(args) -> bool: ) args = parser.parse_args() + # whether to recreate existing binaries + keep = bool(args.keep) + # output path path = Path(args.path) path.mkdir(parents=True, exist_ok=True) @@ -88,11 +97,13 @@ def run_cmd(args) -> bool: raise RuntimeError(f"could not make code.json") # build binaries - if not run_cmd([ + build_args = [ "make-program", ":", f"--appdir={path}", "-fc=ifort", f"-cc={cc}", f"--zip={path}.zip", - "--keep", - ]): + ] + if keep: + build_args.append("--keep") + if not run_cmd(build_args): raise RuntimeError("could not build binaries")