Merge pull request #2504 from o1-labs/chore/test-coverage

CI refactoring, test coverage data gathering and reports generation.

.github/actions/codecov-shared/action.yml

@@ -0,0 +1,31 @@
+name: "Shared Codecov reporting steps"
+description: "Shared Codecov reporting steps"
+inputs:
+  token:
+    description: "Codecov token"
+    required: true
+  files:
+    description: "Files to upload to Codecov"
+    required: false
+    default: "./target/coverage/lcov.info"
+  flags:
+    description: "Flags to pass to Codecov"
+    required: false
+    default: ""
+  name:
+    description: "The report name"
+    required: false
+    default: "proof-systems"
+runs:
+  using: "composite"
+  steps:
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ inputs.token }}
+        files: ${{ inputs.files }}
+        flags: ${{ inputs.flags }}
+        name: ${{ inputs.name }}
+        verbose: true
+        handle_no_reports_found: true
+        fail_ci_if_error: false

.github/actions/coverage-summary-shared/action.yml

@@ -0,0 +1,32 @@
+name: "Shared code coverage summary"
+description: "Shared code coverage summary"
+inputs:
+  html_file:
+    description: "HTML file with the coverage report"
+    required: false
+    default: "target/coverage/index.html"
+  shell:
+    description: "Shell to use"
+    required: false
+    default: "bash"
+runs:
+  using: "composite"
+  steps:
+    - name: Add test coverage summary
+      run: |
+        echo "### Test coverage summary" >> $GITHUB_STEP_SUMMARY
+        # Define the HTML file
+        html_file="${{ inputs.html_file }}"
+        # Extract data for Lines, Functions, and Branches using `sed`
+        lines=$(sed -n '/heading">Lines/{n;n;s/.*<abbr [^>]*>\(.*%\)<\/abbr>.*/\1/p;}' "$html_file")
+        functions=$(sed -n '/heading">Functions/{n;n;s/.*<abbr [^>]*>\(.*%\)<\/abbr>.*/\1/p;}' "$html_file")
+        branches=$(sed -n '/heading">Branches/{n;n;s/.*<abbr [^>]*>\(.*%\)<\/abbr>.*/\1/p;}' "$html_file")
+        # Compose Markdown summary table
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "| Metric    | Coverage |" >> $GITHUB_STEP_SUMMARY
+        echo "|-----------|----------|" >> $GITHUB_STEP_SUMMARY
+        echo "| Lines     | $lines |" >> $GITHUB_STEP_SUMMARY
+        echo "| Functions | $functions |" >> $GITHUB_STEP_SUMMARY
+        echo "| Branches  | $branches |" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+      shell: ${{ inputs.shell }}

.github/actions/ocaml-shared/action.yml

@@ -0,0 +1,15 @@
+name: "Shared OCaml setting up steps"
+description: "Shared OCaml setting up steps"
+inputs:
+  ocaml_version:
+    description: "OCaml version"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Setup OCaml ${{ inputs.ocaml_version }}
+      uses: ocaml/setup-ocaml@v3
+      with:
+        ocaml-compiler: ${{ inputs.ocaml_version }}
+        # https://github.com/ocaml/setup-ocaml/issues/211#issuecomment-1058882386
+        # disable-cache: true

.github/actions/toolchain-shared/action.yml

@@ -0,0 +1,26 @@
+name: "Shared Rust toolchain setting up steps"
+description: "Shared Rust toolchain setting up steps"
+inputs:
+  rust_toolchain_version:
+    description: "Rust toolchain version"
+    required: true
+  shell:
+    description: "Shell to use"
+    required: false
+    default: "bash"
+runs:
+  using: "composite"
+  steps:
+    # As action-rs does not seem to be maintained anymore, building from
+    # scratch the environment using rustup
+    - name: Setup Rust toolchain ${{ inputs.rust_toolchain_version }}
+      run: |
+        curl --proto '=https' --tlsv1.2 -sSf -o rustup-init \
+        https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init
+        chmod +x ./rustup-init
+        ./rustup-init -y --default-toolchain "${{ inputs.rust_toolchain_version }}" --profile default
+        rm ./rustup-init
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+        # overwriting default rust-toolchain
+        echo ${{ inputs.rust_toolchain_version }} > rust-toolchain
+      shell: ${{ inputs.shell }}

.github/workflows/benches.yml

@@ -9,47 +9,37 @@ env:
   OCAML_VERSION: "4.14.0"
   RUST_TOOLCHAIN_VERSION: "1.71"
 
-
 jobs:
   bench:
     runs-on: ubuntu-latest
     name: Run benchmarks
     if: github.event.label.name == 'benchmark'
     steps:
-      - name: Checkout PR
-        uses: actions/[email protected]
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
-      # as action-rs does not seem to be maintained anymore, building from
-      # scratch the environment using rustup
-      - name: Setup Rust toolchain $RUST_TOOLCHAIN_VERSION
-        run:
-          |
-            curl --proto '=https' --tlsv1.2 -sSf -o rustup-init \
-            https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init
-            chmod +x ./rustup-init
-            ./rustup-init -y --default-toolchain "$RUST_TOOLCHAIN_VERSION" --profile default
-            rm ./rustup-init
-            echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-            # overwriting default rust-toolchain
-            echo $RUST_TOOLCHAIN_VERSION > rust-toolchain
+      - name: Use shared Rust toolchain setting up steps
+        uses: ./.github/actions/toolchain-shared
+        with:
+          rust_toolchain_version: ${{ env.RUST_TOOLCHAIN_VERSION }}
 
       - name: Install dependencies
         run: |
           set -x
-          cargo install cargo-criterion   # criterion
+          cargo install cargo-criterion
 
-      - name: Setup OCaml ${{ env.OCAML_VERSION }}
-        uses: ocaml/setup-ocaml@v2
+      - name: Use shared OCaml setting up steps
+        uses: ./.github/actions/ocaml-shared
         with:
-          ocaml-compiler: ${{ env.OCAML_VERSION }}
-          # https://github.com/ocaml/setup-ocaml/issues/211#issuecomment-1058882386
-          # disable-cache: true
+          ocaml_version: ${{ env.OCAML_VERSION }}
 
-#      - name: Run iai bench
-#        run: |
-#          set -x
-#          cargo bench -p kimchi --bench proof_iai > iai_bench
-#          cat iai_bench
+      #      - name: Run iai bench
+      #        run: |
+      #          set -x
+      #          cargo bench -p kimchi --bench proof_iai > iai_bench
+      #          cat iai_bench
 
       - name: Run criterion bench
         run: |
@@ -59,7 +49,7 @@ jobs:
           cat criterion_bench
 
       - name: Write result in PR
-        uses: actions/github-script@v5
+        uses: actions/github-script@v7
         with:
           script: |
             const fs = require('fs');

.github/workflows/ci-nightly.yml

@@ -0,0 +1,72 @@
+#
+# This workflow is triggered by scheduler or on-demand to run all the tests with the code coverage enabled and using the self-hosted GitHub runner.
+# Test coverage report is attached to the current job execution results in a form of Zip archive.
+#
+
+name: Nightly tests with the code coverage
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # every day at midnight
+  workflow_dispatch: {}
+
+env:
+  # https://doc.rust-lang.org/cargo/reference/profiles.html#release
+  RUSTFLAGS: -Coverflow-checks=y -Cdebug-assertions=y
+  # https://doc.rust-lang.org/cargo/reference/profiles.html#incremental
+  CARGO_INCREMENTAL: 1
+  # https://nexte.st/book/pre-built-binaries.html#using-nextest-in-github-actions
+  CARGO_TERM_COLOR: always
+  # 30 MB of stack for Keccak tests
+  RUST_MIN_STACK: 31457280
+
+jobs:
+  run_tests:
+    name: Run all tests with the code coverage
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        rust_toolchain_version: ["1.74"]
+        # FIXME: currently not available for 5.0.0.
+        # It might be related to boxroot dependency, and we would need to bump
+        # up the ocaml-rs dependency
+        ocaml_version: ["4.14"]
+        os: ["hetzner-1"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Use shared Rust toolchain setting up steps
+        uses: ./.github/actions/toolchain-shared
+        with:
+          rust_toolchain_version: ${{ matrix.rust_toolchain_version }}
+
+      - name: Use shared OCaml setting up steps
+        uses: ./.github/actions/ocaml-shared
+        with:
+          ocaml_version: ${{ matrix.ocaml_version }}
+
+      - name: Install test dependencies
+        run: |
+          make install-test-deps
+
+      - name: Run all tests with the code coverage
+        run: |
+          eval $(opam env)
+          make clean
+          make nextest-all-with-coverage
+
+      - name: Use shared code coverage summary
+        uses: ./.github/actions/coverage-summary-shared
+
+      - name: Upload the HTML test coverage report
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        if: always()
+        with:
+          if-no-files-found: ignore
+          name: test-coverage-html-report-${{ matrix.rust_toolchain_version }}-${{ matrix.os }}
+          path: target/coverage/
+          retention-days: 30