Migrate to Github Actions (#119)

* Remove Jenkins coordinator and related components
* Publish existing maven and Elasticsearch services via the ingress
* Migrate to GitHub actions: 
  * Build plan is stored as `workflow_call`, can be called by other jobs 
  * Build config is stored in the repo, not calculated on each build
  * Build is limited to 1000 jobs (with higher number of builds GHA web page is failing to load finished job due to internal timeouts)
  * Each stage is limited to at most 255 jobs (GHA limit for matrix entries)
  * Build plan and config are updated weekly via a scheduled updated
  * Build plan is executed weekly using the detected last version of Scala Nightly
  * Builds can be started manually with a custom version or repo/branch configuration
  * Custom build takes additional arguments allowing to specify additional or disabled scalacOptions (used only by sbt builds)
  * Each build creates artifacts containing 3 reports comparing the current version of compiler, with the results from last stable version, the last RC, and the total report including all failures (not filtering out previously failing builds)
* Raport regressions script is now using public ES instance, no longer requires forwarding k8s pod port
* Upgraded version to v0.2.1

.github/actions/build-project/action.yaml

@@ -0,0 +1,182 @@
+name: "Build projects"
+description: "Used to build a project withing the Open cOmunityu Build"
+inputs:
+  project-name:
+    description: "Name of the project to build"
+    required: true
+  scala-version:
+    description: "Scala Version to use"
+    required: true
+  extra-scalac-options:
+    description: "List of scalacOptions which should be used when building projects. Multiple entires should be seperated by a single comma character `,`"
+    default: ""
+  disabled-scalac-options:
+    description: "List of scalacOptions which should be filted out when building projects."
+    default: ""
+  # Infrastructure inputs / secrets
+  maven-repo-url:
+    description: "Custom Maven repository used to store artifacts"
+    required: true
+  elastic-user:
+    description: "Secret with auth user to elasticsearch"
+    required: true
+  elastic-password:
+    description: "Secret with auth token to elasticsearch"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - uses: coursier/setup-action@v1
+      with:
+        apps: cs
+
+    - name: Check java version
+      shell: bash
+      run: |
+        ConfigFile=".github/workflows/buildConfig.json"
+        DefaultJDK=11
+        javaVersion=$(jq -r ".\"${{ inputs.project-name }}\".config.java.version // ${DefaultJDK}" $ConfigFile)
+        echo "java-version=$javaVersion" >> $GITHUB_ENV
+        echo "JavaVersion set to $javaVersion"
+
+  
+    - name: Check can skip build
+      id: check-history
+      shell: bash
+      run: |
+        ConfigFile=".github/workflows/buildConfig.json"
+        ScalaVersion="${{ inputs.scala-version }}"
+        BinScalaVersion=3
+        CustomMavenRepo="${{ inputs.maven-repo-url }}"
+
+        projectVersion="$(jq -r ".\"${{ inputs.project-name }}\".version" $ConfigFile)"
+
+        canSkip=true
+        if [[ -n "${{ inputs.extra-scalac-options }}" || -n "${{ inputs.disabled-scalac-options }}" ]]; then
+          echo "Using custom scalacOption, cannot skip"
+          canSkip=false
+        else
+          for target in $(jq -r ".\"${{ inputs.project-name }}\".targets" $ConfigFile); do
+            arr=($(echo $target | tr "%" " "))
+            #use offset:length for consistent indexing (zsh starts with 1, bash starts with 0)
+            org=${arr[@]:0:1}
+            name=${arr[@]:1:1}
+            artifact="${org}:${name}_${BinScalaVersion}:${projectVersion}"
+
+            if ! cs resolve --intransitive "$artifact" -r "$CustomMavenRepo" --no-default >> /dev/null ; then 
+              echo "Artifact $artifact not found, would build the project"
+              canSkip=false
+              break;
+            fi
+          done
+        fi
+        echo "Can skip build: $canSkip"
+        echo "can-skip-build=${canSkip}" >> $GITHUB_OUTPUT
+
+    - name: Get current job URL
+      id: job-info
+      if: steps.check-history.outputs.can-skip-build != 'true'
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        GITHUB_API="repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs"
+        JOB_NAME="execute-build-plan / ${{ github.job }} (${{ inputs.project-name }})"
+        PARAMS="?per_page=100"
+        SELECT_URL=".jobs | map(select(.name == \"${JOB_NAME}\")) | .[0].html_url"
+
+        set -o pipefail
+        
+        outFile="gh.out"
+        buildURL=""
+        for i in $(seq 1 10) ;do
+          if gh api "${GITHUB_API}${PARAMS}" --paginate --jq "${SELECT_URL}" 2>&1 | tee $outFile; then 
+            buildURL="$(cat $outFile | xargs)"
+            break
+          elif grep -q 'API rate limit exceeded' "$outFile"; then
+            echo "GitHub API rate limit exceeded, skip resolving build URL"
+            break
+          elif grep -q 'Server Error (HTTP 502)' "$outFile"; then
+            echo "Server error when resolving build URL, retry with backoff"
+            sleep 5
+          else 
+            echo "Unknown error when resolving build URL:"
+            cat $outFile
+            break
+          fi
+        done  
+        
+        echo "Build URL: $buildURL"
+        echo "build-url=${buildURL}" >> $GITHUB_OUTPUT
+
+    - name: Build project
+      uses: addnab/docker-run-action@v3
+      if: steps.check-history.outputs.can-skip-build != 'true'
+      with:
+        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.2.1"
+        options: -v ${{ github.workspace }}:/opencb/ -e  ELASTIC_USERNAME=${{ inputs.elastic-user }} -e ELASTIC_PASSWORD=${{ inputs.elastic-password }}
+        run: |
+          ConfigFile="/opencb/.github/workflows/buildConfig.json"
+          DefaultConfig='{"memoryRequestMb":4096}'
+          config () { 
+            path=".\"${{ inputs.project-name }}\"$@" 
+            jq -c -r "$path" $ConfigFile 
+          }
+
+          touch build-logs.txt  build-summary.txt
+          # Assume failure unless overwritten by a successful build
+          echo 'failure' > build-status.txt 
+
+          /build/build-revision.sh \
+            "$(config .repoUrl)" \
+            "$(config .revision)" \
+            "${{ inputs.scala-version }}" \
+            "$(config .version)" \
+            "$(config .targets)" \
+            "${{ inputs.maven-repo-url }}" \
+            '1.6.2' \
+            "$(config .config // ${DefaultConfig})" \
+            "${{ inputs.extra-scalac-options }}" \
+            "${{ inputs.disabled-scalac-options }}" 2>&1 | tee build-logs.txt
+
+          # Remove ASCII coloring from the indexed logs
+          cat build-logs.txt | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g" > build-logs-uncolored.txt 
+
+          /build/feed-elastic.sh \
+            'https://scala3.westeurope.cloudapp.azure.com/data' \
+            "${{ inputs.project-name }}" \
+            "$(cat build-status.txt)" \
+            "$(date --iso-8601=seconds)" \
+            build-summary.txt \
+            build-logs-uncolored.txt \
+            "$(config .version)" \
+            "${{ inputs.scala-version }}" \
+            "${{ github.run_id }}" \
+            "${{ steps.job-info.outputs.build-url }}"
+          if [ $? != 0 ]; then 
+            echo "::warning title=Indexing failure::Indexing results of ${{ inputs.project-name }} failed" 
+          fi
+          
+          # Store results
+          mv build-logs.txt /opencb/
+          mv build-status.txt /opencb/
+          mv build-summary.txt /opencb
+
+    # - name: Upload artifacts
+    #   uses: actions/upload-artifact@v3
+    #   if: steps.check-history.outputs.can-skip-build != 'true'
+    #   with:
+    #     name: ${{ inputs.project-name }}
+    #     path: ${{ github.workspace }}/build-*.txt
+
+    - name: Check results
+      shell: bash
+      if: steps.check-history.outputs.can-skip-build != 'true'
+      run: |
+        # Set the result of the build
+        if [[ "$(cat build-status.txt)" == "success" ]]; then 
+          echo "Build successful"
+        else 
+          echo "Build failure! Check logs for details"
+          exit 1
+        fi

.github/actions/setup-build/action.yaml

@@ -0,0 +1,81 @@
+name: "Prepare Open Community Build"
+description: "Ensure compiler version is published, resolve url for custom Maven repository"
+inputs:
+  scala-version:
+    description: "Scala version to check"
+  repository-url:
+    description: "GitHub repository URL for compiler to build"
+    required: true
+  repository-branch:
+    description: "GitHub repository branch for compiler to build"
+    required: true
+outputs:
+  scala-version:
+    description: "Effective Scala version, input value of scala-version if using published version or version of builded compiler"
+    value: ${{ steps.calc-version.outputs.effective-scala-version }}
+  maven-repo-url:
+    description: "Effective Maven repository subdirectory to use"
+    value: ${{ steps.calc-version.outputs.effective-maven-url }}
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v3
+      with:
+        repository: ${{ inputs.repository-url }}
+        ref: ${{ inputs.repository-branch }}
+        path: "compiler"
+
+    - name: Calculate version if missing
+      id: calc-version
+      shell: bash
+      run: |
+        commitHash=$(git rev-parse HEAD)
+
+        scalaVersion=${{ inputs.scala-version }}
+        if [[ -z $scalaVersion ]]; then
+          baseVersion=$(cat compiler/project/Build.scala | grep 'val baseVersion =' | xargs | awk '{ print $4 }') 
+          scalaVersion="${baseVersion}-bin-${commitHash}"
+        fi
+
+        baseMavenRepo="https://scala3.westeurope.cloudapp.azure.com/maven2"
+        buildMavenRepo="${baseMavenRepo}/${scalaVersion}"
+
+        echo "Effective Scala version for this build: $scalaVersion"
+        echo "effective-scala-version=${scalaVersion}" >> $GITHUB_OUTPUT
+
+        echo "Effective Maven repository for this build: $buildMavenRepo"
+        echo "effective-maven-url=${buildMavenRepo}" >> $GITHUB_OUTPUT
+
+    - name: Install coursier
+      uses: coursier/setup-action@v1
+      with:
+        apps: cs
+    - name: Check version is published
+      shell: bash
+      id: check-published
+      run: |
+        Version="${{ steps.calc-version.outputs.effective-scala-version }}"
+        CustomMavenRepo="${{ steps.calc-version.outputs.effective-maven-url }}"
+        isPublished=false
+        # Download jar instead of checking only for pom to ensure that it's complete
+        if cs fetch org.scala-lang:scala3-compiler_3:${Version} -r $CustomMavenRepo ; then
+          isPublished=true
+        elif [[ ! -z "${{ inputs.scala-version }}" ]]; then
+          echo "::error title=Compiler version unavailable::Requested compiler version ${{ inputs.scala-version }} is unavailable" 
+          exit 1
+        fi
+        echo "Can skip compiler build: ${isPublished}"
+        echo "is-compiler-published=${isPublished}" >> $GITHUB_OUTPUT
+
+    - name: Build compiler
+      uses: addnab/docker-run-action@v3
+      if: steps.check-published.outputs.is-compiler-published == 'false'
+      with:
+        image: "virtuslab/scala-community-build-compiler-builder:v0.2.1"
+        options: -v ${{ github.workspace }}/compiler:/compiler/
+        run: |
+          Version="${{ steps.calc-version.outputs.effective-scala-version }}"
+          CustomMavenRepo="${{ steps.calc-version.outputs.effective-maven-url }}"
+          echo "Building Scala compiler, version ${Version}"
+          /build/build.sh /compiler "${Version}" "${CustomMavenRepo}"

.github/workflows/buildAutoUpdatePlan.yaml

@@ -0,0 +1,58 @@
+name: Auto update build plan
+
+permissions:
+  contents: write
+  pull-requests: write
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Every friday at 4 PM
+    - cron: "0 16 * * 5"
+  push:
+    paths:
+      - coordinator/**
+      - env/**
+jobs:
+  build-plan:
+    runs-on: ubuntu-22.04
+    if: github.event_name == 'schedule' && github.repository == 'VirtusLab/community-build3'
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v3
+
+      - name: Install coursier
+        uses: coursier/setup-action@v1
+        with:
+          apps: scala-cli
+
+      - name: Get Date
+        id: get-date
+        run: echo "week-start=$(date -dmonday +%Y%m%d)" >> $GITHUB_OUTPUT
+
+      - name: Cache coordinator data
+        uses: actions/cache@v3
+        with:
+          # coordinator stores cache in current working directory
+          path: data/
+          key: coordinator-data-cache-${{ steps.get-date.outputs.week-start}}
+
+      - name: Build plan
+        # Limit to 1000 most starred project. GitHub actions has problems when rendering workflow of more then 1k jobs
+        run: scala-cli run coordinator/ -- 3 -1 1000 "" env/prod/config/replaced-projects.txt env/prod/config/projects-config.conf env/prod/config/filtered-projects.txt
+
+      - name: Create PR
+        uses: peter-evans/create-pull-request@v4
+        id: cpr
+        with:
+          token: ${{ secrets.OPENCB_CONFIG_UPDATE_TOKEN }}
+          commit-message: Update build config
+          assignees: WojciechMazur
+          branch: build/scheduled-update
+          branch-suffix: timestamp
+          delete-branch: true
+          title: "[Bot] Update build config"
+          draft: false
+          add-paths: |
+            .github/workflows/buildPlan.yaml
+            .github/workflows/buildConfig.json