From 886d3f6a3f4f374b41832de91575fca8ec191a66 Mon Sep 17 00:00:00 2001 From: Tayfun Oztemel Date: Fri, 20 Sep 2024 10:35:36 +0200 Subject: [PATCH 1/5] Update ci.yml Upgrade github actions to version 4 --- .github/workflows/ci.yml | 151 ++++++++++----------------------------- 1 file changed, 38 insertions(+), 113 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3705a62..547aaa43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,130 +5,55 @@ # change your sbt build configuration to revise the workflow description # to meet your needs, then regenerate this file. -name: Continuous Integration +name: Clean -on: - pull_request: - branches: ['**'] - push: - branches: ['**'] - -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +on: push jobs: - build: - name: Build and Test - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - scala: [2.13.13, 3.3.1] - java: [temurin@17] - clickhouse: [21.3, 21.8.14, 22.3] - runs-on: ${{ matrix.os }} + delete-artifacts: + name: Delete Artifacts + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Checkout current branch (full) - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Docker Compose Action - uses: isbang/compose-action@v1.0.0 - env: - CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} - with: - compose-file: './docker-compose.yml' - down-flags: '--volumes' - - - name: Setup Java (temurin@17) - if: matrix.java == 'temurin@17' - uses: actions/setup-java@v2 - with: - distribution: temurin - java-version: 17 - - - name: Cache sbt - uses: actions/cache@v2 - with: - path: | - ~/.sbt - ~/.ivy2/cache - ~/.coursier/cache/v1 - ~/.cache/coursier/v1 - ~/AppData/Local/Coursier/Cache/v1 - ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - -# - name: Check that workflows are up to date -# run: sbt ++${{ matrix.scala }} githubWorkflowCheck + - name: Delete artifacts + run: | + # Customize those three lines with your repository and credentials: + REPO=${GITHUB_API_URL}/repos/${{ github.repository }} - - name: Build project - run: sbt ++${{ matrix.scala }} test it:test + # A shortcut to call GitHub API. + ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } - - name: Compress target directories - run: tar cf targets.tar target client/target dsl/target testkit/target project/target + # A temporary file which receives HTTP response headers. + TMPFILE=/tmp/tmp.$$ - - name: Upload target directories - uses: actions/upload-artifact@v2 - with: - name: target-${{ matrix.os }}-${{ matrix.scala }}-${{ matrix.java }} - path: targets.tar + # An associative array, key: artifact name, value: number of artifacts of that name. + declare -A ARTCOUNT - publish: - name: Publish Artifacts - needs: [build] - if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main') - strategy: - matrix: - os: [ubuntu-latest] - scala: [2.13.13, 3.3.1] - java: [temurin@17] - runs-on: ${{ matrix.os }} - steps: - - name: Checkout current branch (full) - uses: actions/checkout@v2 - with: - fetch-depth: 0 + # Process all artifacts on this repository, loop on returned "pages". + URL=$REPO/actions/artifacts + while [[ -n "$URL" ]]; do - - name: Setup Java (temurin@17) - if: matrix.java == 'temurin@17' - uses: actions/setup-java@v2 - with: - distribution: temurin - java-version: 17 + # Get current page, get response headers in a temporary file. + JSON=$(ghapi --dump-header $TMPFILE "$URL") - - name: Cache sbt - uses: actions/cache@v2 - with: - path: | - ~/.sbt - ~/.ivy2/cache - ~/.coursier/cache/v1 - ~/.cache/coursier/v1 - ~/AppData/Local/Coursier/Cache/v1 - ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} + # Get URL of next page. Will be empty if we are at the last page. + URL=$(grep '^Link:' "$TMPFILE" | tr ',' '\n' | grep 'rel="next"' | head -1 | sed -e 's/.*.*//') + rm -f $TMPFILE - - name: Download target directories (3.3.1) - uses: actions/download-artifact@v2 - with: - name: target-${{ matrix.os }}-3.3.1-${{ matrix.java }} + # Number of artifacts on this page: + COUNT=$(( $(jq <<<$JSON -r '.artifacts | length') )) - - name: Inflate target directories (3.3.1) - run: | - tar xf targets.tar - rm targets.tar + # Loop on all artifacts on this page. + for ((i=0; $i < $COUNT; i++)); do - - name: Download target directories (2.13.13) - uses: actions/download-artifact@v2 - with: - name: target-${{ matrix.os }}-2.13.13-${{ matrix.java }} - - - name: Inflate target directories (2.13.13) - run: | - tar xf targets.tar - rm targets.tar + # Get name of artifact and count instances of this name. + name=$(jq <<<$JSON -r ".artifacts[$i].name?") + ARTCOUNT[$name]=$(( $(( ${ARTCOUNT[$name]} )) + 1)) - - name: Publish project - run: sbt ++${{ matrix.scala }} +publish + id=$(jq <<<$JSON -r ".artifacts[$i].id?") + size=$(( $(jq <<<$JSON -r ".artifacts[$i].size_in_bytes?") )) + printf "Deleting '%s' #%d, %'d bytes\n" $name ${ARTCOUNT[$name]} $size + ghapi -X DELETE $REPO/actions/artifacts/$id + done + done From 781cf26e3ed954f8e50b81b669f898ca13a3f6b8 Mon Sep 17 00:00:00 2001 From: Tayfun Oztemel Date: Fri, 20 Sep 2024 10:52:39 +0200 Subject: [PATCH 2/5] Update ci.yml --- .github/workflows/ci.yml | 151 +++++++++++++++++++++++++++++---------- 1 file changed, 113 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 547aaa43..7eb6de05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,55 +5,130 @@ # change your sbt build configuration to revise the workflow description # to meet your needs, then regenerate this file. -name: Clean +name: Continuous Integration -on: push +on: + pull_request: + branches: ['**'] + push: + branches: ['**'] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - delete-artifacts: - name: Delete Artifacts - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + build: + name: Build and Test + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + scala: [2.13.13, 3.3.1] + java: [temurin@17] + clickhouse: [21.3, 21.8.14, 22.3] + runs-on: ${{ matrix.os }} steps: - - name: Delete artifacts - run: | - # Customize those three lines with your repository and credentials: - REPO=${GITHUB_API_URL}/repos/${{ github.repository }} + - name: Checkout current branch (full) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Docker Compose Action + uses: isbang/compose-action@v1.0.0 + env: + CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} + with: + compose-file: './docker-compose.yml' + down-flags: '--volumes' + + - name: Setup Java (temurin@17) + if: matrix.java == 'temurin@17' + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache sbt + uses: actions/cache@v4 + with: + path: | + ~/.sbt + ~/.ivy2/cache + ~/.coursier/cache/v1 + ~/.cache/coursier/v1 + ~/AppData/Local/Coursier/Cache/v1 + ~/Library/Caches/Coursier/v1 + key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - # A shortcut to call GitHub API. - ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } +# - name: Check that workflows are up to date +# run: sbt ++${{ matrix.scala }} githubWorkflowCheck - # A temporary file which receives HTTP response headers. - TMPFILE=/tmp/tmp.$$ + - name: Build project + run: sbt ++${{ matrix.scala }} test it:test - # An associative array, key: artifact name, value: number of artifacts of that name. - declare -A ARTCOUNT + - name: Compress target directories + run: tar cf targets.tar target client/target dsl/target testkit/target project/target - # Process all artifacts on this repository, loop on returned "pages". - URL=$REPO/actions/artifacts - while [[ -n "$URL" ]]; do + - name: Upload target directories + uses: actions/upload-artifact@v4 + with: + name: target-${{ matrix.os }}-${{ matrix.scala }}-${{ matrix.java }} + path: targets.tar - # Get current page, get response headers in a temporary file. - JSON=$(ghapi --dump-header $TMPFILE "$URL") + publish: + name: Publish Artifacts + needs: [build] + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main') + strategy: + matrix: + os: [ubuntu-latest] + scala: [2.13.13, 3.3.1] + java: [temurin@17] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout current branch (full) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Java (temurin@17) + if: matrix.java == 'temurin@17' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 - # Get URL of next page. Will be empty if we are at the last page. - URL=$(grep '^Link:' "$TMPFILE" | tr ',' '\n' | grep 'rel="next"' | head -1 | sed -e 's/.*.*//') - rm -f $TMPFILE + - name: Cache sbt + uses: actions/cache@v4 + with: + path: | + ~/.sbt + ~/.ivy2/cache + ~/.coursier/cache/v1 + ~/.cache/coursier/v1 + ~/AppData/Local/Coursier/Cache/v1 + ~/Library/Caches/Coursier/v1 + key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - # Number of artifacts on this page: - COUNT=$(( $(jq <<<$JSON -r '.artifacts | length') )) + - name: Download target directories (3.3.1) + uses: actions/download-artifact@v4 + with: + name: target-${{ matrix.os }}-3.3.1-${{ matrix.java }} - # Loop on all artifacts on this page. - for ((i=0; $i < $COUNT; i++)); do + - name: Inflate target directories (3.3.1) + run: | + tar xf targets.tar + rm targets.tar - # Get name of artifact and count instances of this name. - name=$(jq <<<$JSON -r ".artifacts[$i].name?") - ARTCOUNT[$name]=$(( $(( ${ARTCOUNT[$name]} )) + 1)) + - name: Download target directories (2.13.13) + uses: actions/download-artifact@v4 + with: + name: target-${{ matrix.os }}-2.13.13-${{ matrix.java }} + + - name: Inflate target directories (2.13.13) + run: | + tar xf targets.tar + rm targets.tar - id=$(jq <<<$JSON -r ".artifacts[$i].id?") - size=$(( $(jq <<<$JSON -r ".artifacts[$i].size_in_bytes?") )) - printf "Deleting '%s' #%d, %'d bytes\n" $name ${ARTCOUNT[$name]} $size - ghapi -X DELETE $REPO/actions/artifacts/$id - done - done + - name: Publish project + run: sbt ++${{ matrix.scala }} +publish From 086cbd6446672f42312d891b13b3b0aefb4bf269 Mon Sep 17 00:00:00 2001 From: Tayfun Oztemel Date: Fri, 20 Sep 2024 13:13:30 +0200 Subject: [PATCH 3/5] Update Docker compose action --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7eb6de05..b3115424 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: fetch-depth: 0 - name: Docker Compose Action - uses: isbang/compose-action@v1.0.0 + uses: hoverkraft-tech/compose-action@v2.0.1 env: CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} with: From 81886a7989cb9e8c1248a7c6ac3d9971003e7952 Mon Sep 17 00:00:00 2001 From: Tayfun Oztemel Date: Fri, 20 Sep 2024 14:03:53 +0200 Subject: [PATCH 4/5] Update upload target name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3115424..b273ca62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: os: [ubuntu-latest] scala: [2.13.13, 3.3.1] java: [temurin@17] - clickhouse: [21.3, 21.8.14, 22.3] + clickhouse: [21.3, 21.8.14, 22.3, 23.8] runs-on: ${{ matrix.os }} steps: - name: Checkout current branch (full) @@ -72,7 +72,7 @@ jobs: - name: Upload target directories uses: actions/upload-artifact@v4 with: - name: target-${{ matrix.os }}-${{ matrix.scala }}-${{ matrix.java }} + name: target-${{ matrix.os }}-${{ matrix.scala }}-${{ matrix.java }}-${{matrix.clickhouse}} path: targets.tar publish: From 86b582917ddd8a48f1a3e0874247b0186694a100 Mon Sep 17 00:00:00 2001 From: Tayfun Oztemel Date: Fri, 20 Sep 2024 14:31:46 +0200 Subject: [PATCH 5/5] Drop clickhouse 23.8 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b273ca62..5ce767a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: os: [ubuntu-latest] scala: [2.13.13, 3.3.1] java: [temurin@17] - clickhouse: [21.3, 21.8.14, 22.3, 23.8] + clickhouse: [21.3, 21.8.14, 22.3] runs-on: ${{ matrix.os }} steps: - name: Checkout current branch (full)