From 9cfa48402fa203831e1f2285bb491334556c7c7b Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 8 May 2020 17:00:20 +0900 Subject: [PATCH 01/18] Create ci.yml --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..095c9b253c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,32 @@ +# This workflow will build a Java project with Gradle +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: CI + +on: [ push ] + +jobs: + build: + + runs-on: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c + + strategy: + matrix: + ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Run unit test + run: ./gradlew testClasses + - name: Run tests in Docker + run: ci/run_test.sh + env: + CI_NODE_INDEX: ${{ matrix.ci_node_index }} + CI_NODE_TOTAL: 12 + BUILD_IMAGE: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c From 20443650db3f5c29085a8a94869b5b750ceca793 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 8 May 2020 17:39:30 +0900 Subject: [PATCH 02/18] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 095c9b253c..8f9c53a6f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: [ push ] jobs: build: - runs-on: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c + runs-on: ubuntu-latest strategy: matrix: From ef4f1fe6abf8564e9be1f803bcec1b85d375176c Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 8 May 2020 18:26:04 +0900 Subject: [PATCH 03/18] Show stacktrace in docker run --- ci/run_test_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_test_docker.sh b/ci/run_test_docker.sh index f965d86980..722d19b83f 100755 --- a/ci/run_test_docker.sh +++ b/ci/run_test_docker.sh @@ -27,4 +27,4 @@ docker run \ -e TEST_S3_SECRET_ACCESS_KEY=$TEST_S3_SECRET_ACCESS_KEY \ -e TERM=dumb \ $BUILD_IMAGE \ -./gradlew clean cleanTest test --info --no-daemon "$@" +./gradlew clean cleanTest test --info --stacktrace --no-daemon "$@" From ebb05f2ae4d16e0e6caa0270ffd0290e4866d91d Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 8 May 2020 21:52:50 +0900 Subject: [PATCH 04/18] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f9c53a6f5..61c09e2a6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,8 @@ jobs: java-version: 1.8 - name: Grant execute permission for gradlew run: chmod +x gradlew - - name: Run unit test - run: ./gradlew testClasses + # - name: Run unit test + # run: ./gradlew testClasses - name: Run tests in Docker run: ci/run_test.sh env: From 49bc667ba5e13be9bb728ba8d719d610d74a9903 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 8 May 2020 22:37:07 +0900 Subject: [PATCH 05/18] Correct a description about CI_ACCEPTANCE_TEST --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 40beea84a8..3858fdd35d 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ $ CI_ACCEPTANCE_TEST=true ./gradlew digdag-tests:test --info --tests acceptance. ``` To execute tests in digdag-tests subproject locally, `tests` option that is provided by Gradle is useful. -Environment variable `CI_ACCEPTANCE_TEST=true` is needed to execute digdag-tests only. +Environment variable `CI_ACCEPTANCE_TEST=true` is needed to execute digdag-tests. ### Testing with PostgreSQL From 8399642dd7e487d326eedb6ad7489ed06275db23 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 8 May 2020 22:37:59 +0900 Subject: [PATCH 06/18] Tweak Github Actions workflow config --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61c09e2a6a..e776023064 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7] steps: - uses: actions/checkout@v2 @@ -22,11 +22,9 @@ jobs: java-version: 1.8 - name: Grant execute permission for gradlew run: chmod +x gradlew - # - name: Run unit test - # run: ./gradlew testClasses - name: Run tests in Docker run: ci/run_test.sh env: CI_NODE_INDEX: ${{ matrix.ci_node_index }} - CI_NODE_TOTAL: 12 + CI_NODE_TOTAL: 8 BUILD_IMAGE: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c From 5df1f42707feb7de8f9e8013b7273a59a5e1a38c Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 13 May 2020 14:55:03 +0900 Subject: [PATCH 07/18] Avoid building all CI nodes with UI --- ci/run_test.sh | 9 ++++++++- ci/run_test_h2.sh | 2 +- ci/run_test_pg.sh | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/run_test.sh b/ci/run_test.sh index 6ee70dc9e4..4d8fa41813 100755 --- a/ci/run_test.sh +++ b/ci/run_test.sh @@ -9,6 +9,13 @@ if [[ $(( 1 % ${CI_NODE_TOTAL} )) -eq ${CI_NODE_INDEX} ]]; then "${BASEDIR}/validate.sh" fi +if [[ $(( 2 % ${CI_NODE_TOTAL} )) -eq ${CI_NODE_INDEX} ]]; then + UI_BUILD_OPT="" +else + # Don't need to build UI always + UI_BUILD_OPT="-PwithoutUi" +fi + "${BASEDIR}/run_test_h2.sh" -"${BASEDIR}/run_test_pg.sh" +"${BASEDIR}/run_test_pg.sh ${UI_BUILD_OPT}" diff --git a/ci/run_test_h2.sh b/ci/run_test_h2.sh index 128be47c86..4b9265154d 100755 --- a/ci/run_test_h2.sh +++ b/ci/run_test_h2.sh @@ -1,4 +1,4 @@ #!/bin/bash -xe BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -$BASEDIR/run_test_docker.sh +$BASEDIR/run_test_docker.sh "$@" diff --git a/ci/run_test_pg.sh b/ci/run_test_pg.sh index f3f5330124..d9b9fab977 100755 --- a/ci/run_test_pg.sh +++ b/ci/run_test_pg.sh @@ -13,4 +13,4 @@ minimumPoolSize = 0 export DIGDAG_TEST_REDIS=127.0.0.1 -$BASEDIR/run_test_docker.sh +$BASEDIR/run_test_docker.sh "$@" From 707477e21c9b74ffb5b5c4faf12d9869ae0c4fcd Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 13 May 2020 14:55:43 +0900 Subject: [PATCH 08/18] Remove CI on Travis CI --- .travis.yml | 63 ----------------------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ea0bf0ed45..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,63 +0,0 @@ -sudo: required - -language: java - -dist: trusty - -jdk: - - oraclejdk8 - -services: - - docker - -cache: - directories: - - $HOME/.gradle - - $HOME/.m2 - -env: - global: - - TERM=dumb - - CI_NODE_TOTAL=12 - - BUILD_IMAGE=digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c - matrix: - - CI_NODE_INDEX=0 - - CI_NODE_INDEX=1 - - CI_NODE_INDEX=2 - - CI_NODE_INDEX=3 - - CI_NODE_INDEX=4 - - CI_NODE_INDEX=5 - - CI_NODE_INDEX=6 - - CI_NODE_INDEX=7 - - CI_NODE_INDEX=8 - - CI_NODE_INDEX=9 - - CI_NODE_INDEX=10 - - CI_NODE_INDEX=11 - -before_install: - - sudo service --status-all - - sudo /etc/init.d/mysql stop - - sudo /etc/init.d/redis-server stop - - | - docker run \ - -w /digdag \ - -v `pwd`/:/digdag \ - -v $HOME/.gradle:/root/.gradle \ - $BUILD_IMAGE \ - ./gradlew testClasses --no-daemon - - ci/validate.sh - -install: true - -before_cache: - - sudo chown `id -u`:`id -g` -R $HOME/.gradle $HOME/.m2 # make them cachable - -script: - - travis_wait 40 ci/run_test.sh - -after_success: - - ci/travis_report_coverage.sh - -after_failure: - - sudo free -m -t - - sudo dmesg From 313002bab6c26c48a167f52d8b86d23bbaa3fa32 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 13 May 2020 15:07:38 +0900 Subject: [PATCH 09/18] Fix script --- ci/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_test.sh b/ci/run_test.sh index 4d8fa41813..c226d92396 100755 --- a/ci/run_test.sh +++ b/ci/run_test.sh @@ -17,5 +17,5 @@ else fi "${BASEDIR}/run_test_h2.sh" -"${BASEDIR}/run_test_pg.sh ${UI_BUILD_OPT}" +"${BASEDIR}/run_test_pg.sh" ${UI_BUILD_OPT} From 5106f0df4400016b47fe50a68e606d708d731b20 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 13 May 2020 15:42:02 +0900 Subject: [PATCH 10/18] Integrate with Coveralls on Github Actions --- .github/workflows/ci.yml | 14 +++++++++++--- ci/travis_report_coverage.sh | 17 ----------------- 2 files changed, 11 insertions(+), 20 deletions(-) delete mode 100755 ci/travis_report_coverage.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e776023064..76618a82c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,13 +7,10 @@ on: [ push ] jobs: build: - runs-on: ubuntu-latest - strategy: matrix: ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7] - steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 @@ -28,3 +25,14 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} CI_NODE_TOTAL: 8 BUILD_IMAGE: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c + + finish: + needs: test + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.github_token }} + parallel-finished: true + diff --git a/ci/travis_report_coverage.sh b/ci/travis_report_coverage.sh deleted file mode 100755 index 1ce43945bd..0000000000 --- a/ci/travis_report_coverage.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -xe - -# TRAVIS_* and CI_NAME environment variables are required by -# org.kt3k.gradle.plugin.coveralls.domain.ServiceInfoFactory. - -docker run \ --w /digdag \ --v `pwd`/:/digdag \ --v ~/.gradle:/root/.gradle \ --e TERM=dumb \ --e TRAVIS \ --e TRAVIS_JOB_ID \ --e TRAVIS_JOB_ID \ --e TRAVIS_PULL_REQUEST \ --e CI_NAME \ -digdag-build \ -./gradlew jacocoTestReport coveralls -x test --no-daemon "$@" From 3eb61993739708542f3b98d125809e5bd0db95b0 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 13 May 2020 15:48:36 +0900 Subject: [PATCH 11/18] Update ci.yml --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76618a82c7..e6bba7175b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,9 +25,15 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} CI_NODE_TOTAL: 8 BUILD_IMAGE: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c + - name: Coveralls Parallel + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.github_token }} + flag-name: run-${{ matrix.test_number }} + parallel: true finish: - needs: test + needs: build runs-on: ubuntu-latest steps: - name: Coveralls Finished From 8d3119a222a6132ff62dea4736afc5925dcc6523 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Wed, 13 May 2020 18:42:23 +0900 Subject: [PATCH 12/18] Remove Coveralls integration for now --- .github/workflows/ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6bba7175b..fa20e58080 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,20 +25,4 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} CI_NODE_TOTAL: 8 BUILD_IMAGE: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c - - name: Coveralls Parallel - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.github_token }} - flag-name: run-${{ matrix.test_number }} - parallel: true - - finish: - needs: build - runs-on: ubuntu-latest - steps: - - name: Coveralls Finished - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.github_token }} - parallel-finished: true From db45ac89eba36a9842361635f99a10c0a53d1298 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Thu, 14 May 2020 14:55:48 +0900 Subject: [PATCH 13/18] Update ci.yml --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa20e58080..f9447dbe52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,14 +3,14 @@ name: CI -on: [ push ] +on: [ push, pull_request ] jobs: build: runs-on: ubuntu-latest strategy: matrix: - ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7] + ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 @@ -23,6 +23,6 @@ jobs: run: ci/run_test.sh env: CI_NODE_INDEX: ${{ matrix.ci_node_index }} - CI_NODE_TOTAL: 8 + CI_NODE_TOTAL: 10 BUILD_IMAGE: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c From 5207a3223b7b477e3d016f5420e96dba900b7f58 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 15 May 2020 15:16:01 +0900 Subject: [PATCH 14/18] Avoid UI build in case of h2 tests --- ci/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_test.sh b/ci/run_test.sh index c226d92396..f94a98c05a 100755 --- a/ci/run_test.sh +++ b/ci/run_test.sh @@ -16,6 +16,6 @@ else UI_BUILD_OPT="-PwithoutUi" fi -"${BASEDIR}/run_test_h2.sh" +"${BASEDIR}/run_test_h2.sh" -PwithoutUi "${BASEDIR}/run_test_pg.sh" ${UI_BUILD_OPT} From 0d81ed356a8ed62c85fcdbf40a1c9de959e86efa Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 19 Jun 2020 21:06:34 +0900 Subject: [PATCH 15/18] Use '-x buildUi' instead of '-PwithoutUi' --- ci/run_test.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ci/run_test.sh b/ci/run_test.sh index f94a98c05a..7bfa21b883 100755 --- a/ci/run_test.sh +++ b/ci/run_test.sh @@ -9,13 +9,16 @@ if [[ $(( 1 % ${CI_NODE_TOTAL} )) -eq ${CI_NODE_INDEX} ]]; then "${BASEDIR}/validate.sh" fi +WITH_BUILD_UI="" +WITHOUT_BUILD_UI="-x buildUi" + +# Don't need to build UI always if [[ $(( 2 % ${CI_NODE_TOTAL} )) -eq ${CI_NODE_INDEX} ]]; then - UI_BUILD_OPT="" + BUILD_UI_OPT=${WITH_BUILD_UI} else - # Don't need to build UI always - UI_BUILD_OPT="-PwithoutUi" + BUILD_UI_OPT=${WITHOUT_BUILD_UI} fi -"${BASEDIR}/run_test_h2.sh" -PwithoutUi -"${BASEDIR}/run_test_pg.sh" ${UI_BUILD_OPT} +"${BASEDIR}/run_test_h2.sh" ${WITHOUT_BUILD_UI} +"${BASEDIR}/run_test_pg.sh" ${BUILD_UI_OPT} From 187c48d6614013e6f4e08099f36e24407e927ca9 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Thu, 9 Jul 2020 17:47:41 +0900 Subject: [PATCH 16/18] Update ci.yml Prevent duplicated CI build --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9447dbe52..9e80c37525 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,15 @@ name: CI -on: [ push, pull_request ] +on: + push: + branches: + - master + - v0_10 + pull_request: + branches: + - master + - v0_10 jobs: build: From 38b302f0fdff2f6e084e437042702a6a0469ad7a Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Tue, 21 Jul 2020 15:45:22 +0900 Subject: [PATCH 17/18] Update README.md Replace the badge of Travis CI with Github Action's one --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3858fdd35d..d68dd28830 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![Circle CI](https://circleci.com/gh/treasure-data/digdag.svg?style=svg&circle-token=8ccc5c665022ce4d1ee05cf7b829c84877387a6c)](https://circleci.com/gh/treasure-data/digdag) -[![Travis CI](https://travis-ci.org/treasure-data/digdag.svg?branch=master)](https://travis-ci.org/treasure-data/digdag) +[![CI](https://github.com/treasure-data/digdag/workflows/CI/badge.svg)](https://github.com/treasure-data/digdag/actions) + ## [Documentation](https://docs.digdag.io) From 89f3905195f140dfffb6f26103c00588a8c63f09 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Tue, 21 Jul 2020 16:21:35 +0900 Subject: [PATCH 18/18] Add a comment --- ci/run_test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/run_test.sh b/ci/run_test.sh index 7bfa21b883..a924557972 100755 --- a/ci/run_test.sh +++ b/ci/run_test.sh @@ -19,6 +19,7 @@ else BUILD_UI_OPT=${WITHOUT_BUILD_UI} fi +# It's enough to build UI either with H2 or PostgreSQL not with the both "${BASEDIR}/run_test_h2.sh" ${WITHOUT_BUILD_UI} "${BASEDIR}/run_test_pg.sh" ${BUILD_UI_OPT}