Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor tests #3585

Closed
wants to merge 2 commits into from
Closed

Refactor tests #3585

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2feae34
Refactor tests
yuxiqian Aug 29, 2024
a5c8186
fix workflow path
yuxiqian Aug 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
342 changes: 14 additions & 328 deletions .github/workflows/flink_cdc.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,333 +43,19 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
cancel-in-progress: true

env:
MODULES_CORE: "\
flink-cdc-cli,\
flink-cdc-common,\
flink-cdc-composer,\
flink-cdc-runtime"

MODULES_PIPELINE_CONNECTORS: "\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"

MODULES_MYSQL: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"

MODULES_POSTGRES: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"

MODULES_ORACLE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"

MODULES_MONGODB: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc"

MODULES_SQLSERVER: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc"

MODULES_TIDB: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"

MODULES_OCEANBASE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"

MODULES_DB2: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"

MODULES_VITESS: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"

MODULES_PIPELINE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"

MODULES_SOURCE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"

jobs:
license_check:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
submodules: true
- name: Set up Ruby environment
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.3'
- name: Compiling jar packages
run: mvn --no-snapshot-updates -B package -DskipTests
- name: Run license check
run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb

compile_and_test:
needs: license_check
# Only run the CI pipeline for the flink-cdc-connectors repository
# if: github.repository == 'apache/flink-cdc-connectors'
runs-on: ubuntu-latest
strategy:
matrix:
jdk: [ 8 ]
module: [ "core",
"pipeline_connectors",
"mysql",
"postgres",
"oracle",
"mongodb",
"sqlserver",
"tidb",
"oceanbase",
"db2",
"vitess",
"pipeline_e2e",
"source_e2e"
]
timeout-minutes: 120
env:
MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \
-Dorg.slf4j.simpleLogger.showDateTime=true \
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
--no-snapshot-updates -B \
--settings /home/vsts/work/1/s/tools/ci/google-mirror-settings.xml \
-Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \
-Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties
steps:
- run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}"

- name: Clean up disk space
run: |
set -euo pipefail

echo "Disk space before cleanup"
df -h

echo "Cleaning up disk space"
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force

echo "Disk space after cleanup"
df -h

- name: Check out repository code
uses: actions/checkout@v4
with:
submodules: true

- name: Set JDK
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: 'maven'

- name: Set Maven 3.8.6
uses: stCarolas/setup-maven@v5
with:
maven-version: 3.8.6

- name: Compile and test ${{ matrix.module }}
timeout-minutes: 90
run: |
set -o pipefail

case ${{ matrix.module }} in
("core")
modules=${{ env.MODULES_CORE }}
;;
("pipeline_connectors")
modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
;;
("mysql")
modules=${{ env.MODULES_MYSQL }}
;;
("postgres")
modules=${{ env.MODULES_POSTGRES }}
;;
("oracle")
modules=${{ env.MODULES_ORACLE }}
;;
("mongodb")
modules=${{ env.MODULES_MONGODB }}
;;
("sqlserver")
modules=${{ env.MODULES_SQLSERVER }}
;;
("tidb")
modules=${{ env.MODULES_TIDB }}
;;
("oceanbase")
modules=${{ env.MODULES_OCEANBASE }}
;;
("db2")
modules=${{ env.MODULES_DB2 }}
;;
("vitess")
modules=${{ env.MODULES_VITESS }}
;;
("pipeline_e2e")
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
modules=${{ env.MODULES_PIPELINE_E2E }}
;;
("source_e2e")
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
modules=${{ env.MODULES_SOURCE_E2E }}
;;
esac

if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
compile_modules=$modules
fi

mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B -pl $modules verify

- name: Print JVM thread dumps when cancelled
if: ${{ failure() }}
run: |
# ----------------------------------------------------------------------------
# Copyright 2023 The Netty Project
#
# ----------------------------------------------------------------------------
# Source: https://github.com/netty/netty/blob/main/.github/actions/thread-dump-jvms/action.yml
echo "$OSTYPE"
if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then
echo "Setting up JVM thread dumps"
# use jattach so that Java processes in docker containers are also covered
# download jattach
curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach
if command -v sha256sum &> /dev/null; then
# verify hash of jattach binary
sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e /tmp/jattach") || exit 1
fi
chmod +x /tmp/jattach
for java_pid in $(sudo pgrep java); do
echo "----------------------- pid $java_pid -----------------------"
echo "command line: $(sudo cat /proc/$java_pid/cmdline | xargs -0 echo)"
sudo /tmp/jattach $java_pid jcmd VM.command_line || true
sudo /tmp/jattach $java_pid jcmd "Thread.print -l"
sudo /tmp/jattach $java_pid jcmd GC.heap_info || true
done
else
for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do
echo "----------------------- pid $java_pid -----------------------"
jcmd $java_pid VM.command_line || true
jcmd $java_pid Thread.print -l
jcmd $java_pid GC.heap_info || true
done
fi
exit 0


migration_test_ut:
needs: license_check
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
submodules: true
- name: Compile snapshot CDC version
run: mvn --no-snapshot-updates -B install -DskipTests
- name: Run migration tests
run: cd flink-cdc-migration-tests && mvn clean verify

pipeline_migration_test:
needs: migration_test_ut
runs-on: ubuntu-latest
strategy:
matrix:
java-version: [ '8', '11' ]

steps:
- uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: 3.0
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: temurin
cache: maven
- name: Install dependencies
run: gem install terminal-table
- name: Prepare CDC versions
run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- name: Prepare Flink distro
run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
working-directory: ./tools/mig-test
- name: Patch Flink configs
run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
working-directory: ./tools/mig-test
- name: Start containers
run: cd conf && docker compose up -d
working-directory: ./tools/mig-test
- name: Run migration tests
run: FLINK_HOME=./flink-1.18.1/ ruby run_migration_test.rb
working-directory: ./tools/mig-test
- name: Stop containers
if: always()
run: cd conf && docker compose down
working-directory: ./tools/mig-test

data_stream_migration_test:
needs: migration_test_ut
runs-on: ubuntu-latest
strategy:
matrix:
java-version: [ '8', '11' ]

steps:
- uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: 3.0
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: temurin
cache: maven
- name: Install dependencies
run: gem install terminal-table
- name: Prepare CDC versions
run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- name: Prepare Flink distro
run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
working-directory: ./tools/mig-test
- name: Patch Flink configs
run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
working-directory: ./tools/mig-test
- name: Compile Dummy DataStream Jobs
run: cd datastream && ruby compile_jobs.rb
working-directory: ./tools/mig-test
- name: Start containers
run: cd conf && docker compose up -d
working-directory: ./tools/mig-test
- name: Run migration tests
run: cd datastream && FLINK_HOME=../flink-1.18.1/ ruby run_migration_test.rb
working-directory: ./tools/mig-test
- name: Stop containers
if: always()
run: cd conf && docker compose down
working-directory: ./tools/mig-test
name: License Check
uses: ./.github/workflows/license_check.yml
ut:
name: Unit Tests
uses: ./.github/workflows/ut.yml
pipeline_e2e:
name: Pipeline End-to-end Tests
uses: ./.github/workflows/pipeline_e2e.yml
source_e2e:
name: Source End-to-end Tests
uses: ./.github/workflows/source_e2e.yml
migration_test:
name: Migration Tests
uses: ./.github/workflows/migration_test.yml
Loading
Loading