diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml index 489d7d8c6918..c58c5a714e5f 100644 --- a/.github/workflows/velox_be.yml +++ b/.github/workflows/velox_be.yml @@ -42,350 +42,6 @@ concurrency: cancel-in-progress: true jobs: - ubuntu2004-test-spark32: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name ubuntu2004-test-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/ubuntu:20.04 \ - 'cd /opt/gluten && sleep 14400' - - name: Build Gluten velox third party - run: | - docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --build_tests=ON' - - name: Build Gluten CPP library - run: | - docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_tests=ON --build_examples=ON --build_benchmarks=ON' - - name: Run CPP unit test - run: | - docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/cpp/build && \ - ctest -V' - - name: Run HBM CPP unit test - run: | - docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/cpp/build && \ - cmake -DBUILD_TESTS=ON -DENABLE_HBM=ON .. && \ - cmake --build . --target hbw_allocator_test -- -j && \ - ctest -V -R TestHbw' - - name: Build and run unit test for Spark 3.2.2 (other tests) - run: | - docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \ - mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest' - # Cpp micro benchmarks will use generated files from unit test in backends-velox module. - - name: Run micro benchmarks - run: | - docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/cpp/build/velox/benchmarks && \ - ./generic_benchmark --with-shuffle --threads 1 --iterations 1' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop ubuntu2004-test-$GITHUB_RUN_ID || true - - ubuntu2004-test-spark32-slow: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name ubuntu2004-test-slow-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/ubuntu:20.04 \ - 'cd /opt/gluten && sleep 14400' - - name: Build Gluten velox third party - run: | - docker exec ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON' - - name: Build Gluten CPP library - run: | - docker exec ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox' - - name: Build and run unit test for Spark 3.2.2(slow tests) - run: | - docker exec ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 - run: | - docker exec ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop ubuntu2004-test-slow-$GITHUB_RUN_ID || true - - ubuntu2004-test-spark33-slow: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/ubuntu:20.04 \ - 'cd /opt/gluten && sleep 14400' - - name: Build Gluten velox third party - run: | - docker exec ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID bash -l -c ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON' - - name: Build Gluten CPP library - run: | - docker exec ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID bash -l -c ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox' - - name: Build and Run unit test for Spark 3.3.1(slow tests) - run: | - docker exec ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID bash -l -c 'cd /opt/gluten && \ - mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 - run: | - docker exec ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID bash -l -c 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.3 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID || true - - ubuntu2004-test-spark33: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name ubuntu2004-test-spark33-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/ubuntu:20.04 \ - 'cd /opt/gluten && sleep 14400' - - name: Build Gluten velox third party - run: | - docker exec ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON' - - name: Build Gluten CPP library - run: | - docker exec ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_examples=ON' - - name: Build and Run unit test for Spark 3.3.1(other tests) - run: | - docker exec ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \ - mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \ - mvn test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop ubuntu2004-test-spark33-$GITHUB_RUN_ID || true - - ubuntu2004-test-spark34-slow: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/ubuntu:20.04 \ - 'cd /opt/gluten && sleep 14400' - - name: Build Gluten velox third party - run: | - docker exec ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID bash -l -c ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON' - - name: Build Gluten CPP library - run: | - docker exec ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID bash -l -c ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox ' - - name: Build and Run unit test for Spark 3.4.1(slow tests) - run: | - docker exec ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID bash -l -c 'cd /opt/gluten && \ - mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.4 - run: | - docker exec ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID bash -l -c 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.4 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID || true - - ubuntu2004-test-spark34: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name ubuntu2004-test-spark34-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/ubuntu:20.04 \ - 'cd /opt/gluten && sleep 14400' - - name: Build Gluten velox third party - run: | - docker exec ubuntu2004-test-spark34-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON' - - name: Build Gluten CPP library - run: | - docker exec ubuntu2004-test-spark34-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_examples=ON' - - name: Build and Run unit test for Spark 3.4.1(other tests) - run: | - docker exec ubuntu2004-test-spark34-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \ - mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \ - mvn test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop ubuntu2004-test-spark34-$GITHUB_RUN_ID || true - - ubuntu2204-test: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name ubuntu2204-test-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/ubuntu:22.04 \ - 'cd /opt/gluten && sleep 14400' - - name: Build Gluten velox third party - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' - - name: Build Gluten CPP library - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' - - name: Build for Spark 3.2.2 - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=40g -s=10.0 --threads=32 --iterations=1' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 with Celeborn - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c \ - 'mv /opt/apache-celeborn-0.3.0-incubating-bin/conf/celeborn-env.sh.template /opt/apache-celeborn-0.3.0-incubating-bin/conf/celeborn-env.sh && \ - echo -e "CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g" > /opt/apache-celeborn-0.3.0-incubating-bin/conf/celeborn-env.sh && \ - echo -e "celeborn.worker.commitFiles.threads 128\nceleborn.worker.sortPartition.threads 64" > /opt/apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf \ - && bash /opt/apache-celeborn-0.3.0-incubating-bin/sbin/start-master.sh && bash /opt/apache-celeborn-0.3.0-incubating-bin/sbin/start-worker.sh && \ - cd /opt/gluten/tools/gluten-it && mvn clean install -Pspark-3.2,rss \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-celeborn --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-celeborn --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 && \ - bash /opt/apache-celeborn-0.3.0-incubating-bin/sbin/stop-worker.sh \ - && bash /opt/apache-celeborn-0.3.0-incubating-bin/sbin/stop-master.sh' - - name: Build for Spark 3.3.1 - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.3 - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.3 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=30g -s=10.0 --threads=32 --iterations=1' - - name: Build for Spark 3.4.1 - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.4 - run: | - docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.4 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=30g -s=10.0 --threads=32 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop ubuntu2204-test-$GITHUB_RUN_ID || true - - centos8-test: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name centos8-test-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/centos:8 \ - bash -c 'cd /opt/gluten && sleep 14400' - - name: Build Gluten velox third party - run: | - docker exec centos8-test-$GITHUB_RUN_ID bash -c ' - source /env.sh && \ - sudo yum -y install patch && \ - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' - - name: Build Gluten CPP library - run: | - docker exec centos8-test-$GITHUB_RUN_ID bash -c ' - source /env.sh && \ - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' - - name: Build for Spark 3.2.2 - run: | - docker exec centos8-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 - run: | - docker exec centos8-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=50g -s=30.0 --threads=32 --iterations=1' - - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 random kill tasks - run: | - docker exec centos8-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 --skip-data-gen --random-kill-tasks \ - && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh queries \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=50g -s=30.0 --threads=32 --iterations=1 --skip-data-gen --random-kill-tasks' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop centos8-test-$GITHUB_RUN_ID || true centos7-test: runs-on: velox-self-hosted @@ -393,21 +49,22 @@ jobs: - uses: actions/checkout@v4 - name: Setup docker container run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name centos7-test-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/centos:7 \ - bash -c 'cd /opt/gluten && sleep 14400' + EXTRA_DOCKER_OPTIONS="--name centos7-test-$GITHUB_RUN_ID -e NUM_THREADS=30 --detach" \ + NON_INTERACTIVE=ON \ + MOUNT_MAVEN_CACHE=OFF \ + OS_IMAGE=centos:7 \ + OS_VERSION=7 \ + tools/gluten-te/centos/cbash.sh sleep 14400 - name: Build Gluten velox third party run: | docker exec centos7-test-$GITHUB_RUN_ID bash -c ' - source /env.sh && \ - sudo yum -y install patch && \ + yum -y install epel-release centos-release-scl patch sudo && \ cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON' + ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON' - name: Build Gluten CPP library run: | docker exec centos7-test-$GITHUB_RUN_ID bash -c ' - source /env.sh && \ cd /opt/gluten/cpp && \ ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' - name: Build for Spark 3.2.2 @@ -474,59 +131,59 @@ jobs: run: | docker stop centos7-test-$GITHUB_RUN_ID || true - static-build-test: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/centos:7 \ - bash -c 'cd /opt/gluten && sleep 14400' - - name: Build Gluten CPP library - run: | - docker exec -i static-build-test-$GITHUB_RUN_ID bash -c ' - source /env.sh && \ - sudo yum -y install patch && \ - cd /opt/gluten && \ - sudo -E ./dev/vcpkg/setup-build-depends.sh && \ - source ./dev/vcpkg/env.sh && \ - ./dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON' - - name: Build for Spark 3.2.2 - run: | - docker exec static-build-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -DskipTests && \ - cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (centos 8) - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30 10.0.2.4:5000/gluten-dev/centos:8 \ - bash -c 'cd /opt/gluten/tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --threads=32 --iterations=1' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (ubuntu 20.04) - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30 10.0.2.4:5000/gluten-dev/ubuntu:20.04 \ - 'cd /opt/gluten/tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --threads=32 --iterations=1' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (ubuntu 22.04) - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30 10.0.2.4:5000/gluten-dev/ubuntu:22.04 \ - 'cd /opt/gluten/tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --threads=32 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop static-build-test-$GITHUB_RUN_ID || true + # static-build-test: + # runs-on: velox-self-hosted + # steps: + # - uses: actions/checkout@v4 + # - name: Setup docker container + # run: | + # docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ + # -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach 10.0.2.4:5000/gluten-dev/centos:7 \ + # bash -c 'cd /opt/gluten && sleep 14400' + # - name: Build Gluten CPP library + # run: | + # docker exec -i static-build-test-$GITHUB_RUN_ID bash -c ' + # source /env.sh && \ + # sudo yum -y install patch && \ + # cd /opt/gluten && \ + # sudo -E ./dev/vcpkg/setup-build-depends.sh && \ + # source ./dev/vcpkg/env.sh && \ + # ./dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON' + # - name: Build for Spark 3.2.2 + # run: | + # docker exec static-build-test-$GITHUB_RUN_ID bash -c ' + # cd /opt/gluten && \ + # mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -DskipTests && \ + # cd /opt/gluten/tools/gluten-it && \ + # mvn clean install -Pspark-3.2' + # - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (centos 8) + # run: | + # docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ + # -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30 10.0.2.4:5000/gluten-dev/centos:8 \ + # bash -c 'cd /opt/gluten/tools/gluten-it \ + # && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ + # --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ + # && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \ + # --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --threads=32 --iterations=1' + # - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (ubuntu 20.04) + # run: | + # docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ + # -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30 10.0.2.4:5000/gluten-dev/ubuntu:20.04 \ + # 'cd /opt/gluten/tools/gluten-it \ + # && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ + # --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ + # && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \ + # --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --threads=32 --iterations=1' + # - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (ubuntu 22.04) + # run: | + # docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ + # -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30 10.0.2.4:5000/gluten-dev/ubuntu:22.04 \ + # 'cd /opt/gluten/tools/gluten-it \ + # && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ + # --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ + # && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \ + # --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --threads=32 --iterations=1' + # - name: Exit docker container + # if: ${{ always() }} + # run: | + # docker stop static-build-test-$GITHUB_RUN_ID || true diff --git a/dev/vcpkg/init.sh b/dev/vcpkg/init.sh index 1b481ccb0883..d5f828e2022c 100755 --- a/dev/vcpkg/init.sh +++ b/dev/vcpkg/init.sh @@ -12,7 +12,7 @@ VCPKG_TRIPLET=x64-linux-avx cd "$SCRIPT_ROOT" if [ ! -d "$VCPKG_ROOT" ] || [ -z "$(ls "$VCPKG_ROOT")" ]; then - git clone https://github.com/microsoft/vcpkg.git --branch master "$VCPKG_ROOT" + git clone https://github.com/microsoft/vcpkg.git --branch 2023.10.19 "$VCPKG_ROOT" fi [ -f "$VCPKG" ] || "$VCPKG_ROOT/bootstrap-vcpkg.sh" -disableMetrics diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json index 09feb4041618..56b924f81df1 100644 --- a/dev/vcpkg/vcpkg.json +++ b/dev/vcpkg/vcpkg.json @@ -2,25 +2,8 @@ "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", "builtin-baseline": "a7b6122f6b6504d16d96117336a0562693579933", "dependencies": ["jemalloc"], - "default-features": ["arrow", "velox", "velox-s3", "velox-gcs", "velox-hdfs"], + "default-features": ["velox", "velox-s3", "velox-gcs", "velox-hdfs"], "features": { - "arrow": { - "description": "Apache Arrow", - "dependencies": [ - "libevent", - "libdwarf", - "openssl", - "orc", - "protobuf", - "rapidjson", - "snappy", - "xsimd", - "zlib", - "zstd", - "thrift", - "jemalloc" - ] - }, "velox": { "description": "Velox backend", "dependencies": [ diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index d805dcecd07c..e3425c4875c1 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -152,7 +152,7 @@ function process_setup_centos7 { sed -i '/^ run_and_time install_fmt/a \ \ run_and_time install_awssdk' scripts/setup-centos7.sh fi if [ $ENABLE_GCS == "ON" ]; then - sed -i '/^cmake_install_deps fmt/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos7.sh + sed -i '/^ run_and_time install_fmt/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos7.sh fi } diff --git a/tools/gluten-te/centos/dockerfile-buildenv b/tools/gluten-te/centos/dockerfile-buildenv index 67fab78942d3..3f36f8c91933 100755 --- a/tools/gluten-te/centos/dockerfile-buildenv +++ b/tools/gluten-te/centos/dockerfile-buildenv @@ -56,13 +56,13 @@ RUN wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven- rm apache-maven-3.8.8-bin.tar.gz && \ mv apache-maven-3.8.8 /usr/lib/maven -# Build & install Spark 3.2.2 -RUN cd /opt && git clone --depth 1 --branch v3.2.2 https://github.com/apache/spark.git spark322 -RUN cd /opt/spark322 && ./build/mvn -Pyarn -DskipTests clean install +# # Build & install Spark 3.2.2 +# RUN cd /opt && git clone --depth 1 --branch v3.2.2 https://github.com/apache/spark.git spark322 +# RUN cd /opt/spark322 && ./build/mvn -Pyarn -DskipTests clean install -# Build & install Spark 3.3.1 -RUN cd /opt && git clone --depth 1 --branch v3.3.1 https://github.com/apache/spark.git spark331 -RUN cd /opt/spark331 && ./build/mvn -Pyarn -DskipTests clean install +# # Build & install Spark 3.3.1 +# RUN cd /opt && git clone --depth 1 --branch v3.3.1 https://github.com/apache/spark.git spark331 +# RUN cd /opt/spark331 && ./build/mvn -Pyarn -DskipTests clean install ENV PATH="$PATH:/usr/lib/maven/bin" ENV LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:/lib64:/lib