From 24d949e617103a98e339ecbedc1c73c35a48a513 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 10 May 2024 04:20:00 +0800 Subject: [PATCH] [VL] Daily Update Velox Version (2024_05_09) (#5664) Upstream Velox's New Commits: f53687563 by zjuwangg, Add more test method in Filter::AlwaysFalse class (9718) 48cfbe3e7 by Zac Wen, Prevent duplicates in writable regions (9746) 87ea7d5fb by Daniel Munoz, Create howMuchToSkip method in UnitLoaderTools (9706) 16e2c1375 by Daniel Munoz, UnitLoader supports initial rowsToSkip (9703) dec2556b8 by Ubuntu, Add barrier at kernel end and scatterBits (9745) 20be1936b by PHILO-HE, Register some re-usable Presto functions for Spark (9425) 798a12736 by Jimmy Lu, Fix crash in ArrayVectorBase::copyRangesImpl in case target offsets or sizes is nullptr (9725) bc6632ac9 by Karteekmurthys, Add decimal support for min_by and max_by functions (8723) 5c4903fe2 by Zac Wen, Handle ssd cache region score overflow (9709) 49b124665 by Sergey Pershin, Improve instrumentation around Zombie Task detection. (9663) --- .github/workflows/velox_docker_cache.yml | 68 +++++++++++++++++++ .../gluten/execution/TestOperator.scala | 2 +- .../Substrait2VeloxPlanConversionTest.cc | 6 +- ep/build-velox/src/get_velox.sh | 2 +- 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 969053657b1a..d79b5e717154 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -51,3 +51,71 @@ jobs: with: path: ./cpp/build/releases/ key: cache-velox-build-${{ hashFiles('./cache-key') }} + + ccache-native-lib-ubuntu-velox-ut: + runs-on: ubuntu-20.04 + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx + steps: + - uses: actions/checkout@v2 + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-ubuntu-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' + - name: Build Gluten velox third party + run: | + rm -rf /opt/miniconda-for-velox/ + cd ep/build-velox/src && \ + ./get_velox.sh + cd ../build/velox_ep/ + git reset --hard + make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" + + - name: CCache after + run: | + ccache -vs + + - uses: actions/cache/save@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-ubuntu-release-default + ccache-native-lib-centos-velox-ut: + runs-on: ubuntu-20.04 + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + container: ghcr.io/facebookincubator/velox-dev:circleci-avx + steps: + - uses: actions/checkout@v2 + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' + - name: Build Gluten velox third party + run: | + rm -rf /opt/miniconda-for-velox/ + cd ep/build-velox/src && \ + ./get_velox.sh + cd ../build/velox_ep/ + source /opt/rh/gcc-toolset-9/enable + git reset --hard + make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" + + - name: CCache after + run: | + ccache -vs + + - uses: actions/cache/save@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default \ No newline at end of file diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index a14a5b7e78de..920b8e2bde9f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -630,7 +630,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } assert(wholeStageTransformers.size == 3) val nativePlanString = wholeStageTransformers.head.nativePlanString() - assert(nativePlanString.contains("Aggregation[SINGLE")) + assert(nativePlanString.contains("Aggregation[1][SINGLE")) assert(nativePlanString.contains("ValueStream")) assert(wholeStageTransformers(1).nativePlanString().contains("ValueStream")) assert(wholeStageTransformers.last.nativePlanString().contains("TableScan")) diff --git a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc index e64331610f9e..841514261859 100644 --- a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc +++ b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc @@ -258,8 +258,8 @@ TEST_F(Substrait2VeloxPlanConversionTest, ifthenTest) { // Convert to Velox PlanNode. auto planNode = planConverter_->toVeloxPlan(substraitPlan, std::vector<::substrait::ReadRel_LocalFiles>{split}); ASSERT_EQ( - "-- Project[expressions: ] -> \n " - "-- TableScan[table: hive_table, range filters: [(hd_demo_sk, Filter(IsNotNull, deterministic, null not allowed))," + "-- Project[1][expressions: ] -> \n " + "-- TableScan[0][table: hive_table, range filters: [(hd_demo_sk, Filter(IsNotNull, deterministic, null not allowed))," " (hd_vehicle_count, BigintRange: [1, 9223372036854775807] no nulls)], remaining filter: " "(and(or(equalto(\"hd_buy_potential\",\">10000\"),equalto(\"hd_buy_potential\",\"unknown\"))," "if(greaterthan(\"hd_vehicle_count\",0),greaterthan(divide(cast \"hd_dep_count\" as DOUBLE," @@ -279,7 +279,7 @@ TEST_F(Substrait2VeloxPlanConversionTest, filterUpper) { // Convert to Velox PlanNode. auto planNode = planConverter_->toVeloxPlan(substraitPlan, std::vector<::substrait::ReadRel_LocalFiles>{split}); ASSERT_EQ( - "-- Project[expressions: ] -> \n -- TableScan[table: hive_table, range filters: " + "-- Project[1][expressions: ] -> \n -- TableScan[0][table: hive_table, range filters: " "[(key, BigintRange: [-2147483648, 2] no nulls)]] -> n0_0:INTEGER\n", planNode->toString(true, true)); } diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 5ef78bb32958..a3013dd49087 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_08 +VELOX_BRANCH=2024_05_09 VELOX_HOME="" #Set on run gluten on HDFS