[VL] Daily Update Velox Version (2024_05_09) (#5664)

Upstream Velox's New Commits: f53687563 by zjuwangg, Add more test method in Filter::AlwaysFalse class (9718) 48cfbe3e7 by Zac Wen, Prevent duplicates in writable regions (9746) 87ea7d5fb by Daniel Munoz, Create howMuchToSkip method in UnitLoaderTools (9706) 16e2c1375 by Daniel Munoz, UnitLoader supports initial rowsToSkip (9703) dec2556b8 by Ubuntu, Add barrier at kernel end and scatterBits (9745) 20be1936b by PHILO-HE, Register some re-usable Presto functions for Spark (9425) 798a12736 by Jimmy Lu, Fix crash in ArrayVectorBase::copyRangesImpl in case target offsets or sizes is nullptr (9725) bc6632ac9 by Karteekmurthys, Add decimal support for min_by and max_by functions (8723) 5c4903fe2 by Zac Wen, Handle ssd cache region score overflow (9709) 49b124665 by Sergey Pershin, Improve instrumentation around Zombie Task detection. (9663)
apache · May 9, 2024 · 24d949e · 24d949e
1 parent 9496509
commit 24d949e
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 5 deletions.
diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml
@@ -51,3 +51,71 @@ jobs:
         with:
           path: ./cpp/build/releases/
           key: cache-velox-build-${{ hashFiles('./cache-key') }}
+
+  ccache-native-lib-ubuntu-velox-ut:
+    runs-on: ubuntu-20.04
+    env:
+      CCACHE_DIR: "${{ github.workspace }}/.ccache"
+    container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get Ccache
+        uses: actions/cache/restore@v3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-ubuntu-release-default
+      - name: Ensure Cache Dirs Exists
+        working-directory: ${{ github.workspace }}
+        run: |
+          mkdir -p '${{ env.CCACHE_DIR }}'
+      - name: Build Gluten velox third party
+        run: |
+          rm -rf /opt/miniconda-for-velox/
+          cd ep/build-velox/src && \
+          ./get_velox.sh
+          cd ../build/velox_ep/
+          git reset --hard
+          make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" 
+
+      - name: CCache after
+        run: |
+          ccache -vs
+
+      - uses: actions/cache/save@v3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-ubuntu-release-default
+  ccache-native-lib-centos-velox-ut:
+    runs-on: ubuntu-20.04
+    env:
+      CCACHE_DIR: "${{ github.workspace }}/.ccache"
+    container: ghcr.io/facebookincubator/velox-dev:circleci-avx
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get Ccache
+        uses: actions/cache/restore@v3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-centos-release-default
+      - name: Ensure Cache Dirs Exists
+        working-directory: ${{ github.workspace }}
+        run: |
+          mkdir -p '${{ env.CCACHE_DIR }}'
+      - name: Build Gluten velox third party
+        run: |
+          rm -rf /opt/miniconda-for-velox/
+          cd ep/build-velox/src && \
+          ./get_velox.sh
+          cd ../build/velox_ep/
+          source /opt/rh/gcc-toolset-9/enable
+          git reset --hard
+          make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" 
+
+      - name: CCache after
+        run: |
+          ccache -vs
+
+      - uses: actions/cache/save@v3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-centos-release-default
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
@@ -630,7 +630,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite {
         }
         assert(wholeStageTransformers.size == 3)
         val nativePlanString = wholeStageTransformers.head.nativePlanString()
-        assert(nativePlanString.contains("Aggregation[SINGLE"))
+        assert(nativePlanString.contains("Aggregation[1][SINGLE"))
         assert(nativePlanString.contains("ValueStream"))
         assert(wholeStageTransformers(1).nativePlanString().contains("ValueStream"))
         assert(wholeStageTransformers.last.nativePlanString().contains("TableScan"))

diff --git a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc
@@ -258,8 +258,8 @@ TEST_F(Substrait2VeloxPlanConversionTest, ifthenTest) {
   // Convert to Velox PlanNode.
   auto planNode = planConverter_->toVeloxPlan(substraitPlan, std::vector<::substrait::ReadRel_LocalFiles>{split});
   ASSERT_EQ(
-      "-- Project[expressions: ] -> \n  "
-      "-- TableScan[table: hive_table, range filters: [(hd_demo_sk, Filter(IsNotNull, deterministic, null not allowed)),"
+      "-- Project[1][expressions: ] -> \n  "
+      "-- TableScan[0][table: hive_table, range filters: [(hd_demo_sk, Filter(IsNotNull, deterministic, null not allowed)),"
       " (hd_vehicle_count, BigintRange: [1, 9223372036854775807] no nulls)], remaining filter: "
       "(and(or(equalto(\"hd_buy_potential\",\">10000\"),equalto(\"hd_buy_potential\",\"unknown\")),"
       "if(greaterthan(\"hd_vehicle_count\",0),greaterthan(divide(cast \"hd_dep_count\" as DOUBLE,"
@@ -279,7 +279,7 @@ TEST_F(Substrait2VeloxPlanConversionTest, filterUpper) {
   // Convert to Velox PlanNode.
   auto planNode = planConverter_->toVeloxPlan(substraitPlan, std::vector<::substrait::ReadRel_LocalFiles>{split});
   ASSERT_EQ(
-      "-- Project[expressions: ] -> \n  -- TableScan[table: hive_table, range filters: "
+      "-- Project[1][expressions: ] -> \n  -- TableScan[0][table: hive_table, range filters: "
       "[(key, BigintRange: [-2147483648, 2] no nulls)]] -> n0_0:INTEGER\n",
       planNode->toString(true, true));
 }

diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
 set -exu
 
 VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_05_08
+VELOX_BRANCH=2024_05_09
 VELOX_HOME=""
 
 #Set on run gluten on HDFS