Removed --arrow_home

apache · Nov 2, 2023 · 5856426 · 5856426
1 parent 0ac328f
commit 5856426
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 60 deletions.
diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml
@@ -62,7 +62,7 @@ jobs:
         run: |
           docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c '
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --build_tests=ON --build_examples=ON --build_benchmarks=ON'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_examples=ON --build_benchmarks=ON'
       - name: Run CPP unit test
         run: |
           docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/cpp/build && \
@@ -108,7 +108,7 @@ jobs:
         run: |
           docker exec ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c '
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox'
       - name: Build and run unit test for Spark 3.2.2(slow tests)
         run: |
           docker exec ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c '
@@ -146,7 +146,7 @@ jobs:
         run: |
           docker exec ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID bash -l -c '
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox'
       - name: Build and Run unit test for Spark 3.3.1(slow tests)
         run: |
           docker exec ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID bash -l -c 'cd /opt/gluten && \
@@ -183,7 +183,7 @@ jobs:
         run: |
           docker exec ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c '
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --build_examples=ON'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_examples=ON'
       - name: Build and Run unit test for Spark 3.3.1(other tests)
         run: |
           docker exec ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \
@@ -213,7 +213,7 @@ jobs:
         run: |
           docker exec ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID bash -l -c '
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox '
       - name: Build and Run unit test for Spark 3.4.1(slow tests)
         run: |
           docker exec ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID bash -l -c 'cd /opt/gluten && \
@@ -250,7 +250,7 @@ jobs:
         run: |
           docker exec ubuntu2004-test-spark34-$GITHUB_RUN_ID bash -c '
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --build_examples=ON'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_examples=ON'
       - name: Build and Run unit test for Spark 3.4.1(other tests)
         run: |
           docker exec ubuntu2004-test-spark34-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \
@@ -280,7 +280,7 @@ jobs:
         run: |
           docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c '
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --enable_hdfs=ON --enable_s3=ON'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON'
       - name: Build for Spark 3.2.2
         run: |
           docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c '
@@ -361,7 +361,7 @@ jobs:
           docker exec centos8-test-$GITHUB_RUN_ID bash -c '
           source /env.sh && \
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --enable_hdfs=ON'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON'
       - name: Build for Spark 3.2.2
         run: |
           docker exec centos8-test-$GITHUB_RUN_ID bash -c '
@@ -410,7 +410,7 @@ jobs:
           docker exec centos7-test-$GITHUB_RUN_ID bash -c '
           source /env.sh && \
           cd /opt/gluten/cpp && \
-          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep'
+          ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox'
       - name: Build for Spark 3.2.2
         run: |
           docker exec centos7-test-$GITHUB_RUN_ID bash -c '

diff --git a/cpp/compile.sh b/cpp/compile.sh
@@ -27,16 +27,11 @@ ENABLE_QAT=OFF
 ENABLE_HBM=OFF
 ENABLE_S3=OFF
 ENABLE_HDFS=OFF
-ARROW_HOME=
 VELOX_HOME=
 NPROC=$(nproc --ignore=2)
 
 for arg in "$@"; do
   case $arg in
-  --arrow_home=*)
-    ARROW_HOME=("${arg#*=}")
-    shift # Remove argument name from processing
-    ;;
   --velox_home=*)
     VELOX_HOME=("${arg#*=}")
     shift # Remove argument name from processing
@@ -97,23 +92,13 @@ CURRENT_DIR=$(
   pwd
 )
 
-#gluten cpp will find arrow/parquet lib from ARROW_HOME
-if [ "$ARROW_HOME" == "" ]; then
-  if [ "$BUILD_TYPE" == "Debug" ]; then
-    ARROW_HOME="$CURRENT_DIR/../ep/build-velox/build/velox_ep/_build/debug/third_party/arrow_ep"
-  else
-    ARROW_HOME="$CURRENT_DIR/../ep/build-velox/build/velox_ep/_build/release/third_party/arrow_ep"
-  fi
-fi
-
 #gluten cpp will find velox lib from VELOX_HOME
 if [ "$VELOX_HOME" == "" ]; then
   VELOX_HOME="$CURRENT_DIR/../ep/build-velox/build/velox_ep"
 fi
 
 echo "Building gluten cpp part..."
 echo "CMAKE Arguments:"
-echo "ARROW_HOME=${ARROW_HOME}"
 echo "VELOX_HOME=${VELOX_HOME}"
 echo "BUILD_TYPE=${BUILD_TYPE}"
 echo "BUILD_VELOX_BACKEND=${BUILD_VELOX_BACKEND}"
@@ -134,7 +119,6 @@ cd build
 cmake .. \
   -DBUILD_TESTS=${BUILD_TESTS} \
   -DBUILD_EXAMPLES=${BUILD_EXAMPLES} \
-  -DARROW_HOME=${ARROW_HOME} \
   -DBUILD_JEMALLOC=${BUILD_JEMALLOC} \
   -DBUILD_VELOX_BACKEND=${BUILD_VELOX_BACKEND} \
   -DVELOX_HOME=${VELOX_HOME} \

diff --git a/docs/get-started/GlutenUsage.md b/docs/get-started/GlutenUsage.md
@@ -8,37 +8,21 @@ parent: Getting-Started
 ## Parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh
 Please set them via `--`, e.g. `--build_type=Release`.
 
-| Parameters       | Description                                                 | Default value |
-|------------------|-------------------------------------------------------------|---------------|
-| build_type       | build type for arrow, velox & gluten cpp, CMAKE_BUILD_TYPE  | Release       |
-| build_tests      | build test code in cpp folder and arrow                     | OFF           |
-| build_benchmarks | build benchmark code in cpp folder and arrow                | OFF           |
-| build_jemalloc   | build with jemalloc                                         | ON            |
-| build_protobuf   | build protobuf lib                                          | ON            |
-| enable_qat       | enable QAT for shuffle data de/compression                  | OFF           |
-| enable_iaa       | enable IAA for shuffle data de/compression                  | OFF           |
-| enable_hbm       | enable HBM allocator                                        | OFF           |
-| enable_s3        | build with s3 lib                                           | OFF           |
-| enable_hdfs      | build with hdfs lib                                         | OFF           |
-| enable_ep_cache  | enable caching for external project build (Arrow and Velox) | OFF           |
-| skip_build_ep    | skip the build of external projects (arrow, velox)          | OFF           |
-| enable_vcpkg     | enable vcpkg for static build                               | OFF           |
-
-## Parameters for get_arrow.sh
-Please set them via `--`, e.g., `--enable_custom_codec=ON`.
-
-| Parameters | Description | Default value |
-| ---------- | ----------- | ------------- |
-| enable_custom_codec | Apply patch to plugin custom codec (used by QAT/IAA) in Arrow cpp IPC module. | OFF |
-
-## Parameters for build_arrow.sh
-Please set them via `--`, e.g., `--arrow_home=/YOUR/PATH`.
-
-| Parameters | Description | Default value |
-| ---------- | ----------- | ------------- |
-| arrow_home | Arrow build path                          | GLUTEN_DIR/ep/build-arrow/build|
-| build_type | ARROW build type, CMAKE_BUILD_TYPE        | Release|
-| build_tests | Build arrow with -DARROW_JSON=ON          | OFF           |
+| Parameters       | Description                                         | Default value |
+|------------------|-----------------------------------------------------|---------------|
+| build_type       | build type for velox & gluten cpp, CMAKE_BUILD_TYPE | Release       |
+| build_tests      | build test code in cpp folder                       | OFF           |
+| build_benchmarks | build benchmark code in cpp folder                  | OFF           |
+| build_jemalloc   | build with jemalloc                                 | ON            |
+| build_protobuf   | build protobuf lib                                  | ON            |
+| enable_qat       | enable QAT for shuffle data de/compression          | OFF           |
+| enable_iaa       | enable IAA for shuffle data de/compression          | OFF           |
+| enable_hbm       | enable HBM allocator                                | OFF           |
+| enable_s3        | build with s3 lib                                   | OFF           |
+| enable_hdfs      | build with hdfs lib                                 | OFF           |
+| enable_ep_cache  | enable caching for external project build (Velox)   | OFF           |
+| skip_build_ep    | skip the build of external projects (velox)         | OFF           |
+| enable_vcpkg     | enable vcpkg for static build                       | OFF           |
 
 ## Parameters for build_velox.sh
 Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`.

diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
@@ -4,7 +4,7 @@ title: Gluten with Velox Backend
 nav_order: 1
 parent: Getting-Started
 ---
-Currently, the mvn script can automatically fetch and build all dependency libraries incluing Velox and Arrow. Our nightly build still use Velox under oap-project. 
+Currently, the mvn script can automatically fetch and build all dependency libraries incluing Velox. Our nightly build still use Velox under oap-project. 
 
 # Prerequisite
 
@@ -61,7 +61,7 @@ cd /path/to/gluten
 ./dev/buildbundle-veloxbe.sh
 
 ## After a complete build, if you need to re-build the project and only some gluten code is changed,
-## you can use the following command to skip building arrow, velox and protobuf.
+## you can use the following command to skip building velox and protobuf.
 # ./dev/buildbundle-veloxbe.sh --skip_build_ep=ON --build_protobuf=OFF
 ```
 
@@ -86,12 +86,12 @@ Velox provides arrow/parquet lib. Gluten cpp module need a required VELOX_HOME p
 cd /path/to/gluten/ep/build-velox/src/
 ## you could use custom ep location by --velox_home=custom_path, make sure specify --velox_home in build_velox.sh too.
 ./get_velox.sh
-## make sure specify --arrow_home or --velox_home if you have specified it in get_xxx.sh.
+## make sure specify --velox_home if you have specified it in get_xxx.sh.
 ./build_velox.sh
 
 ## compile Gluten cpp module
 cd /path/to/gluten/cpp
-## if you use custom velox_home or arrow_home, make sure specified here by --arrow_home or --velox_home 
+## if you use custom velox_home, make sure specified here by --velox_home 
 ./compile.sh --build_velox_backend=ON
 
 ## compile Gluten java module and create package jar