From 5856426d61cb6c5d19736e6075a486d437791cbd Mon Sep 17 00:00:00 2001 From: Surbhi Vijayvargeeya Date: Thu, 2 Nov 2023 16:09:46 +0530 Subject: [PATCH] Removed --arrow_home --- .github/workflows/velox_be.yml | 18 ++++++------- cpp/compile.sh | 16 ------------ docs/get-started/GlutenUsage.md | 46 +++++++++++---------------------- docs/get-started/Velox.md | 8 +++--- 4 files changed, 28 insertions(+), 60 deletions(-) diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml index e8862f1bfb646..ffc41319e0256 100644 --- a/.github/workflows/velox_be.yml +++ b/.github/workflows/velox_be.yml @@ -62,7 +62,7 @@ jobs: run: | docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c ' cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --build_tests=ON --build_examples=ON --build_benchmarks=ON' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_examples=ON --build_benchmarks=ON' - name: Run CPP unit test run: | docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/cpp/build && \ @@ -108,7 +108,7 @@ jobs: run: | docker exec ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c ' cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox' - name: Build and run unit test for Spark 3.2.2(slow tests) run: | docker exec ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c ' @@ -146,7 +146,7 @@ jobs: run: | docker exec ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID bash -l -c ' cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox' - name: Build and Run unit test for Spark 3.3.1(slow tests) run: | docker exec ubuntu2004-test-spark33-slow-$GITHUB_RUN_ID bash -l -c 'cd /opt/gluten && \ @@ -183,7 +183,7 @@ jobs: run: | docker exec ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c ' cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --build_examples=ON' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_examples=ON' - name: Build and Run unit test for Spark 3.3.1(other tests) run: | docker exec ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \ @@ -213,7 +213,7 @@ jobs: run: | docker exec ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID bash -l -c ' cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox ' - name: Build and Run unit test for Spark 3.4.1(slow tests) run: | docker exec ubuntu2004-test-spark34-slow-$GITHUB_RUN_ID bash -l -c 'cd /opt/gluten && \ @@ -250,7 +250,7 @@ jobs: run: | docker exec ubuntu2004-test-spark34-$GITHUB_RUN_ID bash -c ' cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --build_examples=ON' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --build_examples=ON' - name: Build and Run unit test for Spark 3.4.1(other tests) run: | docker exec ubuntu2004-test-spark34-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \ @@ -280,7 +280,7 @@ jobs: run: | docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --enable_hdfs=ON --enable_s3=ON' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON' - name: Build for Spark 3.2.2 run: | docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' @@ -361,7 +361,7 @@ jobs: docker exec centos8-test-$GITHUB_RUN_ID bash -c ' source /env.sh && \ cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep --enable_hdfs=ON' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON' - name: Build for Spark 3.2.2 run: | docker exec centos8-test-$GITHUB_RUN_ID bash -c ' @@ -410,7 +410,7 @@ jobs: docker exec centos7-test-$GITHUB_RUN_ID bash -c ' source /env.sh && \ cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --arrow_home=/opt/velox/_build/release/third_party/arrow_ep' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox' - name: Build for Spark 3.2.2 run: | docker exec centos7-test-$GITHUB_RUN_ID bash -c ' diff --git a/cpp/compile.sh b/cpp/compile.sh index e3733bdf755a9..4f2ec01150d5e 100755 --- a/cpp/compile.sh +++ b/cpp/compile.sh @@ -27,16 +27,11 @@ ENABLE_QAT=OFF ENABLE_HBM=OFF ENABLE_S3=OFF ENABLE_HDFS=OFF -ARROW_HOME= VELOX_HOME= NPROC=$(nproc --ignore=2) for arg in "$@"; do case $arg in - --arrow_home=*) - ARROW_HOME=("${arg#*=}") - shift # Remove argument name from processing - ;; --velox_home=*) VELOX_HOME=("${arg#*=}") shift # Remove argument name from processing @@ -97,15 +92,6 @@ CURRENT_DIR=$( pwd ) -#gluten cpp will find arrow/parquet lib from ARROW_HOME -if [ "$ARROW_HOME" == "" ]; then - if [ "$BUILD_TYPE" == "Debug" ]; then - ARROW_HOME="$CURRENT_DIR/../ep/build-velox/build/velox_ep/_build/debug/third_party/arrow_ep" - else - ARROW_HOME="$CURRENT_DIR/../ep/build-velox/build/velox_ep/_build/release/third_party/arrow_ep" - fi -fi - #gluten cpp will find velox lib from VELOX_HOME if [ "$VELOX_HOME" == "" ]; then VELOX_HOME="$CURRENT_DIR/../ep/build-velox/build/velox_ep" @@ -113,7 +99,6 @@ fi echo "Building gluten cpp part..." echo "CMAKE Arguments:" -echo "ARROW_HOME=${ARROW_HOME}" echo "VELOX_HOME=${VELOX_HOME}" echo "BUILD_TYPE=${BUILD_TYPE}" echo "BUILD_VELOX_BACKEND=${BUILD_VELOX_BACKEND}" @@ -134,7 +119,6 @@ cd build cmake .. \ -DBUILD_TESTS=${BUILD_TESTS} \ -DBUILD_EXAMPLES=${BUILD_EXAMPLES} \ - -DARROW_HOME=${ARROW_HOME} \ -DBUILD_JEMALLOC=${BUILD_JEMALLOC} \ -DBUILD_VELOX_BACKEND=${BUILD_VELOX_BACKEND} \ -DVELOX_HOME=${VELOX_HOME} \ diff --git a/docs/get-started/GlutenUsage.md b/docs/get-started/GlutenUsage.md index 46247c6e9c3bf..2b78d2b84530f 100644 --- a/docs/get-started/GlutenUsage.md +++ b/docs/get-started/GlutenUsage.md @@ -8,37 +8,21 @@ parent: Getting-Started ## Parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh Please set them via `--`, e.g. `--build_type=Release`. -| Parameters | Description | Default value | -|------------------|-------------------------------------------------------------|---------------| -| build_type | build type for arrow, velox & gluten cpp, CMAKE_BUILD_TYPE | Release | -| build_tests | build test code in cpp folder and arrow | OFF | -| build_benchmarks | build benchmark code in cpp folder and arrow | OFF | -| build_jemalloc | build with jemalloc | ON | -| build_protobuf | build protobuf lib | ON | -| enable_qat | enable QAT for shuffle data de/compression | OFF | -| enable_iaa | enable IAA for shuffle data de/compression | OFF | -| enable_hbm | enable HBM allocator | OFF | -| enable_s3 | build with s3 lib | OFF | -| enable_hdfs | build with hdfs lib | OFF | -| enable_ep_cache | enable caching for external project build (Arrow and Velox) | OFF | -| skip_build_ep | skip the build of external projects (arrow, velox) | OFF | -| enable_vcpkg | enable vcpkg for static build | OFF | - -## Parameters for get_arrow.sh -Please set them via `--`, e.g., `--enable_custom_codec=ON`. - -| Parameters | Description | Default value | -| ---------- | ----------- | ------------- | -| enable_custom_codec | Apply patch to plugin custom codec (used by QAT/IAA) in Arrow cpp IPC module. | OFF | - -## Parameters for build_arrow.sh -Please set them via `--`, e.g., `--arrow_home=/YOUR/PATH`. - -| Parameters | Description | Default value | -| ---------- | ----------- | ------------- | -| arrow_home | Arrow build path | GLUTEN_DIR/ep/build-arrow/build| -| build_type | ARROW build type, CMAKE_BUILD_TYPE | Release| -| build_tests | Build arrow with -DARROW_JSON=ON | OFF | +| Parameters | Description | Default value | +|------------------|-----------------------------------------------------|---------------| +| build_type | build type for velox & gluten cpp, CMAKE_BUILD_TYPE | Release | +| build_tests | build test code in cpp folder | OFF | +| build_benchmarks | build benchmark code in cpp folder | OFF | +| build_jemalloc | build with jemalloc | ON | +| build_protobuf | build protobuf lib | ON | +| enable_qat | enable QAT for shuffle data de/compression | OFF | +| enable_iaa | enable IAA for shuffle data de/compression | OFF | +| enable_hbm | enable HBM allocator | OFF | +| enable_s3 | build with s3 lib | OFF | +| enable_hdfs | build with hdfs lib | OFF | +| enable_ep_cache | enable caching for external project build (Velox) | OFF | +| skip_build_ep | skip the build of external projects (velox) | OFF | +| enable_vcpkg | enable vcpkg for static build | OFF | ## Parameters for build_velox.sh Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`. diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md index 57b9f0638f2af..4f654bb44611b 100644 --- a/docs/get-started/Velox.md +++ b/docs/get-started/Velox.md @@ -4,7 +4,7 @@ title: Gluten with Velox Backend nav_order: 1 parent: Getting-Started --- -Currently, the mvn script can automatically fetch and build all dependency libraries incluing Velox and Arrow. Our nightly build still use Velox under oap-project. +Currently, the mvn script can automatically fetch and build all dependency libraries incluing Velox. Our nightly build still use Velox under oap-project. # Prerequisite @@ -61,7 +61,7 @@ cd /path/to/gluten ./dev/buildbundle-veloxbe.sh ## After a complete build, if you need to re-build the project and only some gluten code is changed, -## you can use the following command to skip building arrow, velox and protobuf. +## you can use the following command to skip building velox and protobuf. # ./dev/buildbundle-veloxbe.sh --skip_build_ep=ON --build_protobuf=OFF ``` @@ -86,12 +86,12 @@ Velox provides arrow/parquet lib. Gluten cpp module need a required VELOX_HOME p cd /path/to/gluten/ep/build-velox/src/ ## you could use custom ep location by --velox_home=custom_path, make sure specify --velox_home in build_velox.sh too. ./get_velox.sh -## make sure specify --arrow_home or --velox_home if you have specified it in get_xxx.sh. +## make sure specify --velox_home if you have specified it in get_xxx.sh. ./build_velox.sh ## compile Gluten cpp module cd /path/to/gluten/cpp -## if you use custom velox_home or arrow_home, make sure specified here by --arrow_home or --velox_home +## if you use custom velox_home, make sure specified here by --velox_home ./compile.sh --build_velox_backend=ON ## compile Gluten java module and create package jar