[VL] Move setup-centos7.sh & setup-centos8.sh into Gluten and clean u…

…p some script code (#6559)
apache · Jul 25, 2024 · d60d445 · d60d445
1 parent da2b60c
commit d60d445
Show file tree

Hide file tree

Showing 5 changed files with 623 additions and 171 deletions.
diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh
@@ -81,6 +81,28 @@ function get_cxx_flags {
 
 }
 
+function github_checkout {
+  local REPO=$1
+  shift
+  local VERSION=$1
+  shift
+  local GIT_CLONE_PARAMS=$@
+  local DIRNAME=$(basename $REPO)
+  SUDO="${SUDO:-""}"
+  cd "${DEPENDENCY_DIR}"
+  if [ -z "${DIRNAME}" ]; then
+    echo "Failed to get repo name from ${REPO}"
+    exit 1
+  fi
+  if [ -d "${DIRNAME}" ] && prompt "${DIRNAME} already exists. Delete?"; then
+    ${SUDO} rm -rf "${DIRNAME}"
+  fi
+  if [ ! -d "${DIRNAME}" ]; then
+    git clone -q -b $VERSION $GIT_CLONE_PARAMS "https://github.com/${REPO}.git"
+  fi
+  cd "${DIRNAME}"
+}
+
 function wget_and_untar {
   local URL=$1
   local DIR=$2
@@ -130,15 +152,17 @@ function setup_macos {
 function setup_linux {
   local LINUX_DISTRIBUTION=$(. /etc/os-release && echo ${ID})
   local LINUX_VERSION_ID=$(. /etc/os-release && echo ${VERSION_ID})
+  CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+  GLUTEN_VELOX_SCRIPT_HOME=$CURRENT_DIR/../ep/build-velox/src
 
   if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian" || "$LINUX_DISTRIBUTION" == "pop" ]]; then
     scripts/setup-ubuntu.sh
   elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then
     case "$LINUX_VERSION_ID" in
     9) scripts/setup-centos9.sh ;;
-    8) scripts/setup-centos8.sh ;;
+    8) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
     7)
-      scripts/setup-centos7.sh
+      $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
       set +u
       export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
       source /opt/rh/devtoolset-9/enable
@@ -152,13 +176,13 @@ function setup_linux {
   elif [[ "$LINUX_DISTRIBUTION" == "alinux" ]]; then
     case "${LINUX_VERSION_ID:0:1}" in
     2)
-      scripts/setup-centos7.sh
+      $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
       set +u
       export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
       source /opt/rh/devtoolset-9/enable
       set -u
       ;;
-    3) scripts/setup-centos8.sh ;;
+    3) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
     *)
       echo "Unsupported alinux version: $LINUX_VERSION_ID"
       exit 1
@@ -167,13 +191,13 @@ function setup_linux {
   elif [[ "$LINUX_DISTRIBUTION" == "tencentos" ]]; then
     case "$LINUX_VERSION_ID" in
     2.4)
-        scripts/setup-centos7.sh
+        $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
         set +u
         export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
         source /opt/rh/devtoolset-9/enable
         set -u
         ;;
-    3.2) scripts/setup-centos8.sh ;;
+    3.2) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
     *)
       echo "Unsupported tencentos version: $LINUX_VERSION_ID"
       exit 1
@@ -184,3 +208,9 @@ function setup_linux {
     exit 1
   fi
 }
+
+function install_libhdfs3 {
+  github_checkout oap-project/libhdfs3 master
+  cmake_install
+}
+
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
@@ -223,7 +223,7 @@ function build_velox_backend {
 
 (
   cd $GLUTEN_DIR/ep/build-velox/src
-  ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER
+  ./get_velox.sh $VELOX_PARAMETER
 )
 
 if [ "$VELOX_HOME" == "" ]; then
@@ -232,6 +232,9 @@ fi
 
 OS=`uname -s`
 ARCH=`uname -m`
+DEPENDENCY_DIR=${DEPENDENCY_DIR:-$CURRENT_DIR/../ep/_ep}
+mkdir -p ${DEPENDENCY_DIR}
+
 source $GLUTEN_DIR/dev/build_helper_functions.sh
 if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then
   echo "Start to install dependencies"
@@ -244,6 +247,29 @@ if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then
     echo "Unsupported kernel: $OS"
     exit 1
   fi
+  if [ $ENABLE_S3 == "ON" ]; then
+    if [ $OS == 'Darwin' ]; then
+      echo "S3 is not supported on MacOS."
+      exit 1
+    fi
+    ${VELOX_HOME}/scripts/setup-adapters.sh aws
+  fi
+  if [ $ENABLE_HDFS == "ON" ]; then
+    if [ $OS == 'Darwin' ]; then
+      echo "HDFS is not supported on MacOS."
+      exit 1
+    fi
+    pushd $VELOX_HOME
+    install_libhdfs3
+    popd
+  fi
+  if [ $ENABLE_GCS == "ON" ]; then
+    ${VELOX_HOME}/scripts/setup-adapters.sh gcs
+  fi
+  if [ $ENABLE_ABFS == "ON" ]; then
+    export AZURE_SDK_DISABLE_AUTO_VCPKG=ON
+    ${VELOX_HOME}/scripts/setup-adapters.sh abfs
+  fi
   popd
 fi