Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[VL] Move setup-centos7.sh & setup-centos8.sh into Gluten and clean up some script code #6559

Merged
merged 2 commits into from
Jul 25, 2024
Merged

[VL] Move setup-centos7.sh & setup-centos8.sh into Gluten and clean up some script code #6559

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
96706ba
Initial
PHILO-HE Jul 23, 2024
59b1b7c
Revert change for testing
PHILO-HE Jul 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 36 additions & 6 deletions dev/build_helper_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,28 @@ function get_cxx_flags {

}

function github_checkout {
local REPO=$1
shift
local VERSION=$1
shift
local GIT_CLONE_PARAMS=$@
local DIRNAME=$(basename $REPO)
SUDO="${SUDO:-""}"
cd "${DEPENDENCY_DIR}"
if [ -z "${DIRNAME}" ]; then
echo "Failed to get repo name from ${REPO}"
exit 1
fi
if [ -d "${DIRNAME}" ] && prompt "${DIRNAME} already exists. Delete?"; then
${SUDO} rm -rf "${DIRNAME}"
fi
if [ ! -d "${DIRNAME}" ]; then
git clone -q -b $VERSION $GIT_CLONE_PARAMS "https://github.com/${REPO}.git"
fi
cd "${DIRNAME}"
}

function wget_and_untar {
local URL=$1
local DIR=$2
Expand Down Expand Up @@ -130,15 +152,17 @@ function setup_macos {
function setup_linux {
local LINUX_DISTRIBUTION=$(. /etc/os-release && echo ${ID})
local LINUX_VERSION_ID=$(. /etc/os-release && echo ${VERSION_ID})
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
GLUTEN_VELOX_SCRIPT_HOME=$CURRENT_DIR/../ep/build-velox/src

if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian" || "$LINUX_DISTRIBUTION" == "pop" ]]; then
scripts/setup-ubuntu.sh
elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then
case "$LINUX_VERSION_ID" in
9) scripts/setup-centos9.sh ;;
8) scripts/setup-centos8.sh ;;
8) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
7)
scripts/setup-centos7.sh
$GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
set +u
export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
source /opt/rh/devtoolset-9/enable
Expand All @@ -152,13 +176,13 @@ function setup_linux {
elif [[ "$LINUX_DISTRIBUTION" == "alinux" ]]; then
case "${LINUX_VERSION_ID:0:1}" in
2)
scripts/setup-centos7.sh
$GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
set +u
export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
source /opt/rh/devtoolset-9/enable
set -u
;;
3) scripts/setup-centos8.sh ;;
3) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
*)
echo "Unsupported alinux version: $LINUX_VERSION_ID"
exit 1
Expand All @@ -167,13 +191,13 @@ function setup_linux {
elif [[ "$LINUX_DISTRIBUTION" == "tencentos" ]]; then
case "$LINUX_VERSION_ID" in
2.4)
scripts/setup-centos7.sh
$GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
set +u
export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
source /opt/rh/devtoolset-9/enable
set -u
;;
3.2) scripts/setup-centos8.sh ;;
3.2) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
*)
echo "Unsupported tencentos version: $LINUX_VERSION_ID"
exit 1
Expand All @@ -184,3 +208,9 @@ function setup_linux {
exit 1
fi
}

function install_libhdfs3 {
github_checkout oap-project/libhdfs3 master
cmake_install
}

28 changes: 27 additions & 1 deletion dev/builddeps-veloxbe.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ function build_velox_backend {

(
cd $GLUTEN_DIR/ep/build-velox/src
./get_velox.sh --enable_hdfs=$ENABLE_HDFS --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER
./get_velox.sh $VELOX_PARAMETER
)

if [ "$VELOX_HOME" == "" ]; then
Expand All @@ -232,6 +232,9 @@ fi

OS=`uname -s`
ARCH=`uname -m`
DEPENDENCY_DIR=${DEPENDENCY_DIR:-$CURRENT_DIR/../ep/_ep}
mkdir -p ${DEPENDENCY_DIR}

source $GLUTEN_DIR/dev/build_helper_functions.sh
if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then
echo "Start to install dependencies"
Expand All @@ -244,6 +247,29 @@ if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then
echo "Unsupported kernel: $OS"
exit 1
fi
if [ $ENABLE_S3 == "ON" ]; then
if [ $OS == 'Darwin' ]; then
echo "S3 is not supported on MacOS."
exit 1
fi
${VELOX_HOME}/scripts/setup-adapters.sh aws
fi
if [ $ENABLE_HDFS == "ON" ]; then
if [ $OS == 'Darwin' ]; then
echo "HDFS is not supported on MacOS."
exit 1
fi
pushd $VELOX_HOME
install_libhdfs3
popd
fi
if [ $ENABLE_GCS == "ON" ]; then
${VELOX_HOME}/scripts/setup-adapters.sh gcs
fi
if [ $ENABLE_ABFS == "ON" ]; then
export AZURE_SDK_DISABLE_AUTO_VCPKG=ON
${VELOX_HOME}/scripts/setup-adapters.sh abfs
fi
popd
fi

Expand Down
Loading
Loading