From d60d4458b9d38d0eff565266cb1c2bd02510d4a9 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 25 Jul 2024 11:08:21 +0800 Subject: [PATCH] [VL] Move setup-centos7.sh & setup-centos8.sh into Gluten and clean up some script code (#6559) --- dev/build_helper_functions.sh | 42 ++++- dev/builddeps-veloxbe.sh | 28 ++- ep/build-velox/src/get_velox.sh | 191 +++---------------- ep/build-velox/src/setup-centos7.sh | 283 ++++++++++++++++++++++++++++ ep/build-velox/src/setup-centos8.sh | 250 ++++++++++++++++++++++++ 5 files changed, 623 insertions(+), 171 deletions(-) create mode 100755 ep/build-velox/src/setup-centos7.sh create mode 100755 ep/build-velox/src/setup-centos8.sh diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh index 92ae5c29a884..b69a36c29b1c 100644 --- a/dev/build_helper_functions.sh +++ b/dev/build_helper_functions.sh @@ -81,6 +81,28 @@ function get_cxx_flags { } +function github_checkout { + local REPO=$1 + shift + local VERSION=$1 + shift + local GIT_CLONE_PARAMS=$@ + local DIRNAME=$(basename $REPO) + SUDO="${SUDO:-""}" + cd "${DEPENDENCY_DIR}" + if [ -z "${DIRNAME}" ]; then + echo "Failed to get repo name from ${REPO}" + exit 1 + fi + if [ -d "${DIRNAME}" ] && prompt "${DIRNAME} already exists. Delete?"; then + ${SUDO} rm -rf "${DIRNAME}" + fi + if [ ! -d "${DIRNAME}" ]; then + git clone -q -b $VERSION $GIT_CLONE_PARAMS "https://github.com/${REPO}.git" + fi + cd "${DIRNAME}" +} + function wget_and_untar { local URL=$1 local DIR=$2 @@ -130,15 +152,17 @@ function setup_macos { function setup_linux { local LINUX_DISTRIBUTION=$(. /etc/os-release && echo ${ID}) local LINUX_VERSION_ID=$(. /etc/os-release && echo ${VERSION_ID}) + CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) + GLUTEN_VELOX_SCRIPT_HOME=$CURRENT_DIR/../ep/build-velox/src if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian" || "$LINUX_DISTRIBUTION" == "pop" ]]; then scripts/setup-ubuntu.sh elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then case "$LINUX_VERSION_ID" in 9) scripts/setup-centos9.sh ;; - 8) scripts/setup-centos8.sh ;; + 8) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;; 7) - scripts/setup-centos7.sh + $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh set +u export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH source /opt/rh/devtoolset-9/enable @@ -152,13 +176,13 @@ function setup_linux { elif [[ "$LINUX_DISTRIBUTION" == "alinux" ]]; then case "${LINUX_VERSION_ID:0:1}" in 2) - scripts/setup-centos7.sh + $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh set +u export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH source /opt/rh/devtoolset-9/enable set -u ;; - 3) scripts/setup-centos8.sh ;; + 3) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;; *) echo "Unsupported alinux version: $LINUX_VERSION_ID" exit 1 @@ -167,13 +191,13 @@ function setup_linux { elif [[ "$LINUX_DISTRIBUTION" == "tencentos" ]]; then case "$LINUX_VERSION_ID" in 2.4) - scripts/setup-centos7.sh + $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh set +u export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH source /opt/rh/devtoolset-9/enable set -u ;; - 3.2) scripts/setup-centos8.sh ;; + 3.2) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;; *) echo "Unsupported tencentos version: $LINUX_VERSION_ID" exit 1 @@ -184,3 +208,9 @@ function setup_linux { exit 1 fi } + +function install_libhdfs3 { + github_checkout oap-project/libhdfs3 master + cmake_install +} + diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 38f3978efe47..82c9cfc8d6c0 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -223,7 +223,7 @@ function build_velox_backend { ( cd $GLUTEN_DIR/ep/build-velox/src - ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER + ./get_velox.sh $VELOX_PARAMETER ) if [ "$VELOX_HOME" == "" ]; then @@ -232,6 +232,9 @@ fi OS=`uname -s` ARCH=`uname -m` +DEPENDENCY_DIR=${DEPENDENCY_DIR:-$CURRENT_DIR/../ep/_ep} +mkdir -p ${DEPENDENCY_DIR} + source $GLUTEN_DIR/dev/build_helper_functions.sh if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then echo "Start to install dependencies" @@ -244,6 +247,29 @@ if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then echo "Unsupported kernel: $OS" exit 1 fi + if [ $ENABLE_S3 == "ON" ]; then + if [ $OS == 'Darwin' ]; then + echo "S3 is not supported on MacOS." + exit 1 + fi + ${VELOX_HOME}/scripts/setup-adapters.sh aws + fi + if [ $ENABLE_HDFS == "ON" ]; then + if [ $OS == 'Darwin' ]; then + echo "HDFS is not supported on MacOS." + exit 1 + fi + pushd $VELOX_HOME + install_libhdfs3 + popd + fi + if [ $ENABLE_GCS == "ON" ]; then + ${VELOX_HOME}/scripts/setup-adapters.sh gcs + fi + if [ $ENABLE_ABFS == "ON" ]; then + export AZURE_SDK_DISABLE_AUTO_VCPKG=ON + ${VELOX_HOME}/scripts/setup-adapters.sh abfs + fi popd fi diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 30e13926b11b..164bf88a4cb1 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -16,20 +16,10 @@ set -exu - VELOX_REPO=https://github.com/oap-project/velox.git VELOX_BRANCH=2024_07_24 VELOX_HOME="" -#Set on run gluten on HDFS -ENABLE_HDFS=OFF -#Set on run gluten on S3 -ENABLE_S3=OFF -#Set on run gluten on GCS -ENABLE_GCS=OFF -#Set on run gluten on ABFS -ENABLE_ABFS=OFF - OS=`uname -s` for arg in "$@"; do @@ -46,22 +36,6 @@ for arg in "$@"; do VELOX_HOME=("${arg#*=}") shift # Remove argument name from processing ;; - --enable_hdfs=*) - ENABLE_HDFS=("${arg#*=}") - shift # Remove argument name from processing - ;; - --enable_s3=*) - ENABLE_S3=("${arg#*=}") - shift # Remove argument name from processing - ;; - --enable_gcs=*) - ENABLE_GCS=("${arg#*=}") - shift # Remove argument name from processing - ;; - --enable_abfs=*) - ENABLE_ABFS=("${arg#*=}") - shift # Remove argument name from processing - ;; *) OTHER_ARGUMENTS+=("$1") shift # Remove generic argument from processing @@ -112,24 +86,9 @@ function process_setup_ubuntu { sed -i '/^function install_folly.*/i function install_protobuf {\n wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh ensure_pattern_matched ' run_and_time install_folly' scripts/setup-ubuntu.sh sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh - if [ $ENABLE_HDFS == "ON" ]; then - sed -i '/^function install_folly.*/i function install_libhdfs3 {\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-ubuntu.sh - ensure_pattern_matched ' run_and_time install_protobuf' scripts/setup-ubuntu.sh - sed -i '/^ run_and_time install_protobuf/a \ \ run_and_time install_libhdfs3' scripts/setup-ubuntu.sh - ensure_pattern_matched 'ccache ' scripts/setup-ubuntu.sh - sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh - fi - ensure_pattern_matched 'apt install -y' scripts/setup-adapters.sh - sed -i "s/apt install -y/sudo apt install -y/" scripts/setup-adapters.sh - if [ $ENABLE_S3 == "ON" ]; then - sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh - fi - if [ $ENABLE_GCS == "ON" ]; then - sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-ubuntu.sh - fi - if [ $ENABLE_ABFS == "ON" ]; then - sed -i '/^ run_and_time install_folly/a \ \ export AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-ubuntu.sh - fi + # Required by lib hdfs. + ensure_pattern_matched 'ccache ' scripts/setup-ubuntu.sh + sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh ensure_pattern_matched 'run_and_time install_conda' scripts/setup-ubuntu.sh sed -i '/run_and_time install_conda/d' scripts/setup-ubuntu.sh # Just depends on Gluten to install arrow libs since Gluten will apply some patches to Arrow source and uses different build options. @@ -158,113 +117,28 @@ function process_setup_centos9 { sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos9.sh sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh - if [ $ENABLE_HDFS == "ON" ]; then - sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n cmake_install\n}\n' scripts/setup-centos9.sh - sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_libhdfs3' scripts/setup-centos9.sh - sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos9.sh - fi - sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh - if [ $ENABLE_S3 == "ON" ]; then - sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos9.sh - fi - if [ $ENABLE_GCS == "ON" ]; then - sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos9.sh - fi - if [ $ENABLE_ABFS == "ON" ]; then - sed -i '/^ run_and_time install_fbthrift/a \ \ export AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-centos9.sh - fi -} - - -function process_setup_centos8 { - # Allows other version of git already installed. - if [ -z "$(which git)" ]; then - dnf install -y -q --setopt=install_weak_deps=False git - fi - # make this function Reentrant - git checkout scripts/setup-centos8.sh - # need set BUILD_SHARED_LIBS flag for thrift - sed -i "/cd fbthrift/{n;s/cmake_install -Denable_tests=OFF/cmake_install -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF/;}" scripts/setup-centos8.sh - # No need to re-install git. - sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-9 git/dnf_install ninja-build cmake curl ccache gcc-toolset-9/' scripts/setup-centos8.sh - sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}' scripts/setup-centos8.sh - sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos8.sh - sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos8.sh - sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos8.sh - sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos8.sh - - if [ $ENABLE_HDFS == "ON" ]; then - sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n cmake_install\n}\n' scripts/setup-centos8.sh - sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_libhdfs3' scripts/setup-centos8.sh - sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos8.sh - fi - - sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh - if [ $ENABLE_S3 == "ON" ]; then - sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos8.sh - fi - if [ $ENABLE_GCS == "ON" ]; then - sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos8.sh - fi - if [ $ENABLE_ABFS == "ON" ]; then - sed -i '/^ run_and_time install_fbthrift/a \ \ export AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-centos8.sh - fi -} - -function process_setup_centos7 { - # Allows other version of git already installed. - if [ -z "$(which git)" ]; then - dnf install -y -q --setopt=install_weak_deps=False git - fi - # make this function Reentrant - git checkout scripts/setup-centos7.sh - - # No need to re-install git. - sed -i 's/dnf_install ccache git/dnf_install ccache/' scripts/setup-centos7.sh - - # install gtest - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_gtest' scripts/setup-centos7.sh - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-centos7.sh - sed -i 's/https:\/\/cmake.org\/files\/v3.25\/cmake-3.25.1.tar.gz/https:\/\/cmake.org\/files\/v3.28\/cmake-3.28.3.tar.gz/' scripts/setup-centos7.sh - if [ $ENABLE_HDFS = "ON" ]; then - sed -i '/^function install_protobuf.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-centos7.sh - sed -i '/^ run_and_time install_protobuf/a \ \ run_and_time install_libhdfs3' scripts/setup-centos7.sh - sed -i '/^dnf_install ccache/a\ \ yasm \\' scripts/setup-centos7.sh - fi - sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh - if [ $ENABLE_S3 == "ON" ]; then - sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos7.sh - fi - if [ $ENABLE_GCS == "ON" ]; then - sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos7.sh - fi - if [ $ENABLE_ABFS == "ON" ]; then - sed -i '/^function cmake_install_deps.*/i function install_curl {\n cd "${DEPENDENCY_DIR}" \n github_checkout curl/curl curl-7_68_0 \n cmake -DCMAKE_BUILD_TYPE=Release -DOPENSSL_USE_STATIC_LIBS=TRUE -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON && cmake --build . && sudo cmake --install . \n}\n' scripts/setup-centos7.sh - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_curl' scripts/setup-centos7.sh - sed -i '/^ run_and_time install_curl/a \ \ export AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-centos7.sh - fi + # Required by lib hdfs. + ensure_pattern_matched 'dnf_install ninja-build' scripts/setup-centos9.sh + sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos9.sh } function process_setup_alinux3 { - process_setup_centos8 - sed -i "s/.*dnf_install epel-release/#&/" scripts/setup-centos8.sh - sed -i "s/.*run_and_time install_conda/#&/" scripts/setup-centos8.sh - sed -i "s/.*dnf config-manager --set-enabled powertools/#&/" scripts/setup-centos8.sh - sed -i "s/gcc-toolset-9 //" scripts/setup-centos8.sh - sed -i "s/.*source \/opt\/rh\/gcc-toolset-9\/enable/#&/" scripts/setup-centos8.sh - sed -i 's|^export CC=/opt/rh/gcc-toolset-9/root/bin/gcc|# &|' scripts/setup-centos8.sh - sed -i 's|^export CXX=/opt/rh/gcc-toolset-9/root/bin/g++|# &|' scripts/setup-centos8.sh - sed -i 's/python39 python39-devel python39-pip //g' scripts/setup-centos8.sh - sed -i "s/.*pip.* install/#&/" scripts/setup-centos8.sh + sed -i "s/.*dnf_install epel-release/#&/" ${CURRENT_DIR}/setup-centos8.sh + sed -i "s/.*run_and_time install_conda/#&/" ${CURRENT_DIR}/setup-centos8.sh + sed -i "s/.*dnf config-manager --set-enabled powertools/#&/" ${CURRENT_DIR}/setup-centos8.sh + sed -i "s/gcc-toolset-9 //" ${CURRENT_DIR}/setup-centos8.sh + sed -i "s/.*source \/opt\/rh\/gcc-toolset-9\/enable/#&/" ${CURRENT_DIR}/setup-centos8.sh + sed -i 's|^export CC=/opt/rh/gcc-toolset-9/root/bin/gcc|# &|' ${CURRENT_DIR}/setup-centos8.sh + sed -i 's|^export CXX=/opt/rh/gcc-toolset-9/root/bin/g++|# &|' ${CURRENT_DIR}/setup-centos8.sh + sed -i 's/python39 python39-devel python39-pip //g' ${CURRENT_DIR}/setup-centos8.sh + sed -i "s/.*pip.* install/#&/" ${CURRENT_DIR}/setup-centos8.sh } function process_setup_tencentos32 { - process_setup_centos8 - sed -i "s/.*dnf config-manager --set-enabled powertools/#&/" scripts/setup-centos8.sh + sed -i "s/.*dnf config-manager --set-enabled powertools/#&/" ${CURRENT_DIR}/setup-centos8.sh } echo "Preparing Velox source code..." -echo "ENABLE_HDFS=${ENABLE_HDFS}" CURRENT_DIR=$( cd "$(dirname "$BASH_SOURCE")" @@ -327,44 +201,33 @@ function setup_linux { elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then case "$LINUX_VERSION_ID" in 9) process_setup_centos9 ;; - 8) process_setup_centos8 ;; - 7) process_setup_centos7 ;; + 8) ;; + 7) ;; *) - echo "Unsupport centos version: $LINUX_VERSION_ID" + echo "Unsupported centos version: $LINUX_VERSION_ID" exit 1 ;; esac elif [[ "$LINUX_DISTRIBUTION" == "alinux" ]]; then case "${LINUX_VERSION_ID:0:1}" in - 2) process_setup_centos7 ;; + 2) ;; 3) process_setup_alinux3 ;; *) - echo "Unsupport alinux version: $LINUX_VERSION_ID" + echo "Unsupported alinux version: $LINUX_VERSION_ID" exit 1 ;; esac elif [[ "$LINUX_DISTRIBUTION" == "tencentos" ]]; then case "$LINUX_VERSION_ID" in - 2.4) process_setup_centos7 ;; + 2.4) ;; 3.2) process_setup_tencentos32 ;; *) - echo "Unsupport tencentos version: $LINUX_VERSION_ID" + echo "Unsupported tencentos version: $LINUX_VERSION_ID" exit 1 ;; esac else - echo "Unsupport linux distribution: $LINUX_DISTRIBUTION" - exit 1 - fi -} - -function setup_macos { - if [ $ENABLE_HDFS == "ON" ]; then - echo "Unsupport hdfs" - exit 1 - fi - if [ $ENABLE_S3 == "ON" ]; then - echo "Unsupport s3" + echo "Unsupported linux distribution: $LINUX_DISTRIBUTION" exit 1 fi } @@ -372,12 +235,12 @@ function setup_macos { if [ $OS == 'Linux' ]; then setup_linux elif [ $OS == 'Darwin' ]; then - setup_macos + : else - echo "Unsupport kernel: $OS" + echo "Unsupported kernel: $OS" exit 1 fi apply_compilation_fixes $CURRENT_DIR $VELOX_SOURCE_DIR -echo "Velox-get finished." +echo "Finished getting Velox code" diff --git a/ep/build-velox/src/setup-centos7.sh b/ep/build-velox/src/setup-centos7.sh new file mode 100755 index 000000000000..6f76112ae6a4 --- /dev/null +++ b/ep/build-velox/src/setup-centos7.sh @@ -0,0 +1,283 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -efx -o pipefail +# Some of the packages must be build with the same compiler flags +# so that some low level types are the same size. Also, disable warnings. +SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")/../build/velox_ep/scripts +source $SCRIPTDIR/setup-helper-functions.sh +DEPENDENCY_DIR=${DEPENDENCY_DIR:-/tmp/velox-deps} +CPU_TARGET="${CPU_TARGET:-avx}" +NPROC=$(getconf _NPROCESSORS_ONLN) +FMT_VERSION=10.1.1 +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. +export CXXFLAGS=$CFLAGS # Used by boost. +export CPPFLAGS=$CFLAGS # Used by LZO. +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH +FB_OS_VERSION=v2024.02.26.00 + +# shellcheck disable=SC2037 +SUDO="sudo -E" + +function run_and_time { + time "$@" + { echo "+ Finished running $*"; } 2> /dev/null +} + +function dnf_install { + $SUDO dnf install -y -q --setopt=install_weak_deps=False "$@" +} + +function yum_install { + $SUDO yum install -y "$@" +} + +function wget_and_untar { + local URL=$1 + local DIR=$2 + mkdir -p "${DIR}" + wget -q --max-redirect 3 -O - "${URL}" | tar -xz -C "${DIR}" --strip-components=1 +} + +function install_cmake { + cd "${DEPENDENCY_DIR}" + wget_and_untar https://cmake.org/files/v3.28/cmake-3.28.3.tar.gz cmake-3 + cd cmake-3 + ./bootstrap --prefix=/usr/local + make -j$(nproc) + $SUDO make install + cmake --version +} + +function install_ninja { + cd "${DEPENDENCY_DIR}" + github_checkout ninja-build/ninja v1.11.1 + ./configure.py --bootstrap + cmake -Bbuild-cmake + cmake --build build-cmake + $SUDO cp ninja /usr/local/bin/ +} + +function install_folly { + cd "${DEPENDENCY_DIR}" + github_checkout facebook/folly "${FB_OS_VERSION}" + cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON +} + +function install_conda { + cd "${DEPENDENCY_DIR}" + mkdir -p conda && cd conda + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + MINICONDA_PATH=/opt/miniconda-for-velox + bash Miniconda3-latest-Linux-x86_64.sh -b -u $MINICONDA_PATH +} + +function install_openssl { + cd "${DEPENDENCY_DIR}" + wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl + cd openssl + ./config no-shared + make depend + make + $SUDO make install +} + +function install_gflags { + cd "${DEPENDENCY_DIR}" + wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags + cd gflags + cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local +} + +function install_glog { + cd "${DEPENDENCY_DIR}" + wget_and_untar https://github.com/google/glog/archive/v0.5.0.tar.gz glog + cd glog + cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr/local +} + +function install_snappy { + cd "${DEPENDENCY_DIR}" + wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy + cd snappy + cmake_install -DSNAPPY_BUILD_TESTS=OFF +} + +function install_dwarf { + cd "${DEPENDENCY_DIR}" + wget_and_untar https://github.com/davea42/libdwarf-code/archive/refs/tags/20210528.tar.gz dwarf + cd dwarf + #local URL=https://github.com/davea42/libdwarf-code/releases/download/v0.5.0/libdwarf-0.5.0.tar.xz + #local DIR=dwarf + #mkdir -p "${DIR}" + #wget -q --max-redirect 3 "${URL}" + #tar -xf libdwarf-0.5.0.tar.xz -C "${DIR}" + #cd dwarf/libdwarf-0.5.0 + ./configure --enable-shared=no + make + make check + $SUDO make install +} + +function install_re2 { + cd "${DEPENDENCY_DIR}" + wget_and_untar https://github.com/google/re2/archive/refs/tags/2023-03-01.tar.gz re2 + cd re2 + $SUDO make install +} + +function install_flex { + cd "${DEPENDENCY_DIR}" + wget_and_untar https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz flex + cd flex + ./autogen.sh + ./configure + $SUDO make install +} + +function install_lzo { + cd "${DEPENDENCY_DIR}" + wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo + cd lzo + ./configure --prefix=/usr/local --enable-shared --disable-static --docdir=/usr/local/share/doc/lzo-2.10 + make "-j$(nproc)" + $SUDO make install +} + +function install_boost { + # Remove old version. + sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_* /opt/rh/devtoolset-9/root/usr/lib64/dyninst/libboost_* + sudo rm -rf /tmp/velox-deps/boost/ /usr/local/include/boost/ /usr/local/lib/cmake/Boost-1.72.0/ + cd "${DEPENDENCY_DIR}" + wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost + cd boost + ./bootstrap.sh --prefix=/usr/local --with-python=/usr/bin/python3 --with-python-root=/usr/lib/python3.6 --without-libraries=python + $SUDO ./b2 "-j$(nproc)" -d0 install threading=multi +} + +function install_libhdfs3 { + cd "${DEPENDENCY_DIR}" + github_checkout apache/hawq master + cd depends/libhdfs3 + sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt + sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake + sed -i "s/dfs.domain.socket.path\", \"\"/dfs.domain.socket.path\", \"\/var\/lib\/hadoop-hdfs\/dn_socket\"/g" src/common/SessionConfig.cpp + sed -i "s/pos < endOfCurBlock/pos \< endOfCurBlock \&\& pos \- cursor \<\= 128 \* 1024/g" src/client/InputStreamImpl.cpp + cmake_install +} + +function install_protobuf { + cd "${DEPENDENCY_DIR}" + wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz + tar -xzf protobuf-all-21.4.tar.gz + cd protobuf-21.4 + ./configure CXXFLAGS="-fPIC" --prefix=/usr/local + make "-j$(nproc)" + $SUDO make install +} + +function install_gtest { + cd "${DEPENDENCY_DIR}" + wget https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz + tar -xzf release-1.12.1.tar.gz + cd googletest-release-1.12.1 + mkdir -p build && cd build && cmake -DBUILD_GTEST=ON -DBUILD_GMOCK=ON -DINSTALL_GTEST=ON -DINSTALL_GMOCK=ON -DBUILD_SHARED_LIBS=ON .. + make "-j$(nproc)" + $SUDO make install +} + +function install_fmt { + rm -rf /usr/local/lib64/libfmt.a + rm -rf /usr/local/lib64/cmake/fmt + rm -rf /usr/local/include/fmt + rm -rf fmt + wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt + cmake_install fmt -DFMT_TEST=OFF +} + +function install_duckdb { + if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb + ( + cd duckdb + cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release + ) + fi +} + +function install_prerequisites { + run_and_time install_lzo + run_and_time install_boost + run_and_time install_re2 + run_and_time install_flex + run_and_time install_openssl + run_and_time install_gflags + run_and_time install_glog + run_and_time install_snappy + run_and_time install_dwarf +} + +function install_velox_deps { + run_and_time install_fmt + run_and_time install_folly + run_and_time install_protobuf + run_and_time install_gtest + run_and_time install_conda + run_and_time install_duckdb +} + +$SUDO dnf makecache + +# dnf install dependency libraries +dnf_install epel-release dnf-plugins-core # For ccache, ninja +# PowerTools only works on CentOS8 +# dnf config-manager --set-enabled powertools +dnf_install ccache wget which libevent-devel \ + yasm \ + openssl-devel libzstd-devel lz4-devel double-conversion-devel \ + curl-devel libxml2-devel libgsasl-devel libuuid-devel patch + +$SUDO dnf remove -y gflags + +# Required for Thrift +dnf_install autoconf automake libtool bison python3 python3-devel + +# Required for build flex +dnf_install gettext-devel texinfo help2man + +# dnf_install conda + +# Activate gcc9; enable errors on unset variables afterwards. +# GCC9 install via yum and devtoolset +# dnf install gcc-toolset-9 only works on CentOS8 + +$SUDO yum makecache +yum_install centos-release-scl +yum_install devtoolset-9 +source /opt/rh/devtoolset-9/enable || exit 1 +gcc --version +set -u + +# Build from source +[ -d "$DEPENDENCY_DIR" ] || mkdir -p "$DEPENDENCY_DIR" + +run_and_time install_cmake +run_and_time install_ninja + +install_prerequisites +install_velox_deps diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh new file mode 100755 index 000000000000..c8078893ae68 --- /dev/null +++ b/ep/build-velox/src/setup-centos8.sh @@ -0,0 +1,250 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script documents setting up a Centos8 host for Velox +# development. Running it should make you ready to compile. +# +# Environment variables: +# * INSTALL_PREREQUISITES="N": Skip installation of packages for build. +# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts. +# Use "n" to never wipe directories. +# +# You can also run individual functions below by specifying them as arguments: +# $ ./setup-centos8.sh install_googletest install_fmt +# + +set -efx -o pipefail +# Some of the packages must be build with the same compiler flags +# so that some low level types are the same size. Also, disable warnings. +SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")/../build/velox_ep/scripts +source $SCRIPTDIR/setup-helper-functions.sh +CPU_TARGET="${CPU_TARGET:-avx}" +NPROC=$(getconf _NPROCESSORS_ONLN) +export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. +export CXXFLAGS=$CFLAGS # Used by boost. +export CPPFLAGS=$CFLAGS # Used by LZO. +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +export CC=/opt/rh/gcc-toolset-9/root/bin/gcc +export CXX=/opt/rh/gcc-toolset-9/root/bin/g++ + +DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} +function dnf_install { + dnf install -y -q --setopt=install_weak_deps=False "$@" +} + +# Install packages required for build. +function install_build_prerequisites { + dnf update -y + dnf_install epel-release dnf-plugins-core # For ccache, ninja + dnf config-manager --set-enabled powertools + dnf update -y + dnf_install ninja-build curl ccache gcc-toolset-9 git wget which + dnf_install yasm + dnf_install autoconf automake python39 python39-devel python39-pip libtool + pip3.9 install cmake==3.28.3 +} + +# Install dependencies from the package managers. +function install_velox_deps_from_dnf { + dnf_install libevent-devel \ + openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ + libdwarf-devel curl-devel libicu-devel bison flex libsodium-devel + + # install sphinx for doc gen + pip3.9 install sphinx sphinx-tabs breathe sphinx_rtd_theme +} + +function install_conda { + dnf_install conda +} + +function install_openssl { + wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl + cd openssl + ./config no-shared && make depend && make && sudo make install + cd .. +} + +function install_gflags { + # Remove an older version if present. + dnf remove -y gflags + wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags + ( + cd gflags + cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 + ) +} + +function install_glog { + wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog + ( + cd glog + cmake_install -DBUILD_SHARED_LIBS=ON + ) +} + +function install_lzo { + wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo + ( + cd lzo + ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10 + make "-j$(nproc)" + make install + ) +} + +function install_boost { + wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost + ( + cd boost + ./bootstrap.sh --prefix=/usr/local + ./b2 "-j$(nproc)" -d0 install threading=multi --without-python + ) +} + +function install_snappy { + wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy + ( + cd snappy + cmake_install -DSNAPPY_BUILD_TESTS=OFF + ) +} + +function install_fmt { + wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt + ( + cd fmt + cmake_install -DFMT_TEST=OFF + ) +} + +function install_protobuf { + wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protobuf-all-21.8.tar.gz protobuf + ( + cd protobuf + ./configure CXXFLAGS="-fPIC" --prefix=/usr/local + make "-j${NPROC}" + make install + ldconfig + ) +} + +FB_OS_VERSION="v2024.05.20.00" + +function install_fizz { + wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz + ( + cd fizz/fizz + cmake_install -DBUILD_TESTS=OFF + ) +} + +function install_folly { + wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly + ( + cd folly + cmake_install -DFOLLY_HAVE_INT128_T=ON + ) +} + +function install_wangle { + wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle + ( + cd wangle/wangle + cmake_install -DBUILD_TESTS=OFF + ) +} + +function install_fbthrift { + wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift + ( + cd fbthrift + cmake_install -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + ) +} + +function install_mvfst { + wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst + ( + cd mvfst + cmake_install -DBUILD_TESTS=OFF + ) +} + +function install_duckdb { + if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb + ( + cd duckdb + cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release + ) + fi +} + +function install_cuda { + # See https://developer.nvidia.com/cuda-downloads + dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo + yum install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-devel-$(echo $1 | tr '.' '-') +} + +function install_velox_deps { + run_and_time install_velox_deps_from_dnf + run_and_time install_conda + run_and_time install_gflags + run_and_time install_glog + run_and_time install_lzo + run_and_time install_snappy + run_and_time install_boost + run_and_time install_protobuf + run_and_time install_fmt + run_and_time install_folly + run_and_time install_fizz + run_and_time install_wangle + run_and_time install_mvfst + run_and_time install_fbthrift + run_and_time install_openssl + run_and_time install_duckdb +} + +(return 2> /dev/null) && return # If script was sourced, don't run commands. + +( + if [[ $# -ne 0 ]]; then + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u + for cmd in "$@"; do + run_and_time "${cmd}" + done + echo "All specified dependencies installed!" + else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u + install_velox_deps + echo "All dependencies for Velox installed!" + dnf clean all + fi +) +