From fc0b9fa6ac74a719e29c206c05b042e2bc747b7d Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Tue, 25 Jun 2024 00:45:08 +0000 Subject: [PATCH] Revert "Remove setup-centos8.sh (#10249)" This reverts commit 82a12e1653964b927be88085a87a0bb643c7bf47. --- scripts/setup-centos8.sh | 239 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100755 scripts/setup-centos8.sh diff --git a/scripts/setup-centos8.sh b/scripts/setup-centos8.sh new file mode 100755 index 000000000000..93dfef052dc1 --- /dev/null +++ b/scripts/setup-centos8.sh @@ -0,0 +1,239 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script documents setting up a Centos8 host for Velox +# development. Running it should make you ready to compile. +# +# Environment variables: +# * INSTALL_PREREQUISITES="N": Skip installation of packages for build. +# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts. +# Use "n" to never wipe directories. +# +# You can also run individual functions below by specifying them as arguments: +# $ scripts/setup-centos8.sh install_googletest install_fmt +# + +set -efx -o pipefail +# Some of the packages must be build with the same compiler flags +# so that some low level types are the same size. Also, disable warnings. +SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") +source $SCRIPTDIR/setup-helper-functions.sh +CPU_TARGET="${CPU_TARGET:-avx}" +NPROC=$(getconf _NPROCESSORS_ONLN) +export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. +export CXXFLAGS=$CFLAGS # Used by boost. +export CPPFLAGS=$CFLAGS # Used by LZO. +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +export CC=/opt/rh/gcc-toolset-9/root/bin/gcc +export CXX=/opt/rh/gcc-toolset-9/root/bin/g++ + +function dnf_install { + dnf install -y -q --setopt=install_weak_deps=False "$@" +} + +# Install packages required for build. +function install_build_prerequisites { + dnf update -y + dnf_install epel-release dnf-plugins-core # For ccache, ninja + dnf config-manager --set-enabled powertools + dnf update -y + dnf_install ninja-build curl ccache gcc-toolset-9 git wget which + dnf_install autoconf automake python39 python39-devel python39-pip libtool + pip3.9 install cmake==3.28.3 +} + +# Install dependencies from the package managers. +function install_velox_deps_from_dnf { + dnf_install libevent-devel \ + openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ + libdwarf-devel curl-devel libicu-devel bison flex libsodium-devel + + # install sphinx for doc gen + pip3.9 install sphinx sphinx-tabs breathe sphinx_rtd_theme +} + +function install_conda { + dnf_install conda +} + +function install_gflags { + # Remove an older version if present. + dnf remove -y gflags + wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags + ( + cd gflags + cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 + ) +} + +function install_glog { + wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog + ( + cd glog + cmake_install -DBUILD_SHARED_LIBS=ON + ) +} + +function install_lzo { + wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo + ( + cd lzo + ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10 + make "-j$(nproc)" + make install + ) +} + +function install_boost { + wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost + ( + cd boost + ./bootstrap.sh --prefix=/usr/local + ./b2 "-j$(nproc)" -d0 install threading=multi --without-python + ) +} + +function install_snappy { + wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy + ( + cd snappy + cmake_install -DSNAPPY_BUILD_TESTS=OFF + ) +} + +function install_fmt { + wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt + ( + cd fmt + cmake_install -DFMT_TEST=OFF + ) +} + +function install_protobuf { + wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protobuf-all-21.8.tar.gz protobuf + ( + cd protobuf + ./configure --prefix=/usr + make "-j${NPROC}" + make install + ldconfig + ) +} + +FB_OS_VERSION="v2024.05.20.00" + +function install_fizz { + wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz + ( + cd fizz/fizz + cmake_install -DBUILD_TESTS=OFF + ) +} + +function install_folly { + wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly + ( + cd folly + cmake_install -DFOLLY_HAVE_INT128_T=ON + ) +} + +function install_wangle { + wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle + ( + cd wangle/wangle + cmake_install -DBUILD_TESTS=OFF + ) +} + +function install_fbthrift { + wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift + ( + cd fbthrift + cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + ) +} + +function install_mvfst { + wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst + ( + cd mvfst + cmake_install -DBUILD_TESTS=OFF + ) +} + +function install_duckdb { + if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb + ( + cd duckdb + cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release + ) + fi +} + +function install_cuda { + # See https://developer.nvidia.com/cuda-downloads + dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo + yum install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-devel-$(echo $1 | tr '.' '-') +} + +function install_velox_deps { + run_and_time install_velox_deps_from_dnf + run_and_time install_conda + run_and_time install_gflags + run_and_time install_glog + run_and_time install_lzo + run_and_time install_snappy + run_and_time install_boost + run_and_time install_protobuf + run_and_time install_fmt + run_and_time install_folly + run_and_time install_fizz + run_and_time install_wangle + run_and_time install_mvfst + run_and_time install_fbthrift + run_and_time install_duckdb +} + +(return 2> /dev/null) && return # If script was sourced, don't run commands. + +( + if [[ $# -ne 0 ]]; then + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u + for cmd in "$@"; do + run_and_time "${cmd}" + done + echo "All specified dependencies installed!" + else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u + install_velox_deps + echo "All dependencies for Velox installed!" + dnf clean all + fi +) +