Skip to content

Commit

Permalink
Update the MacOS/Linux build scripts that build/install protobuf from…
Browse files Browse the repository at this point in the history
… source (microsoft#16906)

### Description
1. As a follow-up of microsoft#16761, this PR allows build ORT on iOS/Android
without the need to explicitly specify a protoc path. microsoft#16761 is for
WASM. This one is for iOS/Android
2. Update the MacOS/Linux build scripts that build/install protobuf from
source. Make them be more flexible. Add the support for
RedHatEnterprise(ubi), which will needed for upgrading the base image
from centos:7 to ubi:8.
3. Update tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile :
the docker file's base image has preinstalled protobuf in /usr/local, we
should uninstall them to avoid conflicts.
  • Loading branch information
snnn authored Jul 31, 2023
1 parent 28a099f commit 73ddba9
Show file tree
Hide file tree
Showing 11 changed files with 177 additions and 82 deletions.
2 changes: 1 addition & 1 deletion cmake/external/onnxruntime_external_deps.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ if(CMAKE_CROSSCOMPILING AND NOT ONNX_CUSTOM_PROTOC_EXECUTABLE)
set(ONNX_CUSTOM_PROTOC_EXECUTABLE ${protoc_binary_SOURCE_DIR}/bin/protoc)
set(PROTOC_EXECUTABLE ${ONNX_CUSTOM_PROTOC_EXECUTABLE})
endif()
elseif (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
elseif ((CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR CMAKE_SYSTEM_NAME STREQUAL "Android" OR CMAKE_SYSTEM_NAME STREQUAL "iOS") AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
FetchContent_Declare(protoc_binary URL ${DEP_URL_protoc_mac_universal} URL_HASH SHA1=${DEP_SHA1_protoc_mac_universal})
FetchContent_Populate(protoc_binary)
if(protoc_binary_SOURCE_DIR)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,39 +11,83 @@ d) DEP_FILE_PATH=${OPTARG};;
esac
done

EXTRA_CMAKE_ARGS=""


EXTRA_CMAKE_ARGS="-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_CXX_STANDARD=17"

case "$(uname -s)" in
Darwin*)
echo 'Building ONNX Runtime on Mac OS X'
EXTRA_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=x86_64;arm64"
EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DCMAKE_OSX_ARCHITECTURES=x86_64;arm64"
GCC_PATH=$(which clang)
GPLUSPLUS_PATH=$(which clang++)
;;
Linux*)
# Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version.
GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1)
#-fstack-clash-protection prevents attacks based on an overlapping heap and stack.
if [ "$GCC_VERSION" -ge 8 ]; then
SYS_LONG_BIT=$(getconf LONG_BIT)
DISTRIBUTOR=$(lsb_release -i -s)

if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then
LIBDIR="lib64"
else
LIBDIR="lib"
fi
EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=$LIBDIR"
# Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version.
GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1)
#-fstack-clash-protection prevents attacks based on an overlapping heap and stack.
if [ "$GCC_VERSION" -ge 8 ]; then
CFLAGS="$CFLAGS -fstack-clash-protection"
CXXFLAGS="$CXXFLAGS -fstack-clash-protection"
fi
ARCH=$(uname -m)

if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then
fi
ARCH=$(uname -m)
GCC_PATH=$(which gcc)
GPLUSPLUS_PATH=$(which g++)
if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then
CFLAGS="$CFLAGS -fcf-protection"
CXXFLAGS="$CXXFLAGS -fcf-protection"
fi
export CFLAGS
export CXXFLAGS
;;
*)
fi
export CFLAGS
export CXXFLAGS
;;
*)
exit 1
esac
mkdir -p $INSTALL_PREFIX
mkdir -p "$INSTALL_PREFIX"

if [ -x "$(command -v ninja)" ]; then
EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -G Ninja"
fi
echo "Installing abseil ..."
pushd .
absl_url=$(grep '^abseil_cpp' "$DEP_FILE_PATH" | cut -d ';' -f 2 )
if [[ "$absl_url" = https* ]]; then
absl_url=$(echo $absl_url | sed 's/\.zip$/\.tar.gz/')
curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o absl_src.tar.gz $absl_url
mkdir abseil
cd abseil
tar -zxf ../absl_src.tar.gz --strip=1
else
cp $absl_url absl_src.zip
unzip absl_src.zip
cd *
fi

CC=$GCC_PATH CXX=$GPLUSPLUS_PATH cmake "." "-DABSL_PROPAGATE_CXX_STD=ON" "-DCMAKE_BUILD_TYPE=Release" "-DBUILD_TESTING=OFF" "-DABSL_USE_EXTERNAL_GOOGLETEST=ON" "-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" "-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" $EXTRA_CMAKE_ARGS
if [ -x "$(command -v ninja)" ]; then
ninja
ninja install
else
make -j$(getconf _NPROCESSORS_ONLN)
make install
fi
popd

pushd .
echo "Installing protobuf ..."
protobuf_url=$(grep '^protobuf' $DEP_FILE_PATH | cut -d ';' -f 2 )
if [[ "$protobuf_url" = https* ]]; then
protobuf_url=$(echo $protobuf_url | sed 's/\.zip$/\.tar.gz/')
curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o protobuf_src.tar.gz $protobuf_url
protobuf_url=$(echo "$protobuf_url" | sed 's/\.zip$/\.tar.gz/')
curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o protobuf_src.tar.gz "$protobuf_url"
mkdir protobuf
cd protobuf
tar -zxf ../protobuf_src.tar.gz --strip=1
Expand All @@ -53,7 +97,12 @@ else
cd protobuf-*
fi

cmake . -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF $EXTRA_CMAKE_ARGS
make -j$(getconf _NPROCESSORS_ONLN)
make install
cd ..
CC=$GCC_PATH CXX=$GPLUSPLUS_PATH cmake . "-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF "-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" $EXTRA_CMAKE_ARGS -Dprotobuf_ABSL_PROVIDER=package
if [ -x "$(command -v ninja)" ]; then
ninja
ninja install
else
make -j$(getconf _NPROCESSORS_ONLN)
make install
fi
popd
8 changes: 4 additions & 4 deletions tools/ci_build/github/linux/docker/scripts/install_os_deps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ set -e -x

SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )"
INSTALL_DEPS_DISTRIBUTED_SETUP=false

INSTALL_PREFIX='/usr'
while getopts p:d:v:tmur parameter_Option
do case "${parameter_Option}"
in
p) echo "Python version is no longer accepted as an input to this script. Ignoring the input argument -p.";;
p) INSTALL_PREFIX=${OPTARG};;
d) DEVICE_TYPE=${OPTARG};;
v) echo "Cuda version is no longer accepted as an input to this script. Ignoring the input argument -v.";;
t) echo "Installing python training dependencies argument is no longer accepted as an input to this script. Ignoring the input argument -t.";;
Expand Down Expand Up @@ -59,7 +59,7 @@ GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.)

DISTRIBUTOR=$(lsb_release -i -s)

if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then
if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then
LIBDIR="lib64"
else
LIBDIR="lib"
Expand Down Expand Up @@ -91,7 +91,7 @@ fi
cd /tmp/src

if ! [ -x "$(command -v protoc)" ]; then
source ${0/%install_os_deps\.sh/install_protobuf\.sh}
$SCRIPT_DIR/install_protobuf.sh -p $INSTALL_PREFIX
fi

if [ $DEVICE_TYPE = "gpu" ]; then
Expand Down
109 changes: 83 additions & 26 deletions tools/ci_build/github/linux/docker/scripts/install_protobuf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,41 +11,98 @@ d) DEP_FILE_PATH=${OPTARG};;
esac
done

EXTRA_CMAKE_ARGS=""


EXTRA_CMAKE_ARGS="-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_CXX_STANDARD=17"

case "$(uname -s)" in
Darwin*)
echo 'Building ONNX Runtime on Mac OS X'
EXTRA_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=x86_64;arm64"
EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DCMAKE_OSX_ARCHITECTURES=x86_64;arm64"
GCC_PATH=$(which clang)
GPLUSPLUS_PATH=$(which clang++)
;;
Linux*)
# Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version.
GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1)
#-fstack-clash-protection prevents attacks based on an overlapping heap and stack.
if [ "$GCC_VERSION" -ge 8 ]; then
SYS_LONG_BIT=$(getconf LONG_BIT)
DISTRIBUTOR=$(lsb_release -i -s)

if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then
LIBDIR="lib64"
else
LIBDIR="lib"
fi
EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=$LIBDIR"
# Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version.
GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1)
#-fstack-clash-protection prevents attacks based on an overlapping heap and stack.
if [ "$GCC_VERSION" -ge 8 ]; then
CFLAGS="$CFLAGS -fstack-clash-protection"
CXXFLAGS="$CXXFLAGS -fstack-clash-protection"
fi
ARCH=$(uname -m)

if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then
fi
ARCH=$(uname -m)
GCC_PATH=$(which gcc)
GPLUSPLUS_PATH=$(which g++)
if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then
CFLAGS="$CFLAGS -fcf-protection"
CXXFLAGS="$CXXFLAGS -fcf-protection"
fi
export CFLAGS
export CXXFLAGS
;;
*)
exit -1
fi
export CFLAGS
export CXXFLAGS
;;
*)
exit 1
esac
mkdir -p $INSTALL_PREFIX
mkdir -p "$INSTALL_PREFIX"

if [ -x "$(command -v ninja)" ]; then
EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -G Ninja"
fi
echo "Installing abseil ..."
pushd .
absl_url=$(grep '^abseil_cpp' "$DEP_FILE_PATH" | cut -d ';' -f 2 )
if [[ "$absl_url" = https* ]]; then
absl_url=$(echo $absl_url | sed 's/\.zip$/\.tar.gz/')
curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o absl_src.tar.gz $absl_url
mkdir abseil
cd abseil
tar -zxf ../absl_src.tar.gz --strip=1
else
cp $absl_url absl_src.zip
unzip absl_src.zip
cd *
fi

CC=$GCC_PATH CXX=$GPLUSPLUS_PATH cmake "." "-DABSL_PROPAGATE_CXX_STD=ON" "-DCMAKE_BUILD_TYPE=Release" "-DBUILD_TESTING=OFF" "-DABSL_USE_EXTERNAL_GOOGLETEST=ON" "-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" "-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" $EXTRA_CMAKE_ARGS
if [ -x "$(command -v ninja)" ]; then
ninja
ninja install
else
make -j$(getconf _NPROCESSORS_ONLN)
make install
fi
popd

pushd .
echo "Installing protobuf ..."
protobuf_url=$(grep '^protobuf' $DEP_FILE_PATH | cut -d ';' -f 2 | sed 's/\.zip$/\.tar.gz/')
curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o protobuf_src.tar.gz $protobuf_url
mkdir protobuf
cd protobuf
tar -zxf ../protobuf_src.tar.gz --strip=1
cmake ./cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF $EXTRA_CMAKE_ARGS
make -j$(getconf _NPROCESSORS_ONLN)
make install
cd ..
protobuf_url=$(grep '^protobuf' $DEP_FILE_PATH | cut -d ';' -f 2 )
if [[ "$protobuf_url" = https* ]]; then
protobuf_url=$(echo "$protobuf_url" | sed 's/\.zip$/\.tar.gz/')
curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o protobuf_src.tar.gz "$protobuf_url"
mkdir protobuf
cd protobuf
tar -zxf ../protobuf_src.tar.gz --strip=1
else
cp $protobuf_url protobuf_src.zip
unzip protobuf_src.zip
cd protobuf-*
fi

CC=$GCC_PATH CXX=$GPLUSPLUS_PATH cmake . "-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF "-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" $EXTRA_CMAKE_ARGS -Dprotobuf_ABSL_PROVIDER=package
if [ -x "$(command -v ninja)" ]; then
ninja
ninja install
else
make -j$(getconf _NPROCESSORS_ONLN)
make install
fi
popd
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,14 @@ set -e
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)

echo "installing for os major version : $os_major_version"
yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel

if [ "$os_major_version" -gt 7 ]; then
PACKAGE_MANAGER="dnf"
$PACKAGE_MANAGER install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel perl-IPC-Cmd openssl-devel wget
else
PACKAGE_MANAGER="yum"
$PACKAGE_MANAGER install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel perl-IPC-Cmd openssl-devel wget
fi

# Install Java
# Install automatic documentation generation dependencies
yum install -y java-11-openjdk-devel graphviz
$PACKAGE_MANAGER install -y java-11-openjdk-devel graphviz
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.)

DISTRIBUTOR=$(lsb_release -i -s)

if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then
if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then
LIBDIR="lib64"
else
LIBDIR="lib"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.)

DISTRIBUTOR=$(lsb_release -i -s)

if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then
if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then
LIBDIR="lib64"
else
LIBDIR="lib"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,7 @@ set -e -x
yum -y install \
graphviz

os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)

SYS_LONG_BIT=$(getconf LONG_BIT)
mkdir -p /tmp/src
GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.)

DISTRIBUTOR=$(lsb_release -i -s)

if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then
LIBDIR="lib64"
else
LIBDIR="lib"
fi

cd /tmp/src
source $(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/install_shared_deps.sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,16 +66,7 @@ def main():
[sys.executable, str(REPO_ROOT / "tools/ci_build/build.py"), *build_params]
+ (["--cmake_extra_defines", "ADD_DEBUG_INFO_TO_MINIMAL_BUILD=ON"] if args.with_debug_info else [])
# put the following options last so they don't get overridden by build_params
+ [
f"--build_dir={args.build_dir}",
f"--config={build_config}",
"--update",
"--build",
"--parallel",
"--test",
"--path_to_protoc_exe",
str(pathlib.Path(args.build_dir) / "installed" / "bin" / "protoc"),
]
+ [f"--build_dir={args.build_dir}", f"--config={build_config}", "--update", "--build", "--parallel", "--test"]
)

subprocess.run(build_command, check=True)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ python3 $ORT_ROOT/tools/ci_build/build.py \
--disable_ml_ops \
--disable_exceptions \
--include_ops_by_config $ORT_ROOT/onnxruntime/test/testdata/required_ops_and_types.config \
--path_to_protoc_exe $ORT_ROOT/protobuf_install/bin/protoc \
--skip_tests

# Push onnxruntime_test_all and testdata to emulator
Expand Down
8 changes: 7 additions & 1 deletion tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
FROM rocm/cupy:rocm5.5.0_ubuntu20.04_py3.8_pytorch2.0.0_cupy13.0.0

RUN apt-get update -y && apt-get upgrade -y && apt-get autoremove -y && apt-get clean -y

RUN apt-get update -y && apt-get upgrade -y && apt-get autoremove -y libprotobuf\* protobuf-compiler\* && \
rm -f /usr/local/bin/protoc && apt-get install -y locales unzip && apt-get clean -y
RUN locale-gen en_US.UTF-8
RUN update-locale LANG=en_US.UTF-8
ENV LC_ALL C.UTF-8
ENV LANG C.UTF-8

WORKDIR /stage

Expand Down

0 comments on commit 73ddba9

Please sign in to comment.