From 8cd90245e414099354640e829b95649e477a9e64 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 16 Jul 2024 08:24:20 +0800 Subject: [PATCH] [CI][VL] Re-enable a build job running on clean dockers weekly (#6424) This is to make sure no build issue in Gluten + Velox. Other CI jobs use dockers with dependencies pre-installed, which doesn't uncover some build issues sometimes. This job is scheduled to be triggered per week. Also fixes some issues found by this job. --- ..._nightly.yml.disabled => velox_weekly.yml} | 33 +++++++++++-------- dev/builddeps-veloxbe.sh | 23 +++++++------ ep/build-velox/src/get_velox.sh | 5 +-- 3 files changed, 34 insertions(+), 27 deletions(-) rename .github/workflows/{velox_nightly.yml.disabled => velox_weekly.yml} (79%) diff --git a/.github/workflows/velox_nightly.yml.disabled b/.github/workflows/velox_weekly.yml similarity index 79% rename from .github/workflows/velox_nightly.yml.disabled rename to .github/workflows/velox_weekly.yml index 90073e8a3465..513730c84de9 100644 --- a/.github/workflows/velox_nightly.yml.disabled +++ b/.github/workflows/velox_weekly.yml @@ -13,14 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Velox backend nightly job +name: Velox backend weekly job on: pull_request: paths: - - '.github/workflows/velox_nightly.yml' + - '.github/workflows/velox_weekly.yml' schedule: - - cron: '0 20 * * *' + - cron: '0 20 * * 0' + +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -37,7 +40,6 @@ jobs: steps: - uses: actions/checkout@v2 - name: Update mirror list - if: matrix.os == 'centos:8' run: | sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true @@ -45,17 +47,21 @@ jobs: run: | yum update -y yum install -y epel-release sudo dnf - if [ "${{ matrix.os }}" = "centos:8" ]; then + if [ "${{ matrix.os }}" = "centos:7" ]; then + yum install -y centos-release-scl + rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f + sed -i \ + -e 's/^mirrorlist/#mirrorlist/' \ + -e 's/^#baseurl/baseurl/' \ + -e 's/mirror\.centos\.org/vault.centos.org/' \ + /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo + else dnf install -y --setopt=install_weak_deps=False gcc-toolset-9 source /opt/rh/gcc-toolset-9/enable || exit 1 - else - yum install -y centos-release-scl - yum install -y devtoolset-9 - source /opt/rh/devtoolset-9/enable || exit 1 fi - yum install -y java-1.8.0-openjdk-devel patch wget git && \ + yum install -y java-1.8.0-openjdk-devel patch wget git export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ - export PATH=$JAVA_HOME/bin:$PATH && \ + export PATH=$JAVA_HOME/bin:$PATH wget --no-check-certificate https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \ tar -xvf apache-maven-3.8.8-bin.tar.gz && \ mv apache-maven-3.8.8 /usr/lib/maven && \ @@ -76,7 +82,8 @@ jobs: - name: build run: | # To avoid the prompt for region selection during installing tzdata. - export DEBIAN_FRONTEND="noninteractive" - apt-get update && apt-get install -y sudo openjdk-8-jdk maven wget git + export DEBIAN_FRONTEND=noninteractive + apt-get update && apt-get install -y sudo maven wget git + sudo apt-get install -y openjdk-8-jdk export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 cd $GITHUB_WORKSPACE/ && ./dev/package.sh diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index ec20f43b6863..c0f2290b7dee 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -240,18 +240,17 @@ fi OS=`uname -s` source $GLUTEN_DIR/dev/build_helper_functions.sh if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then - ( - echo "Start to install dependencies" - cd $VELOX_HOME - if [ $OS == 'Linux' ]; then - setup_linux - elif [ $OS == 'Darwin' ]; then - setup_macos - else - echo "Unsupported kernel: $OS" - exit 1 - fi - ) + echo "Start to install dependencies" + pushd $VELOX_HOME + if [ $OS == 'Linux' ]; then + setup_linux + elif [ $OS == 'Darwin' ]; then + setup_macos + else + echo "Unsupported kernel: $OS" + exit 1 + fi + popd fi commands_to_run=${OTHER_ARGUMENTS:-} diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 3b496a9bcae7..8bec8b0a4212 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -94,7 +94,7 @@ function process_setup_ubuntu { sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh if [ $ENABLE_HDFS == "ON" ]; then sed -i '/^function install_folly.*/i function install_libhdfs3 {\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-ubuntu.sh - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_libhdfs3' scripts/setup-ubuntu.sh + sed -i '/^ run_and_time install_protobuf/a \ \ run_and_time install_libhdfs3' scripts/setup-ubuntu.sh sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh fi sed -i "s/apt install -y/sudo apt install -y/" ${VELOX_HOME}/scripts/setup-adapters.sh @@ -166,9 +166,10 @@ function process_setup_centos7 { # install gtest sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_gtest' scripts/setup-centos7.sh sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-centos7.sh + sed -i 's/https:\/\/cmake.org\/files\/v3.25\/cmake-3.25.1.tar.gz/https:\/\/cmake.org\/files\/v3.28\/cmake-3.28.3.tar.gz/' scripts/setup-centos7.sh if [ $ENABLE_HDFS = "ON" ]; then sed -i '/^function install_protobuf.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-centos7.sh - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_libhdfs3' scripts/setup-centos7.sh + sed -i '/^ run_and_time install_protobuf/a \ \ run_and_time install_libhdfs3' scripts/setup-centos7.sh sed -i '/^dnf_install ccache/a\ \ yasm \\' scripts/setup-centos7.sh fi sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh