Rebase on master 23-Jul-2024

.ci/dpdk-build.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -o errexit
+set -x
+
+function build_dpdk()
+{
+    local DPDK_VER=$1
+    local DPDK_OPTS=""
+    local DPDK_INSTALL_DIR="$(pwd)/dpdk-dir"
+    local VERSION_FILE="$DPDK_INSTALL_DIR/cached-version"
+
+    rm -rf dpdk-src
+    rm -rf $DPDK_INSTALL_DIR
+
+    if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then
+        git clone --single-branch $DPDK_GIT dpdk-src -b "${DPDK_VER##refs/*/}"
+        pushd dpdk-src
+        git log -1 --oneline
+    else
+        wget https://fast.dpdk.org/rel/dpdk-$1.tar.xz
+        tar xvf dpdk-$1.tar.xz > /dev/null
+        DIR_NAME=$(tar -tf dpdk-$1.tar.xz | head -1 | cut -f1 -d"/")
+        mv ${DIR_NAME} dpdk-src
+        pushd dpdk-src
+    fi
+
+    # Switching to 'generic' platform to make the dpdk cache usable on
+    # different CPUs. We can't be sure that all CI machines are exactly same.
+    DPDK_OPTS="$DPDK_OPTS -Dplatform=generic"
+
+    # Disable building DPDK unit tests. Not needed for OVS build or tests.
+    DPDK_OPTS="$DPDK_OPTS -Dtests=false"
+
+    # Disable DPDK developer mode, this results in less build checks and less
+    # meson verbose outputs.
+    DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled"
+
+    # OVS compilation and "normal" unit tests (run in the CI) do not depend on
+    # any DPDK driver.
+    # check-dpdk unit tests requires testpmd and some net/ driver.
+    DPDK_OPTS="$DPDK_OPTS -Denable_apps=test-pmd"
+    enable_drivers="net/null,net/af_xdp,net/tap,net/virtio,net/pcap"
+    DPDK_OPTS="$DPDK_OPTS -Denable_drivers=$enable_drivers"
+    # OVS depends on the vhost library (and its dependencies).
+    # net/tap depends on the gso library.
+    DPDK_OPTS="$DPDK_OPTS -Denable_libs=cryptodev,dmadev,gso,vhost"
+
+    # Install DPDK using prefix.
+    DPDK_OPTS="$DPDK_OPTS --prefix=$DPDK_INSTALL_DIR"
+
+    meson setup $DPDK_OPTS build
+    ninja -C build
+    ninja -C build install
+    popd
+
+    # Remove examples sources.
+    rm -rf $DPDK_INSTALL_DIR/share/dpdk/examples
+
+    echo "Installed DPDK in $DPDK_INSTALL_DIR"
+    echo "${DPDK_VER}" > ${VERSION_FILE}
+}
+
+build_dpdk $DPDK_VER

.ci/dpdk-prepare.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ev
+
+# Installing wheel separately because it may be needed to build some
+# of the packages during dependency backtracking and pip >= 22.0 will
+# abort backtracking on build failures:
+#     https://github.com/pypa/pip/issues/10655
+pip3 install --disable-pip-version-check --user wheel
+pip3 install --disable-pip-version-check --user pyelftools
+pip3 install --user  'meson>=1.4,<1.5'

.ci/linux-build.sh

@@ -6,111 +6,13 @@ set -x
 CFLAGS_FOR_OVS="-g -O2"
 SPARSE_FLAGS=""
 EXTRA_OPTS="--enable-Werror"
-
-on_exit() {
-    if [ $? = 0 ]; then
-        exit
-    fi
-    FILES_TO_PRINT="config.log"
-    FILES_TO_PRINT="$FILES_TO_PRINT */_build/sub/tests/testsuite.log"
-
-    for pr_file in $FILES_TO_PRINT; do
-        cat "$pr_file" 2>/dev/null
-    done
-}
-# We capture the error logs as artifacts in Github Actions, no need to dump
-# them via a EXIT handler.
-[ -n "$GITHUB_WORKFLOW" ] || trap on_exit EXIT
-
-function install_kernel()
-{
-    if [[ "$1" =~ ^5.* ]]; then
-        PREFIX="v5.x"
-    elif [[ "$1" =~ ^4.* ]]; then
-        PREFIX="v4.x"
-    elif [[ "$1" =~ ^3.* ]]; then
-        PREFIX="v3.x"
-    else
-        PREFIX="v2.6/longterm/v2.6.32"
-    fi
-
-    base_url="https://cdn.kernel.org/pub/linux/kernel/${PREFIX}"
-    # Download page with list of all available kernel versions.
-    wget ${base_url}/
-    # Uncompress in case server returned gzipped page.
-    (file index* | grep ASCII) || (mv index* index.new.gz && gunzip index*)
-    # Get version of the latest stable release.
-    hi_ver=$(echo ${1} | sed 's/\./\\\./')
-    lo_ver=$(cat ./index* | grep -P -o "${hi_ver}\.[0-9]+" | \
-             sed 's/.*\..*\.\(.*\)/\1/' | sort -h | tail -1)
-    version="${1}.${lo_ver}"
-
-    rm -rf index* linux-*
-
-    url="${base_url}/linux-${version}.tar.xz"
-    # Download kernel sources. Try direct link on CDN failure.
-    wget ${url} ||
-    (rm -f linux-${version}.tar.xz && wget ${url}) ||
-    (rm -f linux-${version}.tar.xz && wget ${url/cdn/www})
-
-    tar xvf linux-${version}.tar.xz > /dev/null
-    pushd linux-${version}
-    make allmodconfig
-
-    # Cannot use CONFIG_KCOV: -fsanitize-coverage=trace-pc is not supported by compiler
-    sed -i 's/CONFIG_KCOV=y/CONFIG_KCOV=n/' .config
-
-    # stack validation depends on tools/objtool, but objtool does not compile on travis.
-    # It is giving following error.
-    #  >>> GEN      arch/x86/insn/inat-tables.c
-    #  >>> Semantic error at 40: Unknown imm opnd: AL
-    # So for now disable stack-validation for the build.
-
-    sed -i 's/CONFIG_STACK_VALIDATION=y/CONFIG_STACK_VALIDATION=n/' .config
-    make oldconfig
-
-    # Older kernels do not include openvswitch
-    if [ -d "net/openvswitch" ]; then
-        make net/openvswitch/
-    else
-        make net/bridge/
-    fi
-
-    if [ "$AFXDP" ]; then
-        sudo make headers_install INSTALL_HDR_PATH=/usr
-        pushd tools/lib/bpf/
-        # Bulding with gcc because there are some issues in make files
-        # that breaks building libbpf with clang on Travis.
-        CC=gcc sudo make install
-        CC=gcc sudo make install_headers
-        sudo ldconfig
-        popd
-        # The Linux kernel defines __always_inline in stddef.h (283d7573), and
-        # sys/cdefs.h tries to re-define it.  Older libc-dev package in xenial
-        # doesn't have a fix for this issue.  Applying it manually.
-        sudo sed -i '/^# define __always_inline .*/i # undef __always_inline' \
-                    /usr/include/x86_64-linux-gnu/sys/cdefs.h || true
-        EXTRA_OPTS="${EXTRA_OPTS} --enable-afxdp"
-    fi
-    popd
-}
+JOBS=${JOBS:-"-j4"}
 
 function install_dpdk()
 {
-    local DPDK_VER=$1
-    local VERSION_FILE="dpdk-dir/travis-dpdk-cache-version"
-    local DPDK_OPTS=""
-    local DPDK_LIB=""
-
-    if [ -z "$TRAVIS_ARCH" ] ||
-       [ "$TRAVIS_ARCH" == "amd64" ]; then
-        DPDK_LIB=$(pwd)/dpdk-dir/build/lib/x86_64-linux-gnu
-    elif [ "$TRAVIS_ARCH" == "aarch64" ]; then
-        DPDK_LIB=$(pwd)/dpdk-dir/build/lib/aarch64-linux-gnu
-    else
-        echo "Target is unknown"
-        exit 1
-    fi
+    local DPDK_INSTALL_DIR="$(pwd)/dpdk-dir"
+    local VERSION_FILE="${DPDK_INSTALL_DIR}/cached-version"
+    local DPDK_LIB=${DPDK_INSTALL_DIR}/lib/x86_64-linux-gnu
 
     if [ "$DPDK_SHARED" ]; then
         EXTRA_OPTS="$EXTRA_OPTS --with-dpdk=shared"
@@ -122,58 +24,17 @@ function install_dpdk()
     # Export the following path for pkg-config to find the .pc file.
     export PKG_CONFIG_PATH=$DPDK_LIB/pkgconfig/:$PKG_CONFIG_PATH
 
-    if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then
-        # Avoid using cache for git tree build.
-        rm -rf dpdk-dir
+    # Expose dpdk binaries.
+    export PATH=$(pwd)/dpdk-dir/bin:$PATH
 
-        DPDK_GIT=${DPDK_GIT:-https://dpdk.org/git/dpdk}
-        git clone --single-branch $DPDK_GIT dpdk-dir -b "${DPDK_VER##refs/*/}"
-        pushd dpdk-dir
-        git log -1 --oneline
-    else
-        if [ -f "${VERSION_FILE}" ]; then
-            VER=$(cat ${VERSION_FILE})
-            if [ "${VER}" = "${DPDK_VER}" ]; then
-                # Update the library paths.
-                sudo ldconfig
-                echo "Found cached DPDK ${VER} build in $(pwd)/dpdk-dir"
-                return
-            fi
-        fi
-        # No cache or version mismatch.
-        rm -rf dpdk-dir
-        wget https://fast.dpdk.org/rel/dpdk-$1.tar.xz
-        tar xvf dpdk-$1.tar.xz > /dev/null
-        DIR_NAME=$(tar -tf dpdk-$1.tar.xz | head -1 | cut -f1 -d"/")
-        mv ${DIR_NAME} dpdk-dir
-        pushd dpdk-dir
+    if [ ! -f "${VERSION_FILE}" ]; then
+        echo "Could not find DPDK in $DPDK_INSTALL_DIR"
+        return 1
     fi
 
-    # Switching to 'default' machine to make dpdk-dir cache usable on
-    # different CPUs. We can't be sure that all CI machines are exactly same.
-    DPDK_OPTS="$DPDK_OPTS -Dmachine=default"
-
-    # Disable building DPDK unit tests. Not needed for OVS build or tests.
-    DPDK_OPTS="$DPDK_OPTS -Dtests=false"
-
-    # Disable DPDK developer mode, this results in less build checks and less
-    # meson verbose outputs.
-    DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled"
-
-    # Install DPDK using prefix.
-    DPDK_OPTS="$DPDK_OPTS --prefix=$(pwd)/build"
-
-    CC=gcc meson $DPDK_OPTS build
-    ninja -C build
-    ninja -C build install
-
     # Update the library paths.
     sudo ldconfig
-
-
-    echo "Installed DPDK source in $(pwd)"
-    popd
-    echo "${DPDK_VER}" > ${VERSION_FILE}
+    echo "Found cached DPDK $(cat ${VERSION_FILE}) build in $DPDK_INSTALL_DIR"
 }
 
 function configure_ovs()
@@ -187,7 +48,32 @@ function build_ovs()
     configure_ovs $OPTS
     make selinux-policy
 
-    make -j4
+    make ${JOBS}
+}
+
+function clang_analyze()
+{
+    [ -d "./base-clang-analyzer-results" ] && cache_build=false \
+                                           || cache_build=true
+    if [ "$cache_build" = true ]; then
+        # If this is a cache build, proceed to the base branch's directory.
+        pushd base_ovs_main
+    fi;
+
+    configure_ovs $OPTS
+
+    make clean
+    scan-build -o ./clang-analyzer-results -sarif --use-cc=${CC} make ${JOBS}
+
+    if [ "$cache_build" = true ]; then
+        # Move results, so it will be picked up by the cache.
+        mv ./clang-analyzer-results ../base-clang-analyzer-results
+        popd
+    else
+        # Only do the compare on the none cache builds.
+        sarif --check note diff ./base-clang-analyzer-results \
+                                ./clang-analyzer-results
+    fi;
 }
 
 if [ "$DEB_PACKAGE" ]; then
@@ -222,15 +108,12 @@ assert ovs.json.from_string('{\"a\": 42}') == {'a': 42}"
     exit 0
 fi
 
-if [ "$KERNEL" ]; then
-    install_kernel $KERNEL
+if [ "$DPDK" ] || [ "$DPDK_SHARED" ]; then
+    install_dpdk
 fi
 
-if [ "$DPDK" ] || [ "$DPDK_SHARED" ]; then
-    if [ -z "$DPDK_VER" ]; then
-        DPDK_VER="21.11.2"
-    fi
-    install_dpdk $DPDK_VER
+if [ "$STD" ]; then
+    CFLAGS_FOR_OVS="${CFLAGS_FOR_OVS} -std=$STD"
 fi
 
 if [ "$CC" = "clang" ]; then
@@ -240,40 +123,53 @@ elif [ "$M32" ]; then
     # Adding m32 flag directly to CC to avoid any posiible issues with API/ABI
     # difference on 'configure' and 'make' stages.
     export CC="$CC -m32"
-elif [ "$TRAVIS_ARCH" != "aarch64" ]; then
-    OPTS="--enable-sparse"
-    if [ "$AFXDP" ]; then
-        # netdev-afxdp uses memset for 64M for umem initialization.
-        SPARSE_FLAGS="${SPARSE_FLAGS} -Wno-memcpy-max-count"
-    fi
+else
+    EXTRA_OPTS="$EXTRA_OPTS --enable-sparse"
     CFLAGS_FOR_OVS="${CFLAGS_FOR_OVS} ${SPARSE_FLAGS}"
 fi
 
-if [ "$ASAN" ]; then
-    # This will override default option configured in tests/atlocal.in.
+if [ "$SANITIZERS" ]; then
+    # This will override default ASAN options configured in tests/atlocal.in.
     export ASAN_OPTIONS='detect_leaks=1'
-    CFLAGS_ASAN="-fno-omit-frame-pointer -fno-common -fsanitize=address"
-    CFLAGS_FOR_OVS="${CFLAGS_FOR_OVS} ${CFLAGS_ASAN}"
-fi
-
-if [ "$UBSAN" ]; then
-    # Use the default options configured in tests/atlocal.in, in UBSAN_OPTIONS.
-    CFLAGS_UBSAN="-fno-omit-frame-pointer -fno-common -fsanitize=undefined"
-    CFLAGS_FOR_OVS="${CFLAGS_FOR_OVS} ${CFLAGS_UBSAN}"
+    CFLAGS_FOR_SAN="-fno-omit-frame-pointer -fno-common -fsanitize=$SANITIZERS"
+    CFLAGS_FOR_OVS="${CFLAGS_FOR_OVS} ${CFLAGS_FOR_SAN}"
 fi
 
 OPTS="${EXTRA_OPTS} ${OPTS} $*"
 
-if [ "$TESTSUITE" ]; then
+if [ "$CLANG_ANALYZE" ]; then
+    clang_analyze
+    exit 0
+fi
+
+if [ "$TESTSUITE" = 'test' ]; then
     # 'distcheck' will reconfigure with required options.
     # Now we only need to prepare the Makefile without sparse-wrapped CC.
     configure_ovs
 
     export DISTCHECK_CONFIGURE_FLAGS="$OPTS"
-    make distcheck -j4 CFLAGS="${CFLAGS_FOR_OVS}" \
-        TESTSUITEFLAGS=-j4 RECHECK=yes
+    make distcheck ${JOBS} CFLAGS="${CFLAGS_FOR_OVS}" \
+        TESTSUITEFLAGS=${JOBS} RECHECK=yes
 else
     build_ovs
+    for testsuite in $TESTSUITE; do
+        run_as_root=
+        if [ "$testsuite" != "check" ] && \
+           [ "$testsuite" != "check-ovsdb-cluster" ] ; then
+            run_as_root="sudo -E PATH=$PATH GITHUB_ACTIONS=$GITHUB_ACTIONS"
+            sudo ip netns add ovs-system-test-ns
+            # Some system tests may rely on traffic loopback.
+            sudo ip -netns ovs-system-test-ns link set dev lo up
+            run_as_root="${run_as_root} ip netns exec ovs-system-test-ns"
+        fi
+        if [ "${testsuite##*dpdk}" != "$testsuite" ]; then
+            sudo sh -c 'echo 1024 > /proc/sys/vm/nr_hugepages' || true
+            [ "$(cat /proc/sys/vm/nr_hugepages)" = '1024' ]
+            export DPDK_EAL_OPTIONS="--lcores 0@1,1@1,2@1"
+        fi
+        $run_as_root make $testsuite TESTSUITEFLAGS="${JOBS} ${TEST_RANGE}" \
+                                     RECHECK=yes
+    done
 fi
 
 exit 0