Moves to manylinux_2_28

- adjusts package installation to account for the next
  manylinux
- silents false warnings from the gcc 12.x
- updates clang to 17.x

Signed-off-by: Janusz Lisiecki <[email protected]>

CMakeLists.txt

@@ -348,7 +348,7 @@ set(CMAKE_C_STANDARD 11)
 set(CMAKE_CUDA_STANDARD 17)
 
 # CXX flags
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-free-nonheap-object -Wno-unused-variable -Wno-unused-function -Wno-strict-overflow -fno-strict-aliasing -fPIC -fvisibility=hidden")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-free-nonheap-object -Wno-unused-variable -Wno-unused-function -Wno-strict-overflow -Wno-array-bounds -fno-strict-aliasing -fPIC -fvisibility=hidden")
 
 if (WERROR)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")

dali/kernels/test/scatter_gather_test.cc

@@ -15,6 +15,7 @@
 #include <gtest/gtest.h>
 #include <vector>
 #include <algorithm>
+#include <random>
 
 #include "dali/core/cuda_error.h"
 #include "dali/core/mm/memory.h"
@@ -132,8 +133,10 @@ class ScatterGatherTest : public testing::Test {
       j += l;
     }
 
-    std::random_shuffle(ranges.begin(), ranges.end());
-    std::random_shuffle(back_ranges.begin(), back_ranges.end());
+    std::random_device rd;
+    std::mt19937 g(rd());
+    std::shuffle(ranges.begin(), ranges.end(), g);
+    std::shuffle(back_ranges.begin(), back_ranges.end(), g);
 
     this->template Memcpy<kind>(in_ptr.get(), in.data(), in.size(), cudaMemcpyHostToDevice);
     this->template Memset<kind>(out_ptr.get(), 0, out.size());

dali/operators/image/crop/bbox_crop.cc

@@ -678,7 +678,9 @@ class RandomBBoxCropImpl : public OpImplBase<CPUBackend> {
 
     std::array<int, ndim> order;
     std::iota(order.begin(), order.end(), 0);
-    std::random_shuffle(order.begin(), order.end());
+    std::random_device rd;
+    std::mt19937 g(rd());
+    std::shuffle(order.begin(), order.end(), g);
 
     float max_extent = 0.0f;
     for (int d = 0; d < ndim; d++) {

dali/operators/reader/video_reader_decoder_op_test.cc

@@ -137,6 +137,7 @@ class VideoReaderDecoderBaseTest : public VideoTestBase {
 
     pipe.Build({{"frames", backend}});
 
+    // ToFix
     std::vector<int> expected_order = {29, 46, 33, 6, 37};
 
     int num_sequences = 5;

dali/pipeline/executor/executor_test.cc

@@ -166,6 +166,15 @@ TYPED_TEST(ExecutorTest, DISABLED_TestDataSetup) {
   vector<string> outputs = {"data3_gpu"};
   exe->Build(&graph, outputs);
 
+#pragma GCC diagnostic push
+// most recent gcc seems to be incorrectly report this as being null in this test
+#if defined(__has_warning)
+  #if __has_warning("-Wnonnull")
+    #pragma GCC diagnostic ignored "-Wnonnull"
+  #endif
+#else
+  #pragma GCC diagnostic ignored "-Wnonnull"
+#endif
   // Verify the data has been setup correctly
   for (int i = 0; i < 2; ++i) {
     auto host_workspaces = this->CPUData(exe.get(), i);
@@ -193,6 +202,7 @@ TYPED_TEST(ExecutorTest, DISABLED_TestDataSetup) {
     ASSERT_EQ(dws.NumOutput(), 1);
     ASSERT_TRUE(dws.OutputIsType<GPUBackend>(0));
   }
+#pragma GCC diagnostic pop
 }
 
 TYPED_TEST(ExecutorTest, TestRunBasicGraph) {

docker/Dockerfile

@@ -13,9 +13,9 @@ ENV PYVER=${PYVER} \
     PYTHONPATH=/opt/python/v \
     PYBIN=${PYTHONPATH}/bin \
     PYLIB=${PYTHONPATH}/lib \
-    PATH=/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:${PYBIN}:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin:${PYBIN}:/opt/python/cp312-cp312/bin:${PYBIN}:${PATH} \
-    LD_LIBRARY_PATH=/usr/local/lib:/opt/dali/${DALI_BUILD_DIR}:/opt/python/cp38-cp38/lib:/opt/python/cp39-cp39/lib:/opt/python/cp310-cp310/lib:/opt/python/cp311-cp311/lib:${PYLIB}:/opt/python/cp312-cp312/lib:${PYLIB}:${LD_LIBRARY_PATH} \
-    LIBRARY_PATH=/usr/local/lib:/opt/dali/${DALI_BUILD_DIR}:/opt/python/cp38-cp38/lib:/opt/python/cp39-cp39/lib:/opt/python/cp310-cp310/lib:/opt/python/cp311-cp311/lib:${PYLIB}:/opt/python/cp312-cp312/lib:${PYLIB}:${LIBRARY_PATH}
+    PATH=/opt/python/cp39-cp39/bin:${PYBIN}:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin:${PYBIN}:/opt/python/cp312-cp312/bin:${PYBIN}:${PATH} \
+    LD_LIBRARY_PATH=/usr/local/lib:/opt/dali/${DALI_BUILD_DIR}:opt/python/cp39-cp39/lib:/opt/python/cp310-cp310/lib:/opt/python/cp311-cp311/lib:${PYLIB}:/opt/python/cp312-cp312/lib:${PYLIB}:${LD_LIBRARY_PATH} \
+    LIBRARY_PATH=/usr/local/lib:/opt/dali/${DALI_BUILD_DIR}:/opt/python/cp39-cp39/lib:/opt/python/cp310-cp310/lib:/opt/python/cp311-cp311/lib:${PYLIB}:/opt/python/cp312-cp312/lib:${PYLIB}:${LIBRARY_PATH}
 
 RUN ln -s /opt/python/cp${PYV}* /opt/python/v
 

docker/Dockerfile.cuda118.aarch64.deps

@@ -9,7 +9,8 @@ RUN apt update && apt install -y libxml2 curl perl gcc && \
 RUN curl -LO https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux_sbsa.run && \
     chmod +x cuda_*.run && \
     ./cuda_*.run --silent --no-opengl-libs --toolkit && \
-    rm -f cuda_*.run;
+    rm -f cuda_*.run && \
+    sed -i "s/__GNUC__ > 11/__GNUC__ > 13/" /usr/local/cuda/bin/../targets/x86_64-linux/include/crt/host_config.h;
 
 RUN NVJPEG2K_VERSION=0.8.0.38-1 && \
     CUDA_VERSION_MAJOR=11 && \

docker/Dockerfile.cuda118.x86_64.deps

@@ -9,7 +9,8 @@ RUN apt update && apt install -y libxml2 curl perl gcc && \
 RUN curl -LO https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run && \
     chmod +x cuda_*.run && \
     ./cuda_*.run --silent --no-opengl-libs --toolkit && \
-    rm -f cuda_*.run;
+    rm -f cuda_*.run && \
+    sed -i "s/__GNUC__ > 11/__GNUC__ > 13/" /usr/local/cuda/bin/../targets/x86_64-linux/include/crt/host_config.h;
 
 RUN NVJPEG2K_VERSION=0.8.0.38-1 && \
     CUFILE_VERSION=1.4.0.31-1 && \

docker/Dockerfile.deps

@@ -2,42 +2,29 @@
 ##  Build DALI dependencies on top of manylinux2014
 ##  DALI is based on "manylinux2014", official page https://github.com/pypa/manylinux
 #########################################################################################
-ARG FROM_IMAGE_NAME=quay.io/pypa/manylinux2014_x86_64
+ARG FROM_IMAGE_NAME=quay.io/pypa/manylinux_2_28_x86_64
 ARG BUILDER_EXTRA_DEPS=scratch
 FROM ${BUILDER_EXTRA_DEPS} as extra_deps
 FROM ${FROM_IMAGE_NAME}
 
 # Install yum Dependencies
 RUN yum install -y wget doxygen graphviz gettext xz openssl autogen zip \
-                   devtoolset-10-libasan-devel devtoolset-10-liblsan-devel \
-                   devtoolset-10-libtsan-devel devtoolset-10-libubsan-devel \
-                   perl perl-IPC-Cmd
+                   gcc-toolset-12-libasan-devel gcc-toolset-12-liblsan-devel \
+                   gcc-toolset-12-libtsan-devel gcc-toolset-12-libubsan-devel \
+                   perl perl-IPC-Cmd nasm
 
 ENV ACLOCAL_PATH=/usr/share/aclocal/:/usr/local/share/aclocal
 
-# Don't want the short-unicode version for Python 2.7
-RUN rm -f /opt/python/cp27-cp27m
-
 RUN CMAKE_VERSION=3.20.1 && CMAKE_ARCH=$(uname -m) && \
     wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.sh && \
     test -e /bin/sh || ln -s /usr/bin/sh /bin/sh && \
     chmod +x cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.sh && \
     ./cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.sh --prefix=/usr/local --skip-license && \
     rm -rf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.sh
 
-# We need newer NASM than manylinux2014 offers
-RUN cd /tmp && \
-    curl -O -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2  && \
-    tar xjvf nasm-2.15.05.tar.bz2 && \
-    cd nasm-2.15.05 && \
-    ./autogen.sh && \
-    ./configure && \
-    make && \
-    make install
-
 # Clang, build it before deps as deps changes more frequently
-RUN CLANG_VERSION=15.0.2                                                                           && \
-    cd /tmp                                                                                         && \
+RUN CLANG_VERSION=17.0.6                                                                           && \
+    cd /tmp                                                                                        && \
     wget https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-${CLANG_VERSION}.tar.gz    && \
     tar -xf llvmorg-*.tar.gz                                                                       && \
     rm -rf llvmorg-*.tar.gz                                                                        && \
@@ -47,13 +34,7 @@ RUN CLANG_VERSION=15.0.2
     cmake -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DCMAKE_BUILD_TYPE=Release -G "Unix Makefiles" ../llvm && \
     make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install                                      && \
     cd /tmp                                                                                        && \
-    rm -rf llvm-*                                                                                  && \
-    sed -i 's/#if __cplusplus >= 201103L \&\& CUDA_VERSION >= 9000/\#if __cplusplus >= 201103L \&\& CUDA_VERSION >= 9000 \&\& CUDA_VERSION < 12000/g' \
-        /usr/local/lib/clang/${CLANG_VERSION}/include/__clang_cuda_runtime_wrapper.h && \
-    sed -i -z 's/#if CUDA_VERSION >= 9000\n\/\/ Provide a hint that texture support needs C++11./#if CUDA_VERSION >= 9000 \&\& CUDA_VERSION < 12000\n\/\/ Provide a hint that texture support needs C++11./g' \
-        /usr/local/lib/clang/${CLANG_VERSION}/include/__clang_cuda_runtime_wrapper.h && \
-    sed -i 's/#include "crt\/device_functions.hpp"/#include "crt\/device_functions.h"/g' \
-        /usr/local/lib/clang/${CLANG_VERSION}/include/__clang_cuda_runtime_wrapper.h
+    rm -rf llvm-*
 
 COPY DALI_DEPS_VERSION /tmp
 

include/dali/core/small_vector.h

@@ -23,6 +23,16 @@
 #include "dali/core/util.h"
 #include "dali/core/cuda_utils.h"
 
+#pragma GCC diagnostic push
+// most recent gcc seems to be confused by some things in small vector raising false warnings
+#if defined(__has_warning)
+  #if __has_warning("-Wuse-after-free")
+    #pragma GCC diagnostic ignored "-Wuse-after-free"
+  #endif
+#else
+  #pragma GCC diagnostic ignored "-Wuse-after-free"
+#endif
+
 namespace dali {
 
 template <typename T, typename Allocator, bool Contextless = std::is_empty<Allocator>::value>
@@ -732,4 +742,6 @@ class SmallVector : SmallVectorAlloc<T, allocator>, SmallVectorBase<T> {
 
 }  // namespace dali
 
+#pragma GCC diagnostic pop
+
 #endif  // DALI_CORE_SMALL_VECTOR_H_