Upgrade docker buildx 0.9.1 and fix Python compilation issues (#3643)

Signed-off-by: Peter Zhu <[email protected]>

docker/ci/build-image-multi-arch.sh

@@ -95,6 +95,8 @@ trap cleanup_all TERM INT EXIT
 DIR=`Temp_Folder_Create`
 echo "New workspace $DIR"
 echo -e "\n* Prepare docker buildx"
+docker buildx rm --all-inactive --force
+docker buildx prune --all --force
 docker buildx use default
 docker buildx create --name ${BUILDER_NAME} --use
 docker buildx inspect --bootstrap

docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile

@@ -68,7 +68,7 @@ ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
 RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
     cd Python-3.9.17 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python links
 RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \

docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile

@@ -77,7 +77,7 @@ ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
 RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
     cd Python-3.9.17 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python links
 RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \

docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile

@@ -54,9 +54,14 @@ ENV JAVA_HOME=/opt/java/openjdk-11
 ENV PATH=$PATH:$JAVA_HOME/bin
 
 # Install docker buildx
-# Stays on 0.6.3 as that is the stable version we used since the introduction of OpenSearch Multi-Arch Docker Images
+# 2023-06-20 Upgrade from 0.6.3 to 0.9.1 due to binary translation speedup in emulation mode during multi-arch image generation
+# https://github.com/docker/buildx/releases/tag/v0.9.1
+# Avoid upgrading to 0.10.0+ due to this change:
+#   Buildx v0.10 enables support for a minimal SLSA Provenance attestation, which requires support for OCI-compliant multi-platform images.
+#   This may introduce issues with registry and runtime support (e.g. Google Cloud Run and Lambda).
+#   You can optionally disable the default provenance attestation functionality using --provenance=false.
 RUN mkdir -p ~/.docker/cli-plugins && \
-    curl -SL https://github.com/docker/buildx/releases/download/v0.6.3/buildx-v0.6.3.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx  && \
+    curl -SL https://github.com/docker/buildx/releases/download/v0.9.1/buildx-v0.9.1.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx  && \
     chmod 775 ~/.docker/cli-plugins/docker-buildx && \
     docker buildx version
 

docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile

@@ -41,11 +41,11 @@ RUN if [[ `uname -m` = 'aarch64' ]]; then mkdir -p aarch64-builds && cd aarch64-
     ln -sfn /lib64/libstdc++.so.6.0.29 /lib64/libstdc++.so.6 && \
     echo "Installing glibc 2.18" && \
     curl -SLO https://ftp.gnu.org/gnu/glibc/glibc-2.18.tar.gz && tar -xzvf glibc-2.18.tar.gz && cd glibc-2.18 && mkdir -p build && cd build && \
-    ../configure --prefix=/usr && make -j $(( `nproc` / 2 )) && make install && cd ../../ && \
+    ../configure --prefix=/usr && make && make install && cd ../../ && \
     echo "Installing libicu 53+" && \
     rpm -e --nodeps libicu && \
     curl -SLO https://github.com/unicode-org/icu/releases/download/release-53-2/icu4c-53_2-src.tgz && tar -xzvf icu4c-53_2-src.tgz && cd icu && \
-    cd source && ./configure --prefix=/usr && make -j $(( `nproc` / 2 )) && make install && \
+    cd source && ./configure --prefix=/usr && make && make install && \
     cd ../../../ && rm -rf aarch64-builds; fi
 
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
@@ -75,7 +75,7 @@ RUN chmod -R 777 /dev/shm
 RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
     cd Python-3.9.17 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python links
 RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
@@ -92,7 +92,7 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib
 RUN yum install -y curl libcurl-devel libfaketime perl-core pcre-devel && yum remove -y openssl-devel && yum clean all && \
     mkdir -p /tmp/openssl && cd /tmp/openssl && \
     curl -sSL -o- https://www.openssl.org/source/openssl-1.1.1g.tar.gz | tar -xz --strip-components 1 && \
-    ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make -j $(( `nproc` / 2 )) && make install && \
+    ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make && make install && \
     echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib" > /etc/profile.d/openssl.sh && openssl version
 
 # Installing osslsigncode

docker/ci/dockerfiles/current/test.centos7.performance-test.x64.arm64.dockerfile

@@ -38,7 +38,7 @@ RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqli
 RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
     cd Python-3.7.7 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python37 links
 RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \

docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile

@@ -120,7 +120,7 @@ RUN chmod -R 777 /dev/shm
 RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
     cd Python-3.7.7 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python37 links
 RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \

docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile

@@ -131,7 +131,7 @@ RUN chmod -R 777 /dev/shm
 RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
     cd Python-3.7.7 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python37 links
 RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \

docker/release/build-image-multi-arch.sh

@@ -135,6 +135,8 @@ trap cleanup_all TERM INT EXIT
 DIR=`Temp_Folder_Create`
 echo New workspace $DIR
 echo -e "\n* Prepare docker buildx"
+docker buildx rm --all-inactive --force
+docker buildx prune --all --force
 docker buildx use default
 docker buildx create --name $BUILDER_NAME --use
 docker buildx inspect --bootstrap

jenkins/docker/docker-build.jenkinsfile

@@ -38,7 +38,7 @@ pipeline {
             agent {
                 docker {
                     label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder'
-                    image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2'
+                    image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1'
                     args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
                     registryUrl 'https://public.ecr.aws/'
                     alwaysPull true

jenkins/docker/docker-copy.jenkinsfile

@@ -40,7 +40,7 @@ pipeline {
             agent {
                 docker {
                     label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
-                    image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2'
+                    image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1'
                     args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
                     registryUrl 'https://public.ecr.aws/'
                     alwaysPull true

jenkins/docker/docker-scan.jenkinsfile

@@ -20,7 +20,7 @@ pipeline {
             agent {
                 docker {
                     label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
-                    image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2'
+                    image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1'
                     args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
                     registryUrl 'https://public.ecr.aws/'
                     alwaysPull true

jenkins/packer/packer-build.jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
     agent {
         docker {
             label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder'
-            image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2'
+            image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1'
             registryUrl 'https://public.ecr.aws/'
             alwaysPull true
         }

jenkins/promotion/promote-docker-ecr.jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
     agent {
         docker {
             label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
-            image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2'
+            image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1'
             registryUrl 'https://public.ecr.aws/'
             args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
         }

tests/jenkins/jenkinsjob-regression-files/docker/docker-build.jenkinsfile.txt

@@ -5,7 +5,7 @@
          docker-build.stage(Parameters Check, groovy.lang.Closure)
             docker-build.script(groovy.lang.Closure)
          docker-build.stage(docker-build, groovy.lang.Closure)
-            docker-build.echo(Executing on agent [docker:[alwaysPull:true, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+            docker-build.echo(Executing on agent [docker:[alwaysPull:true, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
             docker-build.script(groovy.lang.Closure)
                docker-build.echo(The docker-build workflow will only push docker images to staging, please use docker-copy to move the image to other repositories)
                docker-build.checkout({$class=GitSCM, branches=[{name=main}], userRemoteConfigs=[{url=https://github.com/opensearch-project/opensearch-build}]})

tests/jenkins/jenkinsjob-regression-files/packer/packer-build.jenkinsfile.txt

@@ -3,7 +3,7 @@
       packer-build.library({[email protected], retriever=null})
       packer-build.pipeline(groovy.lang.Closure)
          packer-build.timeout({time=4, unit=HOURS})
-         packer-build.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         packer-build.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          packer-build.stage(Parameters Check, groovy.lang.Closure)
             packer-build.script(groovy.lang.Closure)
          packer-build.stage(packer-build, groovy.lang.Closure)

tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDocker.jenkinsfile.txt

@@ -3,7 +3,7 @@
       promote-docker-ecr.library({[email protected], retriever=null})
       promote-docker-ecr.pipeline(groovy.lang.Closure)
          promote-docker-ecr.timeout({time=1, unit=HOURS})
-         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure)
             promote-docker-ecr.script(groovy.lang.Closure)
          promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure)

tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerECRLatestMajor.jenkinsfile.txt

@@ -3,7 +3,7 @@
       promote-docker-ecr.library({[email protected], retriever=null})
       promote-docker-ecr.pipeline(groovy.lang.Closure)
          promote-docker-ecr.timeout({time=1, unit=HOURS})
-         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure)
             promote-docker-ecr.script(groovy.lang.Closure)
          promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure)

tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatest.jenkinsfile.txt

@@ -3,7 +3,7 @@
       promote-docker-ecr.library({[email protected], retriever=null})
       promote-docker-ecr.pipeline(groovy.lang.Closure)
          promote-docker-ecr.timeout({time=1, unit=HOURS})
-         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure)
             promote-docker-ecr.script(groovy.lang.Closure)
          promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure)

tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatestMajor.jenkinsfile.txt

@@ -3,7 +3,7 @@
       promote-docker-ecr.library({[email protected], retriever=null})
       promote-docker-ecr.pipeline(groovy.lang.Closure)
          promote-docker-ecr.timeout({time=1, unit=HOURS})
-         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure)
             promote-docker-ecr.script(groovy.lang.Closure)
          promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure)

tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerMajor.jenkinsfile.txt

@@ -3,7 +3,7 @@
       promote-docker-ecr.library({[email protected], retriever=null})
       promote-docker-ecr.pipeline(groovy.lang.Closure)
          promote-docker-ecr.timeout({time=1, unit=HOURS})
-         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure)
             promote-docker-ecr.script(groovy.lang.Closure)
          promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure)

tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToECRLatestMajor.jenkinsfile.txt

@@ -3,7 +3,7 @@
       promote-docker-ecr.library({[email protected], retriever=null})
       promote-docker-ecr.pipeline(groovy.lang.Closure)
          promote-docker-ecr.timeout({time=1, unit=HOURS})
-         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         promote-docker-ecr.echo(Executing on agent [docker:[alwaysPull:false, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure)
             promote-docker-ecr.script(groovy.lang.Closure)
          promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure)