[Python 3.9 Upgrade] Add the test docker images upgrades to Python 3.…

…9 (Part 2) (#3680)

Signed-off-by: Peter Zhu <[email protected]>

docker/ci/dockerfiles/current/test.centos7.performance-test.x64.arm64.dockerfile

@@ -13,11 +13,6 @@ FROM centos:7
 # Ensure localedef running correct with root permission
 USER 0
 
-# Setup ENV to prevent ASCII data issues with Python3
-RUN echo "export LC_ALL=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
-    echo "export LANG=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
-    localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || echo set locale
-
 # Add normal dependencies
 RUN yum clean all && yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \
     yum install epel-release -y && \
@@ -30,25 +25,25 @@ RUN groupadd -g 1000 opensearch && \
     mkdir -p /usr/share/opensearch && \
     chown -R 1000:1000 /usr/share/opensearch
 
-# Add Python37 dependencies
+# Add Python dependencies
 RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils && \
     yum clean all
 
-# Install Python37 binary
-RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
-    cd Python-3.7.7 && \
+# Install Python binary
+RUN curl https://www.python.org/ftp/python/3.9.7/Python-3.9.7.tgz | tar xzvf - && \
+    cd Python-3.9.7 && \
     ./configure --enable-optimizations && \
     make altinstall
 
-# Setup Python37 links
-RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/local/bin/pip && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip3 && \
-    pip3 install pipenv && pipenv --version
+# Setup Python links
+RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
+    ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip && \
+    ln -sfn /usr/local/bin/pip3.9 /usr/local/bin/pip && \
+    ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip3 && \
+    pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12
 
 # Add performance dependencies
-RUN pip3 install pip==21.3.1 awscli==1.22.12 dataclasses_json~=0.5 aws_requests_auth~=0.4 json2html~=1.3.0 aws-cdk.core~=1.143.0 aws_cdk.aws_ec2~=1.143.0 \
+RUN pip3 install dataclasses_json~=0.5 aws_requests_auth~=0.4 json2html~=1.3.0 aws-cdk.core~=1.143.0 aws_cdk.aws_ec2~=1.143.0 \
                  aws_cdk.aws_iam~=1.143.0 boto3~=1.18 setuptools~=57.4 retry~=0.9
 
 # install yq

docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile

@@ -16,7 +16,7 @@ USER 0
 # Add normal dependencies
 RUN yum clean all && \
     yum update -y && \
-    yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip
+    yum install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip
 
 # Add Dashboards dependencies (mainly for cypress)
 RUN yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
@@ -81,7 +81,7 @@ USER 0
 # Add normal dependencies
 RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \
     dnf update -y && \
-    dnf install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip gh
+    dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip gh
 
 # Create user group
 RUN groupadd -g 1000 opensearch && \
@@ -101,7 +101,7 @@ ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
 # Check dirs
 RUN source $NVM_DIR/nvm.sh && ls -al /usr/share/opensearch && echo $NODE_VERSION $NVM_DIR && nvm use $NODE_VERSION
 
-# Add Python37 dependencies
+# Add Python dependencies
 RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils
 
 # Add Dashboards dependencies (mainly for cypress)
@@ -116,24 +116,14 @@ RUN yum groupinstall -y "Development Tools" && yum clean all && rm -rf /var/cach
 # Setup Shared Memory
 RUN chmod -R 777 /dev/shm
 
-# Install Python37 binary
-RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
-    cd Python-3.7.7 && \
-    ./configure --enable-optimizations && \
-    make altinstall
-
-# Setup Python37 links
-RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/local/bin/pip && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip3
+# Install Python binary
+RUN update-alternatives --set python /usr/bin/python3.9 && \
+    update-alternatives --set python3 /usr/bin/python3.9 && \
+    pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12
 
 # Add other dependencies
 RUN yum install -y epel-release && yum clean all && yum install -y chromium jq && yum clean all && rm -rf /var/cache/yum/* && \
-    pip3 install pip==21.3.1 && \
-    pip3 install cmake==3.21.3 && \
-    pip3 install awscli==1.22.12 && \
-    pip3 install pipenv
+    pip3 install cmake==3.21.3
 
 # Change User
 USER 1000

docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile

@@ -21,7 +21,7 @@ USER 0
 # Add normal dependencies
 RUN dnf clean all && \
     dnf update -y && \
-    dnf install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq
+    dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq
 
 # Add Dashboards dependencies (mainly for cypress)
 RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
@@ -86,7 +86,7 @@ USER 0
 # Add normal dependencies
 RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \
     dnf update -y && \
-    dnf install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq gh
+    dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq gh
 
 # Create user group
 RUN dnf install -y sudo && \
@@ -108,7 +108,7 @@ ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
 # Check dirs
 RUN source $NVM_DIR/nvm.sh && ls -al /usr/share/opensearch && echo $NODE_VERSION $NVM_DIR && nvm use $NODE_VERSION
 
-# Add Python37 dependencies
+# Add Python dependencies
 RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils
 
 # Add Dashboards dependencies (mainly for cypress)
@@ -127,24 +127,14 @@ RUN /tmp/jdk-setup.sh && /tmp/yq-setup.sh
 # Setup Shared Memory
 RUN chmod -R 777 /dev/shm
 
-# Install Python37 binary
-RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
-    cd Python-3.7.7 && \
-    ./configure --enable-optimizations && \
-    make altinstall
-
-# Setup Python37 links
-RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/local/bin/pip && \
-    ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip3
+# Install Python binary
+RUN update-alternatives --set python /usr/bin/python3.9 && \
+    update-alternatives --set python3 /usr/bin/python3.9 && \
+    pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12
 
 # Add other dependencies
 RUN dnf install -y epel-release && dnf clean all && dnf install -y chromium jq && dnf clean all && \
-    pip3 install pip==21.3.1 && \
-    pip3 install cmake==3.21.3 && \
-    pip3 install awscli==1.22.12 && \
-    pip3 install pipenv
+    pip3 install cmake==3.21.3
 
 # We use the version test to check if packages installed correctly
 # And get added to the PATH

docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.dockerfile

@@ -85,15 +85,14 @@ USER 0
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-# Install python37 dependencies and chromium dependencies
-RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa -y && \
-    add-apt-repository ppa:saiarcot895/chromium-beta -y
+# Install python dependencies and chromium dependencies
+RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:saiarcot895/chromium-beta -y
 
-# Install python37 binaries
+# Install python binaries
 RUN apt-get update -y && apt-get install python3 && \
-    apt-get install -y python3.7-full python3.7-dev && \
-    update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 && \
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+    apt-get install -y python3.9-full python3.9-dev && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
 
 # Install necessary packages
 RUN apt-get update -y && apt-get upgrade -y && apt-get install -y curl git gnupg2 tar procps build-essential cmake zip unzip jq && \
@@ -103,10 +102,8 @@ RUN apt-get update -y && apt-get upgrade -y && apt-get install -y curl git gnupg
 
 # Install pip packages
 RUN curl -SL https://bootstrap.pypa.io/get-pip.py | python && \
-    pip3 install pip==21.3.1 && \
-    pip3 install cmake==3.21.3 && \
-    pip3 install awscli==1.22.12 && \
-    pip3 install pipenv
+    pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12 && \
+    pip3 install cmake==3.21.3
 
 # Create user group
 RUN apt-get install -y sudo && \