Skip to content

Commit

Permalink
[Python 3.9 Update] Docker CI Images update 3.7 to 3.9 (Part 1) (#3634)
Browse files Browse the repository at this point in the history 
Signed-off-by: Peter Zhu <[email protected]>
  • Loading branch information
@peterzhuamazon
peterzhuamazon authored Jun 16, 2023
1 parent 8c9ab7d commit 9fc03ae
Show file tree
Hide file tree
Showing 6 changed files with 62 additions and 83 deletions.
29 changes: 14 additions & 15 deletions docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,6 @@ FROM centos:7
# Ensure localedef running correct with root permission
USER 0

# Setup ENV to prevent ASCII data issues with Python3
RUN echo "export LC_ALL=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
echo "export LANG=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || echo set locale

# Add normal dependencies
RUN yum clean all && yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \
yum install epel-release -y && \
Expand All @@ -37,7 +32,7 @@ RUN groupadd -g 1000 opensearch && \
mkdir -p /usr/share/opensearch && \
chown -R 1000:1000 /usr/share/opensearch

# Add Python37 dependencies
# Add Python dependencies
RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils

# Add Dashboards dependencies
Expand Down Expand Up @@ -69,18 +64,22 @@ ENV GEM_HOME=/usr/share/opensearch/.gem
ENV GEM_PATH=$GEM_HOME
ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH

# Install Python37 binary
RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
cd Python-3.7.7 && \
# Install Python binary
RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
cd Python-3.9.17 && \
./configure --enable-optimizations && \
make altinstall -j $(( `nproc` / 2 ))

# Setup Python37 links
RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \
ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip && \
ln -sfn /usr/local/bin/pip3.7 /usr/local/bin/pip && \
ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip3 && \
pip3 install pipenv && pipenv --version
# Setup Python links
RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip && \
ln -sfn /usr/local/bin/pip3.9 /usr/local/bin/pip && \
ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip3 && \
pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12

# Preparation for awscliv2
#RUN pip3 install git+https://github.com/aws/[email protected]
#ENV AWS_CLI_FILE_ENCODING=UTF-8

# Upgrade gcc8
# The setup part is partially based on Austin Dewey's article:
Expand Down
27 changes: 10 additions & 17 deletions docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,6 @@ ARG MAVEN_DIR=/usr/local/apache-maven
# Ensure localedef running correct with root permission
USER 0

# Setup ENV to prevent ASCII data issues with Python3
RUN echo "export LC_ALL=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
echo "export LANG=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || echo set locale

# Add normal dependencies
RUN yum clean all && yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \
yum install epel-release -y && \
Expand All @@ -33,7 +28,7 @@ RUN groupadd -g 1000 opensearch && \
mkdir -p /usr/share/opensearch && \
chown -R 1000:1000 /usr/share/opensearch

# Add Python37 dependencies
# Add Python dependencies
RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils

# Add Dashboards dependencies
Expand Down Expand Up @@ -78,24 +73,22 @@ ENV GEM_HOME=/usr/share/opensearch/.gem
ENV GEM_PATH=$GEM_HOME
ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH

# Install Python37 binary
RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
cd Python-3.7.7 && \
# Install Python binary
RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
cd Python-3.9.17 && \
./configure --enable-optimizations && \
make altinstall -j $(( `nproc` / 2 ))

# Setup Python37 links
RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \
ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip && \
ln -sfn /usr/local/bin/pip3.7 /usr/local/bin/pip && \
ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip3 && \
pip3 install pipenv && pipenv --version
# Setup Python links
RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip && \
ln -sfn /usr/local/bin/pip3.9 /usr/local/bin/pip && \
ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip3 && \
pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12

# Add k-NN Library dependencies
RUN yum install epel-release -y && yum repolist && yum install openblas-static lapack gcc-gfortran -y
RUN pip3 install pip==21.3.1
RUN pip3 install cmake==3.21.3
RUN pip3 install awscli==1.22.12

# Change User
USER 1000
Expand Down
27 changes: 6 additions & 21 deletions docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,18 @@ ARG MAVEN_DIR=/usr/local/apache-maven
# Ensure localedef running correct with root permission
USER 0

# Setup ENV to prevent ASCII data issues with Python3
RUN echo "export LC_ALL=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
echo "export LANG=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || echo set locale

# Add normal dependencies
RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \
dnf update -y && \
dnf install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq gh
dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq gh

# Create user group
RUN groupadd -g 1000 opensearch && \
useradd -u 1000 -g 1000 -d /usr/share/opensearch opensearch && \
mkdir -p /usr/share/opensearch && \
chown -R 1000:1000 /usr/share/opensearch

# Add Python37 dependencies
# Add Python dependencies
RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils

# Add Dashboards dependencies
Expand Down Expand Up @@ -77,28 +72,18 @@ ENV GEM_HOME=/usr/share/opensearch/.gem
ENV GEM_PATH=$GEM_HOME
ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH

# Install Python37 binary
RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
cd Python-3.7.7 && \
./configure --enable-optimizations && \
make altinstall -j $(( `nproc` / 2 ))

# Setup Python37 links
RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \
ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip && \
ln -sfn /usr/local/bin/pip3.7 /usr/local/bin/pip && \
ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip3 && \
pip3 install pipenv && pipenv --version
# Install Python binary
RUN update-alternatives --set python /usr/bin/python3.9 && \
update-alternatives --set python3 /usr/bin/python3.9 && \
pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12

# Add k-NN Library dependencies
# EL8 requires install config-manager and enable powertools to consume openblas-static
RUN dnf install epel-release -y && dnf repolist && \
dnf install -y 'dnf-command(config-manager)' && \
dnf config-manager --set-enabled powertools && \
dnf install openblas-static lapack gcc-gfortran -y && dnf clean all
RUN pip3 install pip==21.3.1
RUN pip3 install cmake==3.21.3
RUN pip3 install awscli==1.22.12

# Change User
USER 1000
Expand Down
16 changes: 8 additions & 8 deletions docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@ FROM ubuntu:20.04

ARG DEBIAN_FRONTEND=noninteractive

# Install python37 dependencies
RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa -y
# Install python dependencies
RUN apt-get update -y && apt-get install -y software-properties-common

# Install python37 binaries
# Install python binaries
RUN apt-get update -y && apt-get install python3 && \
apt-get install -y python3.7-full python3.7-dev && \
update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 && \
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
apt-get install -y python3.9-full python3.9-dev && \
apt-get install -y python-is-python3 && \
update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 && \
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1

# Install necessary packages
RUN apt-get update -y && apt-get upgrade -y && apt-get install -y docker.io curl build-essential git jq && \
Expand All @@ -29,8 +30,7 @@ RUN apt-get update -y && apt-get upgrade -y && apt-get install -y docker.io curl

# Install pip packages
RUN curl -SL https://bootstrap.pypa.io/get-pip.py | python && \
pip3 install pipenv && pipenv --version && \
pip3 install awscli==1.22.12 && aws --version
pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12

# Install aptly and required changes to debmake
# Remove lintian for now due to it takes nearly 20 minutes for OpenSearch as well as nearly an hour for OpenSearch-Dashboards during debmake
Expand Down
11 changes: 9 additions & 2 deletions docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

FROM ubuntu:20.04

ARG DEBIAN_FRONTEND=noninteractive

# Import necessary repository for installing qemu 5.0
RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:jacob/virtualisation -y

Expand Down Expand Up @@ -52,22 +54,27 @@ ENV JAVA_HOME=/opt/java/openjdk-11
ENV PATH=$PATH:$JAVA_HOME/bin

# Install docker buildx
# Stays on 0.6.3 as that is the stable version we used since the introduction of OpenSearch Multi-Arch Docker Images
RUN mkdir -p ~/.docker/cli-plugins && \
curl -SL https://github.com/docker/buildx/releases/download/v0.6.3/buildx-v0.6.3.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx && \
chmod 775 ~/.docker/cli-plugins/docker-buildx && \
docker buildx version

# Install gcrane
RUN curl -SL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz -o go-containerregistry.tar.gz && \
# Stays on 0.15.2 due to --all-tags was introduced in 0.15.1 and several bugs are fixed in 0.15.2: https://github.com/google/go-containerregistry/pull/1682
RUN curl -SL https://github.com/google/go-containerregistry/releases/download/v0.15.2/go-containerregistry_Linux_x86_64.tar.gz -o go-containerregistry.tar.gz && \
tar -zxvf go-containerregistry.tar.gz && \
chmod +x gcrane crane krane && \
mv -v gcrane crane krane /usr/local/bin/ && \
rm -v go-containerregistry.tar.gz && \
gcrane version && crane version && krane version

# Install packer
# Stays on 1.8.7 version due to 1.8.7 fixed the JSON regression: https://github.com/hashicorp/packer/issues/12281
# As well as 1.9.0+ includes major changes. A lot of plugins are removed since 1.9.0: https://github.com/hashicorp/packer/releases/tag/v1.9.0
RUN curl -SL -o- https://apt.releases.hashicorp.com/gpg | gpg --dearmor > /usr/share/keyrings/hashicorp-archive-keyring.gpg && \
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/hashicorp.list && \
apt-get update && \
apt-get install packer && \
apt-get install packer=1.8.7* && \
packer --version && \
apt-get clean
35 changes: 15 additions & 20 deletions docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,12 @@ ARG MAVEN_DIR=/usr/local/apache-maven
# Ensure localedef running correct with root permission
USER 0

# Setup ENV to prevent ASCII data issues with Python3
RUN echo "export LC_ALL=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
echo "export LANG=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || echo set locale

# Add normal dependencies
RUN yum clean all && yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \
yum update -y && \
yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq gh epel-release

# Add Python37 dependencies
# Add Python dependencies
RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils

# Add Yarn dependencies
Expand All @@ -46,11 +41,11 @@ RUN if [[ `uname -m` = 'aarch64' ]]; then mkdir -p aarch64-builds && cd aarch64-
ln -sfn /lib64/libstdc++.so.6.0.29 /lib64/libstdc++.so.6 && \
echo "Installing glibc 2.18" && \
curl -SLO https://ftp.gnu.org/gnu/glibc/glibc-2.18.tar.gz && tar -xzvf glibc-2.18.tar.gz && cd glibc-2.18 && mkdir -p build && cd build && \
../configure --prefix=/usr && make && make install && cd ../../ && \
../configure --prefix=/usr && make -j $(( `nproc` / 2 )) && make install && cd ../../ && \
echo "Installing libicu 53+" && \
rpm -e --nodeps libicu && \
curl -SLO https://github.com/unicode-org/icu/releases/download/release-53-2/icu4c-53_2-src.tgz && tar -xzvf icu4c-53_2-src.tgz && cd icu && \
cd source && ./configure --prefix=/usr && make && make install && \
cd source && ./configure --prefix=/usr && make -j $(( `nproc` / 2 )) && make install && \
cd ../../../ && rm -rf aarch64-builds; fi

ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
Expand All @@ -76,28 +71,28 @@ RUN export MAVEN_URL=`curl -s https://maven.apache.org/download.cgi | grep -Eo '
# Setup Shared Memory
RUN chmod -R 777 /dev/shm

# Installing Python37 binary
RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
cd Python-3.7.7 && \
# Installing Python binary
RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
cd Python-3.9.17 && \
./configure --enable-optimizations && \
make altinstall -j $(( `nproc` / 2 ))

# Setup Python37 links
RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \
ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip && \
ln -sfn /usr/local/bin/pip3.7 /usr/local/bin/pip && \
ln -sfn /usr/local/bin/pip3.7 /usr/bin/pip3 && \
pip3 install pipenv && pipenv --version && pip install --upgrade pip
# Setup Python links
RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip && \
ln -sfn /usr/local/bin/pip3.9 /usr/local/bin/pip && \
ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip3 && \
pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12

# Installing twine
RUN pip3 install twine cmake==3.24.1.1
# Installing pip packages
RUN pip3 install twine==4.0.2 cmake==3.24.1.1

# Installing openssl1.1.1
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib
RUN yum install -y curl libcurl-devel libfaketime perl-core pcre-devel && yum remove -y openssl-devel && yum clean all && \
mkdir -p /tmp/openssl && cd /tmp/openssl && \
curl -sSL -o- https://www.openssl.org/source/openssl-1.1.1g.tar.gz | tar -xz --strip-components 1 && \
./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make && make install && \
./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make -j $(( `nproc` / 2 )) && make install && \
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib" > /etc/profile.d/openssl.sh && openssl version

# Installing osslsigncode
Expand Down

0 comments on commit 9fc03ae

Please sign in to comment.