Rework PyPy installation

README.rst

@@ -173,7 +173,7 @@ Image content
 
 All images currently contain:
 
-- CPython 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and PyPy 3.7, 3.8, 3.9 installed in
+- CPython 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and PyPy 3.7, 3.8, 3.9, 3.10 installed in
   ``/opt/python/<python tag>-<abi tag>``. The directories are named
   after the PEP 425 tags for each environment --
   e.g. ``/opt/python/cp37-cp37m`` contains a CPython 3.7 build, and
@@ -184,6 +184,49 @@ All images currently contain:
 
 - The `auditwheel <https://pypi.python.org/pypi/auditwheel>`_ tool
 
+- The manylinux-interpreters tool which allows to list all available interpreters & install ones missing from the image
+
+  3 commands are available:
+
+  - ``manylinux-interpreters list``
+
+    .. code-block:: bash
+
+      usage: manylinux-interpreters list [-h] [-v] [-i] [--format {text,json}]
+
+      list available or installed interpreters
+
+      options:
+        -h, --help            show this help message and exit
+        -v, --verbose         display additional information (--format=text only, ignored for --format=json)
+        -i, --installed       only list installed interpreters
+        --format {text,json}  text is not meant to be machine readable (i.e. the format is not stable)
+
+  - ``manylinux-interpreters ensure-all``
+
+    .. code-block:: bash
+
+      usage: manylinux-interpreters ensure-all [-h]
+
+      make sure all interpreters are installed
+
+      options:
+        -h, --help  show this help message and exit
+
+  - ``manylinux-interpreters ensure``
+
+    .. code-block:: bash
+
+      usage: manylinux-interpreters ensure [-h] TAG [TAG ...]
+
+      make sure a list of interpreters are installed
+
+      positional arguments:
+        TAG         tag with format '<python tag>-<abi tag>' e.g. 'pp310-pypy310_pp73'
+
+      options:
+        -h, --help  show this help message and exit
+
 Note that less common or virtually unheard of flag combinations
 (such as ``--with-pydebug`` (``d``) and ``--without-pymalloc`` (absence of ``m``)) are not provided.
 

docker/Dockerfile

@@ -138,43 +138,31 @@ FROM build_cpython AS build_cpython312
 COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.0rc2
 
-FROM build_cpython AS all_python
-COPY build_scripts/install-pypy.sh \
-     build_scripts/pypy.sha256 \
-     build_scripts/finalize-python.sh \
-     /build_scripts/
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.11
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.12
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.10 7.3.12
-COPY --from=build_cpython36 /opt/_internal /opt/_internal/
-COPY --from=build_cpython37 /opt/_internal /opt/_internal/
-COPY --from=build_cpython38 /opt/_internal /opt/_internal/
-COPY --from=build_cpython39 /opt/_internal /opt/_internal/
-COPY --from=build_cpython310 /opt/_internal /opt/_internal/
-COPY --from=build_cpython311 /opt/_internal /opt/_internal/
-COPY --from=build_cpython312 /opt/_internal /opt/_internal/
-RUN manylinux-entrypoint /build_scripts/finalize-python.sh
+FROM build_cpython AS all_cpython
+COPY build_scripts/finalize-python.sh /build_scripts/
+RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
+    --mount=type=bind,target=/build_cpython37,from=build_cpython37 \
+    --mount=type=bind,target=/build_cpython38,from=build_cpython38 \
+    --mount=type=bind,target=/build_cpython39,from=build_cpython39 \
+    --mount=type=bind,target=/build_cpython310,from=build_cpython310 \
+    --mount=type=bind,target=/build_cpython311,from=build_cpython311 \
+    --mount=type=bind,target=/build_cpython312,from=build_cpython312 \
+    mkdir -p /opt/_internal && \
+    cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
+    manylinux-entrypoint /build_scripts/finalize-python.sh
 
 
 FROM runtime_base
 COPY --from=build_git /manylinux-rootfs /
 COPY --from=build_cpython /manylinux-rootfs /
-COPY --from=all_python /opt/_internal /opt/_internal/
-COPY build_scripts/finalize.sh \
-     build_scripts/update-system-packages.sh \
-     build_scripts/python-tag-abi-tag.py \
-     build_scripts/requirements3.6.txt \
-     build_scripts/requirements3.7.txt \
-     build_scripts/requirements3.8.txt \
-     build_scripts/requirements3.9.txt \
-     build_scripts/requirements3.10.txt \
-     build_scripts/requirements3.11.txt \
-     build_scripts/requirements3.12.txt \
-     build_scripts/requirements-base-tools.txt \
-     /build_scripts/
-COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
-RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
+COPY build_scripts /opt/_internal/build_scripts/
+RUN --mount=type=bind,target=/all_cpython,from=all_cpython \
+    cp -rf /all_cpython/opt/_internal/* /opt/_internal/ && \
+    manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \
+    pp37-pypy37_pp73 \
+    pp38-pypy38_pp73 \
+    pp39-pypy39_pp73 \
+    pp310-pypy310_pp73
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 

docker/build_scripts/download-and-install-interpreter.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+ABI_TAG=$1
+DOWNLOAD_URL=$2
+SHA256=$3
+
+PREFIX="/opt/_internal/${ABI_TAG}"
+
+case ${DOWNLOAD_URL} in
+  *.tar) COMP=;;
+  *.tar.gz) COMP=z;;
+  *.tar.bz2) COMP=j;;
+  *.tar.xz) COMP=J;;
+  *) echo "unsupported archive"; exit 1;;
+esac
+
+mkdir ${PREFIX}
+
+curl -fsSL ${DOWNLOAD_URL} | tee >(tar -C ${PREFIX} --strip-components 1 -x${COMP}f -) | sha256sum -c <(echo "${SHA256} -")
+
+# remove debug symbols if any
+find ${PREFIX}/bin -name '*.debug' -delete
+
+${MY_DIR}/finalize-one.sh ${PREFIX}

docker/build_scripts/finalize-one.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+PREFIX=$1
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# Some python's install as bin/python3. Make them available as
+# bin/python.
+if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
+	ln -s python3 ${PREFIX}/bin/python
+fi
+PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
+PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)")
+
+# Install pinned packages for this python version.
+# Use the already intsalled cpython pip to bootstrap pip if available
+if [ -f /usr/local/bin/python${PY_VER} ]; then
+	/usr/local/bin/python${PY_VER} -m pip --python ${PREFIX}/bin/python install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
+else
+	${PREFIX}/bin/python -m ensurepip
+	${PREFIX}/bin/python -m pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
+fi
+if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
+	ln -s pip3 ${PREFIX}/bin/pip
+fi
+# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
+ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
+ln -s ${PREFIX} /opt/python/${ABI_TAG}
+# Make versioned python commands available directly in environment.
+if [[ "${PY_IMPL}" == "cpython" ]]; then
+	ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
+fi
+ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}

docker/build_scripts/finalize.sh

@@ -11,30 +11,21 @@ source $MY_DIR/build_utils.sh
 
 mkdir /opt/python
 for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do
-	# Some python's install as bin/python3. Make them available as
-	# bin/python.
-	if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
-		ln -s python3 ${PREFIX}/bin/python
-	fi
-	${PREFIX}/bin/python -m ensurepip
-	if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
-		ln -s pip3 ${PREFIX}/bin/pip
-	fi
-	PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
-	# Since we fall back on a canned copy of pip, we might not have
-	# the latest pip and friends. Upgrade them to make sure.
-	${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
-	# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
-	ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
-	ln -s ${PREFIX} /opt/python/${ABI_TAG}
-	# Make versioned python commands available directly in environment.
-	if [[ "${PREFIX}" == *"/pypy"* ]]; then
-		ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}
-	else
-		ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
-	fi
+	${MY_DIR}/finalize-one.sh ${PREFIX}
 done
 
+# create manylinux-interpreters script
+cat <<EOF > /usr/local/bin/manylinux-interpreters
+#!/bin/bash
+
+set -euo pipefail
+
+/opt/python/cp310-cp310/bin/python $MY_DIR/manylinux-interpreters.py "\$@"
+EOF
+chmod 755 /usr/local/bin/manylinux-interpreters
+
+MANYLINUX_INTERPRETERS_NO_CHECK=1 /usr/local/bin/manylinux-interpreters ensure "$@"
+
 # Create venv for auditwheel & certifi
 TOOLS_PATH=/opt/_internal/tools
 /opt/python/cp310-cp310/bin/python -m venv $TOOLS_PATH
@@ -84,6 +75,7 @@ clean_pyc /opt/_internal
 
 # remove cache
 rm -rf /root/.cache
+rm -rf /tmp/* || true
 
 hardlink -cv /opt/_internal