From d5ba8a82f3b12a803772944c43121fb685d4ebfb Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 3 Nov 2024 10:40:07 +0100 Subject: [PATCH] feat: create base image for manylinux2014 With CentOS 7 having reach EOL, packages versions are now immutable. Creating a base image with manylinux runtime packages allows to reduce image size and improve cache efficiency. --- .github/workflows/build.yml | 84 ++++------- .github/workflows/update-dependencies.yml | 58 -------- .travis.yml | 93 ------------ docker/Dockerfile | 164 ++++------------------ 4 files changed, 51 insertions(+), 348 deletions(-) delete mode 100644 .github/workflows/update-dependencies.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d1354ac9..7d53c8a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,74 +28,42 @@ concurrency: cancel-in-progress: true jobs: - build_manylinux: - name: ${{ matrix.policy }}_${{ matrix.platform }} + build_manylinux2014_base: runs-on: ubuntu-22.04 - permissions: - actions: write # this permission is needed to delete cache - strategy: - fail-fast: false - matrix: - policy: ["manylinux2014", "musllinux_1_2"] - platform: ["i686", "x86_64"] - include: - - policy: "manylinux_2_28" - platform: "x86_64" - - env: - POLICY: ${{ matrix.policy }} - PLATFORM: ${{ matrix.platform }} - COMMIT_SHA: ${{ github.sha }} - steps: - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: 50 - - name: Set up emulation - if: matrix.platform != 'i686' && matrix.platform != 'x86_64' + - name: Get tag name + id: tag + run: | + COMMIT_DATE=$(git show -s --format=%cd --date=short ${{ github.sha }}) + if $(git rev-parse --is-shallow-repository); then + git fetch --unshallow + fi + BUILD_NUMBER=$(git rev-list --since=${COMMIT_DATE}T00:00:00Z --first-parent --count ${{ github.sha }}) + BUILD_ID2=${COMMIT_DATE//-/.}-${BUILD_NUMBER} + echo "tag=${BUILD_ID2}" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - with: - platforms: ${{ matrix.platform }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Restore cache - if: github.event_name != 'workflow_dispatch' || fromJSON(github.event.inputs.useCache) - uses: actions/cache/restore@v4 + - name: Login to Quay.io + if: github.event_name == 'push' + uses: docker/login-action@v3 with: - path: .buildx-cache-${{ matrix.policy }}_${{ matrix.platform }}/* - key: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }} + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} - - name: Build - run: ./build.sh - - - name: Delete cache - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - run: | - KEY="buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}" - gh cache delete ${KEY} || true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Save cache - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: actions/cache/save@v4 + - name: Build image + uses: docker/build-push-action@v6 with: - path: .buildx-cache-${{ matrix.policy }}_${{ matrix.platform }}/* - key: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }} - - - name: Deploy - if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'pypa/manylinux' - run: ./deploy.sh - env: - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} - - all_passed: - needs: [build_manylinux] - runs-on: ubuntu-latest - steps: - - run: echo "All jobs passed" + context: ./docker + platforms: linux/arm64,linux/x86_64,linux/386,linux/ppc64le,linux/s390x + load: false + push: true + tags: quay.io/pypa/manylinux2014_base:latest,quay.io/pypa/manylinux2014_base:${{ steps.tag.outputs.tag }} diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml deleted file mode 100644 index 71bfd073..00000000 --- a/.github/workflows/update-dependencies.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Update dependencies - -on: - pull_request: - paths: - - '.github/workflows/update-dependencies.yml' - workflow_dispatch: - schedule: - - cron: '0 18 * * 5' # "At 18:00 on Friday." - -env: - FORCE_COLOR: '1' - -jobs: - update-dependencies: - name: Update dependencies - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: wntrblm/nox@2024.10.09 - with: - python-versions: "3.12" - - name: "Install uv" - run: pipx install uv - - name: "Setup bot user" - run: | - git config --global user.name "manylinux-bot[bot]" - git config --global user.email "89297709+manylinux-bot[bot]@users.noreply.github.com" - # we use this step to grab a Github App auth token, so that lastversion can query GitHub API - # without rate-limit and PRs get run by GHA. - - uses: actions/create-github-app-token@v1 - id: generate-token - if: github.ref == 'refs/heads/main' && github.repository == 'pypa/manylinux' - with: - app_id: ${{ secrets.MANYLINUX_BOT_APP_ID }} - private_key: ${{ secrets.MANYLINUX_BOT_APP_PRIVATE_KEY }} - - name: "Run update native dependencies" - run: nox -s update_native_dependencies - env: - GITHUB_API_TOKEN: ${{ steps.generate-token.outputs.token || github.token }} - - name: "Run update downloaded interpreters" - run: nox -s update_interpreters_download - - name: "Run update python dependencies" - run: nox -s update_python_dependencies - - name: Create Pull Request - if: github.ref == 'refs/heads/main' && github.repository == 'pypa/manylinux' - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update python dependencies - title: '[Bot] Update dependencies' - body: | - Update the versions of our dependencies. - - PR generated by "Update dependencies" [workflow](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}). - branch: update-dependencies-pr - sign-commits: true - token: ${{ steps.generate-token.outputs.token }} - delete-branch: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 32963ecb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,93 +0,0 @@ -language: c -os: linux -dist: focal -addons: - apt: - sources: - - sourceline: 'deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable' - key_url: 'https://download.docker.com/linux/ubuntu/gpg' - packages: - - docker-ce docker-ce-cli containerd.io docker-buildx-plugin -services: - - docker - -# Don't build the update-dependencies-pr branch; it's redundant -# with the PR builds that Travis also does. -branches: - except: - - /^update-dependencies-pr/ - -cache: - directories: - - ${HOME}/buildx-cache/ - -env: - global: - # QUAY_USERNAME and QUAY_PASSWORD for docker image upload - - secure: "lKaTzEL6UNiEfp+BWLOUILG9BMtjwEMpwt6Yag0cQGHix7qJ/ElZ0t3oFw6ZwuDmA5qceAXIdxHLUK9HGVI2MloLk8czGhjvtfJ4XhOxtEJRQ0VkDGPsKN4cfhB4ZjGo6GAPtNqStMyNiY7BZuTrZa7coDLCoUeYcOmTpi6pmd1rrkk725B9QCTuhFHbPhkuL2yu/Jk6WxkHJBKjmuZek+iQa7lRItgMrG0/319PXLvwIGGl00nLFy+Ly5Ciwzux4wuHLTySZQKu0H9FX81A7smM0FW/42kg3ckGa2qLxRw/Pi8Nm/aIk8LD0QXzI5N7HhFfidOTgDS8Mt1HgfxmTk4wUXZ/KvCCshqjimzMc/s9i9wPZX9UqqcfrpZkmwz8dzhm1bndN45ZOCy6xAYT6dzf8T4mLMDjVWSW4+DUoW4sYHRLVujjcMk7ybcwGV43VruPTJnc8XVAhT+VIMQkoPjhQmTOn8h82LRNGYtLa5RReCh9OPKVYB2Quz18FXMWgFt7A6VWudL0c7/8CusLvuo+pLcxt9pnV40rvu1YEohpEj8qR/qTSaDUBZM0J9SVf5zrZR80pZUnXkDF8nm+mcLOTley3YWipU19lCR7dzVyCAiQdVAuNPdnyem3Yk8enGkAJbfLd6eaIDs+p73D0JXh1Nx1px1movVLQH3ohIw=" - - secure: "w1614pomHLltkBhqWM2bOvbymFWIWKqSqqIBDvaNn9tbQScioItJoELBT7g7+cD7nyU7OvpQ1U2fk0xVkCeNvYU0xS1vP4o/VnZRpup7f7Tkiq+2rf4fjwYr3HHnJjwak1l9bsw6FkgzKaVvSdiUJHMVxiIuLd3fVozR7qjBBhTDxSlWGOpSgd+ttpgMZwU5zQjdaVQr1D7E8M0979ZnWMrNRyLiAUeHaPILS815b+ijgqR+i5nmu0/FTCGM9Ik4KIzIfWq8AdfPdbRiq8c+LrrTPfyKcIQJaHmfduYRM4LycGWwzkXFBNtLrJ7uFLG9RDVemOHuHOWIJX8qCUIV4XuESXxH3fUQr6r+yxquTJbzXxNtoaLa6tBOTQWKDrRjT4z9Mf9Im14F2V59EUDoQowHx5bjunOH5wg3ruYNKYYBFRYra5kx0CkKrqFBzyl8fTUEQLyx1HWTVUC1WTXEeD/aFKOSIxW5DxZr5W4LLlW2+Raa52ZzY28Q6AdueFQCRzoJ70/GsJRlSsBdWNOHN4gSp1cZuToLWY15y64QhAMVDpikB+V4hmkbceLiTqeWzTStNL1sa32RHr6i/9zeFZw1pMD1+eOg9x6fgODfh2sqr/zPbu2oONsHnc4D2jwsEax4o+Dv5QHLvK7jdyWUmu47a9QReoexXK60jZXs3CA=" - -jobs: - include: - - arch: arm64-graviton2 - virt: vm - group: edge - env: POLICY="manylinux2014" PLATFORM="aarch64" - - arch: s390x - env: POLICY="manylinux2014" PLATFORM="s390x" - - arch: ppc64le - env: POLICY="manylinux2014" PLATFORM="ppc64le" - - arch: arm64-graviton2 - virt: vm - group: edge - env: POLICY="manylinux_2_28" PLATFORM="aarch64" - - arch: s390x - env: POLICY="manylinux_2_28" PLATFORM="s390x" - - arch: ppc64le - env: POLICY="manylinux_2_28" PLATFORM="ppc64le" - - arch: arm64-graviton2 - virt: vm - group: edge - env: POLICY="musllinux_1_2" PLATFORM="aarch64" - - arch: arm64-graviton2 - virt: vm - group: edge - env: POLICY="musllinux_1_2" PLATFORM="armv7l" - - arch: s390x - env: POLICY="musllinux_1_2" PLATFORM="s390x" - - arch: ppc64le - env: POLICY="musllinux_1_2" PLATFORM="ppc64le" - -before_install: - - if [ -d "${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM}" ]; then cp -rlf ${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM} ./; fi - -install: - - docker version - - docker buildx version - - docker buildx create --name builder-manylinux --driver docker-container --use - - docker buildx inspect --bootstrap --builder builder-manylinux 2>&1 | tee /dev/null - -script: | - BUILD_STATUS=success - (while true; do echo "travis_wait"; docker stats --no-stream; free; df -h; sleep 30; done) & - WAIT_PID=$! - COMMIT_SHA=${TRAVIS_COMMIT} ./build.sh || BUILD_STATUS=failed - kill -9 ${WAIT_PID} - if [ "${BUILD_STATUS}" != "success" ]; then - exit 1 - fi - if [ -d "${HOME}/buildx-cache" ]; then - rm -rf ${HOME}/buildx-cache - fi - mkdir ${HOME}/buildx-cache - if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then - cp -rlf ./.buildx-cache-* ${HOME}/buildx-cache/ - fi - -deploy: - provider: script - dpl_version: 1.10.16 - script: COMMIT_SHA=${TRAVIS_COMMIT} ./deploy.sh - on: - branch: main - repo: pypa/manylinux diff --git a/docker/Dockerfile b/docker/Dockerfile index 9488bade..7a5cc96a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,25 +1,22 @@ -# default to latest supported policy, x86_64 -ARG BASEIMAGE=amd64/almalinux:8 -ARG POLICY=manylinux_2_28 -ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-12/root -ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst -ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin: - -FROM $BASEIMAGE AS runtime_base +ARG POLICY=manylinux2014 +ARG PLATFORM=${TARGETARCH} +ARG PLATFORM=${PLATFORM/amd64/x86_64} +ARG PLATFORM=${PLATFORM/arm64/aarch64} +ARG PLATFORM=${PLATFORM/386/i686} +ARG BASEIMAGE=${PLATFORM} +ARG BASEIMAGE=${BASEIMAGE/i686/centos} +ARG BASEIMAGE=${BASEIMAGE/x86_64/centos} +ARG BASEIMAGE=${BASEIMAGE/aarch64/centos} +ARG BASEIMAGE=${BASEIMAGE/ppc64le/centos} +ARG BASEIMAGE=${BASEIMAGE/s390x/clefos} + +FROM ${BASEIMAGE}:7 AS runtime_base ARG POLICY ARG PLATFORM -ARG DEVTOOLSET_ROOTPATH -ARG LD_LIBRARY_PATH_ARG -ARG PREPEND_PATH LABEL maintainer="The ManyLinux project" ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} -ENV PATH=${PREPEND_PATH}${PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig # first copy the fixup mirrors script, keep the script around COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors @@ -40,127 +37,16 @@ COPY build_scripts/install-runtime-packages.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ -COPY build_scripts/build_utils.sh /build_scripts/ - -COPY build_scripts/install-autoconf.sh /build_scripts/ -RUN export AUTOCONF_ROOT=autoconf-2.72 && \ - export AUTOCONF_HASH=afb181a76e1ee72832f6581c0eddf8df032b83e2e0239ef79ebedc4467d92d6e && \ - export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ - manylinux-entrypoint /build_scripts/install-autoconf.sh - -COPY build_scripts/install-automake.sh /build_scripts/ -RUN export AUTOMAKE_ROOT=automake-1.17 && \ - export AUTOMAKE_HASH=397767d4db3018dd4440825b60c64258b636eaf6bf99ac8b0897f06c89310acd && \ - export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ - manylinux-entrypoint /build_scripts/install-automake.sh - -COPY build_scripts/install-libtool.sh /build_scripts/ -RUN export LIBTOOL_ROOT=libtool-2.5.3 && \ - export LIBTOOL_HASH=9322bd8f6bc848fda3e385899dd1934957169652acef716d19d19d24053abb95 && \ - export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ - manylinux-entrypoint /build_scripts/install-libtool.sh - -COPY build_scripts/install-libxcrypt.sh /build_scripts/ -RUN export LIBXCRYPT_VERSION=4.4.36 && \ - export LIBXCRYPT_HASH=b979838d5f1f238869d467484793b72b8bca64c4eae696fdbba0a9e0b6c28453 && \ - export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ - manylinux-entrypoint /build_scripts/install-libxcrypt.sh - -FROM runtime_base AS build_base -COPY build_scripts/install-build-packages.sh /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-build-packages.sh - - -FROM build_base AS build_git -COPY build_scripts/build-git.sh /build_scripts/ -RUN export GIT_ROOT=git-2.45.2 && \ - export GIT_HASH=98b26090ed667099a3691b93698d1e213e1ded73d36a2fde7e9125fce28ba234 && \ - export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \ - manylinux-entrypoint /build_scripts/build-git.sh - - -FROM build_base AS build_cpython_system_ssl -COPY build_scripts/build-sqlite3.sh /build_scripts/ -RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3470000 && \ - export SQLITE_AUTOCONF_HASH=83eb21a6f6a649f506df8bd3aab85a08f7556ceed5dbd8dea743ea003fc3a957 && \ - export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2024 && \ - manylinux-entrypoint /build_scripts/build-sqlite3.sh - -COPY build_scripts/build-tcltk.sh /build_scripts/ -RUN export TCL_ROOT=tcl8.6.14 && \ - export TCL_HASH=5880225babf7954c58d4fb0f5cf6279104ce1cd6aa9b71e9a6322540e1c4de66 && \ - export TCL_DOWNLOAD_URL=https://prdownloads.sourceforge.net/tcl && \ - export TK_ROOT=tk8.6.14 && \ - export TK_HASH=8ffdb720f47a6ca6107eac2dd877e30b0ef7fac14f3a84ebbd0b3612cee41a94 && \ - manylinux-entrypoint /build_scripts/build-tcltk.sh - -COPY build_scripts/build-cpython.sh /build_scripts/ - - -FROM build_cpython_system_ssl AS build_cpython -COPY build_scripts/build-openssl.sh /build_scripts/ -RUN export OPENSSL_ROOT=openssl-3.0.15 && \ - export OPENSSL_HASH=23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533 && \ - export OPENSSL_DOWNLOAD_URL=https://github.com/openssl/openssl/releases/download/${OPENSSL_ROOT} && \ - manylinux-entrypoint /build_scripts/build-openssl.sh - - -FROM build_cpython_system_ssl AS build_cpython36 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.15 - -FROM build_cpython_system_ssl AS build_cpython37 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.17 - - -FROM build_cpython AS build_cpython38 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.20 - -FROM build_cpython AS build_cpython39 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.20 - -FROM build_cpython AS build_cpython310 -COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.15 - -FROM build_cpython AS build_cpython311 -COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.11.10 - -FROM build_cpython AS build_cpython312 -COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.7 - -FROM build_cpython AS build_cpython313 -COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.13.0 - -FROM build_cpython AS build_cpython313_nogil -COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.13.0 nogil - - -FROM runtime_base -COPY --from=build_git /manylinux-rootfs / -COPY --from=build_cpython_system_ssl /manylinux-rootfs / -COPY build_scripts /opt/_internal/build_scripts/ -RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \ - --mount=type=bind,target=/build_cpython37,from=build_cpython37 \ - --mount=type=bind,target=/build_cpython38,from=build_cpython38 \ - --mount=type=bind,target=/build_cpython39,from=build_cpython39 \ - --mount=type=bind,target=/build_cpython310,from=build_cpython310 \ - --mount=type=bind,target=/build_cpython311,from=build_cpython311 \ - --mount=type=bind,target=/build_cpython312,from=build_cpython312 \ - --mount=type=bind,target=/build_cpython313,from=build_cpython313 \ - --mount=type=bind,target=/build_cpython313_nogil,from=build_cpython313_nogil \ - mkdir -p /opt/_internal && \ - cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \ - manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \ - pp310-pypy310_pp73 - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - +FROM scratch +COPY --from=runtime_base / / +LABEL \ + org.label-schema.schema-version="1.0" \ + org.label-schema.name="ManyLinux 2014 Base Image" \ + org.label-schema.vendor="The ManyLinux project" \ + org.label-schema.license="GPLv2" \ + org.label-schema.build-date="20241102" \ + org.opencontainers.image.title="ManyLinux 2014 Base Image" \ + org.opencontainers.image.vendor="The ManyLinux project" \ + org.opencontainers.image.licenses="GPL-2.0-only" \ + org.opencontainers.image.created="2024-11-02 00:00:00+00:00" CMD ["/bin/bash"]