From 8bc2696445dcf2fd39b2fa60033796dc0d6ba7e1 Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Mon, 9 Sep 2024 10:06:04 -0700 Subject: [PATCH] chore(ci): use unified Gitlab pipeline for APM libraries [backport 2.9] (#10529) **What does this PR do?** Converts the Gitlab workflow to use the unified pipeline. By using the unified pipeline across all APM libraries, there is less maintenance hassle, fewer mistakes, and a more uniform creation of artifacts. Additionally, this change creates OCI packages and lib inject images for every `dd-trace-py` build. Previously, only released versions to pypi were built and tested. The changes can be summarized as follows: * Removing github actions that are now taken over by the unified pipeline * Removing Gitlab jobs taken over by the unified pipeline * Adding a job to download artifacts from the github build * Merging the download artifacts with dependencies from pypi `.gitlab-ci.yml` is best reviewed in its final form rather than as a diff. **How to test the change?** The changes were tested manually. CI/CD workflows are notoriously hard to test. Final products the final lib-init image was run through automated onboarding tests. `dd-trace-py` doesn't have automated tests for OCI images ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Laplie Anderson --- .github/workflows/build_deploy.yml | 15 -- .github/workflows/lib-inject-publish.yml | 38 ----- .github/workflows/lib-injection.yml | 167 --------------------- .gitlab-ci.yml | 163 ++++++-------------- .gitlab/build-deb-rpm.sh | 49 ------ .gitlab/build-lib-init.sh | 116 -------------- .gitlab/build-oci.sh | 57 ------- .gitlab/download-dependency-wheels.sh | 26 ++++ .gitlab/download-wheels-from-gh-actions.sh | 50 ++++++ .gitlab/prepare-oci-package.sh | 32 ++++ lib-injection/Dockerfile | 38 ----- lib-injection/copy-lib.sh | 7 - lib-injection/dl_wheels.py | 25 ++- 13 files changed, 180 insertions(+), 603 deletions(-) delete mode 100644 .github/workflows/lib-inject-publish.yml delete mode 100644 .github/workflows/lib-injection.yml delete mode 100755 .gitlab/build-deb-rpm.sh delete mode 100755 .gitlab/build-lib-init.sh delete mode 100755 .gitlab/build-oci.sh create mode 100755 .gitlab/download-dependency-wheels.sh create mode 100755 .gitlab/download-wheels-from-gh-actions.sh create mode 100755 .gitlab/prepare-oci-package.sh delete mode 100644 lib-injection/Dockerfile delete mode 100755 lib-injection/copy-lib.sh diff --git a/.github/workflows/build_deploy.yml b/.github/workflows/build_deploy.yml index 4a6775f33bf..ca8259da1d1 100644 --- a/.github/workflows/build_deploy.yml +++ b/.github/workflows/build_deploy.yml @@ -2,12 +2,6 @@ name: Build on: push: - branches: - - '[0-9].[0-9]*' - - '[0-9].x' - # special branches used to test this workflow - # before merging/releasing - - build_deploy* pull_request: release: types: @@ -128,12 +122,3 @@ jobs: # due to a duplicate wheel being present which will ensure that the rest # of the wheels will be uploaded if some are uploaded manually. skip_existing: true - - build-and-publish-init-image: - needs: [upload_pypi] - uses: ./.github/workflows/lib-inject-publish.yml - secrets: - token: ${{ secrets.GITHUB_TOKEN }} - with: - ddtrace-version: ${{ github.ref_name }} - image-tag: ${{ github.ref_name }} diff --git a/.github/workflows/lib-inject-publish.yml b/.github/workflows/lib-inject-publish.yml deleted file mode 100644 index 6d32ca91a6b..00000000000 --- a/.github/workflows/lib-inject-publish.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Build and publish library injection images - -on: - workflow_call: - inputs: - ddtrace-version: - required: true - type: string - image-tag: - required: true - type: string - secrets: - token: - required: true - -jobs: - wait_for_package: - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Wait for package to be available from PyPI - run: | - until pip install ddtrace==${{ inputs.ddtrace-version }} - do - sleep 20 - done - build_push: - needs: [wait_for_package] - uses: ./.github/workflows/build-and-publish-image.yml - with: - tags: ghcr.io/datadog/dd-trace-py/dd-lib-python-init:${{ inputs.image-tag }} - build-args: 'DDTRACE_PYTHON_VERSION=${{ inputs.ddtrace-version }}' - platforms: 'linux/amd64,linux/arm64/v8' - context: ./lib-injection - secrets: - token: ${{ secrets.token }} diff --git a/.github/workflows/lib-injection.yml b/.github/workflows/lib-injection.yml deleted file mode 100644 index 4cd8fb65432..00000000000 --- a/.github/workflows/lib-injection.yml +++ /dev/null @@ -1,167 +0,0 @@ -name: "Library Injection" -on: - # Build each branch for testing - push: - pull_request: - paths: - - ddtrace/** - - lib-injection/** - - setup* - - pyproject.toml - - .github/workflows/lib-injection.yml -jobs: - build-and-publish-test-image: - uses: ./.github/workflows/lib-inject-publish.yml - secrets: - token: ${{ secrets.GITHUB_TOKEN }} - with: - ddtrace-version: v2.6.3 - image-tag: ${{ github.sha }} - - test-runner-test: - needs: - - build-and-publish-test-image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - matrix: - weblog-variant: ['dd-lib-python-init-test-django','dd-lib-python-init-test-django-gunicorn','dd-lib-python-init-test-django-uvicorn'] - fail-fast: false - env: - TEST_LIBRARY: python - WEBLOG_VARIANT: ${{ matrix.weblog-variant }} - DOCKER_REGISTRY_IMAGES_PATH: ghcr.io/datadog - DOCKER_IMAGE_TAG: ${{ github.sha }} - BUILDX_PLATFORMS: linux/amd64 - steps: - - name: Checkout system tests - uses: actions/checkout@v3 - with: - repository: 'DataDog/system-tests' - - - name: Install runner - uses: ./.github/actions/install_runner - - - name: Run K8s Lib Injection Tests - run: ./run.sh K8S_LIB_INJECTION_BASIC - - - name: Compress logs - id: compress_logs - if: always() - run: tar -czvf artifact.tar.gz $(ls | grep logs) - - - name: Upload artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: logs_k8s_lib_injection_${{ matrix.weblog-variant }} - path: artifact.tar.gz - - test_unit: - runs-on: ubuntu-latest - strategy: - matrix: - variant: [ - 'dd-lib-python-init-test-django', - 'dd-lib-python-init-test-django-gunicorn', - 'dd-lib-python-init-test-django-gunicorn-alpine', - 'dd-lib-python-init-test-django-uvicorn', - 'dd-lib-python-init-test-django-no-perms', - 'dd-lib-python-init-test-django-pre-installed', - 'dd-lib-python-init-test-django-unsupported-package-force', - ] - fail-fast: false - steps: - - uses: actions/checkout@v3 - - name: Build and run the app - run: | - SRC="$(pwd)" - cd lib-injection - export DDTRACE_PYTHON_VERSION="v2.6.3" - export APP_CONTEXT="${SRC}/tests/lib-injection/${{matrix.variant}}" - export TEMP_DIR="${SRC}/tmp/ddtrace" - mkdir -p "${TEMP_DIR}" - # Give the temp dir permissions, by default the docker user doesn't have permissions - # to write to the filesystem. - chmod 777 $TEMP_DIR - # Start the lib_inject to get the files copied. This avoids a race condition with the startup of the - # application. - docker compose up --build lib_inject - docker compose up --build -d - # Wait for the app to start - sleep 60 - docker compose logs - - name: Check Permissions on ddtrace pkgs - run: | - cd lib-injection - # Ensure /datadog-lib/ddtrace_pkgs is a valid directory that is not empty - docker compose run lib_inject find /datadog-init/ddtrace_pkgs -maxdepth 0 -empty | wc -l && if [ $? -ne 0 ]; then exit 1; fi - # Ensure files are not world writeable - docker compose run lib_inject find /datadog-init/ddtrace_pkgs ! -perm /o+w | wc -l && if [ $? -ne 0 ]; then exit 1; fi - # Ensure all users have read and execute permissions to files stored in /datadog-lib/ddtrace_pkgs - docker compose run lib_inject find /datadog-init/ddtrace_pkgs ! -perm u=rwx,o=rx | wc -l && if [ $? -ne 0 ]; then exit 1; fi - - name: Test the app - run: | - curl http://localhost:18080 - sleep 1 # wait for traces to be sent - - name: Print traces - run: curl http://localhost:8126/test/traces - - name: Check test agent received a trace - run: | - N=$(curl http://localhost:8126/test/traces | jq -r -e 'length') - [[ $N == "1" ]] - - name: Output app logs (LOOK HERE IF THE JOB FAILS) - if: success() || failure() - run: | - docker compose logs - - test_unit_no_instrumentation: - runs-on: ubuntu-latest - strategy: - matrix: - variant: [ - 'dd-lib-python-init-test-django-unsupported-python', - 'dd-lib-python-init-test-django-unsupported-package', - ] - fail-fast: false - steps: - - uses: actions/checkout@v4 - - name: Build and run the app - run: | - SRC="$(pwd)" - cd lib-injection - export DDTRACE_PYTHON_VERSION="v2.6.3" - export APP_CONTEXT="${SRC}/tests/lib-injection/${{matrix.variant}}" - export TEMP_DIR="${SRC}/tmp/ddtrace" - mkdir -p "${TEMP_DIR}" - # Give the temp dir permissions, by default the docker user doesn't have permissions - # to write to the filesystem. - chmod 777 $TEMP_DIR - # Start the lib_inject to get the files copied. This avoids a race condition with the startup of the - # application. - docker compose up --build lib_inject - docker compose up --build -d - # Wait for the app to start - sleep 60 - docker compose logs - - name: Check Permissions on ddtrace pkgs - run: | - cd lib-injection - # Ensure /datadog-lib/ddtrace_pkgs is a valid directory that is not empty - docker compose run lib_inject find /datadog-init/ddtrace_pkgs -maxdepth 0 -empty | wc -l && if [ $? -ne 0 ]; then exit 1; fi - # Ensure files are not world writeable - docker compose run lib_inject find /datadog-init/ddtrace_pkgs ! -perm /o+w | wc -l && if [ $? -ne 0 ]; then exit 1; fi - # Ensure all users have read and execute permissions to files stored in /datadog-lib/ddtrace_pkgs - docker compose run lib_inject find /datadog-init/ddtrace_pkgs ! -perm u=rwx,o=rx | wc -l && if [ $? -ne 0 ]; then exit 1; fi - - name: Test the app - run: | - curl http://localhost:18080 - sleep 1 # wait for traces to be sent - - name: Print traces - run: curl http://localhost:8126/test/traces - - name: Check test agent received no trace - run: | - N=$(curl http://localhost:8126/test/traces | jq -r -e 'length') - [[ $N == "0" ]] diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c51a3f6bb8d..88409503afb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,73 +1,72 @@ stages: - package - - deploy + - shared-pipeline + - dogfood - benchmarks - benchmarks-pr-comment - macrobenchmarks include: - - remote: https://gitlab-templates.ddbuild.io/apm/packaging.yml + - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/one-pipeline.yml - local: ".gitlab/benchmarks.yml" - local: ".gitlab/macrobenchmarks.yml" variables: - DOWNSTREAM_BRANCH: - value: "master" - description: "Run a specific datadog-reliability-env branch downstream" - DOWNSTREAM_MBP_BRANCH: - value: "dd-trace-py" - description: "Run a specific relenv-microbenchmarking-platform branch downstream" - PYTHON_PACKAGE_VERSION: - description: "Version to build for .deb and .rpm. Must be already published in PyPi" + REPO_LANG: python # "python" is used everywhere rather than "py" -.common: &common - tags: [ "runner:main", "size:large" ] - -package: - extends: .package - rules: - - if: $PYTHON_PACKAGE_VERSION - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success +download_ddtrace_wheels: + image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0 + tags: [ "arch:amd64" ] + stage: package script: - - ../.gitlab/build-deb-rpm.sh - - find . -iregex '.*\.\(deb\|rpm\)' -printf '%f\0' | xargs -0 dd-pkg lint + - aws ssm get-parameter --region us-east-1 --name ci.$CI_PROJECT_NAME.gh_token --with-decryption --query "Parameter.Value" --out text > token + - gh auth login --with-token < token + - rm token + - .gitlab/download-wheels-from-gh-actions.sh + artifacts: + paths: + - "pywheels/*.whl" -package-arm: - extends: .package-arm - rules: - - if: $PYTHON_PACKAGE_VERSION - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success +download_dependency_wheels: + image: registry.ddbuild.io/images/mirror/python:$PYTHON_IMAGE_TAG + tags: [ "arch:amd64" ] + stage: package + needs: [ download_ddtrace_wheels ] + parallel: + matrix: # The image tags that are mirrored are in: https://github.com/DataDog/images/blob/master/mirror.yaml + - PYTHON_IMAGE_TAG: "3.7" + PYTHON_VERSION: "3.7" + - PYTHON_IMAGE_TAG: "3.8" + PYTHON_VERSION: "3.8" + - PYTHON_IMAGE_TAG: "3.9.13" + PYTHON_VERSION: "3.9" + - PYTHON_IMAGE_TAG: "3.10.13" + PYTHON_VERSION: "3.10" + - PYTHON_IMAGE_TAG: "3.11.6" + PYTHON_VERSION: "3.11" + - PYTHON_IMAGE_TAG: "3.12.0" + PYTHON_VERSION: "3.12" script: - - ../.gitlab/build-deb-rpm.sh - - find . -iregex '.*\.\(deb\|rpm\)' -printf '%f\0' | xargs -0 dd-pkg lint + - .gitlab/download-dependency-wheels.sh + artifacts: + paths: + - "pywheels-dep/" -.release-package: - stage: deploy - variables: - PRODUCT_NAME: auto_inject-python +package-oci: + needs: [ download_dependency_wheels, download_ddtrace_wheels ] -deploy_to_reliability_env: - stage: deploy - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - when: on_success - - when: manual - allow_failure: true - trigger: - project: DataDog/apm-reliability/datadog-reliability-env - branch: $DOWNSTREAM_BRANCH - variables: - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME - UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA +onboarding_tests_installer: + parallel: + matrix: + - ONBOARDING_FILTER_WEBLOG: [test-app-python,test-app-python-container,test-app-python-alpine-libgcc] + +onboarding_tests_k8s_injection: + parallel: + matrix: + - WEBLOG_VARIANT: [dd-lib-python-init-test-django, dd-lib-python-init-test-django-gunicorn, dd-lib-python-init-test-django-uvicorn] deploy_to_di_backend:manual: - stage: deploy + stage: shared-pipeline rules: - when: manual allow_failure: true @@ -82,65 +81,3 @@ deploy_to_di_backend:manual: UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR UPSTREAM_TAG: $CI_COMMIT_TAG UPSTREAM_PACKAGE_JOB: build - -generate-lib-init-tag-values: - tags: ["arch:amd64"] - image: registry.ddbuild.io/ci/auto_inject/gitlab:current - stage: deploy - rules: - - if: '$POPULATE_CACHE' - when: never - # We don't tag prerelease versions - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - - when: manual - allow_failure: true - variables: - IMG_DESTINATION_BASE: dd-lib-python-init - script: - - ./.gitlab/build-lib-init.sh - artifacts: - reports: - dotenv: build.env - -deploy-lib-init-trigger: - stage: deploy - # needs the version from the generate-tag-values job - needs: - - job: generate-lib-init-tag-values - artifacts: true - trigger: -# project: DataDog/dd-trace-dotnet-gitlab-test # can be used for testing - project: DataDog/public-images - branch: main - strategy: depend - variables: - IMG_SOURCES: ghcr.io/datadog/dd-trace-py/dd-lib-python-init:$CI_COMMIT_TAG - IMG_DESTINATIONS: $IMG_DESTINATIONS - IMG_SIGNING: "false" - # Wait 4 hours to trigger the downstream job. - # This is a work-around since there isn't a way to trigger - # Gitlab from the Github workflow (build_deploy.yml:upload_pypi). - # - # The caveat here is that if there is a failure to build to PyPI and it - # isn't fixed in the retry period then this job will fail and images will - # not be published. - RETRY_DELAY: 14400 - RETRY_COUNT: 3 - -package-oci: - stage: package - extends: .package-oci - rules: - - if: $PYTHON_PACKAGE_VERSION - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(-prerelease)?$/' - when: manual - allow_failure: false - script: - - ../.gitlab/build-oci.sh - parallel: - matrix: - - ARCH: - - arm64 - - amd64 diff --git a/.gitlab/build-deb-rpm.sh b/.gitlab/build-deb-rpm.sh deleted file mode 100755 index 894ee034341..00000000000 --- a/.gitlab/build-deb-rpm.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -if [ -n "$CI_COMMIT_TAG" ] && [ -z "$PYTHON_PACKAGE_VERSION" ]; then - PYTHON_PACKAGE_VERSION=${CI_COMMIT_TAG##v} -fi - -echo -n $PYTHON_PACKAGE_VERSION > auto_inject-python.version - -source common_build_functions.sh - -mkdir -p dd-trace.dir/lib - -# Install known compatible pip as default version shipped in Ubuntu (20.0.2) -# does not work. -python3 -m pip install -U "pip>=22.0" -python3 -m pip install packaging - -echo `pwd` - -../lib-injection/dl_wheels.py \ - --python-version=3.12 \ - --python-version=3.11 \ - --python-version=3.10 \ - --python-version=3.9 \ - --python-version=3.8 \ - --python-version=3.7 \ - --ddtrace-version=$PYTHON_PACKAGE_VERSION \ - --arch x86_64 \ - --arch aarch64 \ - --platform musllinux_1_1 \ - --platform manylinux2014 \ - --output-dir dd-trace.dir/lib/ddtrace_pkgs \ - --verbose - -cp ../lib-injection/sitecustomize.py dd-trace.dir/lib/ - -chmod -R o-w dd-trace.dir/lib -chmod -R g-w dd-trace.dir/lib - -cp auto_inject-python.version dd-trace.dir/lib/version - -fpm_wrapper "datadog-apm-library-python" "$PYTHON_PACKAGE_VERSION" \ - --input-type dir \ - --url "https://github.com/DataDog/dd-trace-py" \ - --description "Datadog APM client library for python" \ - --license "BSD-3-Clause" \ - --chdir=dd-trace.dir/lib \ - --prefix "$LIBRARIES_INSTALL_BASE/python" \ - .=. diff --git a/.gitlab/build-lib-init.sh b/.gitlab/build-lib-init.sh deleted file mode 100755 index 6f47cfaaee1..00000000000 --- a/.gitlab/build-lib-init.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -set -e - -# Safety checks to make sure we have required values -if [ -z "$CI_COMMIT_TAG" ]; then - echo "Error: CI_COMMIT_TAG was not provided" - exit 1 -fi - -if [ -z "$CI_COMMIT_SHA" ]; then - echo "Error: CI_COMMIT_SHA was not provided" - exit 1 -fi - -if [ -z "$IMG_DESTINATION_BASE" ]; then - echo "Error: IMG_DESTINATION_BASE. This should be set to the destination docker image, excluding the tag name, e.g. dd-lib-dotnet-init" - exit 1 -fi - -# If this is a pre-release release, we don't publish -if echo "$CI_COMMIT_TAG" | grep -q "-" > /dev/null; then - echo "Error: This is a pre-release version, should not publish images: $CI_COMMIT_TAG" - exit 1 -fi - -# Calculate the tags we use for floating major and minor versions -MAJOR_MINOR_VERSION="$(sed -nE 's/^(v[0-9]+\.[0-9]+)\.[0-9]+$/\1/p' <<< ${CI_COMMIT_TAG})" -MAJOR_VERSION="$(sed -nE 's/^(v[0-9]+)\.[0-9]+\.[0-9]+$/\1/p' <<< ${CI_COMMIT_TAG})" - -# Make sure we have all the tags -git fetch --tags - -# We need to determine whether this is is the latest tag and whether it's the latest major or not -# So we fetch all tags and sort them to find both the latest, and the latest in this major. -# 'sort' technically gets prerelease versions in the wrong order here, but we explicitly -# exclude them anyway, as they're ignored for the purposes of determining the 'latest' tags. -LATEST_TAG="$(git tag | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V -r | head -n 1)" -LATEST_MAJOR_TAG="$(git tag -l "$MAJOR_VERSION.*" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V -r | head -n 1)" -echo "This tag: $CI_COMMIT_TAG" -echo "Latest repository tag: $LATEST_TAG" -echo "Latest repository tag for this major: $LATEST_MAJOR_TAG" -echo "---------" - -# GNU sort -C (silent) reports via exit code whether the data is already in sorted order -# We use this to check whether the current tag is greater than (or equal to) the latest tag -if printf '%s\n' "$LATEST_TAG" "$CI_COMMIT_TAG" | sort -C -V; then - # The current tag is the latest in the repository - IS_LATEST_TAG=1 -else - IS_LATEST_TAG=0 -fi - -if printf '%s\n' "$LATEST_MAJOR_TAG" "$CI_COMMIT_TAG" | sort -C -V; then - # The current tag is the latest for this major version in the repository - IS_LATEST_MAJOR_TAG=1 -else - IS_LATEST_MAJOR_TAG=0 -fi - -# print everything for debugging purposes -echo "Calculated values:" -echo "MAJOR_MINOR_VERSION=${MAJOR_MINOR_VERSION}" -echo "MAJOR_VERSION=${MAJOR_VERSION}" -echo "IS_LATEST_TAG=${IS_LATEST_TAG}" -echo "IS_LATEST_MAJOR_TAG=${IS_LATEST_MAJOR_TAG}" -echo "---------" - -# Final check that everything is ok -# We should have a major_minor version -if [ -z "$MAJOR_MINOR_VERSION" ]; then - echo "Error: Could not determine major_minor version for stable release, this should not happen" - exit 1 -fi - -# if this is a latest major tag, we should have a major version -if [ "$IS_LATEST_MAJOR_TAG" -eq 1 ] && [ -z "$MAJOR_VERSION" ]; then - echo "Error: Could not determine major version for latest major release, this should not happen" - exit 1 -fi - -# Generate the final variables, and save them into build.env so they can be read by the trigger job -set_image_tags() { - SUFFIX="$1" - VARIABLE_SUFFIX="${SUFFIX:+_$SUFFIX}" # add a '_' prefix - TAG_SUFFIX="${SUFFIX:+-$SUFFIX}" # add a '-' prefix - - # We always add this tag, regardless of the version - DESTINATIONS="${IMG_DESTINATION_BASE}:${CI_COMMIT_TAG}${TAG_SUFFIX}" - - # We always add the major_minor tag (we never release 2.5.2 _after_ 2.5.3, for example) - DESTINATIONS="${DESTINATIONS},${IMG_DESTINATION_BASE}:${MAJOR_MINOR_VERSION}${TAG_SUFFIX}" - - # Only latest-major releases get the major tag - if [ "$IS_LATEST_MAJOR_TAG" -eq 1 ]; then - DESTINATIONS="${DESTINATIONS},${IMG_DESTINATION_BASE}:${MAJOR_VERSION}${TAG_SUFFIX}" - fi - - # Only latest releases get the latest tag - if [ "$IS_LATEST_TAG" -eq 1 ]; then - DESTINATIONS="${DESTINATIONS},${IMG_DESTINATION_BASE}:latest${TAG_SUFFIX}" - fi - - # Save the value to the build.env file - echo "IMG_DESTINATIONS${VARIABLE_SUFFIX}=${DESTINATIONS}" - echo "IMG_DESTINATIONS${VARIABLE_SUFFIX}=${DESTINATIONS}" >> build.env -} - -# Calculate the non-suffixed tags -set_image_tags - -# For each suffix, calculate the tags -for ADDITIONAL_TAG_SUFFIX in ${ADDITIONAL_TAG_SUFFIXES//,/ } -do - set_image_tags "$ADDITIONAL_TAG_SUFFIX" -done \ No newline at end of file diff --git a/.gitlab/build-oci.sh b/.gitlab/build-oci.sh deleted file mode 100755 index 5db2d459e45..00000000000 --- a/.gitlab/build-oci.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -source common_build_functions.sh - -if [ -n "$CI_COMMIT_TAG" ] && [ -z "$PYTHON_PACKAGE_VERSION" ]; then - PYTHON_PACKAGE_VERSION=${CI_COMMIT_TAG##v} -fi - -if [ -z "$ARCH" ]; then - ARCH=amd64 -fi - - -TMP_DIR=$(mktemp --dir) -BUILD_DIR=$TMP_DIR/datadog-python-apm.build -mkdir $TMP_DIR/datadog-python-apm.build - -# Install known compatible pip as default version shipped in Ubuntu (20.0.2) -# does not work. -python3 -m pip install -U "pip>=22.0" -python3 -m pip install packaging - -WHEEL_ARCH="x86_64" -if [ "$ARCH" = "arm64" ]; then - WHEEL_ARCH="aarch64" -fi - -../lib-injection/dl_wheels.py \ - --python-version=3.12 \ - --python-version=3.11 \ - --python-version=3.10 \ - --python-version=3.9 \ - --python-version=3.8 \ - --python-version=3.7 \ - --ddtrace-version=$PYTHON_PACKAGE_VERSION \ - --arch=$WHEEL_ARCH \ - --platform=musllinux_1_1 \ - --platform=manylinux2014 \ - --output-dir=$BUILD_DIR/ddtrace_pkgs \ - --verbose -echo -n $PYTHON_PACKAGE_VERSION > auto_inject-python.version -cp ../lib-injection/sitecustomize.py $BUILD_DIR/ -cp auto_inject-python.version $BUILD_DIR/version -cp ../min_compatible_versions.csv $BUILD_DIR/ -chmod -R +r $BUILD_DIR -chmod -R o-w $BUILD_DIR -chmod -R g-w $BUILD_DIR - -# Build packages -datadog-package create \ - --version="$PYTHON_PACKAGE_VERSION" \ - --package="datadog-apm-library-python" \ - --archive=true \ - --archive-path="datadog-apm-library-python-$PYTHON_PACKAGE_VERSION-$ARCH.tar" \ - --arch "$ARCH" \ - --os "linux" \ - $BUILD_DIR diff --git a/.gitlab/download-dependency-wheels.sh b/.gitlab/download-dependency-wheels.sh new file mode 100755 index 00000000000..431e662e4c7 --- /dev/null +++ b/.gitlab/download-dependency-wheels.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -eo pipefail + +if [ -z "$CI_COMMIT_SHA" ]; then + echo "Error: CI_COMMIT_SHA was not provided" + exit 1 +fi + +python3 -m pip install -U "pip>=22.0" +python3 -m pip install packaging + +mkdir pywheels-dep + +cd pywheels + +export PYTHONUNBUFFERED=TRUE + +../lib-injection/dl_wheels.py \ + --python-version=$PYTHON_VERSION \ + --local-ddtrace \ + --arch x86_64 \ + --arch aarch64 \ + --platform musllinux_1_1 \ + --platform manylinux2014 \ + --output-dir ../pywheels-dep \ + --verbose diff --git a/.gitlab/download-wheels-from-gh-actions.sh b/.gitlab/download-wheels-from-gh-actions.sh new file mode 100755 index 00000000000..2803558d784 --- /dev/null +++ b/.gitlab/download-wheels-from-gh-actions.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -eo pipefail + +if [ -z "$CI_COMMIT_SHA" ]; then + echo "Error: CI_COMMIT_SHA was not provided" + exit 1 +fi + +echo "Querying for RUN_ID" + +timeout=600 # 10 minutes +start_time=$(date +%s) +end_time=$((start_time + timeout)) +# Loop for 10 minutes waiting for run to appear in github +while [ $(date +%s) -lt $end_time ]; do + RUN_ID=$(gh run ls --repo DataDog/dd-trace-py --commit=$CI_COMMIT_SHA --workflow=build_deploy.yml --json databaseId --jq "first (.[]) | .databaseId") + if [ -n "$RUN_ID" ]; then + break; + fi + echo "Waiting for RUN_ID" + sleep 20 +done + +if [ -z "$RUN_ID" ]; then + echo "RUN_ID not found" + exit 1 +fi + +echo "Found RUN_ID: $RUN_ID" +echo "Waiting for workflow to finish" + +# wait for run to finish +gh run watch $RUN_ID --interval 45 --exit-status 1 --repo DataDog/dd-trace-py + +mkdir pywheels +cd pywheels + +echo "Github workflow finished. Downloading wheels" + +# download all wheels +gh run download $RUN_ID --repo DataDog/dd-trace-py + +cd .. + +echo "Finished downloading wheels. Fixing directory structure" + +# Flatten directory structure so all wheels are top level +find pywheels -type f -exec mv {} pywheels \; + +echo "Done" diff --git a/.gitlab/prepare-oci-package.sh b/.gitlab/prepare-oci-package.sh new file mode 100755 index 00000000000..5b3380bd4d9 --- /dev/null +++ b/.gitlab/prepare-oci-package.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +if [ -n "$CI_COMMIT_TAG" ] && [ -z "$PYTHON_PACKAGE_VERSION" ]; then + PYTHON_PACKAGE_VERSION=${CI_COMMIT_TAG##v} +fi + +if [ -z "$PYTHON_PACKAGE_VERSION" ]; then + # Get the version from the filename of the first wheel + # wheels have the form: + # ddtrace-2.11.0.dev41+g50bf57680-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + # the version is everything between the first and second "-" + WHEELS_LIST=(../pywheels/*.whl) + FIRST_WHEEL=${WHEELS_LIST[1]} + + #everything before - + WITHOUT_BEGINNING=${FIRST_WHEEL#*-} + + #strip after - + PYTHON_PACKAGE_VERSION=${WITHOUT_BEGINNING%%-*} +fi + +mkdir -p sources/ddtrace_pkgs + +BUILD_DIR=sources + +echo -n "$PYTHON_PACKAGE_VERSION" > sources/version + +cp -r ../pywheels-dep/site-packages* sources/ddtrace_pkgs + +cp ../lib-injection/sitecustomize.py sources/ +cp ../min_compatible_versions.csv sources/ +cp ../lib-injection/telemetry-forwarder.sh sources/ diff --git a/lib-injection/Dockerfile b/lib-injection/Dockerfile deleted file mode 100644 index 8e046a55896..00000000000 --- a/lib-injection/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# This image provides the files needed to install the ddtrace Python package -# and auto instrument Python applications in containerized environments. -FROM python:3.10 -WORKDIR /build -ARG DDTRACE_PYTHON_VERSION -RUN python3 -m pip install -U pip==23.0.1 -RUN python3 -m pip install packaging==23.0 -RUN mkdir -p pkgs -ADD ./dl_wheels.py . -# Note that we only get Python >= 3.7. This is to keep the size of the image -# as small as possible. -RUN python3 dl_wheels.py \ - --python-version=3.12 \ - --python-version=3.11 \ - --python-version=3.10 \ - --python-version=3.9 \ - --python-version=3.8 \ - --python-version=3.7 \ - --ddtrace-version=${DDTRACE_PYTHON_VERSION} \ - --arch x86_64 \ - --arch aarch64 \ - --platform musllinux_1_1 \ - --platform manylinux2014 \ - --output-dir /build/pkgs \ - --verbose - -FROM alpine:3.20 -COPY --from=0 /build/pkgs /datadog-init/ddtrace_pkgs -ARG UID=10000 -RUN addgroup -g 10000 -S datadog && \ - adduser -u ${UID} -S datadog -G datadog -RUN chown -R datadog:datadog /datadog-init/ddtrace_pkgs -RUN chmod -R 755 /datadog-init/ddtrace_pkgs -USER ${UID} -WORKDIR /datadog-init -ADD min_compatible_versions.csv /datadog-init/min_compatible_versions.csv -ADD sitecustomize.py /datadog-init/sitecustomize.py -ADD copy-lib.sh /datadog-init/copy-lib.sh diff --git a/lib-injection/copy-lib.sh b/lib-injection/copy-lib.sh deleted file mode 100755 index 2a17b77d763..00000000000 --- a/lib-injection/copy-lib.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -# This script is used by the admission controller to install the library from the -# init container into the application container. -cp sitecustomize.py "$1/sitecustomize.py" -cp min_compatible_versions.csv "$1/min_compatible_versions.csv" -cp -r ddtrace_pkgs "$1/ddtrace_pkgs" diff --git a/lib-injection/dl_wheels.py b/lib-injection/dl_wheels.py index 22d7fa8f702..e10d8e53e0e 100755 --- a/lib-injection/dl_wheels.py +++ b/lib-injection/dl_wheels.py @@ -37,7 +37,7 @@ if pip_version < MIN_PIP_VERSION: print( "WARNING: using known incompatible version, %r, of pip. The minimum compatible pip version is %r" - % (pip_version, MIN_PIP_VERSION) + % (pip_version, MIN_PIP_VERSION), ) # Supported Python versions lists all python versions that can install at least one version of the ddtrace library. @@ -64,7 +64,8 @@ action="append", required=True, ) -parser.add_argument("--ddtrace-version", type=str, required=True) +parser.add_argument("--ddtrace-version", type=str) +parser.add_argument("--local-ddtrace", action="store_true") parser.add_argument("--output-dir", type=str, required=True) parser.add_argument("--dry-run", action="store_true") parser.add_argument("--verbose", action="store_true") @@ -82,6 +83,22 @@ if python_version in ["2.7", "3.5", "3.6", "3.7"]: abi += "m" + if args.ddtrace_version: + ddtrace_specifier = "ddtrace==%s" % args.ddtrace_version + elif args.local_ddtrace: + wheel_files = [ + f for f in os.listdir(".") if f.endswith(".whl") and abi in f and platform in f and arch in f + ] + + if len(wheel_files) > 1: + print("More than one matching file found %s" % wheel_files) + sys.exit(1) + + ddtrace_specifier = wheel_files[0] + else: + print("--ddtrace-version or --local-ddtrace must be specified") + sys.exit(1) + # See the docs for an explanation of all the options used: # https://pip.pypa.io/en/stable/cli/pip_download/ # only-binary=:all: is specified to ensure we get all the dependencies of ddtrace as well. @@ -90,7 +107,7 @@ "-m", "pip", "download", - "ddtrace==%s" % args.ddtrace_version, + ddtrace_specifier, "--platform", "%s_%s" % (platform, arch), "--python-version", @@ -98,6 +115,8 @@ "--abi", abi, "--only-binary=:all:", + "--exists-action", + "i", # ignore redownloads of same wheel "--dest", dl_dir, ]