Merge branch 'main' into upgrade-dramatiq-version

DataDog · Aug 26, 2024 · bac3df8 · bac3df8
2 parents 2ea8745 + 769579f
commit bac3df8
Show file tree

Hide file tree

Showing 62 changed files with 1,704 additions and 1,569 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -113,18 +113,22 @@ ddtrace/internal/datadog/profiling  @DataDog/profiling-python
 tests/profiling                     @DataDog/profiling-python
 
 # MLObs
-ddtrace/llmobs/                                     @DataDog/ml-observability
-ddtrace/contrib/openai                              @DataDog/ml-observability
-ddtrace/contrib/langchain                           @DataDog/ml-observability
-ddtrace/contrib/botocore/services/bedrock.py        @DataDog/ml-observability
-ddtrace/contrib/anthropic                           @DataDog/ml-observability
-tests/llmobs                                        @DataDog/ml-observability
-tests/contrib/openai                                @DataDog/ml-observability
-tests/contrib/langchain                             @DataDog/ml-observability
-tests/contrib/botocore/test_bedrock.py              @DataDog/ml-observability
-tests/contrib/botocore/test_bedrock_llmobs.py       @DataDog/ml-observability
-tests/contrib/botocore/bedrock_cassettes            @DataDog/ml-observability
-tests/contrib/anthropic                             @DataDog/ml-observability
+ddtrace/llmobs/                                         @DataDog/ml-observability
+ddtrace/contrib/internal/openai                         @DataDog/ml-observability
+ddtrace/contrib/openai                                  @DataDog/ml-observability
+ddtrace/contrib/internal/langchain                      @DataDog/ml-observability
+ddtrace/contrib/langchain                               @DataDog/ml-observability
+ddtrace/contrib/internal/botocore/services/bedrock.py   @DataDog/ml-observability
+ddtrace/contrib/botocore/services/bedrock.py            @DataDog/ml-observability
+ddtrace/contrib/internal/anthropic                      @DataDog/ml-observability
+ddtrace/contrib/anthropic                               @DataDog/ml-observability
+tests/llmobs                                            @DataDog/ml-observability
+tests/contrib/openai                                    @DataDog/ml-observability
+tests/contrib/langchain                                 @DataDog/ml-observability
+tests/contrib/botocore/test_bedrock.py                  @DataDog/ml-observability
+tests/contrib/botocore/test_bedrock_llmobs.py           @DataDog/ml-observability
+tests/contrib/botocore/bedrock_cassettes                @DataDog/ml-observability
+tests/contrib/anthropic                                 @DataDog/ml-observability
 
 # Remote Config
 ddtrace/internal/remoteconfig       @DataDog/remote-config @DataDog/apm-core-python

diff --git a/.github/workflows/generate-package-versions.yml b/.github/workflows/generate-package-versions.yml
@@ -1,10 +1,6 @@
 name: Generate Package Versions
 
 on:
-  push:
-    branches:
-      - main
-  pull_request:
   workflow_dispatch: # can be triggered manually
   schedule:
     - cron: '0 0 * * 0' # weekly on Sunday at midnight

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
@@ -24,7 +24,37 @@ jobs:
           python -c "import os,sys,fnmatch;sys.exit(not bool([_ for pattern in {'ddtrace/*', 'setup*', 'pyproject.toml', '.github/workflows/system-tests.yml'} for _ in fnmatch.filter(os.environ['PATHS'].splitlines(), pattern)]))"
         continue-on-error: true
 
-  system-tests-build:
+  system-tests-build-agent:
+    runs-on: ubuntu-latest
+    needs: needs-run
+    steps:
+
+      - name: Checkout system tests
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        uses: actions/checkout@v4
+        with:
+          repository: 'DataDog/system-tests'
+
+      - name: Build agent
+        id: build
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        run: ./build.sh -i agent
+
+      - name: Save
+        id: save
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        run: |
+          docker image save system_tests/agent:latest | gzip > agent_${{ github.sha }}.tar.gz
+
+      - uses: actions/upload-artifact@v4
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        with:
+          name: agent_${{ github.sha }}
+          path: |
+            agent_${{ github.sha }}.tar.gz
+          retention-days: 2
+
+  system-tests-build-weblog:
     runs-on: ubuntu-latest
     needs: needs-run
     strategy:
@@ -46,11 +76,6 @@ jobs:
       DD_API_KEY: 1234567890abcdef1234567890abcdef
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     steps:
-      - name: Setup python 3.12
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
 
       - name: Checkout system tests
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
@@ -71,28 +96,25 @@ jobs:
       - name: Build
         id: build
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./build.sh
+        run: ./build.sh -i weblog
 
       - name: Save
         id: save
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         run: |
           docker image save system_tests/weblog:latest | gzip > ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
-          docker image save system_tests/agent:latest | gzip > ${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
 
       - uses: actions/upload-artifact@v4
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         with:
           name: ${{ matrix.weblog-variant }}_${{ github.sha }}
           path: |
             ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
-            ${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
-            venv
           retention-days: 2
 
   system-tests:
     runs-on: ubuntu-latest
-    needs: [needs-run, system-tests-build]
+    needs: [needs-run, system-tests-build-agent, system-tests-build-weblog]
     strategy:
       matrix:
         weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12]
@@ -108,35 +130,34 @@ jobs:
       DD_API_KEY: 1234567890abcdef1234567890abcdef
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     steps:
-      - name: Setup python 3.12
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
 
       - name: Checkout system tests
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         uses: actions/checkout@v4
         with:
           repository: 'DataDog/system-tests'
 
+      - name: Build runner
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        uses: ./.github/actions/install_runner
+
       - uses: actions/download-artifact@v4
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         with:
           name: ${{ matrix.weblog-variant }}_${{ github.sha }}
-          path: ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz
+          path: images_artifacts/
 
-      - name: docker load
+      - uses: actions/download-artifact@v4
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: |
-          docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
-          docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
+        with:
+          name: agent_${{ github.sha }}
+          path: images_artifacts/
 
-      - name: move venv
+      - name: docker load
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         run: |
-          mv ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/venv venv
-          chmod -R +x venv/bin/*
+          docker load < images_artifacts/${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
+          docker load < images_artifacts/agent_${{ github.sha }}.tar.gz
 
       - name: Run DEFAULT
         if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
@@ -237,7 +258,8 @@ jobs:
 
 
   parametric:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: "APM Larger Runners"
     needs: needs-run
     env:
       TEST_LIBRARY: python
@@ -254,14 +276,10 @@ jobs:
           path: 'binaries/dd-trace-py'
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
-      - uses: actions/setup-python@v5
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        with:
-          python-version: '3.12'
 
-      - name: Build
+      - name: Build runner
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./build.sh -i runner
+        uses: ./.github/actions/install_runner
 
       - name: Run
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -56,16 +56,3 @@ deploy_to_di_backend:manual:
     UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
     UPSTREAM_TAG: $CI_COMMIT_TAG
     UPSTREAM_PACKAGE_JOB: build
-
-
-# Final step which only runs when a pipeline has finished successfully.
-# This gives us something block on/wait for in GitHub
-finished:
-  image: registry.ddbuild.io/images/mirror/library/alpine:3.19.3
-  tags: [ "arch:amd64" ]
-  stage: .post
-  rules:
-    - when: on_success
-  script:
-    # TODO: Can we get this to reflect the status from the whole pipeline?
-    - exit 0
diff --git a/.riot/requirements/1050efa.txt b/.riot/requirements/1050efa.txt
@@ -2,19 +2,19 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --no-annotate .riot/requirements/1050efa.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/1050efa.in
 #
-attrs==23.1.0
-coverage[toml]==7.3.4
+attrs==24.2.0
+coverage[toml]==7.6.1
 hypothesis==6.45.0
 iniconfig==2.0.0
-mariadb==1.1.8
+mariadb==1.1.10
 mock==5.1.0
 opentracing==2.4.0
-packaging==23.2
-pluggy==1.3.0
-pytest==7.4.3
-pytest-cov==4.1.0
-pytest-mock==3.12.0
+packaging==24.1
+pluggy==1.5.0
+pytest==8.3.2
+pytest-cov==5.0.0
+pytest-mock==3.14.0
 pytest-randomly==3.15.0
 sortedcontainers==2.4.0
diff --git a/.riot/requirements/10aef09.txt b/.riot/requirements/10aef09.txt
@@ -2,20 +2,20 @@
 # This file is autogenerated by pip-compile with Python 3.7
 # by the following command:
 #
-#    pip-compile --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/10aef09.in
+#    pip-compile --allow-unsafe --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/10aef09.in
 #
-attrs==23.1.0
+attrs==24.2.0
 coverage[toml]==7.2.7
-exceptiongroup==1.2.0
+exceptiongroup==1.2.2
 hypothesis==6.45.0
 importlib-metadata==6.7.0
 iniconfig==2.0.0
 mariadb==1.1.6
 mock==5.1.0
 opentracing==2.4.0
-packaging==23.2
+packaging==24.0
 pluggy==1.2.0
-pytest==7.4.3
+pytest==7.4.4
 pytest-cov==4.1.0
 pytest-mock==3.11.1
 pytest-randomly==3.12.0

diff --git a/.riot/requirements/12c10e8.txt b/.riot/requirements/12c10e8.txt
@@ -2,23 +2,23 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --no-annotate .riot/requirements/12c10e8.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/12c10e8.in
 #
-attrs==23.1.0
-coverage[toml]==7.3.4
-exceptiongroup==1.2.0
+attrs==24.2.0
+coverage[toml]==7.6.1
+exceptiongroup==1.2.2
 hypothesis==6.45.0
-importlib-metadata==7.0.0
+importlib-metadata==8.4.0
 iniconfig==2.0.0
-mariadb==1.1.8
+mariadb==1.1.10
 mock==5.1.0
 opentracing==2.4.0
-packaging==23.2
-pluggy==1.3.0
-pytest==7.4.3
-pytest-cov==4.1.0
-pytest-mock==3.12.0
+packaging==24.1
+pluggy==1.5.0
+pytest==8.3.2
+pytest-cov==5.0.0
+pytest-mock==3.14.0
 pytest-randomly==3.15.0
 sortedcontainers==2.4.0
 tomli==2.0.1
-zipp==3.17.0
+zipp==3.20.0
diff --git a/.riot/requirements/12cb0e7.txt b/.riot/requirements/12cb0e7.txt
@@ -2,19 +2,19 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --no-annotate .riot/requirements/12cb0e7.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/12cb0e7.in
 #
-attrs==23.1.0
-coverage[toml]==7.3.4
+attrs==24.2.0
+coverage[toml]==7.6.1
 hypothesis==6.45.0
 iniconfig==2.0.0
-mariadb==1.1.8
+mariadb==1.1.10
 mock==5.1.0
 opentracing==2.4.0
-packaging==23.2
-pluggy==1.3.0
-pytest==7.4.3
-pytest-cov==4.1.0
-pytest-mock==3.12.0
+packaging==24.1
+pluggy==1.5.0
+pytest==8.3.2
+pytest-cov==5.0.0
+pytest-mock==3.14.0
 pytest-randomly==3.15.0
 sortedcontainers==2.4.0
diff --git a/.riot/requirements/147bedb.txt b/.riot/requirements/147bedb.txt
@@ -2,23 +2,23 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --no-annotate .riot/requirements/147bedb.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/147bedb.in
 #
-attrs==23.1.0
-coverage[toml]==7.3.4
-exceptiongroup==1.2.0
+attrs==24.2.0
+coverage[toml]==7.6.1
+exceptiongroup==1.2.2
 hypothesis==6.45.0
-importlib-metadata==7.0.0
+importlib-metadata==8.4.0
 iniconfig==2.0.0
-mariadb==1.1.8
+mariadb==1.1.10
 mock==5.1.0
 opentracing==2.4.0
-packaging==23.2
-pluggy==1.3.0
-pytest==7.4.3
-pytest-cov==4.1.0
-pytest-mock==3.12.0
+packaging==24.1
+pluggy==1.5.0
+pytest==8.3.2
+pytest-cov==5.0.0
+pytest-mock==3.14.0
 pytest-randomly==3.15.0
 sortedcontainers==2.4.0
 tomli==2.0.1
-zipp==3.17.0
+zipp==3.20.0