Merge branch 'test-mq-main' into add-mq-trigger

.circleci/config.templ.yml

@@ -556,44 +556,12 @@ jobs:
           env: 'dd_coverage'
           snapshot: true
 
-  llmobs:
-    <<: *contrib_job
-    steps:
-      - run_test:
-          pattern: "llmobs"
-
-  subprocess:
-    <<: *contrib_job_small
-    steps:
-      - run_test:
-          pattern: "subprocess"
-
   sourcecode:
     <<: *contrib_job_small
     steps:
       - run_test:
           pattern: "sourcecode"
 
-  debugger:
-    <<: *contrib_job
-    steps:
-      - run_test:
-          pattern: "debugger"
-
-  openai:
-    <<: *machine_executor
-    parallelism: 10
-    steps:
-      - run_test:
-          pattern: 'openai'
-          snapshot: true
-
-  opentracer:
-    <<: *contrib_job_large
-    steps:
-      - run_test:
-          pattern: 'opentracer'
-
   opentelemetry:
     parallelism: 4
     <<: *machine_executor
@@ -611,24 +579,6 @@ jobs:
           snapshot: true
           docker_services: "localstack"
 
-  test_logging:
-    <<: *contrib_job
-    docker:
-      - image: *ddtrace_dev_image
-      - *testagent
-    steps:
-      - run_test:
-          pattern: 'test_logging'
-
-  stdlib:
-    <<: *contrib_job
-    docker:
-      - image: *ddtrace_dev_image
-      - *testagent
-    steps:
-      - run_test:
-          pattern: 'asyncio$|sqlite3$|futures$|dbapi$|dbapi_async$'
-
   asyncpg:
     <<: *machine_executor
     steps:
@@ -709,13 +659,6 @@ jobs:
           snapshot: true
           docker_services: "memcached redis postgres"
 
-  django_hosts:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          pattern: 'django_hosts$'
-          snapshot: true
-
   djangorestframework:
     <<: *machine_executor
     parallelism: 2
@@ -822,27 +765,6 @@ jobs:
           pattern: 'pytest'
           snapshot: true
 
-  pytest_v2:
-    <<: *machine_executor
-    parallelism: 10
-    steps:
-     - run_hatch_env_test:
-          env: 'pytest_plugin_v2'
-          snapshot: true
-
-  unittest:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          pattern: 'unittest'
-          snapshot: true
-
-  asynctest:
-    executor: ddtrace_dev
-    steps:
-      - run_test:
-          pattern: 'asynctest$'
-
   pymemcache:
     <<: *contrib_job
     docker:
@@ -934,65 +856,6 @@ jobs:
           wait: postgres
           pattern: 'aiopg'
 
-  aredis:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          docker_services: 'redis'
-          pattern: 'aredis$'
-          snapshot: true
-
-  yaaredis:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          docker_services: 'redis'
-          pattern: 'yaaredis$'
-          snapshot: true
-
-  redis:
-    <<: *machine_executor
-    parallelism: 5
-    steps:
-      - run_test:
-          docker_services: 'redis rediscluster'
-          pattern: 'redis$'
-          snapshot: true
-
-  rediscluster:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          pattern: 'rediscluster'
-          docker_services: 'rediscluster'
-          snapshot: true
-
-  rq:
-    <<: *machine_executor
-    parallelism: 2
-    steps:
-      - run_test:
-          pattern: "rq"
-          snapshot: true
-          docker_services: "redis"
-
-  urllib3:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          pattern: 'urllib3'
-          snapshot: true
-          docker_services: "httpbin_local"
-
-  kafka:
-    <<: *machine_executor
-    parallelism: 4
-    steps:
-      - run_test:
-          pattern: 'kafka'
-          snapshot: true
-          docker_services: 'kafka'
-
   kombu:
     <<: *contrib_job
     docker:
@@ -1004,36 +867,6 @@ jobs:
           wait: rabbitmq
           pattern: 'kombu'
 
-  langchain:
-    <<: *machine_executor
-    parallelism: 6
-    steps:
-      - run_test:
-          pattern: "langchain"
-          snapshot: true
-
-  anthropic:
-    <<: *machine_executor
-    parallelism: 3
-    steps:
-      - run_test:
-          pattern: "anthropic"
-          snapshot: true
-
-  logbook:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          pattern: 'logbook'
-          snapshot: true
-
-  loguru:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          pattern: 'loguru'
-          snapshot: true
-
   build_docs:
     # build documentation and store as an artifact
     executor: ddtrace_dev

.github/CODEOWNERS

@@ -59,9 +59,10 @@ tests/contrib/pytest                @DataDog/ci-app-libraries
 tests/contrib/pytest_bdd            @DataDog/ci-app-libraries
 tests/contrib/unittest_plugin       @DataDog/ci-app-libraries
 ddtrace/ext/ci.py                   @DataDog/ci-app-libraries
-ddtrace/ext/ci_visibility           @DataDog/ci-app-libraries
+ddtrace/ext/test_visibility         @DataDog/ci-app-libraries
 ddtrace/ext/test.py                 @DataDog/ci-app-libraries
 ddtrace/internal/ci_visibility      @DataDog/ci-app-libraries
+ddtrace/internal/test_visibility    @DataDog/ci-app-libraries
 ddtrace/internal/codeowners.py      @DataDog/apm-core-python @datadog/ci-app-libraries
 ddtrace/internal/coverage           @DataDog/apm-core-python @datadog/ci-app-libraries
 tests/internal/test_codeowners.py   @datadog/ci-app-libraries
@@ -112,6 +113,7 @@ ddtrace/profiling                   @DataDog/profiling-python
 ddtrace/settings/profiling.py       @DataDog/profiling-python
 ddtrace/internal/datadog/profiling  @DataDog/profiling-python
 tests/profiling                     @DataDog/profiling-python
+tests/profiling-v2                  @DataDog/profiling-python
 .gitlab/tests/profiling.yml         @DataDog/profiling-python
 
 # MLObs

.github/workflows/backport.yml

@@ -9,6 +9,9 @@ jobs:
   backport:
     name: Backport
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
     # Only react to merged PRs for security reasons.
     # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
     if: >

.github/workflows/generate-package-versions.yml

@@ -9,6 +9,10 @@ jobs:
   generate-package-versions:
     name: Generate package versions
     runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: write
+      pull-requests: write
     steps:
       - uses: actions/checkout@v4
 
@@ -71,20 +75,27 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install packaging
+          pip install requests
           pip install riot==0.19.1
 
       - name: Run regenerate-riot-latest
         run: scripts/regenerate-riot-latest.sh
 
+      - name: Get latest version
+        id: new-latest
+        run: |
+          NEW_LATEST=$(python scripts/get_latest_version.py ${{ env.VENV_NAME }})
+          echo "NEW_LATEST=$NEW_LATEST" >> $GITHUB_ENV
+
       - name: Create Pull Request
         id: pr
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           branch: "upgrade-latest-${{ env.VENV_NAME }}-version"
-          commit-message: "Test Package Versions update"
+          commit-message: "Update package version"
           delete-branch: true
           base: main
-          title: "chore: update ${{ env.VENV_NAME }} latest version"
+          title: "chore: update ${{ env.VENV_NAME }} latest version to ${{ env.NEW_LATEST }}"
           labels: changelog/no-changelog
           body-path: .github/PULL_REQUEST_TEMPLATE.md

.gitlab/services.yml

@@ -25,3 +25,26 @@
   redis:
     name: registry.ddbuild.io/redis:7.0.7
     alias: redis
+  kafka:
+    name: registry.ddbuild.io/images/mirror/bitnami/kafka:3.4.1
+    alias: kafka
+    variables:
+      ALLOW_PLAINTEXT_LISTENER: yes
+      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:29092
+      KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:29092
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      KAFKA_CFG_BROKER_ID: 1
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_CFGOFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_CFG_NODE_ID: 1
+      KAFKA_CFG_PROCESS_ROLES: controller,broker
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+  httpbin_local:
+    name: registry.ddbuild.io/images/mirror/mccutchen/go-httpbin:v2.14.1
+    alias: httpbin-local
+    variables:
+      PORT: 8001
+  rediscluster:
+    name: registry.ddbuild.io/images/mirror/grokzen/redis-cluster:6.2.0
+    alias: rediscluster

.gitlab/tests.yml

@@ -53,5 +53,6 @@ include:
   - local: ".gitlab/tests/appsec.yml"
   - local: ".gitlab/tests/contrib.yml"
   - local: ".gitlab/tests/core.yml"
+  - local: ".gitlab/tests/debugging.yml"
   - local: ".gitlab/tests/tracer.yml"
   - local: ".gitlab/tests/profiling.yml"