chore: upgrade testrunner image (#10405)

1. Upgrade testrunner image to the one generated by #10373 
2. testrunner now runs python 3.11.9 (upgraded from 3.11.3) and gevent
isn't compatible with python 3.11.9,
python/cpython#117983, so update the test to
run when version < 3.11.9
3. Regenerate riot file for appsec_integrations using python 3.12, as
outdated pydantic doesn't work with 3.12.4, see
pydantic/pydantic#9637. This also resulted in
updating pytest version and `setup_module` is used instead of `setup`.
4. Add `CMAKE_BUILD_PARALLEL_LEVEL` to a few places


## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

.circleci/config.templ.yml

@@ -4,7 +4,7 @@ default_resource_class: &default_resource_class medium
 ubuntu_base_image: &ubuntu_base_img ubuntu-2004:2023.04.2
 cimg_base_image: &cimg_base_image cimg/base:2022.08
 python310_image: &python310_image cimg/python:3.10.12
-ddtrace_dev_image: &ddtrace_dev_image ghcr.io/datadog/dd-trace-py/testrunner@sha256:9b2364afa4d1cd8382f251aca5963113cfc47e6c6b9aaed2c28e8a2a48ac1f75
+ddtrace_dev_image: &ddtrace_dev_image ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
 redis_image: &redis_image redis:4.0-alpine@sha256:3e99741f293147ff406657dda7644c2b88564b80a498cd00da8f905743449c9f
 memcached_image: &memcached_image memcached:1.5-alpine@sha256:48cb7207e3d34871893fa1628f3a4984375153e9942facf82e25935b0a633c8a
 cassandra_image: &cassandra_image cassandra:3.11.7@sha256:495e5752526f7e75d3ad85b6a6bbf3b79714321b17a44255a216c341e3baae11

.github/workflows/requirements-locks.yml

@@ -9,7 +9,7 @@ jobs:
   validate:
     name: Check requirements lockfiles
     runs-on: ubuntu-latest
-    container: ghcr.io/datadog/dd-trace-py/testrunner:latest
+    container: ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
     steps:
       - uses: actions/checkout@v4
         with:

.gitlab-ci.yml

@@ -10,12 +10,13 @@ stages:
 
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
-  TESTRUNNER_IMAGE: ghcr.io/datadog/dd-trace-py/testrunner:81fa854aa9fb16ffc4a089d2d0bd82936c7ea3e9
+  TESTRUNNER_IMAGE: ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
   CI_DEBUG_SERVICES: "true"
 
 .testrunner:
   image: $TESTRUNNER_IMAGE
-  tags: [ "arch:arm64" ]
+  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
+  tags: [ "arch:amd64" ]
   timeout: 20m
   before_script:
     - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev

.riot/requirements/1e2d655.txt

@@ -2,58 +2,59 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --no-annotate .riot/requirements/1e2d655.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/1e2d655.in
 #
-aiohttp==3.9.1
+aiohappyeyeballs==2.4.0
+aiohttp==3.10.5
 aiosignal==1.3.1
-annotated-types==0.6.0
-anyio==4.2.0
-attrs==23.2.0
-blinker==1.7.0
-certifi==2023.11.17
+annotated-types==0.7.0
+anyio==4.4.0
+attrs==24.2.0
+blinker==1.8.2
+certifi==2024.7.4
 charset-normalizer==3.3.2
 click==8.1.7
-coverage[toml]==7.4.0
-dataclasses-json==0.6.3
-flask==3.0.0
+coverage[toml]==7.6.1
+dataclasses-json==0.6.7
+flask==3.0.3
 frozenlist==1.4.1
 greenlet==3.0.3
-gunicorn==21.2.0
+gunicorn==23.0.0
 hypothesis==6.45.0
-idna==3.6
+idna==3.8
 iniconfig==2.0.0
-itsdangerous==2.1.2
-jinja2==3.1.2
+itsdangerous==2.2.0
+jinja2==3.1.4
 jsonpatch==1.33
-jsonpointer==2.4
+jsonpointer==3.0.0
 langchain==0.0.354
-langchain-community==0.0.8
-langchain-core==0.1.5
+langchain-community==0.0.20
+langchain-core==0.1.23
 langchain-experimental==0.0.47
-langsmith==0.0.77
-markupsafe==2.1.3
-marshmallow==3.20.1
+langsmith==0.0.87
+markupsafe==2.1.5
+marshmallow==3.22.0
 mock==5.1.0
-multidict==6.0.4
+multidict==6.0.5
 mypy-extensions==1.0.0
-numpy==1.26.3
+numpy==1.26.4
 opentracing==2.4.0
 packaging==23.2
-pluggy==1.3.0
+pluggy==1.5.0
 psycopg2-binary==2.9.9
-pydantic==2.5.3
-pydantic-core==2.14.6
-pytest==7.4.4
-pytest-cov==4.1.0
-pytest-mock==3.12.0
-pyyaml==6.0.1
-requests==2.31.0
-sniffio==1.3.0
+pydantic==2.8.2
+pydantic-core==2.20.1
+pytest==8.3.2
+pytest-cov==5.0.0
+pytest-mock==3.14.0
+pyyaml==6.0.2
+requests==2.32.3
+sniffio==1.3.1
 sortedcontainers==2.4.0
-sqlalchemy==2.0.25
-tenacity==8.2.3
-typing-extensions==4.9.0
+sqlalchemy==2.0.32
+tenacity==8.5.0
+typing-extensions==4.12.2
 typing-inspect==0.9.0
-urllib3==2.1.0
-werkzeug==3.0.1
+urllib3==2.2.2
+werkzeug==3.0.4
 yarl==1.9.4

docker-compose.yml

@@ -147,7 +147,7 @@ services:
           - "127.0.0.1:5433:5433"
 
     testrunner:
-        image: ghcr.io/datadog/dd-trace-py/testrunner@sha256:9b2364afa4d1cd8382f251aca5963113cfc47e6c6b9aaed2c28e8a2a48ac1f75
+        image: ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
         command: bash
         environment:
             - TOX_SKIP_DIST=True

scripts/ddtest

@@ -12,10 +12,8 @@ fi
 # retry docker pull if fails
 for i in {1..3}; do docker-compose pull -q testrunner && break || sleep 3; done
 
-# DEV: Removing hatch==1.12.0 from this command results in weird failure
-# in which building dd-trace-py fails as it can't find clang compiler for some
-# venvs using Python 3.11.
-FULL_CMD="pip install -q --disable-pip-version-check riot==0.19.1 hatch==1.12.0 && $CMD"
+# TODO(DEV): Install riot in the docker image
+FULL_CMD="pip install -q --disable-pip-version-check riot==0.19.1 && $CMD"
 
 
 # install and upgrade riot in case testrunner image has not been updated

tests/appsec/integrations/test_psycopg2.py

@@ -12,7 +12,7 @@
     from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted
 
 
-def setup():
+def setup_module():
     create_context()
     oce._enabled = True
 

tests/profiling/collector/test_stack.py

@@ -23,7 +23,10 @@
 
 
 # FIXME: remove version limitation when gevent segfaults are fixed on Python 3.12
-TESTING_GEVENT = os.getenv("DD_PROFILE_TEST_GEVENT", False) and sys.version_info < (3, 12)
+# Python 3.11.9 is not compatible with gevent, https://github.com/python/cpython/issues/117983
+# The fix was not backported to 3.11. 3.12 got the fix but was not released yet.
+# Revisit this when 3.12.5 is released to check whether tests can be enabled.
+TESTING_GEVENT = os.getenv("DD_PROFILE_TEST_GEVENT", False) and sys.version_info < (3, 11, 9)
 
 
 def func1():