Merge branch 'main' into refactor/native-periodic-thread

.github/CODEOWNERS

@@ -76,6 +76,7 @@ tests/contrib/*/test*appsec*.py     @DataDog/asm-python
 
 # Profiling
 ddtrace/profiling                   @DataDog/profiling-python @DataDog/apm-core-python
+ddtrace/internal/datadog/profiling  @DataDog/profiling-python @DataDog/apm-core-python
 tests/profiling                     @DataDog/profiling-python @DataDog/apm-core-python
 
 # MLObs

.github/workflows/stale.yml

@@ -1,4 +1,4 @@
-name: 'Label stale issues and PRs'
+name: 'Close stale issues and PRs'
 on:
   schedule:
     # 00:00:000 UTC
@@ -17,16 +17,17 @@ jobs:
           # DEV: GitHub Actions have an API rate limit of 1000 operations per hour per repository
           #      This limit is shared across all actions
           operations-per-run: 200
-          days-before-close: 180
+          days-before-pr-close: 30
+          days-before-issue-close: 90
           exempt-issue-labels: 'proposal'
           exempt-pr-labels: 'proposal'
           close-issue-message: |
-            This issue has been automatically closed after six months of inactivity. If it's a
+            This issue has been automatically closed after a period of inactivity. If it's a
             feature request, it has been added to the maintainers' internal backlog and will be
             included in an upcoming round of feature prioritization. Please comment or reopen
             if you think this issue was closed in error.
           close-pr-message: |
-            This pull request has been automatically closed after six months of inactivity.
+            This pull request has been automatically closed after a period of inactivity.
             After this much time, it will likely be easier to open a new pull request with the
             same changes than to update this one from the base branch. Please comment or reopen
             if you think this pull request was closed in error.

.github/workflows/system-tests.yml

@@ -208,8 +208,8 @@ jobs:
 
       - name: Run APPSEC_BLOCKING
         if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
-
         run: ./run.sh APPSEC_BLOCKING
+
       - name: Run APPSEC_BLOCKING_FULL_DENYLIST
         if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
         run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST
@@ -218,6 +218,10 @@ jobs:
         if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
         run: ./run.sh APPSEC_REQUEST_BLOCKING
 
+      - name: Run APPSEC_RASP
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+        run: ./run.sh APPSEC_RASP
+
       # The compress step speed up a lot the upload artifact process
       - name: Compress artifact
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'

.gitlab-ci.yml

@@ -130,3 +130,20 @@ deploy_latest_tag_to_docker_registries:
     # See above note in the `deploy_to_docker_registries` job.
     RETRY_DELAY: 14400
     RETRY_COUNT: 3
+
+package-oci:
+  stage: package
+  extends: .package-oci
+  rules:
+  - if: $PYTHON_PACKAGE_VERSION
+    when: on_success
+  - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(-prerelease)?$/'
+    when: manual
+    allow_failure: false
+  script:
+    - ../.gitlab/build-oci.sh
+  parallel:
+    matrix:
+      - ARCH:
+        - arm64
+        - amd64

.gitlab/benchmarks.yml

@@ -1,12 +1,12 @@
 variables:
-  BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-py
+  MICROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-py
   SLS_CI_IMAGE: registry.ddbuild.io/ci/serverless-tools:1
 
 .benchmarks:
   stage: benchmarks
   when: on_success
   tags: ["runner:apm-k8s-tweaked-metal"]
-  image: $BASE_CI_IMAGE
+  image: $MICROBENCHMARKS_CI_IMAGE
   interruptible: true
   timeout: 1h
   script:
@@ -15,7 +15,7 @@ variables:
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
     - git clone --branch dd-trace-py https://github.com/DataDog/benchmarking-platform /platform && cd /platform
     - ./steps/capture-hardware-software-info.sh
-    - ./steps/run-benchmarks.sh
+    - '([ $SCENARIO = "flask_simple" ] && BP_SCENARIO=$SCENARIO /benchmarking-platform-tools/bp-runner/bp-runner "$REPORTS_DIR/../.gitlab/benchmarks/bp-runner.yml" --debug -t) || ([ $SCENARIO != "flask_simple" ] && ./steps/run-benchmarks.sh)'
     - ./steps/analyze-results.sh
     - "./steps/upload-results-to-s3.sh || :"
   artifacts:
@@ -37,7 +37,7 @@ benchmarks-pr-comment:
   stage: benchmarks-pr-comment
   when: on_success
   tags: ["arch:amd64"]
-  image: $BASE_CI_IMAGE
+  image: $MICROBENCHMARKS_CI_IMAGE
   script:
     - export REPORTS_DIR="$(pwd)/reports/" && (mkdir "${REPORTS_DIR}" || :)
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
@@ -87,7 +87,7 @@ benchmark-flask-sqli:
   extends: .benchmarks
   variables:
     SCENARIO: "flask_sqli"
-    
+
 benchmark-core-api:
   extends: .benchmarks
   variables:

.gitlab/benchmarks/bp-runner.yml

@@ -0,0 +1,13 @@
+experiments:
+  - name: run-microbenchmarks
+    setup:
+      - name: datadog-agent
+        run: datadog_agent
+        cpus: 24-25
+        config_sh: ./steps/update-dd-agent-config.sh
+
+    steps:
+      - name: benchmarks
+        cpus: 26-47
+        run: shell
+        script: export SCENARIO=$BP_SCENARIO && ./steps/run-benchmarks.sh

.gitlab/build-oci.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+source common_build_functions.sh
+
+if [ -n "$CI_COMMIT_TAG" ] && [ -z "$PYTHON_PACKAGE_VERSION" ]; then
+  PYTHON_PACKAGE_VERSION=${CI_COMMIT_TAG##v}
+fi
+
+if [ -z "$ARCH" ]; then
+  ARCH=amd64
+fi
+
+
+TMP_DIR=$(mktemp --dir)
+BUILD_DIR=$TMP_DIR/datadog-python-apm.build
+mkdir $TMP_DIR/datadog-python-apm.build
+
+# Install known compatible pip as default version shipped in Ubuntu (20.0.2)
+# does not work.
+python3 -m pip install -U "pip>=22.0"
+python3 -m pip install packaging
+
+WHEEL_ARCH="x86_64"
+if [ "$ARCH" = "arm64" ]; then
+  WHEEL_ARCH="aarch64"
+fi
+
+../lib-injection/dl_wheels.py \
+    --python-version=3.12 \
+    --python-version=3.11 \
+    --python-version=3.10 \
+    --python-version=3.9 \
+    --python-version=3.8 \
+    --python-version=3.7 \
+    --ddtrace-version=$PYTHON_PACKAGE_VERSION \
+    --arch=$WHEEL_ARCH \
+    --platform=musllinux_1_1 \
+    --platform=manylinux2014 \
+    --output-dir=$BUILD_DIR/ddtrace_pkgs \
+    --verbose
+echo -n $PYTHON_PACKAGE_VERSION > auto_inject-python.version
+cp ../lib-injection/sitecustomize.py $BUILD_DIR/
+cp auto_inject-python.version $BUILD_DIR/version
+chmod -R o-w $BUILD_DIR
+chmod -R g-w $BUILD_DIR
+
+# Build packages
+datadog-package create \
+    --version="$PYTHON_PACKAGE_VERSION" \
+    --package="datadog-apm-library-python" \
+    --archive=true \
+    --archive-path="datadog-apm-library-python-$PYTHON_PACKAGE_VERSION-$ARCH.tar" \
+    --arch "$ARCH" \
+    --os "linux" \
+    $BUILD_DIR

.gitlab/macrobenchmarks.yml

@@ -1,5 +1,5 @@
 variables:
-  BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-py-macrobenchmarks
+  MACROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-py-macrobenchmarks
 
 .macrobenchmarks:
   stage: macrobenchmarks
@@ -14,7 +14,7 @@ variables:
     # - if: $CI_COMMIT_REF_NAME == "main"
     #   when: always
   # If you have a problem with Gitlab cache, see Troubleshooting section in Benchmarking Platform docs
-  image: $BENCHMARKS_CI_IMAGE
+  image: $MACROBENCHMARKS_CI_IMAGE
   script: |
     git clone --branch python/macrobenchmarks https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
     if [ "$BP_PYTHON_SCENARIO_DIR" == "flask-realworld" ]; then

.riot/requirements/1a3fc04.txt → .riot/requirements/18695ab.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.7
 # by the following command:
 #
-#    pip-compile --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/1a3fc04.in
+#    pip-compile --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/18695ab.in
 #
 astunparse==1.6.3
 attrs==23.2.0
@@ -11,12 +11,12 @@ cffi==1.15.1
 charset-normalizer==3.3.2
 coverage[toml]==7.2.7
 cryptography==42.0.5
-exceptiongroup==1.2.0
+exceptiongroup==1.2.1
 googleapis-common-protos==1.63.0
 greenlet==3.0.3
-grpcio==1.62.1
+grpcio==1.62.2
 hypothesis==6.45.0
-idna==3.6
+idna==3.7
 importlib-metadata==6.7.0
 iniconfig==2.0.0
 mock==5.1.0

.riot/requirements/ec026f0.txt → .riot/requirements/1aedbda.txt

@@ -2,31 +2,31 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile --no-annotate .riot/requirements/ec026f0.in
+#    pip-compile --no-annotate .riot/requirements/1aedbda.in
 #
 astunparse==1.6.3
 attrs==23.2.0
 certifi==2024.2.2
 cffi==1.16.0
 charset-normalizer==3.3.2
-coverage[toml]==7.4.4
+coverage[toml]==7.5.0
 cryptography==42.0.5
-exceptiongroup==1.2.0
+exceptiongroup==1.2.1
 googleapis-common-protos==1.63.0
 greenlet==3.0.3
-grpcio==1.62.1
+grpcio==1.63.0
 hypothesis==6.45.0
-idna==3.6
+idna==3.7
 iniconfig==2.0.0
 mock==5.1.0
 opentracing==2.4.0
 packaging==24.0
-pluggy==1.4.0
+pluggy==1.5.0
 protobuf==4.25.3
 psycopg2-binary==2.9.9
 pycparser==2.22
 pycryptodome==3.20.0
-pytest==8.1.1
+pytest==8.2.0
 pytest-cov==5.0.0
 pytest-mock==3.14.0
 requests==2.31.0

.riot/requirements/1b90fc9.txt

@@ -0,0 +1,29 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --no-annotate --resolver=backtracking .riot/requirements/1b90fc9.in
+#
+attrs==23.2.0
+coverage[toml]==7.5.0
+exceptiongroup==1.2.1
+gevent==24.2.1
+greenlet==3.0.3
+hypothesis==6.45.0
+iniconfig==2.0.0
+mock==5.1.0
+msgpack==1.0.8
+opentracing==2.4.0
+packaging==24.0
+pluggy==1.5.0
+pytest==8.2.0
+pytest-cov==5.0.0
+pytest-mock==3.14.0
+pytest-randomly==3.15.0
+sortedcontainers==2.4.0
+tomli==2.0.1
+zope-event==5.0
+zope-interface==6.3
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools