Merge branch 'main' into romain.komorn/SDTEST-169/ci_visibilty_manual…

…_api_cleanup

.circleci/config.templ.yml

@@ -603,14 +603,6 @@ jobs:
       - run_test:
           pattern: "sourcecode"
 
-  telemetry:
-    parallelism: 6
-    <<: *machine_executor
-    steps:
-      - run_test:
-          pattern: "telemetry"
-          snapshot: true
-
   debugger:
     <<: *contrib_job
     steps:
@@ -639,48 +631,6 @@ jobs:
           pattern: 'opentelemetry'
           snapshot: true
 
-  integration_agent:
-    <<: *machine_executor
-    parallelism: 2
-    steps:
-      - attach_workspace:
-          at: .
-      - checkout
-      - setup_riot
-      - start_docker_services:
-          services: ddagent
-      - run:
-          environment:
-            RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
-          command: |
-            ulimit -c unlimited
-            ./scripts/run-test-suite 'integration-latest*' <<pipeline.parameters.coverage>> 1
-      - run:
-          command: |
-            mkdir -p /tmp/core_dumps
-            cp core.* /tmp/core_dumps || true
-            ./scripts/bt
-          when: on_fail
-      - store_artifacts:
-          name: "Store core dumps"
-          path: /tmp/core_dumps
-
-  integration_testagent:
-    <<: *machine_executor
-    steps:
-      - run_test:
-          snapshot: true
-          pattern: 'integration-snapshot*'
-          trace_agent_url: ""
-
-  vendor:
-    <<: *contrib_job_small
-    docker:
-      - image: *ddtrace_dev_image
-    steps:
-      - run_test:
-          pattern: 'vendor'
-
   botocore:
     <<: *machine_executor
     parallelism: 6
@@ -690,17 +640,6 @@ jobs:
           snapshot: true
           docker_services: "localstack"
 
-  ddtracerun:
-    <<: *contrib_job
-    parallelism: 8
-    docker:
-      - image: *ddtrace_dev_image
-      - image: *redis_image
-    steps:
-      - run_test:
-          pattern: 'ddtracerun'
-          trace_agent_url: ""
-
   test_logging:
     <<: *contrib_job
     docker:
@@ -877,14 +816,6 @@ jobs:
           snapshot: true
           docker_services: "redis"
 
-  fastapi:
-    <<: *machine_executor
-    parallelism: 2
-    steps:
-      - run_test:
-          pattern: "fastapi"
-          snapshot: true
-
   flask:
     <<: *machine_executor
     parallelism: 10

.github/workflows/codeowners.yml

@@ -7,6 +7,8 @@ jobs:
   report_codeowners:
     name: "Report codeowners"
     runs-on: ubuntu-latest
+    permissions:
+       pull-requests: write
     steps:
       - uses: actions/checkout@v4
         with:

.gitlab-ci.yml

@@ -11,7 +11,7 @@ stages:
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
   TESTRUNNER_IMAGE: ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
-  CI_DEBUG_SERVICES: "true"
+  # CI_DEBUG_SERVICES: "true"
 
 .testrunner:
   image: $TESTRUNNER_IMAGE

.gitlab/prepare-oci-package.sh

@@ -30,7 +30,3 @@ cp -r ../pywheels-dep/site-packages* sources/ddtrace_pkgs
 cp ../lib-injection/sitecustomize.py sources/
 cp ../min_compatible_versions.csv sources/
 cp ../lib-injection/telemetry-forwarder.sh sources/
-
-clean-apt install python3
-echo "Deduplicating package files"
-python3 ../lib-injection/dedupe.py sources/ddtrace_pkgs/

.gitlab/release.yml

@@ -22,7 +22,7 @@ variables:
     - python -m pip install twine
     - python -m twine check --strict pywheels/*
   script:
-    - echo "python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/*"
+    - python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/*
   artifacts:
     paths:
       - pywheels/*.whl

.gitlab/tests.yml

@@ -20,16 +20,18 @@ variables:
       alias: ddagent
       variables:
         DD_HOSTNAME: ddagent
+        LOG_LEVEL: ERROR
         DD_REMOTE_CONFIGURATION_ENABLED: true
         DD_SITE: datadoghq.com
         DD_API_KEY: invalid_but_this_is_fine
         DD_REMOTE_CONFIGURATION_KEY: invalid_but_this_is_fine
         DD_REMOTE_CONFIGURATION_REFRESH_INTERVAL: 5s
         DD_DOGSTATSD_NON_LOCAL_TRAFFIC: true
     - name: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.17.0
+      alias: testagent
       variables:
-        LOG_LEVEL: DEBUG
-        SNAPSHOT_DIR: /snapshots
+        LOG_LEVEL: INFO
+        SNAPSHOT_DIR: ${CI_PROJECT_DIR}/tests/snapshots
         SNAPSHOT_CI: 1
         PORT: 9126
         DD_POOL_TRACE_CHECK_FAILURES: true
@@ -50,8 +52,17 @@ variables:
         ${RIOT_RUN_CMD} "${hash}"
       done
 
+.test_base_riot_snapshot:
+  extends: .test_base_riot
+  before_script:
+    - !reference [.testrunner, before_script]
+    # DEV: All job variables get shared with services, setting `DD_TRACE_AGENT_URL` on the testagent will tell it to forward all requests to the
+    # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent
+    - export DD_TRACE_AGENT_URL="http://testagent:9126"
+
 include:
   - local: ".gitlab/tests/appsec.yml"
+  - local: ".gitlab/tests/contrib.yml"
   - local: ".gitlab/tests/core.yml"
   - local: ".gitlab/tests/tracer.yml"
   - local: ".gitlab/tests/profiling.yml"

.gitlab/tests/contrib.yml

@@ -0,0 +1,4 @@
+fastapi:
+  extends: .test_base_riot_snapshot
+  variables:
+    SUITE_NAME: "fastapi"

.gitlab/tests/core.yml

@@ -2,3 +2,34 @@ internal:
   extends: .test_base_riot
   variables:
     SUITE_NAME: "internal"
+
+telemetry:
+  extends: .test_base_riot_snapshot
+  parallel: 4
+  variables:
+    SUITE_NAME: "telemetry"
+
+integration-testagent:
+  extends: .test_base_riot_snapshot
+  variables:
+    SUITE_NAME: "integration-snapshot*"
+
+integration-agent:
+  extends: .test_base_riot
+  variables:
+    SUITE_NAME: "integration-latest*"
+
+vendor:
+  extends: .test_base_riot
+  variables:
+    SUITE_NAME: "vendor"
+
+ddtracerun:
+  extends: .test_base_riot
+  services:
+    - !reference [.test_base_riot, services]
+    - name: registry.ddbuild.io/redis:7.0.7
+      alias: redis
+  variables:
+    SUITE_NAME: "ddtracerun"
+    TEST_REDIS_HOST: "redis"

ddtrace/_trace/tracer.py

@@ -39,7 +39,6 @@
 from ddtrace.internal import forksafe
 from ddtrace.internal import hostname
 from ddtrace.internal.atexit import register_on_exit_signal
-from ddtrace.internal.constants import MAX_UINT_64BITS
 from ddtrace.internal.constants import SAMPLING_DECISION_TRACE_TAG_KEY
 from ddtrace.internal.constants import SPAN_API_DATADOG
 from ddtrace.internal.dogstatsd import get_dogstatsd_client
@@ -58,6 +57,7 @@
 from ddtrace.internal.service import ServiceStatusError
 from ddtrace.internal.utils import _get_metas_to_propagate
 from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning
+from ddtrace.internal.utils.formats import format_trace_id
 from ddtrace.internal.utils.http import verify_url
 from ddtrace.internal.writer import AgentResponse
 from ddtrace.internal.writer import AgentWriter
@@ -423,11 +423,8 @@ def get_log_correlation_context(self, active: Optional[Union[Context, Span]] = N
         span_id = "0"
         trace_id = "0"
         if active:
-            span_id = str(active.span_id if active.span_id else span_id)
-            trace_id = str(active.trace_id if active.trace_id else trace_id)
-            # check if we are using 128 bit ids, and switch trace id to hex since backend needs hex 128 bit ids
-            if active.trace_id and active.trace_id > MAX_UINT_64BITS:
-                trace_id = "{:032x}".format(active.trace_id)
+            span_id = str(active.span_id) if active.span_id else span_id
+            trace_id = format_trace_id(active.trace_id) if active.trace_id else trace_id
 
         return {
             "trace_id": trace_id,

ddtrace/appsec/_handlers.py

@@ -216,53 +216,58 @@ def _on_request_init(wrapped, instance, args, kwargs):
 
 def _on_flask_patch(flask_version):
     if _is_iast_enabled():
-        try:
-            from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
-            from ddtrace.appsec._iast._taint_tracking import OriginType
+        from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
+        from ddtrace.appsec._iast._patch import _patched_dictionary
+        from ddtrace.appsec._iast._patch import try_wrap_function_wrapper
+        from ddtrace.appsec._iast._taint_tracking import OriginType
+
+        try_wrap_function_wrapper(
+            "werkzeug.datastructures",
+            "Headers.items",
+            functools.partial(if_iast_taint_yield_tuple_for, (OriginType.HEADER_NAME, OriginType.HEADER)),
+        )
+        _set_metric_iast_instrumented_source(OriginType.HEADER_NAME)
+        _set_metric_iast_instrumented_source(OriginType.HEADER)
 
-            _w(
-                "werkzeug.datastructures",
-                "Headers.items",
-                functools.partial(if_iast_taint_yield_tuple_for, (OriginType.HEADER_NAME, OriginType.HEADER)),
-            )
-            _set_metric_iast_instrumented_source(OriginType.HEADER_NAME)
-            _set_metric_iast_instrumented_source(OriginType.HEADER)
+        try_wrap_function_wrapper(
+            "werkzeug.datastructures",
+            "ImmutableMultiDict.__getitem__",
+            functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER),
+        )
+        _set_metric_iast_instrumented_source(OriginType.PARAMETER)
 
-            _w(
-                "werkzeug.datastructures",
-                "ImmutableMultiDict.__getitem__",
-                functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER),
-            )
-            _set_metric_iast_instrumented_source(OriginType.PARAMETER)
+        try_wrap_function_wrapper(
+            "werkzeug.datastructures",
+            "EnvironHeaders.__getitem__",
+            functools.partial(if_iast_taint_returned_object_for, OriginType.HEADER),
+        )
+        _set_metric_iast_instrumented_source(OriginType.HEADER)
 
-            _w(
-                "werkzeug.datastructures",
-                "EnvironHeaders.__getitem__",
-                functools.partial(if_iast_taint_returned_object_for, OriginType.HEADER),
-            )
-            _set_metric_iast_instrumented_source(OriginType.HEADER)
+        try_wrap_function_wrapper("werkzeug.wrappers.request", "Request.__init__", _on_request_init)
 
-            _w("werkzeug.wrappers.request", "Request.__init__", _on_request_init)
+        _set_metric_iast_instrumented_source(OriginType.PATH)
+        _set_metric_iast_instrumented_source(OriginType.QUERY)
 
-            _set_metric_iast_instrumented_source(OriginType.PATH)
-            _set_metric_iast_instrumented_source(OriginType.QUERY)
+        try_wrap_function_wrapper(
+            "werkzeug.wrappers.request",
+            "Request.get_data",
+            functools.partial(_patched_dictionary, OriginType.BODY, OriginType.BODY),
+        )
+        try_wrap_function_wrapper(
+            "werkzeug.wrappers.request",
+            "Request.get_json",
+            functools.partial(_patched_dictionary, OriginType.BODY, OriginType.BODY),
+        )
+
+        _set_metric_iast_instrumented_source(OriginType.BODY)
 
+        if flask_version < (2, 0, 0):
             _w(
-                "werkzeug.wrappers.request",
-                "Request.get_data",
-                functools.partial(if_iast_taint_returned_object_for, OriginType.BODY),
+                "werkzeug._internal",
+                "_DictAccessorProperty.__get__",
+                functools.partial(if_iast_taint_returned_object_for, OriginType.QUERY),
             )
-            _set_metric_iast_instrumented_source(OriginType.BODY)
-
-            if flask_version < (2, 0, 0):
-                _w(
-                    "werkzeug._internal",
-                    "_DictAccessorProperty.__get__",
-                    functools.partial(if_iast_taint_returned_object_for, OriginType.QUERY),
-                )
-                _set_metric_iast_instrumented_source(OriginType.QUERY)
-        except Exception:
-            log.debug("Unexpected exception while patch IAST functions", exc_info=True)
+            _set_metric_iast_instrumented_source(OriginType.QUERY)
 
 
 def _on_flask_blocked_request(_):
@@ -345,9 +350,9 @@ def _on_django_patch():
             from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
             from ddtrace.appsec._iast._taint_tracking import OriginType
 
+            # we instrument those sources on _on_django_func_wrapped
             _set_metric_iast_instrumented_source(OriginType.HEADER_NAME)
             _set_metric_iast_instrumented_source(OriginType.HEADER)
-            # we instrument those sources on _on_django_func_wrapped
             _set_metric_iast_instrumented_source(OriginType.PATH_PARAMETER)
             _set_metric_iast_instrumented_source(OriginType.PATH)
             _set_metric_iast_instrumented_source(OriginType.COOKIE)