Merge branch 'main' into romain.komorn/CIVIS-9428/support_pytest_lt_5.1

.github/workflows/lib-inject-prune.yml

@@ -34,21 +34,20 @@ jobs:
       packages: write
     strategy:
       matrix:
-        image: [
-          'dd-lib-python-init-test-django',
-          'dd-lib-python-init-test-django-gunicorn',
-          'dd-lib-python-init-test-django-uvicorn',
-          'dd-lib-python-init-test-django-uwsgi',
-          'dd-lib-python-init-test-app',
-          'dd-python-agent-init',
-        ]
+        image:
+          - 'dd-lib-python-init-test-django'
+          - 'dd-lib-python-init-test-django-gunicorn'
+          - 'dd-lib-python-init-test-django-uvicorn'
+          - 'dd-lib-python-init-test-django-uwsgi'
+          - 'dd-lib-python-init-test-app'
+          - 'dd-python-agent-init'
     steps:
     - name: Prune registry
       uses: vlaurin/action-ghcr-prune@0a539594d122b915e71c59733a5b115bfaaf5d52 #v0.5.0
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         organization: Datadog
-        container: dd-trace-py/{{ matrix.image }}
+        container: dd-trace-py/${{ matrix.image }}
         keep-younger-than: 15 # days
         keep-last: 5
         keep-tags: |

.github/workflows/lib-injection.yml

@@ -18,6 +18,15 @@ jobs:
       ddtrace-version: v2.6.3
       image-tag: ${{ github.sha }}
 
+  build-and-publish-latest-snapshot-image:
+    if: github.ref == 'refs/heads/main'
+    uses: ./.github/workflows/lib-inject-publish.yml
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+    with:
+      ddtrace-version: v2.6.3
+      image-tag: latest_snapshot
+
   test-runner-test:
     needs:
       - build-and-publish-test-image

.github/workflows/test_frameworks.yml

@@ -96,40 +96,60 @@ jobs:
         if: needs.needs-run.outputs.outcome == 'success'
         run: cat debugger-expl.txt
 
-#   sanic-testsuite-22_12:
-#     runs-on: ubuntu-20.04
-#     needs: needs-run
-#     env:
-#       # Regression test for DataDog/dd-trace-py/issues/5722
-#       DD_UNLOAD_MODULES_FROM_SITECUSTOMIZE: true
-#       CMAKE_BUILD_PARALLEL_LEVEL: 12
-#     defaults:
-#       run:
-#         working-directory: sanic
-#     steps:
-#       - uses: actions/checkout@v4
-#         if: needs.needs-run.outputs.outcome == 'success'
-#         with:
-#           path: ddtrace
-#       - uses: actions/checkout@v4
-#         if: needs.needs-run.outputs.outcome == 'success'
-#         with:
-#           repository: sanic-org/sanic
-#           ref: v22.12.0
-#           path: sanic
-#       - uses: actions/setup-python@v5
-#         if: needs.needs-run.outputs.outcome == 'success'
-#         with:
-#           python-version: "3.11"
-#       - name: Install sanic and dependencies required to run tests
-#         if: needs.needs-run.outputs.outcome == 'success'
-#         run: pip3 install '.[test]' aioquic
-#       - name: Install ddtrace
-#         if: needs.needs-run.outputs.outcome == 'success'
-#         run: pip3 install ../ddtrace
-#       - name: Run tests
-#         if: needs.needs-run.outputs.outcome == 'success'
-#         run: ddtrace-run pytest -k "not test_add_signal and not test_ode_removes and not test_skip_touchup and not test_dispatch_signal_triggers and not test_keep_alive_connection_context and not test_redirect_with_params"
+  sanic-testsuite-22_12:
+    strategy:
+      matrix:
+        include:
+          # TODO: profiling fails with a timeout error
+          #- suffix: Profiling
+          #  profiling: 1
+          #  iast: 0
+          #  appsec: 0
+          - suffix: IAST
+            profiling: 0
+            iast: 1
+            appsec: 0
+          - suffix: APPSEC
+            profiling: 0
+            iast: 0
+            appsec: 1
+    name: Sanic 22.12 (with ${{ matrix.suffix }})
+    runs-on: ubuntu-20.04
+    needs: needs-run
+    env:
+      DD_PROFILING_ENABLED: ${{ matrix.profiling }}
+      DD_IAST_ENABLED: ${{ matrix.iast }}
+      DD_APPSEC_ENABLED: ${{ matrix.appsec }}
+      DD_TESTING_RAISE: true
+      CMAKE_BUILD_PARALLEL_LEVEL: 12
+      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
+    defaults:
+      run:
+        working-directory: sanic
+    steps:
+      - uses: actions/checkout@v4
+        if: needs.needs-run.outputs.outcome == 'success'
+        with:
+          path: ddtrace
+      - uses: actions/checkout@v4
+        if: needs.needs-run.outputs.outcome == 'success'
+        with:
+          repository: sanic-org/sanic
+          ref: v22.12.0
+          path: sanic
+      - uses: actions/setup-python@v5
+        if: needs.needs-run.outputs.outcome == 'success'
+        with:
+          python-version: "3.11"
+      - name: Install sanic and dependencies required to run tests
+        if: needs.needs-run.outputs.outcome == 'success'
+        run: pip3 install '.[test]' aioquic
+      - name: Install ddtrace
+        if: needs.needs-run.outputs.outcome == 'success'
+        run: pip3 install ../ddtrace
+      - name: Run tests
+        if: needs.needs-run.outputs.outcome == 'success'
+        run: ddtrace-run pytest -k "not test_no_exceptions_when_cancel_pending_request and not test_add_signal and not test_ode_removes and not test_skip_touchup and not test_dispatch_signal_triggers and not test_keep_alive_connection_context and not test_redirect_with_params"
 
   django-testsuite-3_1:
     strategy:
@@ -675,6 +695,9 @@ jobs:
       - name: MarkupSafe fix
         if: needs.needs-run.outputs.outcome == 'success'
         run: pip install --upgrade MarkupSafe==2.0.1
+      - name: Pytest fix
+        if: needs.needs-run.outputs.outcome == 'success'
+        run: pip install --upgrade pytest==5.4.3
       - name: Run tests
         if: needs.needs-run.outputs.outcome == 'success'
         run: PYTHONPATH=../ddtrace/tests/debugging/exploration/ ddtrace-run pytest -p no:warnings tests
@@ -861,3 +884,53 @@ jobs:
       - name: Debugger exploration results
         if: needs.needs-run.outputs.outcome == 'success'
         run: cat debugger-expl.txt
+
+  beautifulsoup-testsuite-4_12_3:
+    strategy:
+      matrix:
+        include:
+          # TODO: profiling is disabled due to a bug in the profiler paths
+          # - suffix: Profiling
+          #   profiling: 1
+          #   iast: 0
+          #   appsec: 0
+          - suffix: IAST
+            profiling: 0
+            iast: 1
+            appsec: 0
+          - suffix: APPSEC
+            profiling: 0
+            iast: 0
+            appsec: 1
+    name: Beautifulsoup 4.12.3 (with ${{ matrix.suffix }})
+    runs-on: "ubuntu-latest"
+    needs: needs-run
+    env:
+      DD_TESTING_RAISE: true
+      DD_PROFILING_ENABLED: ${{ matrix.profiling }}
+      DD_IAST_ENABLED: ${{ matrix.iast }}
+      DD_APPSEC_ENABLED: ${{ matrix.appsec }}
+      CMAKE_BUILD_PARALLEL_LEVEL: 12
+      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
+    steps:
+      - uses: actions/setup-python@v5
+        if: needs.needs-run.outputs.outcome == 'success'
+        with:
+          python-version: '3.9'
+      - uses: actions/checkout@v4
+        if: needs.needs-run.outputs.outcome == 'success'
+        with:
+          path: ddtrace
+      - name: Checkout beautifulsoup
+        if: needs.needs-run.outputs.outcome == 'success'
+        run:  |
+          git clone -b 4.12.3 https://git.launchpad.net/beautifulsoup
+      - name: Install ddtrace
+        if: needs.needs-run.outputs.outcome == 'success'
+        run: pip install ddtrace
+      - name: Pytest fix
+        if: needs.needs-run.outputs.outcome == 'success'
+        run: pip install pytest==8.2.1
+      - name: Run tests
+        if: needs.needs-run.outputs.outcome == 'success'
+        run: cd beautifulsoup && ddtrace-run pytest

.gitlab-ci.yml

@@ -1,6 +1,7 @@
 stages:
   - package
   - deploy
+  - dogfood
   - benchmarks
   - benchmarks-pr-comment
   - macrobenchmarks
@@ -9,6 +10,7 @@ include:
   - remote: https://gitlab-templates.ddbuild.io/apm/packaging.yml
   - local: ".gitlab/benchmarks.yml"
   - local: ".gitlab/macrobenchmarks.yml"
+  - local: ".gitlab/dogfood.yml"
 
 variables:
   DOWNSTREAM_BRANCH:

.gitlab/dogfood.yml

@@ -0,0 +1,33 @@
+dogfood-dogweb-failed:
+  stage: dogfood
+  tags: ["arch:amd64"]
+  needs: ["dogfood-dogweb-trigger"]
+  when: on_failure
+  script:
+    - exit 1
+
+dogfood-dogweb:
+  stage: dogfood
+  tags: ["arch:amd64"]
+  needs: ["dogfood-dogweb-trigger"]
+  when: on_success
+  script:
+    - exit 0
+
+dogfood-dogweb-trigger:
+  stage: dogfood
+  trigger:
+    project: DataDog/dogweb
+    strategy: depend
+    branch: emmett.butler/ddtrace-unstable-dogfooding
+  variables:
+    UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
+    UPSTREAM_PROJECT_URL: $CI_PROJECT_URL
+    DDTRACE_COMMIT_TO_INSTALL: $CI_COMMIT_SHA
+    UPSTREAM_COMMIT_BRANCH: $CI_COMMIT_BRANCH
+    UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
+    UPSTREAM_COMMIT_TITLE: $CI_COMMIT_TITLE
+    UPSTREAM_COMMIT_TAG: $CI_COMMIT_TAG
+    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
+    UPSTREAM_GITLAB_USER_LOGIN: $GITLAB_USER_LOGIN
+    UPSTREAM_GITLAB_USER_EMAIL: $GITLAB_USER_EMAIL

ddtrace/_trace/span.py

@@ -51,6 +51,24 @@
 _MetaDictType = Dict[_TagNameType, Text]
 _MetricDictType = Dict[_TagNameType, NumericType]
 
+
+class SpanEvent:
+    __slots__ = ["name", "attributes", "time_unix_nano"]
+
+    def __init__(self, name: str, attributes: Optional[Dict[str, str]] = None, time_unix_nano: Optional[int] = None):
+        self.name: str = name
+        self.attributes: Dict[str, str] = attributes or {}
+        if time_unix_nano is None:
+            time_unix_nano = time_ns()
+        self.time_unix_nano: int = time_unix_nano
+
+    def __dict__(self):
+        d = {"name": self.name, "time_unix_nano": self.time_unix_nano}
+        if self.attributes:
+            d["attributes"] = self.attributes
+        return d
+
+
 log = get_logger(__name__)
 
 
@@ -91,6 +109,7 @@ class Span(object):
         "_ignored_exceptions",
         "_on_finish_callbacks",
         "_links",
+        "_events",
         "__weakref__",
     ]
 
@@ -173,6 +192,7 @@ def __init__(
         self._links = {}  # type: Dict[int, SpanLink]
         if links:
             self._links = {link.span_id: link for link in links}
+        self._events = []  # type: List[SpanEvent]
         self._parent = None  # type: Optional[Span]
         self._ignored_exceptions = None  # type: Optional[List[Exception]]
         self._local_root = None  # type: Optional[Span]
@@ -486,6 +506,11 @@ def _set_baggage_item(self, key, value):
         self._context = self.context._with_baggage_item(key, value)
         return self
 
+    def _add_event(self, name, attributes=None, timestamp=None):
+        # type: (str, Optional[Dict[str, str]], Optional[int]) -> None
+        """Add an event to the span."""
+        self._events.append(SpanEvent(name, attributes, timestamp))
+
     def _get_baggage_item(self, key):
         # type: (str) -> Optional[Any]
         """Gets a baggage item from the span context of this span."""
@@ -527,9 +552,13 @@ def set_exc_info(self, exc_type, exc_val, exc_tb):
         self._set_exc_tags(exc_type, exc_val, exc_tb)
 
     def _set_exc_tags(self, exc_type, exc_val, exc_tb):
+        limit = config._span_traceback_max_size
+        if limit is None:
+            limit = 30
+
         # get the traceback
         buff = StringIO()
-        traceback.print_exception(exc_type, exc_val, exc_tb, file=buff, limit=30)
+        traceback.print_exception(exc_type, exc_val, exc_tb, file=buff, limit=limit)
         tb = buff.getvalue()
 
         # readable version of type (e.g. exceptions.ZeroDivisionError)

ddtrace/appsec/_capabilities.py

@@ -29,6 +29,14 @@ class Flags(enum.IntFlag):
     ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
     ASM_TRUSTED_IPS = 1 << 10
     ASM_API_SECURITY_SAMPLE_RATE = 1 << 11
+    ASM_RASP_SQLI = 1 << 21
+    ASM_RASP_LFI = 1 << 22
+    ASM_RASP_SSRF = 1 << 23
+    ASM_RASP_SHI = 1 << 24
+    ASM_RASP_XXE = 1 << 25
+    ASM_RASP_RCE = 1 << 26
+    ASM_RASP_NOSQLI = 1 << 27
+    ASM_RASP_XSS = 1 << 28
 
 
 _ALL_ASM_BLOCKING = (
@@ -43,6 +51,8 @@ class Flags(enum.IntFlag):
     | Flags.ASM_CUSTOM_BLOCKING_RESPONSE
 )
 
+_ALL_RASP = Flags.ASM_RASP_LFI | Flags.ASM_RASP_SSRF
+
 
 def _rc_capabilities(test_tracer: Optional[ddtrace.Tracer] = None) -> Flags:
     tracer = ddtrace.tracer if test_tracer is None else test_tracer
@@ -52,6 +62,8 @@ def _rc_capabilities(test_tracer: Optional[ddtrace.Tracer] = None) -> Flags:
             value |= Flags.ASM_ACTIVATION
         if tracer._appsec_processor and _appsec_rc_file_is_not_static():
             value |= _ALL_ASM_BLOCKING
+            if asm_config._ep_enabled:
+                value |= _ALL_RASP
         if asm_config._api_security_enabled:
             value |= Flags.ASM_API_SECURITY_SAMPLE_RATE
     return value

ddtrace/appsec/_processor.py

@@ -420,7 +420,7 @@ def on_span_finish(self, span: Span) -> None:
 
                 headers_req = core.get_item(SPAN_DATA_NAMES.REQUEST_HEADERS_NO_COOKIES, span=span)
                 if headers_req:
-                    _set_headers(span, headers_req, kind="request", only_asm_enabled=True)
+                    _set_headers(span, headers_req, kind="request", only_asm_enabled=False)
 
                 # this call is only necessary for tests or frameworks that are not using blocking
                 if not has_triggers(span) and _asm_request_context.in_context():

ddtrace/appsec/_trace_utils.py

@@ -130,6 +130,9 @@ def track_user_login_failure_event(
     exists: Optional[bool] = None,
     metadata: Optional[dict] = None,
     login_events_mode: str = LOGIN_EVENTS_MODE.SDK,
+    login: Optional[str] = None,
+    name: Optional[str] = None,
+    email: Optional[str] = None,
 ) -> None:
     """
     Add a new login failure tracking event.
@@ -149,13 +152,17 @@ def track_user_login_failure_event(
     span = _track_user_login_common(tracer, False, metadata, login_events_mode)
     if not span:
         return
-
     if user_id:
         span.set_tag_str("%s.failure.%s" % (APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, user.ID), str(user_id))
-
     if exists is not None:
         exists_str = "true" if exists else "false"
         span.set_tag_str("%s.failure.%s" % (APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, user.EXISTS), exists_str)
+    if login:
+        span.set_tag_str("%s.failure.login" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, login)
+    if email:
+        span.set_tag_str("%s.failure.email" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, email)
+    if name:
+        span.set_tag_str("%s.failure.username" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, name)
 
 
 def track_user_signup_event(