Merge branch '2.12' into backport-10316-to-2.12

.gitlab-ci.yml

@@ -5,19 +5,21 @@ stages:
   - benchmarks
   - macrobenchmarks
   - dogfood
+  - release
 
 include:
   - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/one-pipeline.yml
   - local: ".gitlab/benchmarks.yml"
   - local: ".gitlab/package.yml"
   - local: ".gitlab/macrobenchmarks.yml"
   - local: ".gitlab/dogfood.yml"
+  - local: ".gitlab/release.yml"
 
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
 
 package-oci:
-  needs: [ download_dependency_wheels, download_ddtrace_wheels ]
+  needs: [ download_dependency_wheels, download_ddtrace_artifacts ]
 
 onboarding_tests_installer:
   parallel:

.gitlab/dogfood.yml

@@ -1,24 +1,7 @@
-dogfood-dogweb-failed:
-  stage: dogfood
-  tags: ["arch:amd64"]
-  needs: ["dogfood-dogweb-trigger"]
-  when: on_failure
-  script:
-    - exit 0
-
-dogfood-dogweb:
-  stage: dogfood
-  tags: ["arch:amd64"]
-  needs: ["dogfood-dogweb-trigger"]
-  when: on_success
-  script:
-    - exit 0
-
 dogfood-dogweb-trigger:
   stage: dogfood
   trigger:
     project: DataDog/dogweb
-    strategy: depend
     branch: emmett.butler/ddtrace-unstable-dogfooding
   allow_failure: true
   needs: []

.gitlab/package.yml

@@ -23,7 +23,7 @@ build_base_venvs:
       - ddtrace/**/*.so*
       - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe
 
-download_ddtrace_wheels:
+download_ddtrace_artifacts:
   image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
   tags: [ "arch:amd64" ]
   stage: package
@@ -35,12 +35,13 @@ download_ddtrace_wheels:
   artifacts:
     paths:
       - "pywheels/*.whl"
+      - "pywheels/*.tar.gz"
 
 download_dependency_wheels:
   image: registry.ddbuild.io/images/mirror/python:$PYTHON_IMAGE_TAG
   tags: [ "arch:amd64" ]
   stage: package
-  needs: [ download_ddtrace_wheels ]
+  needs: [ download_ddtrace_artifacts ]
   parallel:
     matrix: # The image tags that are mirrored are in: https://github.com/DataDog/images/blob/master/mirror.yaml
       - PYTHON_IMAGE_TAG: "3.7"

.gitlab/release.yml

@@ -0,0 +1,42 @@
+variables:
+  PYPI_PUBLISH_IMAGE: registry.ddbuild.io/images/mirror/python:3.12.0
+
+.release_base:
+  stage: release
+  only:
+    # v2.10.0
+    # v2.10.1
+    # v2.10.0rc0
+    # v2.10.0rc5
+    - /^v[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$/
+
+.release_pypi:
+  extends: .release_base
+  image: ${PYPI_PUBLISH_IMAGE}
+  tags: [ "arch:amd64" ]
+  variables:
+    TWINE_USERNAME: "__token__"
+    TWINE_NON_INTERACTIVE: "1"
+  before_script:
+    - export TWINE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name "ci.${CI_PROJECT_NAME}.${PYPI_REPOSITORY}_token" --with-decryption --query "Parameter.Value" --out text)
+    - python -m pip install twine
+    - python -m twine check --strict pywheels/*
+  script:
+    - echo "python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/*"
+  artifacts:
+    paths:
+      - pywheels/*.whl
+      - pywheels/*.tar.gz
+
+release_pypi_test:
+  extends: .release_pypi
+  dependencies: [ "download_ddtrace_artifacts" ]
+  variables:
+    PYPI_REPOSITORY: testpypi
+
+# TODO: Replace GitHub Action PyPI upload with this job
+# release_pypi_prod:
+#   extends: .release_pypi
+#   needs: [ "release_pypi_test" ]
+#   variables:
+#     PYPI_REPOSITORY: pypi

ddtrace/appsec/_ddwaf/ddwaf_types.py

@@ -181,7 +181,7 @@ def create_without_limits(cls, struct: DDWafRulesType) -> "ddwaf_object":
     def struct(self) -> DDWafRulesType:
         """Generate a python structure from ddwaf_object"""
         if self.type == DDWAF_OBJ_TYPE.DDWAF_OBJ_STRING:
-            return self.value.stringValue.decode("UTF-8", errors="ignore")
+            return self.value.stringValue[: self.nbEntries].decode("UTF-8", errors="ignore")
         if self.type == DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP:
             return {
                 self.value.array[i].parameterName.decode("UTF-8", errors="ignore"): self.value.array[i].struct
@@ -211,7 +211,7 @@ def __repr__(self):
 
 class ddwaf_value(ctypes.Union):
     _fields_ = [
-        ("stringValue", ctypes.c_char_p),
+        ("stringValue", ctypes.POINTER(ctypes.c_char)),
         ("uintValue", ctypes.c_ulonglong),
         ("intValue", ctypes.c_longlong),
         ("array", ddwaf_object_p),

ddtrace/contrib/internal/django/patch.py

@@ -6,6 +6,7 @@
 `django.apps.registry.Apps.populate` is patched to add instrumentation for any
 specific Django apps like Django Rest Framework (DRF).
 """
+
 import functools
 from inspect import getmro
 from inspect import isclass
@@ -121,8 +122,10 @@ def patch_conn(django, conn):
     settings_dict = getattr(conn, "settings_dict", {})
     for tag, attr in DB_CONN_ATTR_BY_TAG.items():
         if attr in settings_dict:
-            tags[tag] = trace_utils._convert_to_string(conn.settings_dict.get(attr))
-
+            try:
+                tags[tag] = trace_utils._convert_to_string(conn.settings_dict.get(attr))
+            except Exception:
+                tags[tag] = str(conn.settings_dict.get(attr))
     conn._datadog_tags = tags
 
     def cursor(django, pin, func, instance, args, kwargs):

ddtrace/contrib/internal/openai/patch.py

@@ -293,7 +293,12 @@ async def patched_endpoint(func, args, kwargs):
             raise
         finally:
             try:
-                g.send((resp, err))
+                if resp is not None:
+                    # openai responses cannot be None
+                    # if resp is None, it is likely because the context
+                    # of the request was cancelled, so we want that to propagate up properly
+                    # see: https://github.com/DataDog/dd-trace-py/issues/10191
+                    g.send((resp, err))
             except StopIteration as e:
                 if err is None:
                     # This return takes priority over `return resp`

ddtrace/internal/core/crashtracking.py

@@ -35,6 +35,7 @@ def start() -> bool:
     crashtracker.set_runtime_id(get_runtime_id())
     crashtracker.set_library_version(version.get_version())
     crashtracker.set_alt_stack(bool(crashtracker_config.alt_stack))
+    crashtracker.set_wait_for_receiver(bool(crashtracker_config.wait_for_receiver))
     if crashtracker_config.stacktrace_resolver == "fast":
         crashtracker.set_resolve_frames_fast()
     elif crashtracker_config.stacktrace_resolver == "full":

ddtrace/internal/datadog/profiling/crashtracker/_crashtracker.pyi

@@ -11,6 +11,7 @@ def set_library_version(profiler_version: StringType) -> None: ...
 def set_stdout_filename(filename: StringType) -> None: ...
 def set_stderr_filename(filename: StringType) -> None: ...
 def set_alt_stack(alt_stack: bool) -> None: ...
+def set_wait_for_receiver(wait: bool) -> None: ...
 def set_resolve_frames_disable() -> None: ...
 def set_resolve_frames_fast() -> None: ...
 def set_resolve_frames_full() -> None: ...

ddtrace/internal/datadog/profiling/crashtracker/_crashtracker.pyx

@@ -28,6 +28,7 @@ cdef extern from "crashtracker_interface.hpp":
     void crashtracker_set_stdout_filename(string_view filename)
     void crashtracker_set_stderr_filename(string_view filename)
     void crashtracker_set_alt_stack(bint alt_stack)
+    void crashtracker_set_wait_for_receiver(bint wait)
     void crashtracker_set_resolve_frames_disable()
     void crashtracker_set_resolve_frames_fast()
     void crashtracker_set_resolve_frames_full()
@@ -98,6 +99,10 @@ def set_alt_stack(alt_stack: bool) -> None:
     crashtracker_set_alt_stack(alt_stack)
 
 
+def set_wait_for_receiver(wait: bool) -> None:
+    crashtracker_set_wait_for_receiver(wait)
+
+
 def set_resolve_frames_disable() -> None:
     crashtracker_set_resolve_frames_disable()
 

ddtrace/internal/datadog/profiling/dd_wrapper/include/crashtracker.hpp

@@ -32,6 +32,7 @@ class Crashtracker
 {
   private:
     bool create_alt_stack = false;
+    bool wait_for_receiver = true;
     std::optional<std::string> stderr_filename{ std::nullopt };
     std::optional<std::string> stdout_filename{ std::nullopt };
     std::string path_to_receiver_binary;
@@ -71,6 +72,7 @@ class Crashtracker
     void set_library_version(std::string_view _library_version);
     void set_url(std::string_view _url);
     void set_tag(std::string_view _key, std::string_view _value);
+    void set_wait_for_receiver(bool _wait);
 
     void set_create_alt_stack(bool _create_alt_stack);
     void set_stderr_filename(std::string_view _stderr_filename);

ddtrace/internal/datadog/profiling/dd_wrapper/include/crashtracker_interface.hpp

@@ -19,6 +19,7 @@ extern "C"
     void crashtracker_set_stdout_filename(std::string_view filename);
     void crashtracker_set_stderr_filename(std::string_view filename);
     void crashtracker_set_alt_stack(bool alt_stack);
+    void crashtracker_set_wait_for_receiver(bool wait);
     void crashtracker_set_resolve_frames_disable();
     void crashtracker_set_resolve_frames_fast();
     void crashtracker_set_resolve_frames_full();

ddtrace/internal/datadog/profiling/dd_wrapper/src/crashtracker.cpp

@@ -12,6 +12,12 @@ Datadog::Crashtracker::set_create_alt_stack(bool _create_alt_stack)
     create_alt_stack = _create_alt_stack;
 }
 
+void
+Datadog::Crashtracker::set_wait_for_receiver(bool _wait)
+{
+    wait_for_receiver = _wait;
+}
+
 void
 Datadog::Crashtracker::set_env(std::string_view _env)
 {
@@ -122,7 +128,7 @@ Datadog::Crashtracker::get_config()
     config.endpoint = ddog_endpoint_from_url(to_slice(url));
     config.resolve_frames = resolve_frames;
     config.timeout_secs = timeout_secs;
-
+    config.wait_for_receiver = wait_for_receiver;
     return config;
 }
 

ddtrace/internal/datadog/profiling/dd_wrapper/src/crashtracker_interface.cpp

@@ -82,6 +82,12 @@ crashtracker_set_alt_stack(bool alt_stack) // cppcheck-suppress unusedFunction
     crashtracker.set_create_alt_stack(alt_stack);
 }
 
+void
+crashtracker_set_wait_for_receiver(bool wait) // cppcheck-suppress unusedFunction
+{
+    crashtracker.set_wait_for_receiver(wait);
+}
+
 void
 crashtracker_set_resolve_frames_disable() // cppcheck-suppress unusedFunction
 {

ddtrace/llmobs/__init__.py

@@ -5,6 +5,7 @@
     from ddtrace.llmobs import LLMObs
     LLMObs.enable()
 """
+
 from ._llmobs import LLMObs
 
 

ddtrace/llmobs/_integrations/openai.py

@@ -98,11 +98,13 @@ def _logs_tags(cls, span: Span) -> str:
 
     @classmethod
     def _metrics_tags(cls, span: Span) -> List[str]:
+        model_name = span.get_tag("openai.request.model") or ""
         tags = [
             "version:%s" % (config.version or ""),
             "env:%s" % (config.env or ""),
             "service:%s" % (span.service or ""),
-            "openai.request.model:%s" % (span.get_tag("openai.request.model") or ""),
+            "openai.request.model:%s" % model_name,
+            "model:%s" % model_name,
             "openai.request.endpoint:%s" % (span.get_tag("openai.request.endpoint") or ""),
             "openai.request.method:%s" % (span.get_tag("openai.request.method") or ""),
             "openai.organization.id:%s" % (span.get_tag("openai.organization.id") or ""),

ddtrace/llmobs/_llmobs.py

@@ -11,6 +11,7 @@
 from ddtrace import patch
 from ddtrace.ext import SpanTypes
 from ddtrace.internal import atexit
+from ddtrace.internal import forksafe
 from ddtrace.internal import telemetry
 from ddtrace.internal.compat import ensure_text
 from ddtrace.internal.logger import get_logger
@@ -82,11 +83,22 @@ def __init__(self, tracer=None):
             interval=float(os.getenv("_DD_LLMOBS_WRITER_INTERVAL", 1.0)),
             timeout=float(os.getenv("_DD_LLMOBS_WRITER_TIMEOUT", 5.0)),
         )
+        self._trace_processor = LLMObsTraceProcessor(self._llmobs_span_writer)
+        forksafe.register(self._child_after_fork)
+
+    def _child_after_fork(self):
+        self._llmobs_span_writer = self._llmobs_span_writer.recreate()
+        self._trace_processor._span_writer = self._llmobs_span_writer
+        self.tracer.configure(settings={"FILTERS": [self._trace_processor]})
+        try:
+            self._llmobs_span_writer.start()
+        except ServiceStatusError:
+            log.debug("Error starting LLMObs span writer after fork")
 
     def _start_service(self) -> None:
         tracer_filters = self.tracer._filters
         if not any(isinstance(tracer_filter, LLMObsTraceProcessor) for tracer_filter in tracer_filters):
-            tracer_filters += [LLMObsTraceProcessor(self._llmobs_span_writer)]
+            tracer_filters += [self._trace_processor]
         self.tracer.configure(settings={"FILTERS": tracer_filters})
         try:
             self._llmobs_span_writer.start()
@@ -102,6 +114,7 @@ def _stop_service(self) -> None:
             log.debug("Error stopping LLMObs writers")
 
         try:
+            forksafe.unregister(self._child_after_fork)
             self.tracer.shutdown()
         except Exception:
             log.warning("Failed to shutdown tracer", exc_info=True)

ddtrace/llmobs/_writer.py

@@ -292,6 +292,7 @@ def recreate(self):
         return self.__class__(
             interval=self._interval,
             timeout=self._timeout,
+            is_agentless=config._llmobs_agentless_enabled,
         )
 
 

ddtrace/settings/crashtracker.py

@@ -101,5 +101,13 @@ class CrashtrackingConfig(En):
         "This is generally useful only for dd-trace-py development.",
     )
 
+    wait_for_receiver = En.v(
+        bool,
+        "wait_for_receiver",
+        default=True,
+        help_type="Boolean",
+        help="Whether to wait for the crashtracking receiver",
+    )
+
 
 config = CrashtrackingConfig()

releasenotes/notes/fix-llmobs-forked-writer-257b993bcf131af8.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    LLM Observability: This fix resolves an issue where LLM Observability spans were not being submitted in forked processes,  
+    such as when using ``celery`` or ``gunicorn`` workers. The LLM Observability writer thread now automatically restarts 
+    when a forked process is detected.

releasenotes/notes/fix_django_db_connection_instrumentation-c23defb666caff3c.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    tracer: This fix resolves an issue where django db instrumentation could fail.

releasenotes/notes/openai-async-timeout-errors-f9ccc1adbe4ed14e.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    openai: Fixes a bug where `asyncio.TimeoutError`s were not being propagated correctly from canceled OpenAI API requests.

releasenotes/notes/openai-model-tag-2482b3d5b2905db9.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    openai: This introduces 'model' tag for openai integration metrics for consistency with the OpenAI SaaS Integration. It has the same value as `openai.request.model`.