feat(llmobs): add error stack to LLMObs span event (#8634)

This PR adds an `error.stack` field to the span events struct submitted
to LLMObs.

## Checklist

- [x] Change(s) are motivated and described in the PR description
- [x] Testing strategy is described if automated tests are not included
in the PR
- [x] Risks are described (performance impact, potential for breakage,
maintainability)
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed or label `changelog/no-changelog` is set
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/))
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))
- [x] If this PR changes the public interface, I've notified
`@DataDog/apm-tees`.
- [x] If change touches code that signs or publishes builds or packages,
or handles credentials of any kind, I've requested a review from
`@DataDog/security-design-and-guidance`.

## Reviewer Checklist

- [x] Title is accurate
- [x] All changes are related to the pull request's stated goal
- [x] Description motivates each change
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- [x] Testing strategy adequately addresses listed risks
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] Release note makes sense to a user of the library
- [x] Author has acknowledged and discussed the performance implications
of this PR as reported in the benchmarks PR comment
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

ddtrace/llmobs/_llmobs.py

@@ -11,6 +11,7 @@
 from ddtrace import config
 from ddtrace._trace.processor import TraceProcessor
 from ddtrace.constants import ERROR_MSG
+from ddtrace.constants import ERROR_STACK
 from ddtrace.constants import ERROR_TYPE
 from ddtrace.ext import SpanTypes
 from ddtrace.internal import atexit
@@ -360,6 +361,7 @@ def _llmobs_span_event(self, span: Span) -> Dict[str, Any]:
             meta["output"]["value"] = span._meta.pop(OUTPUT_VALUE)
         if span.error:
             meta["error.message"] = span.get_tag(ERROR_MSG)
+            meta["error.stack"] = span.get_tag(ERROR_STACK)
         if not meta["input"]:
             meta.pop("input")
         if not meta["output"]:

tests/contrib/botocore/test_bedrock.py

@@ -578,6 +578,7 @@ def test_llmobs_error(self, ddtrace_global_config, bedrock_client, mock_llmobs_w
                     output_messages=[{"content": ""}],
                     error=span.get_tag("error.type"),
                     error_message=span.get_tag("error.message"),
+                    error_stack=span.get_tag("error.stack"),
                     tags={"service": "aws.bedrock-runtime", "ml_app": "aws.bedrock-runtime"},
                 )
             ),

tests/contrib/openai/test_openai_v0.py

@@ -2436,6 +2436,7 @@ def test_llmobs_completion_error(openai_vcr, openai, ddtrace_global_config, mock
                     token_metrics={},
                     error="openai.error.AuthenticationError",
                     error_message="Incorrect API key provided: <not-a-r****key>. You can find your API key at https://platform.openai.com/account/api-keys.",  # noqa: E501
+                    error_stack=span.get_tag("error.stack"),
                 )
             ),
         ]
@@ -2479,6 +2480,7 @@ def test_llmobs_chat_completion_error(openai_vcr, openai, ddtrace_global_config,
                     token_metrics={},
                     error="openai.error.AuthenticationError",
                     error_message="Incorrect API key provided: <not-a-r****key>. You can find your API key at https://platform.openai.com/account/api-keys.",  # noqa: E501
+                    error_stack=span.get_tag("error.stack"),
                 )
             ),
         ]

tests/contrib/openai/test_openai_v1.py

@@ -2085,6 +2085,7 @@ def test_llmobs_completion_error(openai_vcr, openai, ddtrace_global_config, mock
                     token_metrics={},
                     error="openai.AuthenticationError",
                     error_message="Error code: 401 - {'error': {'message': 'Incorrect API key provided: <not-a-r****key>. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}}",  # noqa: E501
+                    error_stack=span.get_tag("error.stack"),
                 )
             ),
         ]
@@ -2127,6 +2128,7 @@ def test_llmobs_chat_completion_error(openai_vcr, openai, ddtrace_global_config,
                     token_metrics={},
                     error="openai.AuthenticationError",
                     error_message="Error code: 401 - {'error': {'message': 'Incorrect API key provided: <not-a-r****key>. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}}",  # noqa: E501
+                    error_stack=span.get_tag("error.stack"),
                 )
             ),
         ]

tests/llmobs/_utils.py

@@ -52,6 +52,7 @@ def _expected_llmobs_llm_span_event(
     session_id=None,
     error=None,
     error_message=None,
+    error_stack=None,
 ):
     """
     Helper function to create an expected LLM span event.
@@ -66,8 +67,9 @@ def _expected_llmobs_llm_span_event(
     session_id: session ID
     error: error type
     error_message: error message
+    error_stack: error stack
     """
-    span_event = _llmobs_base_span_event(span, span_kind, tags, session_id, error, error_message)
+    span_event = _llmobs_base_span_event(span, span_kind, tags, session_id, error, error_message, error_stack)
     meta_dict = {"input": {}, "output": {}}
     if input_messages is not None:
         meta_dict["input"].update({"messages": input_messages})
@@ -100,6 +102,7 @@ def _expected_llmobs_non_llm_span_event(
     session_id=None,
     error=None,
     error_message=None,
+    error_stack=None,
 ):
     """
     Helper function to create an expected span event of type (workflow, task, tool).
@@ -112,8 +115,9 @@ def _expected_llmobs_non_llm_span_event(
     session_id: session ID
     error: error type
     error_message: error message
+    error_stack: error stack
     """
-    span_event = _llmobs_base_span_event(span, span_kind, tags, session_id, error, error_message)
+    span_event = _llmobs_base_span_event(span, span_kind, tags, session_id, error, error_message, error_stack)
     meta_dict = {"input": {}, "output": {}}
     if input_value is not None:
         meta_dict["input"].update({"value": input_value})
@@ -138,6 +142,7 @@ def _llmobs_base_span_event(
     session_id=None,
     error=None,
     error_message=None,
+    error_stack=None,
 ):
     span_event = {
         "span_id": str(span.span_id),
@@ -154,4 +159,5 @@ def _llmobs_base_span_event(
     }
     if error:
         span_event["meta"]["error.message"] = error_message
+        span_event["meta"]["error.stack"] = error_stack
     return span_event

tests/llmobs/test_llmobs_decorators.py

@@ -176,8 +176,9 @@ def f():
             model_name="test_model",
             model_provider="test_provider",
             session_id="test_session_id",
-            error="builtins.ValueError",
-            error_message="test_error",
+            error=span.get_tag("error.type"),
+            error_message=span.get_tag("error.message"),
+            error_stack=span.get_tag("error.stack"),
         )
     )
 
@@ -197,8 +198,9 @@ def f():
                 span,
                 decorator_name,
                 session_id="test_session_id",
-                error="builtins.ValueError",
-                error_message="test_error",
+                error=span.get_tag("error.type"),
+                error_message=span.get_tag("error.message"),
+                error_stack=span.get_tag("error.stack"),
             )
         )
 

tests/llmobs/test_llmobs_service.py

@@ -317,6 +317,7 @@ def test_llmobs_span_error_sets_error(LLMObs, mock_llmobs_writer):
             model_provider="test_model_provider",
             error="builtins.ValueError",
             error_message="test error message",
+            error_stack=span.get_tag("error.stack"),
         )
     )