chore(telemetry): decouple telemetry writer from global config (#10863)

## Description

Currently, the telemetry writer relies on `ddtrace.config` to enable and
send telemetry payloads. `ddtrace.config` is also used to generate and
queue `app-started` and `app-client-configuration-changed` telemetry
events. This creates a circular dependency that can be painful to
manage. Currently we leverage inlined imports and hope that
`ddtrace.config` is initialized before `ddtrace.internal.telemetry` is
imported. This change resolves this circular dependency by refactoring
components in`ddtrace.internal.telemetry` to access environment
variables directly (instead of using ddtrace.config).

## Motiviation

This change will allow ddtrace contributors to queue telemetry
configurations, metrics, and logs before the global config object is
initialized.

## Changes
- Removes all references to `ddtrace.config` and all envier
configuration objects from `ddtrace.internal.telemetry` package
- Ensures configuration telemetry is queued when configurations are set
(currently this information is collected 10 seconds after ddtrace is
first imported).
- Ensures the origin of a configuration value is recorded (possible
values: remote_config, env_var, code, and unknown). Previously the
origin of all configurations was set to unknown.
- Updates the name of telemetry configurations to match the name of
environment variables (where possible).

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

---------

Co-authored-by: Emmett Butler <[email protected]>

ddtrace/_monkey.py

@@ -79,10 +79,10 @@
     "falcon": True,
     "pyramid": True,
     # Auto-enable logging if the environment variable DD_LOGS_INJECTION is true
-    "logbook": config.logs_injection,
-    "logging": config.logs_injection,
-    "loguru": config.logs_injection,
-    "structlog": config.logs_injection,
+    "logbook": config.logs_injection,  # type: ignore
+    "logging": config.logs_injection,  # type: ignore
+    "loguru": config.logs_injection,  # type: ignore
+    "structlog": config.logs_injection,  # type: ignore
     "pynamodb": True,
     "pyodbc": True,
     "fastapi": True,

ddtrace/bootstrap/sitecustomize.py

@@ -20,6 +20,7 @@
 import sys
 import warnings  # noqa:F401
 
+from ddtrace.internal.telemetry import telemetry_writer
 from ddtrace import config  # noqa:F401
 from ddtrace._logger import _configure_log_injection
 from ddtrace.internal.logger import get_logger  # noqa:F401
@@ -158,7 +159,8 @@ def _(threading):
         else:
             log.debug("additional sitecustomize found in: %s", sys.path)
 
-    config._ddtrace_bootstrapped = True
+    telemetry_writer.add_configuration("ddtrace_bootstrapped", True, "unknown")
+    telemetry_writer.add_configuration("ddtrace_auto_used", "ddtrace.auto" in sys.modules, "unknown")
     # Loading status used in tests to detect if the `sitecustomize` has been
     # properly loaded without exceptions. This must be the last action in the module
     # when the execution ends with a success.

ddtrace/internal/agent.py

@@ -4,8 +4,8 @@
 from typing import TypeVar
 from typing import Union
 
+from ddtrace.internal.constants import DEFAULT_TIMEOUT
 from ddtrace.internal.logger import get_logger
-from ddtrace.settings import _config as ddconfig
 
 from .http import HTTPConnection
 from .http import HTTPSConnection
@@ -44,15 +44,15 @@ def get_trace_url():
 
     Raises a ``ValueError`` if the URL is not supported by the Agent.
     """
-    user_supplied_host = ddconfig._trace_agent_hostname is not None
-    user_supplied_port = ddconfig._trace_agent_port is not None
+    user_supplied_host = os.environ.get("DD_AGENT_HOST", os.environ.get("DD_TRACE_AGENT_HOSTNAME"))
+    user_supplied_port = os.environ.get("DD_AGENT_PORT", os.environ.get("DD_TRACE_AGENT_PORT"))
 
-    url = ddconfig._trace_agent_url
+    url = os.environ.get("DD_TRACE_AGENT_URL")
 
     if not url:
-        if user_supplied_host or user_supplied_port:
-            host = ddconfig._trace_agent_hostname or DEFAULT_HOSTNAME
-            port = ddconfig._trace_agent_port or DEFAULT_TRACE_PORT
+        if user_supplied_host is not None or user_supplied_port is not None:
+            host = user_supplied_host or DEFAULT_HOSTNAME
+            port = user_supplied_port or DEFAULT_TRACE_PORT
             if is_ipv6_hostname(host):
                 host = "[{}]".format(host)
             url = "http://%s:%s" % (host, port)
@@ -66,15 +66,14 @@ def get_trace_url():
 
 def get_stats_url():
     # type: () -> str
-    user_supplied_host = ddconfig._stats_agent_hostname is not None
-    user_supplied_port = ddconfig._stats_agent_port is not None
-
-    url = ddconfig._stats_agent_url
+    user_supplied_host = os.environ.get("DD_AGENT_HOST", os.environ.get("DD_DOGSTATSD_HOST"))
+    user_supplied_port = os.getenv("DD_DOGSTATSD_PORT")
+    url = os.getenv("DD_DOGSTATSD_URL")
 
     if not url:
-        if user_supplied_host or user_supplied_port:
-            port = ddconfig._stats_agent_port or DEFAULT_STATS_PORT
-            host = ddconfig._stats_agent_hostname or DEFAULT_HOSTNAME
+        if user_supplied_host is not None or user_supplied_port is not None:
+            port = user_supplied_port or DEFAULT_STATS_PORT
+            host = user_supplied_host or DEFAULT_HOSTNAME
             if is_ipv6_hostname(host):
                 host = "[{}]".format(host)
             url = "udp://{}:{}".format(host, port)
@@ -87,7 +86,8 @@ def get_stats_url():
 
 def info(url=None):
     agent_url = get_trace_url() if url is None else url
-    _conn = get_connection(agent_url, timeout=ddconfig._agent_timeout_seconds)
+    timeout = float(os.getenv("DD_TRACE_AGENT_TIMEOUT_SECONDS", DEFAULT_TIMEOUT))
+    _conn = get_connection(agent_url, timeout=timeout)
     try:
         _conn.request("GET", "info", headers={"content-type": "application/json"})
         resp = _conn.getresponse()

ddtrace/internal/runtime/runtime_metrics.py

@@ -8,7 +8,6 @@
 from ddtrace.internal import atexit
 from ddtrace.internal import forksafe
 from ddtrace.internal import telemetry
-from ddtrace.internal.telemetry.constants import TELEMETRY_RUNTIMEMETRICS_ENABLED
 from ddtrace.vendor.dogstatsd import DogStatsd
 
 from .. import periodic
@@ -21,6 +20,8 @@
 from .tag_collectors import TracerTagCollector
 
 
+TELEMETRY_RUNTIMEMETRICS_ENABLED = "DD_RUNTIME_METRICS_ENABLED"
+
 log = get_logger(__name__)
 
 

ddtrace/internal/telemetry/constants.py

@@ -21,78 +21,3 @@ class TELEMETRY_APM_PRODUCT(Enum):
     DYNAMIC_INSTRUMENTATION = "dynamic_instrumentation"
     PROFILER = "profiler"
     APPSEC = "appsec"
-
-
-# Configuration names must map to values supported by backend services:
-# https://github.com/DataDog/dd-go/blob/f88e85d64b173e7733ac03e23576d1c9be37f32e/trace/apps/tracer-telemetry-intake/telemetry-payload/static/config_norm_rules.json
-TELEMETRY_DYNAMIC_INSTRUMENTATION_ENABLED = "DD_DYNAMIC_INSTRUMENTATION_ENABLED"
-TELEMETRY_EXCEPTION_REPLAY_ENABLED = "DD_EXCEPTION_REPLAY_ENABLED"
-
-
-# Tracing Features
-
-TELEMETRY_TRACE_DEBUG = "DD_TRACE_DEBUG"
-TELEMETRY_ANALYTICS_ENABLED = "DD_TRACE_ANALYTICS_ENABLED"
-TELEMETRY_STARTUP_LOGS_ENABLED = "DD_TRACE_STARTUP_LOGS"
-TELEMETRY_CLIENT_IP_ENABLED = "DD_TRACE_CLIENT_IP_ENABLED"
-TELEMETRY_128_BIT_TRACEID_GENERATION_ENABLED = "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED"
-TELEMETRY_TRACE_COMPUTE_STATS = "DD_TRACE_COMPUTE_STATS"
-TELEMETRY_OBFUSCATION_QUERY_STRING_PATTERN = "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP"
-TELEMETRY_OTEL_ENABLED = "DD_TRACE_OTEL_ENABLED"
-TELEMETRY_TRACE_HEALTH_METRICS_ENABLED = "DD_TRACE_HEALTH_METRICS_ENABLED"
-TELEMETRY_ENABLED = "DD_INSTRUMENTATION_TELEMETRY_ENABLED"
-TELEMETRY_RUNTIMEMETRICS_ENABLED = "DD_RUNTIME_METRICS_ENABLED"
-TELEMETRY_REMOTE_CONFIGURATION_ENABLED = "DD_REMOTE_CONFIGURATION_ENABLED"
-TELEMETRY_REMOTE_CONFIGURATION_INTERVAL = "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS"
-TELEMETRY_SERVICE_MAPPING = "DD_SERVICE_MAPPING"
-TELEMETRY_SPAN_SAMPLING_RULES = "DD_SPAN_SAMPLING_RULES"
-TELEMETRY_SPAN_SAMPLING_RULES_FILE = "DD_SPAN_SAMPLING_RULES_FILE"
-TELEMETRY_PROPAGATION_STYLE_INJECT = "DD_TRACE_PROPAGATION_STYLE_INJECT"
-TELEMETRY_PROPAGATION_STYLE_EXTRACT = "DD_TRACE_PROPAGATION_STYLE_EXTRACT"
-TELEMETRY_TRACE_SAMPLING_RULES = "DD_TRACE_SAMPLING_RULES"
-TELEMETRY_TRACE_SAMPLING_LIMIT = "DD_TRACE_RATE_LIMIT"
-TELEMETRY_PARTIAL_FLUSH_ENABLED = "DD_TRACE_PARTIAL_FLUSH_ENABLED"
-TELEMETRY_PARTIAL_FLUSH_MIN_SPANS = "DD_TRACE_PARTIAL_FLUSH_MIN_SPANS"
-TELEMETRY_TRACE_SPAN_ATTRIBUTE_SCHEMA = "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA"
-TELEMETRY_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = "DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED"
-TELEMETRY_TRACE_PEER_SERVICE_DEFAULTS_ENABLED = "DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED"
-TELEMETRY_TRACE_PEER_SERVICE_MAPPING = "DD_TRACE_PEER_SERVICE_MAPPING"
-
-TELEMETRY_TRACE_API_VERSION = "DD_TRACE_API_VERSION"
-TELEMETRY_TRACE_WRITER_BUFFER_SIZE_BYTES = "DD_TRACE_WRITER_BUFFER_SIZE_BYTES"
-TELEMETRY_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES = "DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES"
-TELEMETRY_TRACE_WRITER_INTERVAL_SECONDS = "DD_TRACE_WRITER_INTERVAL_SECONDS"
-TELEMETRY_TRACE_WRITER_REUSE_CONNECTIONS = "DD_TRACE_WRITER_REUSE_CONNECTIONS"
-
-TELEMETRY_DOGSTATSD_PORT = "DD_DOGSTATSD_PORT"
-TELEMETRY_DOGSTATSD_URL = "DD_DOGSTATSD_URL"
-
-TELEMETRY_AGENT_HOST = "DD_AGENT_HOST"
-TELEMETRY_AGENT_PORT = "DD_AGENT_PORT"
-TELEMETRY_AGENT_URL = "DD_TRACE_AGENT_URL"
-TELEMETRY_TRACE_AGENT_TIMEOUT_SECONDS = "DD_TRACE_AGENT_TIMEOUT_SECONDS"
-
-# Profiling features
-TELEMETRY_PROFILING_STACK_ENABLED = "DD_PROFILING_STACK_ENABLED"
-TELEMETRY_PROFILING_MEMORY_ENABLED = "DD_PROFILING_MEMORY_ENABLED"
-TELEMETRY_PROFILING_HEAP_ENABLED = "DD_PROFILING_HEAP_ENABLED"
-TELEMETRY_PROFILING_LOCK_ENABLED = "DD_PROFILING_LOCK_ENABLED"
-TELEMETRY_PROFILING_EXPORT_LIBDD_ENABLED = "DD_PROFILING_EXPORT_LIBDD_ENABLED"
-TELEMETRY_PROFILING_CAPTURE_PCT = "DD_PROFILING_CAPTURE_PCT"
-TELEMETRY_PROFILING_UPLOAD_INTERVAL = "DD_PROFILING_UPLOAD_INTERVAL"
-TELEMETRY_PROFILING_MAX_FRAMES = "DD_PROFILING_MAX_FRAMES"
-
-TELEMETRY_INJECT_WAS_ATTEMPTED = "DD_LIB_INJECTION_ATTEMPTED"
-TELEMETRY_LIB_WAS_INJECTED = "DD_LIB_INJECTED"
-TELEMETRY_LIB_INJECTION_FORCED = "DD_INJECT_FORCE"
-
-
-# Crashtracker
-TELEMETRY_CRASHTRACKING_ENABLED = "crashtracking_enabled"  # Env var enabled
-TELEMETRY_CRASHTRACKING_AVAILABLE = "crashtracking_available"  # Feature is available
-TELEMETRY_CRASHTRACKING_STARTED = "crashtracking_started"  # Crashtracking is running
-TELEMETRY_CRASHTRACKING_STDOUT_FILENAME = "crashtracking_stdout_filename"
-TELEMETRY_CRASHTRACKING_STDERR_FILENAME = "crashtracking_stderr_filename"
-TELEMETRY_CRASHTRACKING_ALT_STACK = "crashtracking_alt_stack"
-TELEMETRY_CRASHTRACKING_STACKTRACE_RESOLVER = "crashtracking_stacktrace_resolver"
-TELEMETRY_CRASHTRACKING_DEBUG_URL = "crashtracking_debug_url"

ddtrace/internal/telemetry/data.py

@@ -12,7 +12,6 @@
 from ddtrace.internal.utils.cache import cached
 from ddtrace.version import get_version
 
-from ...settings import _config as config  # noqa:F401
 from ..hostname import get_hostname