diff --git a/ddtrace/_trace/tracer.py b/ddtrace/_trace/tracer.py index 4f359a42d93..7259cdf16a0 100644 --- a/ddtrace/_trace/tracer.py +++ b/ddtrace/_trace/tracer.py @@ -230,7 +230,8 @@ def __init__( self.enabled = config._tracing_enabled self.context_provider = context_provider or DefaultContextProvider() - self._user_sampler: Optional[BaseSampler] = None + # _user_sampler is the backup in case we need to revert from remote config to local + self._user_sampler: Optional[BaseSampler] = DatadogSampler() self._sampler: BaseSampler = DatadogSampler() self._dogstatsd_url = agent.get_stats_url() if dogstatsd_url is None else dogstatsd_url self._compute_stats = config._trace_compute_stats @@ -286,7 +287,7 @@ def __init__( self._shutdown_lock = RLock() self._new_process = False - config._subscribe(["_trace_sample_rate"], self._on_global_config_update) + config._subscribe(["_trace_sample_rate", "_trace_sampling_rules"], self._on_global_config_update) config._subscribe(["logs_injection"], self._on_global_config_update) config._subscribe(["tags"], self._on_global_config_update) config._subscribe(["_tracing_enabled"], self._on_global_config_update) @@ -1125,19 +1126,10 @@ def _is_span_internal(span): def _on_global_config_update(self, cfg, items): # type: (Config, List) -> None - if "_trace_sample_rate" in items: - # Reset the user sampler if one exists - if cfg._get_source("_trace_sample_rate") != "remote_config" and self._user_sampler: - self._sampler = self._user_sampler - return - - if cfg._get_source("_trace_sample_rate") != "default": - sample_rate = cfg._trace_sample_rate - else: - sample_rate = None - sampler = DatadogSampler(default_sample_rate=sample_rate) - self._sampler = sampler + # sampling configs always come as a pair + if "_trace_sample_rate" in items and "_trace_sampling_rules" in items: + self._handle_sampler_update(cfg) if "tags" in items: self._tags = cfg.tags.copy() @@ -1160,3 +1152,42 @@ def _on_global_config_update(self, cfg, items): from ddtrace.contrib.logging import unpatch unpatch() + + def _handle_sampler_update(self, cfg): + # type: (Config) -> None + if ( + cfg._get_source("_trace_sample_rate") != "remote_config" + and cfg._get_source("_trace_sampling_rules") != "remote_config" + and self._user_sampler + ): + # if we get empty configs from rc for both sample rate and rules, we should revert to the user sampler + self.sampler = self._user_sampler + return + + if cfg._get_source("_trace_sample_rate") != "remote_config" and self._user_sampler: + try: + sample_rate = self._user_sampler.default_sample_rate # type: ignore[attr-defined] + except AttributeError: + log.debug("Custom non-DatadogSampler is being used, cannot pull default sample rate") + sample_rate = None + elif cfg._get_source("_trace_sample_rate") != "default": + sample_rate = cfg._trace_sample_rate + else: + sample_rate = None + + if cfg._get_source("_trace_sampling_rules") != "remote_config" and self._user_sampler: + try: + sampling_rules = self._user_sampler.rules # type: ignore[attr-defined] + # we need to chop off the default_sample_rate rule so the new sample_rate can be applied + sampling_rules = sampling_rules[:-1] + except AttributeError: + log.debug("Custom non-DatadogSampler is being used, cannot pull sampling rules") + sampling_rules = None + elif cfg._get_source("_trace_sampling_rules") != "default": + sampling_rules = DatadogSampler._parse_rules_from_str(cfg._trace_sampling_rules) + else: + sampling_rules = None + + sampler = DatadogSampler(rules=sampling_rules, default_sample_rate=sample_rate) + + self._sampler = sampler diff --git a/ddtrace/internal/constants.py b/ddtrace/internal/constants.py index 566bec75dad..50b8e1280e4 100644 --- a/ddtrace/internal/constants.py +++ b/ddtrace/internal/constants.py @@ -90,7 +90,9 @@ class _PRIORITY_CATEGORY: USER = "user" - RULE = "rule" + RULE_DEF = "rule_default" + RULE_CUSTOMER = "rule_customer" + RULE_DYNAMIC = "rule_dynamic" AUTO = "auto" DEFAULT = "default" @@ -99,7 +101,9 @@ class _PRIORITY_CATEGORY: # used to simplify code that selects sampling priority based on many factors _CATEGORY_TO_PRIORITIES = { _PRIORITY_CATEGORY.USER: (USER_KEEP, USER_REJECT), - _PRIORITY_CATEGORY.RULE: (USER_KEEP, USER_REJECT), + _PRIORITY_CATEGORY.RULE_DEF: (USER_KEEP, USER_REJECT), + _PRIORITY_CATEGORY.RULE_CUSTOMER: (USER_KEEP, USER_REJECT), + _PRIORITY_CATEGORY.RULE_DYNAMIC: (USER_KEEP, USER_REJECT), _PRIORITY_CATEGORY.AUTO: (AUTO_KEEP, AUTO_REJECT), _PRIORITY_CATEGORY.DEFAULT: (AUTO_KEEP, AUTO_REJECT), } diff --git a/ddtrace/internal/remoteconfig/client.py b/ddtrace/internal/remoteconfig/client.py index d21081c1d94..c2768e57bc6 100644 --- a/ddtrace/internal/remoteconfig/client.py +++ b/ddtrace/internal/remoteconfig/client.py @@ -75,6 +75,7 @@ class Capabilities(enum.IntFlag): APM_TRACING_HTTP_HEADER_TAGS = 1 << 14 APM_TRACING_CUSTOM_TAGS = 1 << 15 APM_TRACING_ENABLED = 1 << 19 + APM_TRACING_SAMPLE_RULES = 1 << 29 class RemoteConfigError(Exception): @@ -382,6 +383,7 @@ def _build_payload(self, state): | Capabilities.APM_TRACING_HTTP_HEADER_TAGS | Capabilities.APM_TRACING_CUSTOM_TAGS | Capabilities.APM_TRACING_ENABLED + | Capabilities.APM_TRACING_SAMPLE_RULES ) return dict( client=dict( diff --git a/ddtrace/internal/sampling.py b/ddtrace/internal/sampling.py index 0d5aa1a2784..267c575e8a5 100644 --- a/ddtrace/internal/sampling.py +++ b/ddtrace/internal/sampling.py @@ -62,6 +62,16 @@ class SamplingMechanism(object): REMOTE_RATE_USER = 6 REMOTE_RATE_DATADOG = 7 SPAN_SAMPLING_RULE = 8 + REMOTE_USER_RULE = 11 + REMOTE_DYNAMIC_RULE = 12 + + +class PriorityCategory(object): + DEFAULT = "default" + AUTO = "auto" + RULE_DEFAULT = "rule_default" + RULE_CUSTOMER = "rule_customer" + RULE_DYNAMIC = "rule_dynamic" # Use regex to validate trace tag value @@ -278,11 +288,17 @@ def is_single_span_sampled(span): def _set_sampling_tags(span, sampled, sample_rate, priority_category): # type: (Span, bool, float, str) -> None mechanism = SamplingMechanism.TRACE_SAMPLING_RULE - if priority_category == "rule": + if priority_category == PriorityCategory.RULE_DEFAULT: + span.set_metric(SAMPLING_RULE_DECISION, sample_rate) + if priority_category == PriorityCategory.RULE_CUSTOMER: + span.set_metric(SAMPLING_RULE_DECISION, sample_rate) + mechanism = SamplingMechanism.REMOTE_USER_RULE + if priority_category == PriorityCategory.RULE_DYNAMIC: span.set_metric(SAMPLING_RULE_DECISION, sample_rate) - elif priority_category == "default": + mechanism = SamplingMechanism.REMOTE_DYNAMIC_RULE + elif priority_category == PriorityCategory.DEFAULT: mechanism = SamplingMechanism.DEFAULT - elif priority_category == "auto": + elif priority_category == PriorityCategory.AUTO: mechanism = SamplingMechanism.AGENT_RATE span.set_metric(SAMPLING_AGENT_DECISION, sample_rate) priorities = _CATEGORY_TO_PRIORITIES[priority_category] diff --git a/ddtrace/internal/telemetry/writer.py b/ddtrace/internal/telemetry/writer.py index 06cea670c39..836daa5da74 100644 --- a/ddtrace/internal/telemetry/writer.py +++ b/ddtrace/internal/telemetry/writer.py @@ -83,7 +83,6 @@ from .constants import TELEMETRY_TRACE_PEER_SERVICE_MAPPING from .constants import TELEMETRY_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED from .constants import TELEMETRY_TRACE_SAMPLING_LIMIT -from .constants import TELEMETRY_TRACE_SAMPLING_RULES from .constants import TELEMETRY_TRACE_SPAN_ATTRIBUTE_SCHEMA from .constants import TELEMETRY_TRACE_WRITER_BUFFER_SIZE_BYTES from .constants import TELEMETRY_TRACE_WRITER_INTERVAL_SECONDS @@ -386,6 +385,9 @@ def _telemetry_entry(self, cfg_name: str) -> Tuple[str, str, _ConfigSource]: elif cfg_name == "_trace_sample_rate": name = "trace_sample_rate" value = str(item.value()) + elif cfg_name == "_trace_sampling_rules": + name = "trace_sampling_rules" + value = str(item.value()) elif cfg_name == "logs_injection": name = "logs_injection_enabled" value = "true" if item.value() else "false" @@ -428,6 +430,7 @@ def _app_started_event(self, register_app_shutdown=True): self._telemetry_entry("_sca_enabled"), self._telemetry_entry("_dsm_enabled"), self._telemetry_entry("_trace_sample_rate"), + self._telemetry_entry("_trace_sampling_rules"), self._telemetry_entry("logs_injection"), self._telemetry_entry("trace_http_header_tags"), self._telemetry_entry("tags"), @@ -462,7 +465,6 @@ def _app_started_event(self, register_app_shutdown=True): (TELEMETRY_TRACE_SAMPLING_LIMIT, config._trace_rate_limit, "unknown"), (TELEMETRY_SPAN_SAMPLING_RULES, config._sampling_rules, "unknown"), (TELEMETRY_SPAN_SAMPLING_RULES_FILE, config._sampling_rules_file, "unknown"), - (TELEMETRY_TRACE_SAMPLING_RULES, config._trace_sampling_rules, "unknown"), (TELEMETRY_PRIORITY_SAMPLING, config._priority_sampling, "unknown"), (TELEMETRY_PARTIAL_FLUSH_ENABLED, config._partial_flush_enabled, "unknown"), (TELEMETRY_PARTIAL_FLUSH_MIN_SPANS, config._partial_flush_min_spans, "unknown"), diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index 69cc58c73d7..fe558c1f426 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -23,6 +23,8 @@ from .settings import _config as ddconfig +PROVENANCE_ORDER = ["customer", "dynamic", "default"] + try: from json.decoder import JSONDecodeError except ImportError: @@ -158,7 +160,7 @@ def _choose_priority_category(self, sampler): elif isinstance(sampler, _AgentRateSampler): return _PRIORITY_CATEGORY.AUTO else: - return _PRIORITY_CATEGORY.RULE + return _PRIORITY_CATEGORY.RULE_DEF def _make_sampling_decision(self, span): # type: (Span) -> Tuple[bool, BaseSampler] @@ -204,7 +206,7 @@ class DatadogSampler(RateByServiceSampler): per second. """ - __slots__ = ("limiter", "rules") + __slots__ = ("limiter", "rules", "default_sample_rate") NO_RATE_LIMIT = -1 # deprecate and remove the DEFAULT_RATE_LIMIT field from DatadogSampler @@ -228,7 +230,7 @@ def __init__( """ # Use default sample rate of 1.0 super(DatadogSampler, self).__init__() - + self.default_sample_rate = default_sample_rate if default_sample_rate is None: if ddconfig._get_source("_trace_sample_rate") != "default": default_sample_rate = float(ddconfig._trace_sample_rate) @@ -239,7 +241,7 @@ def __init__( if rules is None: env_sampling_rules = ddconfig._trace_sampling_rules if env_sampling_rules: - rules = self._parse_rules_from_env_variable(env_sampling_rules) + rules = self._parse_rules_from_str(env_sampling_rules) else: rules = [] self.rules = rules @@ -268,7 +270,8 @@ def __str__(self): __repr__ = __str__ - def _parse_rules_from_env_variable(self, rules): + @staticmethod + def _parse_rules_from_str(rules): # type: (str) -> List[SamplingRule] sampling_rules = [] try: @@ -283,13 +286,22 @@ def _parse_rules_from_env_variable(self, rules): name = rule.get("name", SamplingRule.NO_RULE) resource = rule.get("resource", SamplingRule.NO_RULE) tags = rule.get("tags", SamplingRule.NO_RULE) + provenance = rule.get("provenance", "default") try: sampling_rule = SamplingRule( - sample_rate=sample_rate, service=service, name=name, resource=resource, tags=tags + sample_rate=sample_rate, + service=service, + name=name, + resource=resource, + tags=tags, + provenance=provenance, ) except ValueError as e: raise ValueError("Error creating sampling rule {}: {}".format(json.dumps(rule), e)) sampling_rules.append(sampling_rule) + + # Sort the sampling_rules list using a lambda function as the key + sampling_rules = sorted(sampling_rules, key=lambda rule: PROVENANCE_ORDER.index(rule.provenance)) return sampling_rules def sample(self, span): @@ -320,7 +332,13 @@ def sample(self, span): def _choose_priority_category_with_rule(self, rule, sampler): # type: (Optional[SamplingRule], BaseSampler) -> str if rule: - return _PRIORITY_CATEGORY.RULE + provenance = rule.provenance + if provenance == "customer": + return _PRIORITY_CATEGORY.RULE_CUSTOMER + if provenance == "dynamic": + return _PRIORITY_CATEGORY.RULE_DYNAMIC + return _PRIORITY_CATEGORY.RULE_DEF + if self.limiter._has_been_configured: return _PRIORITY_CATEGORY.USER return super(DatadogSampler, self)._choose_priority_category(sampler) diff --git a/ddtrace/sampling_rule.py b/ddtrace/sampling_rule.py index aecf03de5ab..72ab1574277 100644 --- a/ddtrace/sampling_rule.py +++ b/ddtrace/sampling_rule.py @@ -34,6 +34,7 @@ def __init__( name=NO_RULE, # type: Any resource=NO_RULE, # type: Any tags=NO_RULE, # type: Any + provenance="default", # type: str ): # type: (...) -> None """ @@ -83,6 +84,7 @@ def __init__( self.service = self.choose_matcher(service) self.name = self.choose_matcher(name) self.resource = self.choose_matcher(resource) + self.provenance = provenance @property def sample_rate(self): @@ -236,13 +238,14 @@ def choose_matcher(self, prop): return GlobMatcher(prop) if prop != SamplingRule.NO_RULE else SamplingRule.NO_RULE def __repr__(self): - return "{}(sample_rate={!r}, service={!r}, name={!r}, resource={!r}, tags={!r})".format( + return "{}(sample_rate={!r}, service={!r}, name={!r}, resource={!r}, tags={!r}, provenance={!r})".format( self.__class__.__name__, self.sample_rate, self._no_rule_or_self(self.service), self._no_rule_or_self(self.name), self._no_rule_or_self(self.resource), self._no_rule_or_self(self.tags), + self.provenance, ) __str__ = __repr__ diff --git a/ddtrace/settings/config.py b/ddtrace/settings/config.py index c49abc83bae..fc0083d6222 100644 --- a/ddtrace/settings/config.py +++ b/ddtrace/settings/config.py @@ -1,4 +1,5 @@ from copy import deepcopy +import json import os import re import sys @@ -282,6 +283,11 @@ def _default_config(): default=1.0, envs=[("DD_TRACE_SAMPLE_RATE", float)], ), + "_trace_sampling_rules": _ConfigItem( + name="trace_sampling_rules", + default=lambda: "", + envs=[("DD_TRACE_SAMPLING_RULES", str)], + ), "logs_injection": _ConfigItem( name="logs_injection", default=False, @@ -384,7 +390,6 @@ def __init__(self): self._startup_logs_enabled = asbool(os.getenv("DD_TRACE_STARTUP_LOGS", False)) self._trace_rate_limit = int(os.getenv("DD_TRACE_RATE_LIMIT", default=DEFAULT_SAMPLING_RATE_LIMIT)) - self._trace_sampling_rules = os.getenv("DD_TRACE_SAMPLING_RULES") self._partial_flush_enabled = asbool(os.getenv("DD_TRACE_PARTIAL_FLUSH_ENABLED", default=True)) self._partial_flush_min_spans = int(os.getenv("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", default=300)) self._priority_sampling = asbool(os.getenv("DD_PRIORITY_SAMPLING", default=True)) @@ -562,7 +567,6 @@ def __init__(self): def __getattr__(self, name) -> Any: if name in self._config: return self._config[name].value() - if name not in self._integration_configs: self._integration_configs[name] = IntegrationConfig(self, name) @@ -753,6 +757,14 @@ def _handle_remoteconfig(self, data, test_tracer=None): if "tracing_sampling_rate" in lib_config: base_rc_config["_trace_sample_rate"] = lib_config["tracing_sampling_rate"] + if "tracing_sampling_rules" in lib_config: + trace_sampling_rules = lib_config["tracing_sampling_rules"] + if trace_sampling_rules: + # returns None if no rules + trace_sampling_rules = self.convert_rc_trace_sampling_rules(trace_sampling_rules) + if trace_sampling_rules: + base_rc_config["_trace_sampling_rules"] = trace_sampling_rules + if "log_injection_enabled" in lib_config: base_rc_config["logs_injection"] = lib_config["log_injection_enabled"] @@ -802,3 +814,60 @@ def enable_remote_configuration(self): remoteconfig_poller.register("APM_TRACING", remoteconfig_pubsub) remoteconfig_poller.register("AGENT_CONFIG", remoteconfig_pubsub) remoteconfig_poller.register("AGENT_TASK", remoteconfig_pubsub) + + def _remove_invalid_rules(self, rc_rules: List) -> List: + """Remove invalid sampling rules from the given list""" + # loop through list of dictionaries, if a dictionary doesn't have certain attributes, remove it + for rule in rc_rules: + if ( + ("service" not in rule and "name" not in rule and "resource" not in rule and "tags" not in rule) + or "sample_rate" not in rule + or "provenance" not in rule + ): + log.debug("Invalid sampling rule from remoteconfig found, rule will be removed: %s", rule) + rc_rules.remove(rule) + + return rc_rules + + def _tags_to_dict(self, tags: List[Dict]): + """ + Converts a list of tag dictionaries to a single dictionary. + """ + if isinstance(tags, list): + return {tag["key"]: tag["value_glob"] for tag in tags} + return tags + + def convert_rc_trace_sampling_rules(self, rc_rules: List[Dict[str, Any]]) -> Optional[str]: + """Example of an incoming rule: + [ + { + "service": "my-service", + "name": "web.request", + "resource": "*", + "provenance": "customer", + "sample_rate": 1.0, + "tags": [ + { + "key": "care_about", + "value_glob": "yes" + }, + { + "key": "region", + "value_glob": "us-*" + } + ] + } + ] + + Example of a converted rule: + '[{"sample_rate":1.0,"service":"my-service","resource":"*","name":"web.request","tags":{"care_about":"yes","region":"us-*"},provenance":"customer"}]' + """ + rc_rules = self._remove_invalid_rules(rc_rules) + for rule in rc_rules: + tags = rule.get("tags") + if tags: + rule["tags"] = self._tags_to_dict(tags) + if rc_rules: + return json.dumps(rc_rules) + else: + return None diff --git a/tests/integration/test_debug.py b/tests/integration/test_debug.py index 0d486355c26..cf5520dcb7c 100644 --- a/tests/integration/test_debug.py +++ b/tests/integration/test_debug.py @@ -336,7 +336,8 @@ def test_startup_logs_sampling_rules(): f = debug.collect(tracer) assert f.get("sampler_rules") == [ - "SamplingRule(sample_rate=1.0, service='NO_RULE', name='NO_RULE', resource='NO_RULE', tags='NO_RULE')" + "SamplingRule(sample_rate=1.0, service='NO_RULE', name='NO_RULE', resource='NO_RULE'," + " tags='NO_RULE', provenance='default')" ] sampler = ddtrace.sampler.DatadogSampler( @@ -346,7 +347,8 @@ def test_startup_logs_sampling_rules(): f = debug.collect(tracer) assert f.get("sampler_rules") == [ - "SamplingRule(sample_rate=1.0, service='xyz', name='abc', resource='NO_RULE', tags='NO_RULE')" + "SamplingRule(sample_rate=1.0, service='xyz', name='abc', resource='NO_RULE'," + " tags='NO_RULE', provenance='default')" ] diff --git a/tests/internal/remoteconfig/test_remoteconfig.py b/tests/internal/remoteconfig/test_remoteconfig.py index deaa2790bde..feb83b775d6 100644 --- a/tests/internal/remoteconfig/test_remoteconfig.py +++ b/tests/internal/remoteconfig/test_remoteconfig.py @@ -10,6 +10,7 @@ from mock.mock import ANY import pytest +from ddtrace import config from ddtrace.internal.remoteconfig._connectors import PublisherSubscriberConnector from ddtrace.internal.remoteconfig._publishers import RemoteConfigPublisherMergeDicts from ddtrace.internal.remoteconfig._pubsub import PubSub @@ -20,6 +21,8 @@ from ddtrace.internal.remoteconfig.worker import RemoteConfigPoller from ddtrace.internal.remoteconfig.worker import remoteconfig_poller from ddtrace.internal.service import ServiceStatus +from ddtrace.sampler import DatadogSampler +from ddtrace.sampling_rule import SamplingRule from tests.internal.test_utils_version import _assert_and_get_version_agent_format from tests.utils import override_global_config @@ -428,3 +431,161 @@ def test_rc_default_products_registered(): assert bool(remoteconfig_poller._client._products.get("APM_TRACING")) == rc_enabled assert bool(remoteconfig_poller._client._products.get("AGENT_CONFIG")) == rc_enabled assert bool(remoteconfig_poller._client._products.get("AGENT_TASK")) == rc_enabled + + +@pytest.mark.parametrize( + "rc_rules,expected_config_rules,expected_sampling_rules", + [ + ( + [ # Test with all fields + { + "service": "my-service", + "name": "web.request", + "resource": "*", + "provenance": "dynamic", + "sample_rate": 1.0, + "tags": [{"key": "care_about", "value_glob": "yes"}, {"key": "region", "value_glob": "us-*"}], + } + ], + '[{"service": "my-service", "name": "web.request", "resource": "*", "provenance": "dynamic",' + ' "sample_rate": 1.0, "tags": {"care_about": "yes", "region": "us-*"}}]', + [ + SamplingRule( + sample_rate=1.0, + service="my-service", + name="web.request", + resource="*", + tags={"care_about": "yes", "region": "us-*"}, + provenance="dynamic", + ) + ], + ), + ( # Test with no service + [ + { + "name": "web.request", + "resource": "*", + "provenance": "customer", + "sample_rate": 1.0, + "tags": [{"key": "care_about", "value_glob": "yes"}, {"key": "region", "value_glob": "us-*"}], + } + ], + '[{"name": "web.request", "resource": "*", "provenance": "customer", "sample_rate": 1.0, "tags": ' + '{"care_about": "yes", "region": "us-*"}}]', + [ + SamplingRule( + sample_rate=1.0, + service=SamplingRule.NO_RULE, + name="web.request", + resource="*", + tags={"care_about": "yes", "region": "us-*"}, + provenance="customer", + ) + ], + ), + ( + # Test with no tags + [ + { + "service": "my-service", + "name": "web.request", + "resource": "*", + "provenance": "customer", + "sample_rate": 1.0, + } + ], + '[{"service": "my-service", "name": "web.request", "resource": "*", "provenance": ' + '"customer", "sample_rate": 1.0}]', + [ + SamplingRule( + sample_rate=1.0, + service="my-service", + name="web.request", + resource="*", + tags=SamplingRule.NO_RULE, + provenance="customer", + ) + ], + ), + ( + # Test with no resource + [ + { + "service": "my-service", + "name": "web.request", + "provenance": "customer", + "sample_rate": 1.0, + "tags": [{"key": "care_about", "value_glob": "yes"}, {"key": "region", "value_glob": "us-*"}], + } + ], + '[{"service": "my-service", "name": "web.request", "provenance": "customer", "sample_rate": 1.0, "tags":' + ' {"care_about": "yes", "region": "us-*"}}]', + [ + SamplingRule( + sample_rate=1.0, + service="my-service", + name="web.request", + resource=SamplingRule.NO_RULE, + tags={"care_about": "yes", "region": "us-*"}, + provenance="customer", + ) + ], + ), + ( + # Test with no name + [ + { + "service": "my-service", + "resource": "*", + "provenance": "customer", + "sample_rate": 1.0, + "tags": [{"key": "care_about", "value_glob": "yes"}, {"key": "region", "value_glob": "us-*"}], + } + ], + '[{"service": "my-service", "resource": "*", "provenance": "customer", "sample_rate": 1.0, "tags":' + ' {"care_about": "yes", "region": "us-*"}}]', + [ + SamplingRule( + sample_rate=1.0, + service="my-service", + name=SamplingRule.NO_RULE, + resource="*", + tags={"care_about": "yes", "region": "us-*"}, + provenance="customer", + ) + ], + ), + ( + # Test with no sample rate + [ + { + "service": "my-service", + "name": "web.request", + "resource": "*", + "provenance": "customer", + "tags": [{"key": "care_about", "value_glob": "yes"}, {"key": "region", "value_glob": "us-*"}], + } + ], + None, + None, + ), + ( + # Test with no service, name, resource, tags + [ + { + "provenance": "customer", + "sample_rate": 1.0, + } + ], + None, + None, + ), + ], +) +def test_trace_sampling_rules_conversion(rc_rules, expected_config_rules, expected_sampling_rules): + trace_sampling_rules = config.convert_rc_trace_sampling_rules(rc_rules) + + assert trace_sampling_rules == expected_config_rules + if trace_sampling_rules is not None: + parsed_rules = DatadogSampler._parse_rules_from_str(trace_sampling_rules) + assert parsed_rules == expected_sampling_rules diff --git a/tests/internal/remoteconfig/test_remoteconfig_client_e2e.py b/tests/internal/remoteconfig/test_remoteconfig_client_e2e.py index ad6a9e4436c..760fa4a2e7b 100644 --- a/tests/internal/remoteconfig/test_remoteconfig_client_e2e.py +++ b/tests/internal/remoteconfig/test_remoteconfig_client_e2e.py @@ -18,7 +18,7 @@ def _expected_payload( rc_client, - capabilities="CPAA", # this was gathered by running the test and observing the payload + capabilities="IAjwAA==", # this was gathered by running the test and observing the payload has_errors=False, targets_version=0, backend_client_state=None, diff --git a/tests/internal/test_settings.py b/tests/internal/test_settings.py index faea554f489..4bae14bfef9 100644 --- a/tests/internal/test_settings.py +++ b/tests/internal/test_settings.py @@ -256,6 +256,216 @@ def test_remoteconfig_sampling_rate_user(run_python_code_in_subprocess): assert status == 0, err.decode("utf-8") +def test_remoteconfig_sampling_rules(run_python_code_in_subprocess): + env = os.environ.copy() + env.update({"DD_TRACE_SAMPLING_RULES": '[{"sample_rate":0.1, "name":"test"}]'}) + + out, err, status, _ = run_python_code_in_subprocess( + """ +from ddtrace import config, tracer +from ddtrace.sampler import DatadogSampler +from tests.internal.test_settings import _base_rc_config, _deleted_rc_config + +with tracer.trace("test") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.1 +assert span.get_tag("_dd.p.dm") == "-3" + +config._handle_remoteconfig(_base_rc_config({"tracing_sampling_rules":[ + { + "service": "*", + "name": "test", + "resource": "*", + "provenance": "customer", + "sample_rate": 0.2, + } + ]})) +with tracer.trace("test") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.2 +assert span.get_tag("_dd.p.dm") == "-11" + +config._handle_remoteconfig(_base_rc_config({})) +with tracer.trace("test") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.1 + +custom_sampler = DatadogSampler(DatadogSampler._parse_rules_from_str('[{"sample_rate":0.3, "name":"test"}]')) +tracer.configure(sampler=custom_sampler) +with tracer.trace("test") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.3 +assert span.get_tag("_dd.p.dm") == "-3" + +config._handle_remoteconfig(_base_rc_config({"tracing_sampling_rules":[ + { + "service": "*", + "name": "test", + "resource": "*", + "provenance": "dynamic", + "sample_rate": 0.4, + } + ]})) +with tracer.trace("test") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.4 +assert span.get_tag("_dd.p.dm") == "-12" + +config._handle_remoteconfig(_base_rc_config({})) +with tracer.trace("test") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.3 +assert span.get_tag("_dd.p.dm") == "-3" + +config._handle_remoteconfig(_base_rc_config({"tracing_sampling_rules":[ + { + "service": "ok", + "name": "test", + "resource": "*", + "provenance": "customer", + "sample_rate": 0.4, + } + ]})) +with tracer.trace(service="ok", name="test") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.4 +assert span.get_tag("_dd.p.dm") == "-11" + +config._handle_remoteconfig(_deleted_rc_config()) +with tracer.trace("test") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.3 +assert span.get_tag("_dd.p.dm") == "-3" + + """, + env=env, + ) + assert status == 0, err.decode("utf-8") + + +def test_remoteconfig_sample_rate_and_rules(run_python_code_in_subprocess): + """There is complex logic regarding the interaction between setting new + sample rates and rules with remote config. + """ + env = os.environ.copy() + env.update({"DD_TRACE_SAMPLING_RULES": '[{"sample_rate":0.9, "name":"rules"}]'}) + env.update({"DD_TRACE_SAMPLE_RATE": "0.8"}) + + out, err, status, _ = run_python_code_in_subprocess( + """ +from ddtrace import config, tracer +from ddtrace.sampler import DatadogSampler +from tests.internal.test_settings import _base_rc_config, _deleted_rc_config + +with tracer.trace("rules") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.9 +assert span.get_tag("_dd.p.dm") == "-3" + +with tracer.trace("sample_rate") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.8 +assert span.get_tag("_dd.p.dm") == "-3" + + +config._handle_remoteconfig(_base_rc_config({"tracing_sampling_rules":[ + { + "service": "*", + "name": "rules", + "resource": "*", + "provenance": "customer", + "sample_rate": 0.7, + } + ]})) + +with tracer.trace("rules") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.7 +assert span.get_tag("_dd.p.dm") == "-11" + +with tracer.trace("sample_rate") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.8 +assert span.get_tag("_dd.p.dm") == "-3" + + +config._handle_remoteconfig(_base_rc_config({"tracing_sampling_rate": 0.2})) + +with tracer.trace("sample_rate") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.2 +assert span.get_tag("_dd.p.dm") == "-3" + +with tracer.trace("rules") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.9 +assert span.get_tag("_dd.p.dm") == "-3" + + +config._handle_remoteconfig(_base_rc_config({"tracing_sampling_rate": 0.3})) + +with tracer.trace("sample_rate") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.3 +assert span.get_tag("_dd.p.dm") == "-3" + +with tracer.trace("rules") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.9 +assert span.get_tag("_dd.p.dm") == "-3" + + +config._handle_remoteconfig(_base_rc_config({})) + +with tracer.trace("rules") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.9 +assert span.get_tag("_dd.p.dm") == "-3" + +with tracer.trace("sample_rate") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.8 +assert span.get_tag("_dd.p.dm") == "-3" + + +config._handle_remoteconfig(_base_rc_config({"tracing_sampling_rules":[ + { + "service": "*", + "name": "rules_dynamic", + "resource": "*", + "provenance": "dynamic", + "sample_rate": 0.1, + }, + { + "service": "*", + "name": "rules_customer", + "resource": "*", + "provenance": "customer", + "sample_rate": 0.6, + } + ]})) + +with tracer.trace("rules_dynamic") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.1 +assert span.get_tag("_dd.p.dm") == "-12" + +with tracer.trace("rules_customer") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.6 +assert span.get_tag("_dd.p.dm") == "-11" + +with tracer.trace("sample_rate") as span: + pass +assert span.get_metric("_dd.rule_psr") == 0.8 +assert span.get_tag("_dd.p.dm") == "-3" + + """, + env=env, + ) + assert status == 0, err.decode("utf-8") + + def test_remoteconfig_custom_tags(run_python_code_in_subprocess): env = os.environ.copy() env.update({"DD_TAGS": "team:apm"}) diff --git a/tests/telemetry/test_writer.py b/tests/telemetry/test_writer.py index fbc56869cb6..18699170152 100644 --- a/tests/telemetry/test_writer.py +++ b/tests/telemetry/test_writer.py @@ -128,7 +128,6 @@ def test_app_started_event(telemetry_writer, test_agent_session, mock_time): {"name": "DD_TRACE_PROPAGATION_STYLE_INJECT", "origin": "unknown", "value": "datadog,tracecontext"}, {"name": "DD_TRACE_RATE_LIMIT", "origin": "unknown", "value": 100}, {"name": "DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED", "origin": "unknown", "value": False}, - {"name": "DD_TRACE_SAMPLING_RULES", "origin": "unknown", "value": None}, {"name": "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", "origin": "unknown", "value": "v0"}, {"name": "DD_TRACE_STARTUP_LOGS", "origin": "unknown", "value": False}, {"name": "DD_TRACE_WRITER_BUFFER_SIZE_BYTES", "origin": "unknown", "value": 20 << 20}, @@ -142,6 +141,7 @@ def test_app_started_event(telemetry_writer, test_agent_session, mock_time): {"name": "data_streams_enabled", "origin": "default", "value": "false"}, {"name": "appsec_enabled", "origin": "default", "value": "false"}, {"name": "trace_sample_rate", "origin": "default", "value": "1.0"}, + {"name": "trace_sampling_rules", "origin": "default", "value": ""}, {"name": "trace_header_tags", "origin": "default", "value": ""}, {"name": "logs_injection_enabled", "origin": "default", "value": "false"}, {"name": "trace_tags", "origin": "default", "value": ""}, @@ -292,11 +292,6 @@ def test_app_started_event_configuration_override( {"name": "DD_TRACE_PROPAGATION_STYLE_INJECT", "origin": "unknown", "value": "tracecontext"}, {"name": "DD_TRACE_RATE_LIMIT", "origin": "unknown", "value": 50}, {"name": "DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED", "origin": "unknown", "value": True}, - { - "name": "DD_TRACE_SAMPLING_RULES", - "origin": "unknown", - "value": '[{"sample_rate":1.0,"service":"xyz","name":"abc"}]', - }, {"name": "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", "origin": "unknown", "value": "v1"}, {"name": "DD_TRACE_STARTUP_LOGS", "origin": "unknown", "value": True}, {"name": "DD_TRACE_WRITER_BUFFER_SIZE_BYTES", "origin": "unknown", "value": 1000}, @@ -310,6 +305,11 @@ def test_app_started_event_configuration_override( {"name": "data_streams_enabled", "origin": "env_var", "value": "true"}, {"name": "appsec_enabled", "origin": "env_var", "value": "true"}, {"name": "trace_sample_rate", "origin": "env_var", "value": "0.5"}, + { + "name": "trace_sampling_rules", + "origin": "env_var", + "value": '[{"sample_rate":1.0,"service":"xyz","name":"abc"}]', + }, {"name": "logs_injection_enabled", "origin": "env_var", "value": "true"}, {"name": "trace_header_tags", "origin": "default", "value": ""}, {"name": "trace_tags", "origin": "env_var", "value": "team:apm,component:web"},