From e1a90376f1d9eb7c450e50b171631f2a8f81dbe0 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Thu, 29 Aug 2024 13:17:29 -0400 Subject: [PATCH 01/17] chore(internal): revert improve generate OCI package size (#10409) Revert #10067 which introduces a symlink deduping script to try and cut down on the OCI package size. There is an edge case where this causes resource consumption to sky rocket for our new k8s injection tests. It is better to revert this for now to ensure our tests are stable, and then we can revisit the optimizations. After this revert the image size will still be ok, but should not be forgotten. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .gitlab/prepare-oci-package.sh | 4 --- lib-injection/dedupe.py | 57 ---------------------------------- 2 files changed, 61 deletions(-) delete mode 100644 lib-injection/dedupe.py diff --git a/.gitlab/prepare-oci-package.sh b/.gitlab/prepare-oci-package.sh index 4f8a3ec13f0..5b3380bd4d9 100755 --- a/.gitlab/prepare-oci-package.sh +++ b/.gitlab/prepare-oci-package.sh @@ -30,7 +30,3 @@ cp -r ../pywheels-dep/site-packages* sources/ddtrace_pkgs cp ../lib-injection/sitecustomize.py sources/ cp ../min_compatible_versions.csv sources/ cp ../lib-injection/telemetry-forwarder.sh sources/ - -clean-apt install python3 -echo "Deduplicating package files" -python3 ../lib-injection/dedupe.py sources/ddtrace_pkgs/ diff --git a/lib-injection/dedupe.py b/lib-injection/dedupe.py deleted file mode 100644 index 405344550b4..00000000000 --- a/lib-injection/dedupe.py +++ /dev/null @@ -1,57 +0,0 @@ -import collections -import glob -import os -from pathlib import Path -import shutil -import subprocess -import sys - - -if len(sys.argv) != 2: - print("Usage: python dedupe.py ") - print("Example: python dedupe.py sources/ddtrace_pkgs") - sys.exit(1) - -source_dir = Path(sys.argv[1]) - -# Find all duplicate files -print("Finding duplicate files") -file_hashes = collections.defaultdict(set) -for src in glob.glob(f"{source_dir}/**/*", recursive=True): - if not os.path.isfile(src): - continue - res = subprocess.check_output(["sha256sum", str(src)]) - file_hash, _, _ = res.decode().partition(" ") - file_hashes[file_hash].add(Path(src)) - - -# Replace shared files with soft links -shared_dir = source_dir / "shared" -try: - shutil.rmtree(shared_dir) -except Exception: - pass -os.makedirs(shared_dir) - -for file_hash in file_hashes: - # Skip unique files that aren't duplicates - if len(file_hashes[file_hash]) <= 1: - continue - - # Copy the first file to the shared directory with the name shared/_ - src = next(iter(file_hashes[file_hash])) - basename = os.path.basename(src) - dest = shared_dir / f"{file_hash}_{basename}" - print(f"Copying {src} to {dest}") - shutil.copy(src, dest) - - for src in file_hashes[file_hash]: - # Replace the duplicate file with a symlink to the shared file - dest_rel = dest.relative_to(source_dir) - # Convert the path to the src file with a relative path to the shared file - # e.g. `site-packages-ddtrace-py3.11-manylinux2014/xmltodict.py` -> `../shared/_xmltodict.py` - # DEV: This means we don't need to know the absolute path of where the files end up on disk - src_rel = Path(*([".." for _ in src.relative_to(source_dir).parts[:-1]] + [dest_rel])) - print(f"Replacing {src} with symlink to {src_rel}") - os.remove(src) - os.symlink(src_rel, src) From a94142011d1c989c89720023beb710580c9d4ace Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Thu, 29 Aug 2024 14:01:45 -0400 Subject: [PATCH 02/17] ci: fix flaky tracer tests (#10436) Previous PR fixed flakiness in one part of this test, but there is a remaining bit which is also flaky. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- tests/tracer/test_rate_limiter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tracer/test_rate_limiter.py b/tests/tracer/test_rate_limiter.py index 3d8b9341e29..d66f980cbc3 100644 --- a/tests/tracer/test_rate_limiter.py +++ b/tests/tracer/test_rate_limiter.py @@ -179,8 +179,8 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): assert limiter.current_window_ns == (now_ns + time_window) assert limiter.prev_window_rate == 0.5 - # Gap of 1.9999 seconds, same window - time_ns = now_ns + (1.9999 * time_window) + # Gap of 1.85 seconds, same window + time_ns = now_ns + (1.85 * time_window) with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is False assert limiter.effective_rate == 0.5 From 47335da4a334fcd74ad668d7df76e6c96a6e33fe Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Thu, 29 Aug 2024 18:36:31 -0400 Subject: [PATCH 03/17] ci: fix permissions for codeowners actions (#10437) The job is currently broken. This should fix it. The job isn't required so no impact to CI/ability to merge. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .github/workflows/codeowners.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codeowners.yml b/.github/workflows/codeowners.yml index 92f64888348..3d7419846d6 100644 --- a/.github/workflows/codeowners.yml +++ b/.github/workflows/codeowners.yml @@ -7,6 +7,8 @@ jobs: report_codeowners: name: "Report codeowners" runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - uses: actions/checkout@v4 with: From d80d22d7650ddd1e858128bf3989084af4f7fb50 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Thu, 29 Aug 2024 18:44:46 -0400 Subject: [PATCH 04/17] ci: make pypi test upload actually upload (#10449) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .gitlab/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/release.yml b/.gitlab/release.yml index 91671ee417b..6d3fc1ad8d0 100644 --- a/.gitlab/release.yml +++ b/.gitlab/release.yml @@ -22,7 +22,7 @@ variables: - python -m pip install twine - python -m twine check --strict pywheels/* script: - - echo "python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/*" + - python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/* artifacts: paths: - pywheels/*.whl From 4f1a3dac0d6cde4435af1898eadaecf99304cbe5 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Fri, 30 Aug 2024 11:57:15 +0100 Subject: [PATCH 05/17] chore(di): make correlation IDs optional (#10442) We make the tracer correlation IDs optional on payloads emitted by Dynamic Instrumentation to align with other runtimes. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/debugging/_encoding.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ddtrace/debugging/_encoding.py b/ddtrace/debugging/_encoding.py index 4371f40f324..771452d6423 100644 --- a/ddtrace/debugging/_encoding.py +++ b/ddtrace/debugging/_encoding.py @@ -104,13 +104,18 @@ def _build_log_track_payload( "debugger": {"snapshot": signal.snapshot}, "host": host, "logger": _logs_track_logger_details(signal.thread, signal.frame), - "dd.trace_id": str(context.trace_id) if context else None, - "dd.span_id": str(context.span_id) if context else None, "ddsource": "dd_debugger", "message": signal.message, "timestamp": int(signal.timestamp * 1e3), # milliseconds, } + + # Add the correlation IDs if available + if context is not None: + payload["dd.trace_id"] = str(context.trace_id) + payload["dd.span_id"] = str(context.span_id) + add_tags(payload) + return payload From 5b394f1d0b8ef3b698e481d4f12fab574aa74951 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Fri, 30 Aug 2024 13:07:28 +0100 Subject: [PATCH 06/17] refactor(di): simplify context capturing API (#10443) We clean-up the internal context capturing API after the recent changes that have introduced support for local variables in function probes and the exposure of globals to expression/condition evaluations. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/debugging/_debugger.py | 6 -- ddtrace/debugging/_probe/model.py | 13 ++-- ddtrace/debugging/_signal/metric_sample.py | 8 +- ddtrace/debugging/_signal/model.py | 26 ++++--- ddtrace/debugging/_signal/snapshot.py | 88 ++++++++++------------ ddtrace/debugging/_signal/tracing.py | 16 ++-- tests/debugging/signal/test_collector.py | 2 - tests/debugging/signal/test_model.py | 14 ++-- tests/debugging/test_debugger.py | 6 +- tests/debugging/test_encoding.py | 41 ++++++---- 10 files changed, 111 insertions(+), 109 deletions(-) diff --git a/ddtrace/debugging/_debugger.py b/ddtrace/debugging/_debugger.py index 6c991dc8711..a93d2077cae 100644 --- a/ddtrace/debugging/_debugger.py +++ b/ddtrace/debugging/_debugger.py @@ -45,7 +45,6 @@ from ddtrace.debugging._probe.remoteconfig import ProbePollerEventType from ddtrace.debugging._probe.remoteconfig import ProbeRCAdapter from ddtrace.debugging._probe.status import ProbeStatusLogger -from ddtrace.debugging._safety import get_args from ddtrace.debugging._signal.collector import SignalCollector from ddtrace.debugging._signal.collector import SignalContext from ddtrace.debugging._signal.metric_sample import MetricSample @@ -175,7 +174,6 @@ def _open_contexts(self) -> None: frame = self.__frame__ assert frame is not None # nosec - args = list(get_args(frame)) thread = threading.current_thread() signal: Optional[Signal] = None @@ -200,7 +198,6 @@ def _open_contexts(self) -> None: probe=probe, frame=frame, thread=thread, - args=args, trace_context=trace_context, meter=self._probe_meter, ) @@ -209,7 +206,6 @@ def _open_contexts(self) -> None: probe=probe, frame=frame, thread=thread, - args=args, trace_context=trace_context, ) elif isinstance(probe, SpanFunctionProbe): @@ -217,7 +213,6 @@ def _open_contexts(self) -> None: probe=probe, frame=frame, thread=thread, - args=args, trace_context=trace_context, ) elif isinstance(probe, SpanDecorationFunctionProbe): @@ -225,7 +220,6 @@ def _open_contexts(self) -> None: probe=probe, frame=frame, thread=thread, - args=args, ) else: log.error("Unsupported probe type: %s", type(probe)) diff --git a/ddtrace/debugging/_probe/model.py b/ddtrace/debugging/_probe/model.py index f6b98e1366f..d96b98eaf46 100644 --- a/ddtrace/debugging/_probe/model.py +++ b/ddtrace/debugging/_probe/model.py @@ -8,6 +8,7 @@ from typing import Callable from typing import Dict from typing import List +from typing import Mapping from typing import Optional from typing import Tuple from typing import Union @@ -198,7 +199,7 @@ class MetricFunctionProbe(Probe, FunctionLocationMixin, MetricProbeMixin, ProbeC @dataclass class TemplateSegment(abc.ABC): @abc.abstractmethod - def eval(self, _locals: Dict[str, Any]) -> str: + def eval(self, scope: Mapping[str, Any]) -> str: pass @@ -206,7 +207,7 @@ def eval(self, _locals: Dict[str, Any]) -> str: class LiteralTemplateSegment(TemplateSegment): str_value: str - def eval(self, _locals: Dict[str, Any]) -> Any: + def eval(self, _scope: Mapping[str, Any]) -> Any: return self.str_value @@ -214,8 +215,8 @@ def eval(self, _locals: Dict[str, Any]) -> Any: class ExpressionTemplateSegment(TemplateSegment): expr: DDExpression - def eval(self, _locals: Dict[str, Any]) -> Any: - return self.expr.eval(_locals) + def eval(self, scope: Mapping[str, Any]) -> Any: + return self.expr.eval(scope) @dataclass @@ -223,11 +224,11 @@ class StringTemplate: template: str segments: List[TemplateSegment] - def render(self, _locals: Dict[str, Any], serializer: Callable[[Any], str]) -> str: + def render(self, scope: Mapping[str, Any], serializer: Callable[[Any], str]) -> str: def _to_str(value): return value if _isinstance(value, str) else serializer(value) - return "".join([_to_str(s.eval(_locals)) for s in self.segments]) + return "".join([_to_str(s.eval(scope)) for s in self.segments]) @dataclass diff --git a/ddtrace/debugging/_signal/metric_sample.py b/ddtrace/debugging/_signal/metric_sample.py index 896735e818d..c14f0173734 100644 --- a/ddtrace/debugging/_signal/metric_sample.py +++ b/ddtrace/debugging/_signal/metric_sample.py @@ -40,14 +40,14 @@ def exit(self, retval, exc_info, duration) -> None: return probe = self.probe - _locals = self._enrich_locals(retval, exc_info, duration) + full_scope = self.get_full_scope(retval, exc_info, duration) - if probe.evaluate_at != ProbeEvaluateTimingForMethod.EXIT: + if probe.evaluate_at is not ProbeEvaluateTimingForMethod.EXIT: return - if not self._eval_condition(_locals): + if not self._eval_condition(full_scope): return - self.sample(_locals) + self.sample(full_scope) self.state = SignalState.DONE def line(self) -> None: diff --git a/ddtrace/debugging/_signal/model.py b/ddtrace/debugging/_signal/model.py index 137568ecc0e..64cb0bff555 100644 --- a/ddtrace/debugging/_signal/model.py +++ b/ddtrace/debugging/_signal/model.py @@ -9,8 +9,8 @@ from typing import Any from typing import Dict from typing import List +from typing import Mapping from typing import Optional -from typing import Tuple from typing import Union from typing import cast from uuid import uuid4 @@ -22,6 +22,8 @@ from ddtrace.debugging._probe.model import LineLocationMixin from ddtrace.debugging._probe.model import Probe from ddtrace.debugging._probe.model import ProbeConditionMixin +from ddtrace.debugging._safety import get_args +from ddtrace.internal.compat import ExcInfoType from ddtrace.internal.rate_limiter import RateLimitExceeded @@ -52,13 +54,12 @@ class Signal(abc.ABC): frame: FrameType thread: Thread trace_context: Optional[Union[Span, Context]] = None - args: Optional[List[Tuple[str, Any]]] = None state: str = SignalState.NONE errors: List[EvaluationError] = field(default_factory=list) timestamp: float = field(default_factory=time.time) uuid: str = field(default_factory=lambda: str(uuid4()), init=False) - def _eval_condition(self, _locals: Optional[Dict[str, Any]] = None) -> bool: + def _eval_condition(self, scope: Optional[Mapping[str, Any]] = None) -> bool: """Evaluate the probe condition against the collected frame.""" probe = cast(ProbeConditionMixin, self.probe) condition = probe.condition @@ -66,7 +67,7 @@ def _eval_condition(self, _locals: Optional[Dict[str, Any]] = None) -> bool: return True try: - if bool(condition.eval(_locals or self.frame.f_locals)): + if bool(condition.eval(scope)): return True except DDExpressionEvaluationError as e: self.errors.append(EvaluationError(expr=e.dsl, message=e.error)) @@ -80,19 +81,22 @@ def _eval_condition(self, _locals: Optional[Dict[str, Any]] = None) -> bool: return False - def _enrich_locals(self, retval, exc_info, duration): + def get_full_scope(self, retval: Any, exc_info: ExcInfoType, duration: float) -> Mapping[str, Any]: frame = self.frame - _locals = dict(frame.f_locals) - _locals["@duration"] = duration / 1e6 # milliseconds + extra: Dict[str, Any] = {"@duration": duration / 1e6} # milliseconds exc = exc_info[1] if exc is not None: - _locals["@exception"] = exc + extra["@exception"] = exc else: - _locals["@return"] = retval + extra["@return"] = retval - # Include the frame globals. - return ChainMap(_locals, frame.f_globals) + # Include the frame locals and globals. + return ChainMap(extra, frame.f_locals, frame.f_globals) + + @property + def args(self): + return dict(get_args(self.frame)) @abc.abstractmethod def enter(self): diff --git a/ddtrace/debugging/_signal/snapshot.py b/ddtrace/debugging/_signal/snapshot.py index 4902ea65721..ef88a63f0a7 100644 --- a/ddtrace/debugging/_signal/snapshot.py +++ b/ddtrace/debugging/_signal/snapshot.py @@ -1,14 +1,16 @@ +from collections import ChainMap from dataclasses import dataclass from dataclasses import field +from itertools import chain import sys +from types import FrameType +from types import FunctionType +from types import ModuleType from typing import Any from typing import Dict -from typing import List from typing import Optional -from typing import Tuple from typing import cast -from ddtrace.debugging import _safety from ddtrace.debugging._expressions import DDExpressionEvaluationError from ddtrace.debugging._probe.model import DEFAULT_CAPTURE_LIMITS from ddtrace.debugging._probe.model import CaptureLimits @@ -22,6 +24,9 @@ from ddtrace.debugging._probe.model import TemplateSegment from ddtrace.debugging._redaction import REDACTED_PLACEHOLDER from ddtrace.debugging._redaction import DDRedactedExpressionError +from ddtrace.debugging._safety import get_args +from ddtrace.debugging._safety import get_globals +from ddtrace.debugging._safety import get_locals from ddtrace.debugging._signal import utils from ddtrace.debugging._signal.model import EvaluationError from ddtrace.debugging._signal.model import LogSignal @@ -35,11 +40,16 @@ CAPTURE_TIME_BUDGET = 0.2 # seconds +_NOTSET = object() + + +EXCLUDE_GLOBAL_TYPES = (ModuleType, type, FunctionType) + + def _capture_context( - arguments: List[Tuple[str, Any]], - _locals: List[Tuple[str, Any]], - _globals: List[Tuple[str, Any]], + frame: FrameType, throwable: ExcInfoType, + retval: Any = _NOTSET, limits: CaptureLimits = DEFAULT_CAPTURE_LIMITS, ) -> Dict[str, Any]: with HourGlass(duration=CAPTURE_TIME_BUDGET) as hg: @@ -47,6 +57,16 @@ def _capture_context( def timeout(_): return not hg.trickling() + arguments = get_args(frame) + _locals = get_locals(frame) + _globals = ((n, v) for n, v in get_globals(frame) if not isinstance(v, EXCLUDE_GLOBAL_TYPES)) + + _, exc, _ = throwable + if exc is not None: + _locals = chain(_locals, [("@exception", exc)]) + elif retval is not _NOTSET: + _locals = chain(_locals, [("@return", retval)]) + return { "arguments": utils.capture_pairs( arguments, limits.max_level, limits.max_len, limits.max_size, limits.max_fields, timeout @@ -67,13 +87,7 @@ def timeout(_): } -_EMPTY_CAPTURED_CONTEXT = _capture_context( - arguments=[], - _locals=[], - _globals=[], - throwable=(None, None, None), - limits=DEFAULT_CAPTURE_LIMITS, -) +_EMPTY_CAPTURED_CONTEXT: Dict[str, Any] = {"arguments": {}, "locals": {}, "staticFields": {}, "throwable": None} @dataclass @@ -117,12 +131,13 @@ def enter(self): probe = self.probe frame = self.frame - _args = list(self.args or _safety.get_args(frame)) if probe.evaluate_at == ProbeEvaluateTimingForMethod.EXIT: return - if not self._eval_condition(dict(_args)): + scope = ChainMap(self.args, frame.f_globals) + + if not self._eval_condition(scope): return if probe.limiter.limit() is RateLimitExceeded: @@ -130,16 +145,10 @@ def enter(self): return if probe.take_snapshot: - self.entry_capture = _capture_context( - _args, - [], - [], - (None, None, None), - limits=probe.limits, - ) + self.entry_capture = _capture_context(frame, (None, None, None), limits=probe.limits) if probe.evaluate_at == ProbeEvaluateTimingForMethod.ENTER: - self._eval_message(dict(_args)) + self._eval_message(scope) self.state = SignalState.DONE def exit(self, retval, exc_info, duration): @@ -147,10 +156,10 @@ def exit(self, retval, exc_info, duration): return probe = self.probe - _locals = self._enrich_locals(retval, exc_info, duration) + full_scope = self.get_full_scope(retval, exc_info, duration) if probe.evaluate_at == ProbeEvaluateTimingForMethod.EXIT: - if not self._eval_condition(_locals): + if not self._eval_condition(full_scope): return if probe.limiter.limit() is RateLimitExceeded: self.state = SignalState.SKIP_RATE @@ -158,30 +167,19 @@ def exit(self, retval, exc_info, duration): elif self.state not in {SignalState.NONE, SignalState.DONE}: return - _pure_locals = list(_safety.get_locals(self.frame)) - _, exc, tb = exc_info - if exc is None: - _pure_locals.append(("@return", retval)) - else: - _pure_locals.append(("@exception", exc)) - if probe.take_snapshot: - self.return_capture = _capture_context( - self.args or _safety.get_args(self.frame), - _pure_locals, - _safety.get_globals(self.frame), - exc_info, - limits=probe.limits, - ) + self.return_capture = _capture_context(self.frame, exc_info, retval=retval, limits=probe.limits) + self.duration = duration self.state = SignalState.DONE if probe.evaluate_at != ProbeEvaluateTimingForMethod.ENTER: - self._eval_message(dict(_locals)) + self._eval_message(full_scope) stack = utils.capture_stack(self.frame) # Fix the line number of the top frame. This might have been mangled by # the instrumented exception handling of function probes. + tb = exc_info[2] while tb is not None: frame = tb.tb_frame if frame == self.frame: @@ -206,15 +204,9 @@ def line(self): self.state = SignalState.SKIP_RATE return - self.line_capture = _capture_context( - self.args or _safety.get_args(frame), - _safety.get_locals(frame), - _safety.get_globals(frame), - sys.exc_info(), - limits=probe.limits, - ) + self.line_capture = _capture_context(frame, sys.exc_info(), limits=probe.limits) - self._eval_message(frame.f_locals) + self._eval_message(ChainMap(frame.f_locals, frame.f_globals)) self._stack = utils.capture_stack(frame) diff --git a/ddtrace/debugging/_signal/tracing.py b/ddtrace/debugging/_signal/tracing.py index 94d73015287..226e0230d90 100644 --- a/ddtrace/debugging/_signal/tracing.py +++ b/ddtrace/debugging/_signal/tracing.py @@ -44,7 +44,7 @@ def enter(self) -> None: log.debug("Dynamic span entered with non-span probe: %s", self.probe) return - if not self._eval_condition(dict(self.args) if self.args else {}): + if not self._eval_condition(self.args): return self._span_cm = ddtrace.tracer.trace( @@ -78,7 +78,7 @@ def line(self): class SpanDecoration(LogSignal): """Decorate a span.""" - def _decorate_span(self, _locals: t.Dict[str, t.Any]) -> None: + def _decorate_span(self, scope: t.Mapping[str, t.Any]) -> None: probe = t.cast(SpanDecorationMixin, self.probe) if probe.target_span == SpanDecorationTargetSpan.ACTIVE: @@ -93,7 +93,7 @@ def _decorate_span(self, _locals: t.Dict[str, t.Any]) -> None: log.debug("Decorating span %r according to span decoration probe %r", span, probe) for d in probe.decorations: try: - if not (d.when is None or d.when(_locals)): + if not (d.when is None or d.when(scope)): continue except DDExpressionEvaluationError as e: self.errors.append( @@ -102,7 +102,7 @@ def _decorate_span(self, _locals: t.Dict[str, t.Any]) -> None: continue for tag in d.tags: try: - tag_value = tag.value.render(_locals, serialize) + tag_value = tag.value.render(scope, serialize) except DDExpressionEvaluationError as e: span.set_tag_str( "_dd.di.%s.evaluation_error" % tag.name, ", ".join([serialize(v) for v in e.args]) @@ -117,8 +117,8 @@ def enter(self) -> None: log.debug("Span decoration entered with non-span decoration probe: %s", self.probe) return - if probe.evaluate_at == ProbeEvaluateTimingForMethod.ENTER: - self._decorate_span(dict(self.args) if self.args else {}) + if probe.evaluate_at is ProbeEvaluateTimingForMethod.ENTER: + self._decorate_span(self.args) self.state = SignalState.DONE def exit(self, retval: t.Any, exc_info: ExcInfoType, duration: float) -> None: @@ -128,8 +128,8 @@ def exit(self, retval: t.Any, exc_info: ExcInfoType, duration: float) -> None: log.debug("Span decoration exited with non-span decoration probe: %s", self.probe) return - if probe.evaluate_at == ProbeEvaluateTimingForMethod.EXIT: - self._decorate_span(self._enrich_locals(retval, exc_info, duration)) + if probe.evaluate_at is ProbeEvaluateTimingForMethod.EXIT: + self._decorate_span(self.get_full_scope(retval, exc_info, duration)) self.state = SignalState.DONE def line(self): diff --git a/tests/debugging/signal/test_collector.py b/tests/debugging/signal/test_collector.py index dd39726f7dc..c7b9e752662 100644 --- a/tests/debugging/signal/test_collector.py +++ b/tests/debugging/signal/test_collector.py @@ -36,7 +36,6 @@ def foo(a=42): condition=DDExpression("a not null", lambda _: _["a"] is not None), ), frame=sys._getframe(), - args=[("a", 42)], thread=threading.current_thread(), ) snapshot.line() @@ -53,7 +52,6 @@ def bar(b=None): condition=DDExpression("b not null", lambda _: _["b"] is not None), ), frame=sys._getframe(), - args=[("b", None)], thread=threading.current_thread(), ) snapshot.line() diff --git a/tests/debugging/signal/test_model.py b/tests/debugging/signal/test_model.py index ad58773d0c9..d99500e093f 100644 --- a/tests/debugging/signal/test_model.py +++ b/tests/debugging/signal/test_model.py @@ -7,7 +7,7 @@ def test_enriched_args_locals_globals(): duration = 123456 - _locals = dict( + full_scope = dict( Snapshot( probe=create_log_function_probe( probe_id="test_duration_millis", @@ -18,19 +18,19 @@ def test_enriched_args_locals_globals(): ), frame=sys._getframe(), thread=current_thread(), - )._enrich_locals(None, (None, None, None), duration) + ).get_full_scope(None, (None, None, None), duration) ) # Check for globals - assert "__file__" in _locals + assert "__file__" in full_scope # Check for locals - assert "duration" in _locals + assert "duration" in full_scope def test_duration_millis(): duration = 123456 - _locals = Snapshot( + full_scope = Snapshot( probe=create_log_function_probe( probe_id="test_duration_millis", module="foo", @@ -40,6 +40,6 @@ def test_duration_millis(): ), frame=sys._getframe(), thread=current_thread(), - )._enrich_locals(None, (None, None, None), duration) + ).get_full_scope(None, (None, None, None), duration) - assert _locals["@duration"] == duration / 1e6 + assert full_scope["@duration"] == duration / 1e6 diff --git a/tests/debugging/test_debugger.py b/tests/debugging/test_debugger.py index 18ecc280f7e..fc990e2fd47 100644 --- a/tests/debugging/test_debugger.py +++ b/tests/debugging/test_debugger.py @@ -1174,7 +1174,7 @@ def test_debugger_redacted_identifiers(): "size": 3, }, }, - "staticFields": {"SensitiveData": {"type": "type", "value": ""}}, + "staticFields": {}, "throwable": None, } @@ -1203,9 +1203,7 @@ def test_debugger_redacted_identifiers(): }, "@return": {"type": "str", "value": "'top secret'"}, # TODO: Ouch! }, - "staticFields": { - "SensitiveData": {"type": "type", "value": ""} - }, + "staticFields": {}, "throwable": None, }, } diff --git a/tests/debugging/test_encoding.py b/tests/debugging/test_encoding.py index 0c150da503b..489c996ceda 100644 --- a/tests/debugging/test_encoding.py +++ b/tests/debugging/test_encoding.py @@ -135,15 +135,23 @@ def c(): assert serialized["message"] == "'bad'" +def capture_context(*args, **kwargs): + return _capture_context(sys._getframe(1), *args, **kwargs) + + def test_capture_context_default_level(): - context = _capture_context([("self", tree)], [], [], (None, None, None), CaptureLimits(max_level=0)) - self = context["arguments"]["self"] + def _(self=tree): + return capture_context((None, None, None), limits=CaptureLimits(max_level=0)) + + self = _()["arguments"]["self"] assert self["fields"]["root"]["notCapturedReason"] == "depth" def test_capture_context_one_level(): - context = _capture_context([("self", tree)], [], [], (None, None, None), CaptureLimits(max_level=1)) - self = context["arguments"]["self"] + def _(self=tree): + return capture_context((None, None, None), limits=CaptureLimits(max_level=1)) + + self = _()["arguments"]["self"] assert self["fields"]["root"]["fields"]["left"] == {"notCapturedReason": "depth", "type": "Node"} @@ -152,13 +160,18 @@ def test_capture_context_one_level(): def test_capture_context_two_level(): - context = _capture_context([("self", tree)], [], [], (None, None, None), CaptureLimits(max_level=2)) - self = context["arguments"]["self"] + def _(self=tree): + return capture_context((None, None, None), limits=CaptureLimits(max_level=2)) + + self = _()["arguments"]["self"] assert self["fields"]["root"]["fields"]["left"]["fields"]["right"] == {"notCapturedReason": "depth", "type": "Node"} def test_capture_context_three_level(): - context = _capture_context([("self", tree)], [], [], (None, None, None), CaptureLimits(max_level=3)) + def _(self=tree): + return capture_context((None, None, None), limits=CaptureLimits(max_level=3)) + + context = _() self = context["arguments"]["self"] assert self["fields"]["root"]["fields"]["left"]["fields"]["right"]["fields"]["right"]["isNull"], context assert self["fields"]["root"]["fields"]["left"]["fields"]["right"]["fields"]["left"]["isNull"], context @@ -169,13 +182,15 @@ def test_capture_context_exc(): try: raise Exception("test", "me") except Exception: - context = _capture_context([], [], [], sys.exc_info()) + + def _(): + return capture_context(sys.exc_info()) + + context = _() + exc = context.pop("throwable") - assert context == { - "arguments": {}, - "locals": {}, - "staticFields": {}, - } + assert context["arguments"] == {} + assert context["locals"] == {"@exception": {"type": "Exception", "fields": {}}} assert exc["message"] == "'test', 'me'" assert exc["type"] == "Exception" From 4e1420181e0b83e1016d879c6c0dc9b33e643e05 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Fri, 30 Aug 2024 15:01:47 +0100 Subject: [PATCH 07/17] refactor: introduce trace ID format utility (#10444) We encapsulate the trace ID formatting logic for the backend in a helper function and use it where appropriate. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/_trace/tracer.py | 9 +++------ ddtrace/debugging/_encoding.py | 9 ++++++--- ddtrace/internal/utils/formats.py | 7 +++++++ tests/debugging/test_debugger.py | 7 ++++--- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ddtrace/_trace/tracer.py b/ddtrace/_trace/tracer.py index f47b93e9a09..493d161f058 100644 --- a/ddtrace/_trace/tracer.py +++ b/ddtrace/_trace/tracer.py @@ -39,7 +39,6 @@ from ddtrace.internal import forksafe from ddtrace.internal import hostname from ddtrace.internal.atexit import register_on_exit_signal -from ddtrace.internal.constants import MAX_UINT_64BITS from ddtrace.internal.constants import SAMPLING_DECISION_TRACE_TAG_KEY from ddtrace.internal.constants import SPAN_API_DATADOG from ddtrace.internal.dogstatsd import get_dogstatsd_client @@ -58,6 +57,7 @@ from ddtrace.internal.service import ServiceStatusError from ddtrace.internal.utils import _get_metas_to_propagate from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.internal.utils.formats import format_trace_id from ddtrace.internal.utils.http import verify_url from ddtrace.internal.writer import AgentResponse from ddtrace.internal.writer import AgentWriter @@ -423,11 +423,8 @@ def get_log_correlation_context(self, active: Optional[Union[Context, Span]] = N span_id = "0" trace_id = "0" if active: - span_id = str(active.span_id if active.span_id else span_id) - trace_id = str(active.trace_id if active.trace_id else trace_id) - # check if we are using 128 bit ids, and switch trace id to hex since backend needs hex 128 bit ids - if active.trace_id and active.trace_id > MAX_UINT_64BITS: - trace_id = "{:032x}".format(active.trace_id) + span_id = str(active.span_id) if active.span_id else span_id + trace_id = format_trace_id(active.trace_id) if active.trace_id else trace_id return { "trace_id": trace_id, diff --git a/ddtrace/debugging/_encoding.py b/ddtrace/debugging/_encoding.py index 771452d6423..fd3d9988a89 100644 --- a/ddtrace/debugging/_encoding.py +++ b/ddtrace/debugging/_encoding.py @@ -19,6 +19,7 @@ from ddtrace.internal import forksafe from ddtrace.internal._encoding import BufferFull from ddtrace.internal.logger import get_logger +from ddtrace.internal.utils.formats import format_trace_id log = get_logger(__name__) @@ -110,9 +111,11 @@ def _build_log_track_payload( } # Add the correlation IDs if available - if context is not None: - payload["dd.trace_id"] = str(context.trace_id) - payload["dd.span_id"] = str(context.span_id) + if context is not None and context.trace_id is not None: + payload["dd"] = { + "trace_id": format_trace_id(context.trace_id), + "span_id": str(context.span_id), + } add_tags(payload) diff --git a/ddtrace/internal/utils/formats.py b/ddtrace/internal/utils/formats.py index e9e3b85f434..63090b44e13 100644 --- a/ddtrace/internal/utils/formats.py +++ b/ddtrace/internal/utils/formats.py @@ -8,6 +8,8 @@ from typing import TypeVar # noqa:F401 from typing import Union # noqa:F401 +from ddtrace.internal.constants import MAX_UINT_64BITS # noqa:F401 + from ..compat import ensure_text @@ -189,3 +191,8 @@ def flatten_key_value(root_key, value): else: flattened[key] = item return flattened + + +def format_trace_id(trace_id: int) -> str: + """Translate a trace ID to a string format supported by the backend.""" + return "{:032x}".format(trace_id) if trace_id > MAX_UINT_64BITS else str(trace_id) diff --git a/tests/debugging/test_debugger.py b/tests/debugging/test_debugger.py index fc990e2fd47..4d453b888ce 100644 --- a/tests/debugging/test_debugger.py +++ b/tests/debugging/test_debugger.py @@ -24,6 +24,7 @@ from ddtrace.debugging._signal.utils import redacted_value from ddtrace.internal.remoteconfig.worker import remoteconfig_poller from ddtrace.internal.service import ServiceStatus +from ddtrace.internal.utils.formats import format_trace_id from ddtrace.internal.utils.inspection import linenos from tests.debugging.mocking import debugger from tests.debugging.utils import compile_template @@ -336,13 +337,13 @@ def test_debugger_tracer_correlation(): ) with d._tracer.trace("test-span") as span: - trace_id = str(span.trace_id) + trace_id = format_trace_id(span.trace_id) span_id = str(span.span_id) Stuff().instancestuff() snapshots = d.uploader.wait_for_payloads() - assert all(snapshot["dd.trace_id"] == trace_id for snapshot in snapshots) - assert all(snapshot["dd.span_id"] == span_id for snapshot in snapshots) + assert all(snapshot["dd"]["trace_id"] == trace_id for snapshot in snapshots) + assert all(snapshot["dd"]["span_id"] == span_id for snapshot in snapshots) def test_debugger_captured_exception(): From b75448fba1c3a71fead023d01bff803f10ed3363 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Martinez Date: Fri, 30 Aug 2024 16:16:12 +0200 Subject: [PATCH 08/17] fix(iast): fix leak on slice aspect. Add cleanup code to the trycatch macro (#10457) ## Description - Fix a memory leak on slice_aspect. - Add a parameter to the TRY_CATCH_ASPECT macro to optionally add cleanup code to the exception handlers. ## Checklist - [X] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Signed-off-by: Juanjo Alvarez --- .../_iast/_taint_tracking/Aspects/AspectIndex.cpp | 2 +- .../_iast/_taint_tracking/Aspects/AspectModulo.cpp | 2 +- .../_taint_tracking/Aspects/AspectOperatorAdd.cpp | 4 ++-- .../_iast/_taint_tracking/Aspects/AspectSlice.cpp | 3 ++- .../_iast/_taint_tracking/Aspects/AspectsOsPath.cpp | 10 +++++----- ddtrace/appsec/_iast/_taint_tracking/Aspects/Helpers.h | 4 +++- .../notes/slice-aspect-leak-95e264c4f1aa851c.yaml | 4 ++++ 7 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/slice-aspect-leak-95e264c4f1aa851c.yaml diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectIndex.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectIndex.cpp index e73c828f198..9641ad6c89e 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectIndex.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectIndex.cpp @@ -52,7 +52,7 @@ api_index_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs) if (!is_text(candidate_text) or !is_some_number(idx)) { return result_o; } - TRY_CATCH_ASPECT("index_aspect", { + TRY_CATCH_ASPECT("index_aspect", , { const auto ctx_map = Initializer::get_tainting_map(); if (not ctx_map or ctx_map->empty()) { return result_o; diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp index 81e3d2a6a1d..2f0f66d822f 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp @@ -19,7 +19,7 @@ api_modulo_aspect(StrType candidate_text, py::object candidate_tuple) py::tuple parameters = py::isinstance(candidate_tuple) ? candidate_tuple : py::make_tuple(candidate_tuple); - TRY_CATCH_ASPECT("modulo_aspect", { + TRY_CATCH_ASPECT("modulo_aspect", , { auto [ranges_orig, candidate_text_ranges] = are_all_text_all_ranges(candidate_text.ptr(), parameters); if (ranges_orig.empty()) { diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectOperatorAdd.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectOperatorAdd.cpp index 35a35b92847..5f70776920f 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectOperatorAdd.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectOperatorAdd.cpp @@ -85,7 +85,7 @@ api_add_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs) // PyNumber_Add actually works for any type! result_o = PyNumber_Add(candidate_text, text_to_add); - TRY_CATCH_ASPECT("add_aspect", { + TRY_CATCH_ASPECT("add_aspect", , { const auto tx_map = Initializer::get_tainting_map(); if (not tx_map or tx_map->empty()) { return result_o; @@ -119,7 +119,7 @@ api_add_inplace_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs) result_o = PyNumber_InPlaceAdd(candidate_text, text_to_add); - TRY_CATCH_ASPECT("add_inplace_aspect", { + TRY_CATCH_ASPECT("add_inplace_aspect", , { const auto tx_map = Initializer::get_tainting_map(); if (not tx_map or tx_map->empty()) { return result_o; diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectSlice.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectSlice.cpp index 1933cc33a62..7f3892cc9ef 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectSlice.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectSlice.cpp @@ -145,12 +145,13 @@ api_slice_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs) PyObject* result_o = PyObject_GetItem(candidate_text, slice); - TRY_CATCH_ASPECT("slice_aspect", { + TRY_CATCH_ASPECT("slice_aspect", Py_XDECREF(slice), { // If no result or the params are not None|Number or the result is the same as the candidate text, nothing // to taint if (result_o == nullptr or (!is_text(candidate_text)) or (start != Py_None and !PyLong_Check(start)) or (stop != Py_None and !PyLong_Check(stop)) or (step != Py_None and !PyLong_Check(step)) or (get_unique_id(result_o) == get_unique_id(candidate_text))) { + Py_XDECREF(slice); return result_o; } diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectsOsPath.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectsOsPath.cpp index fcc5547714a..4f7e5b3b885 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectsOsPath.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectsOsPath.cpp @@ -23,7 +23,7 @@ api_ospathjoin_aspect(StrType& first_part, const py::args& args) return result_o; } - TRY_CATCH_ASPECT("ospathjoin_aspect", { + TRY_CATCH_ASPECT("ospathjoin_aspect", , { const auto separator = ospath.attr("sep").cast(); const auto sepsize = separator.size(); @@ -105,7 +105,7 @@ api_ospathbasename_aspect(const StrType& path) auto basename = ospath.attr("basename"); auto result_o = basename(path); - TRY_CATCH_ASPECT("ospathbasename_aspect", { + TRY_CATCH_ASPECT("ospathbasename_aspect", , { const auto tx_map = Initializer::get_tainting_map(); if (not tx_map or tx_map->empty() or py::len(result_o) == 0) { return result_o; @@ -138,7 +138,7 @@ api_ospathdirname_aspect(const StrType& path) auto dirname = ospath.attr("dirname"); auto result_o = dirname(path); - TRY_CATCH_ASPECT("ospathdirname_aspect", { + TRY_CATCH_ASPECT("ospathdirname_aspect", , { const auto tx_map = Initializer::get_tainting_map(); if (not tx_map or tx_map->empty() or py::len(result_o) == 0) { return result_o; @@ -171,7 +171,7 @@ forward_to_set_ranges_on_splitted(const char* function_name, const StrType& path auto function = ospath.attr(function_name); auto result_o = function(path); - TRY_CATCH_ASPECT("forward_to_set_ranges_on_splitted", { + TRY_CATCH_ASPECT("forward_to_set_ranges_on_splitted", , { const auto tx_map = Initializer::get_tainting_map(); if (not tx_map or tx_map->empty() or py::len(result_o) == 0) { return result_o; @@ -223,7 +223,7 @@ api_ospathnormcase_aspect(const StrType& path) auto normcase = ospath.attr("normcase"); auto result_o = normcase(path); - TRY_CATCH_ASPECT("ospathnormcase_aspect", { + TRY_CATCH_ASPECT("ospathnormcase_aspect", , { const auto tx_map = Initializer::get_tainting_map(); if (not tx_map or tx_map->empty()) { return result_o; diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/Helpers.h b/ddtrace/appsec/_iast/_taint_tracking/Aspects/Helpers.h index 6ee64d0c4d0..a4c15f9ebe2 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/Helpers.h +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/Helpers.h @@ -200,15 +200,17 @@ exception_wrapper(Func func, const char* aspect_name, Args... args) -> std::opti } */ -#define TRY_CATCH_ASPECT(NAME, ...) \ +#define TRY_CATCH_ASPECT(NAME, CLEANUP, ...) \ try { \ __VA_ARGS__; \ } catch (const std::exception& e) { \ const std::string error_message = "IAST propagation error in " NAME ". " + std::string(e.what()); \ iast_taint_log_error(error_message); \ + CLEANUP; \ return result_o; \ } catch (...) { \ const std::string error_message = "Unknown IAST propagation error in " NAME ". "; \ iast_taint_log_error(error_message); \ + CLEANUP; \ return result_o; \ } diff --git a/releasenotes/notes/slice-aspect-leak-95e264c4f1aa851c.yaml b/releasenotes/notes/slice-aspect-leak-95e264c4f1aa851c.yaml new file mode 100644 index 00000000000..c31dad5f28c --- /dev/null +++ b/releasenotes/notes/slice-aspect-leak-95e264c4f1aa851c.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix a memory leak on the native slice aspect. From 62bb9030aaa45d758c8e9f37d8d65273faf1ff26 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 30 Aug 2024 16:19:04 +0200 Subject: [PATCH 09/17] chore(iast): fastapi path parameter support (#10426) Path parameter sources Previous sources tasks: - https://github.com/DataDog/dd-trace-py/pull/10421 - https://github.com/DataDog/dd-trace-py/pull/10331 ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/appsec/_iast/_patch.py | 17 ++++++ ddtrace/contrib/internal/starlette/patch.py | 7 +++ .../fastapi/test_fastapi_appsec_iast.py | 56 +++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/ddtrace/appsec/_iast/_patch.py b/ddtrace/appsec/_iast/_patch.py index 607c28f7784..d77259c2c4d 100644 --- a/ddtrace/appsec/_iast/_patch.py +++ b/ddtrace/appsec/_iast/_patch.py @@ -170,3 +170,20 @@ def _on_iast_fastapi_patch(): functools.partial(_patched_fastapi_function, OriginType.HEADER), ) _set_metric_iast_instrumented_source(OriginType.HEADER) + + # Instrumented on _iast_starlette_scope_taint + _set_metric_iast_instrumented_source(OriginType.PATH_PARAMETER) + + +def _iast_instrument_starlette_scope(scope): + from ddtrace.appsec._iast._taint_tracking import OriginType + from ddtrace.appsec._iast._taint_tracking import taint_pyobject + + if scope.get("path_params"): + try: + for k, v in scope["path_params"].items(): + scope["path_params"][k] = taint_pyobject( + v, source_name=k, source_value=v, source_origin=OriginType.PATH_PARAMETER + ) + except Exception: + log.debug("IAST: Unexpected exception while tainting path parameters", exc_info=True) diff --git a/ddtrace/contrib/internal/starlette/patch.py b/ddtrace/contrib/internal/starlette/patch.py index 40112b9dd47..2ccd19cf5d3 100644 --- a/ddtrace/contrib/internal/starlette/patch.py +++ b/ddtrace/contrib/internal/starlette/patch.py @@ -15,6 +15,7 @@ from ddtrace import Pin from ddtrace import config from ddtrace._trace.span import Span # noqa:F401 +from ddtrace.appsec._iast import _is_iast_enabled from ddtrace.contrib import trace_utils from ddtrace.contrib.asgi import TraceMiddleware from ddtrace.contrib.trace_utils import with_traced_module @@ -154,7 +155,13 @@ def traced_handler(wrapped, instance, args, kwargs): if name == b"cookie": request_cookies = value.decode("utf-8", errors="ignore") break + if request_spans: + if _is_iast_enabled(): + from ddtrace.appsec._iast._patch import _iast_instrument_starlette_scope + + _iast_instrument_starlette_scope(scope) + trace_utils.set_http_meta( request_spans[0], "starlette", diff --git a/tests/contrib/fastapi/test_fastapi_appsec_iast.py b/tests/contrib/fastapi/test_fastapi_appsec_iast.py index 2c1afc6cf03..b732cebd1cb 100644 --- a/tests/contrib/fastapi/test_fastapi_appsec_iast.py +++ b/tests/contrib/fastapi/test_fastapi_appsec_iast.py @@ -50,15 +50,18 @@ def test_query_param_source(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html") async def test_route(request: Request): from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking import origin_to_str query_params = request.query_params.get("iast_queryparam") ranges_result = get_tainted_ranges(query_params) + return JSONResponse( { "result": query_params, "is_tainted": len(ranges_result), "ranges_start": ranges_result[0].start, "ranges_length": ranges_result[0].length, + "ranges_origin": origin_to_str(ranges_result[0].source.origin), } ) @@ -78,6 +81,7 @@ async def test_route(request: Request): assert result["is_tainted"] == 1 assert result["ranges_start"] == 0 assert result["ranges_length"] == 8 + assert result["ranges_origin"] == "http.request.parameter" @pytest.mark.usefixtures("setup_core_ok_after_test") @@ -85,15 +89,18 @@ def test_header_value_source(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html") async def test_route(request: Request): from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking import origin_to_str query_params = request.headers.get("iast_header") ranges_result = get_tainted_ranges(query_params) + return JSONResponse( { "result": query_params, "is_tainted": len(ranges_result), "ranges_start": ranges_result[0].start, "ranges_length": ranges_result[0].length, + "ranges_origin": origin_to_str(ranges_result[0].source.origin), } ) @@ -113,6 +120,7 @@ async def test_route(request: Request): assert result["is_tainted"] == 1 assert result["ranges_start"] == 0 assert result["ranges_length"] == 8 + assert result["ranges_origin"] == "http.request.header" @pytest.mark.usefixtures("setup_core_ok_after_test") @@ -122,14 +130,17 @@ def test_header_value_source_typing_param(fastapi_application, client, tracer, t @fastapi_application.get("/index.html") async def test_route(iast_header: typing.Annotated[str, Header()] = None): from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking import origin_to_str ranges_result = get_tainted_ranges(iast_header) + return JSONResponse( { "result": iast_header, "is_tainted": len(ranges_result), "ranges_start": ranges_result[0].start, "ranges_length": ranges_result[0].length, + "ranges_origin": origin_to_str(ranges_result[0].source.origin), } ) @@ -149,6 +160,7 @@ async def test_route(iast_header: typing.Annotated[str, Header()] = None): assert result["is_tainted"] == 1 assert result["ranges_start"] == 0 assert result["ranges_length"] == 8 + assert result["ranges_origin"] == "http.request.header" @pytest.mark.usefixtures("setup_core_ok_after_test") @@ -156,6 +168,7 @@ def test_cookies_source(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html") async def test_route(request: Request): from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking import origin_to_str query_params = request.cookies.get("iast_cookie") ranges_result = get_tainted_ranges(query_params) @@ -165,6 +178,7 @@ async def test_route(request: Request): "is_tainted": len(ranges_result), "ranges_start": ranges_result[0].start, "ranges_length": ranges_result[0].length, + "ranges_origin": origin_to_str(ranges_result[0].source.origin), } ) @@ -184,6 +198,7 @@ async def test_route(request: Request): assert result["is_tainted"] == 1 assert result["ranges_start"] == 0 assert result["ranges_length"] == 8 + assert result["ranges_origin"] == "http.request.cookie.value" @pytest.mark.usefixtures("setup_core_ok_after_test") @@ -193,14 +208,17 @@ def test_cookies_source_typing_param(fastapi_application, client, tracer, test_s @fastapi_application.get("/index.html") async def test_route(iast_cookie: typing.Annotated[str, Cookie()] = "ddd"): from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking import origin_to_str ranges_result = get_tainted_ranges(iast_cookie) + return JSONResponse( { "result": iast_cookie, "is_tainted": len(ranges_result), "ranges_start": ranges_result[0].start, "ranges_length": ranges_result[0].length, + "ranges_origin": origin_to_str(ranges_result[0].source.origin), } ) @@ -220,3 +238,41 @@ async def test_route(iast_cookie: typing.Annotated[str, Cookie()] = "ddd"): assert result["is_tainted"] == 1 assert result["ranges_start"] == 0 assert result["ranges_length"] == 8 + assert result["ranges_origin"] == "http.request.cookie.value" + + +@pytest.mark.usefixtures("setup_core_ok_after_test") +def test_path_param_source(fastapi_application, client, tracer, test_spans): + @fastapi_application.get("/index.html/{item_id}") + async def test_route(item_id): + from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking import origin_to_str + + ranges_result = get_tainted_ranges(item_id) + + return JSONResponse( + { + "result": item_id, + "is_tainted": len(ranges_result), + "ranges_start": ranges_result[0].start, + "ranges_length": ranges_result[0].length, + "ranges_origin": origin_to_str(ranges_result[0].source.origin), + } + ) + + # test if asgi middleware is ok without any callback registered + core.reset_listeners(event_id="asgi.request.parse.body") + + with override_global_config(dict(_iast_enabled=True)), override_env(IAST_ENV): + # disable callback + _aux_appsec_prepare_tracer(tracer) + resp = client.get( + "/index.html/test1234/", + ) + assert resp.status_code == 200 + result = json.loads(get_response_body(resp)) + assert result["result"] == "test1234" + assert result["is_tainted"] == 1 + assert result["ranges_start"] == 0 + assert result["ranges_length"] == 8 + assert result["ranges_origin"] == "http.request.path.parameter" From 7d710e1c18e8b40d625a43b502d09872fd6979b1 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 30 Aug 2024 16:21:35 +0200 Subject: [PATCH 10/17] feat(iast): flask json body (#10448) - Taint the json of a Flask request for Flask 2.0.0. `get_json` method - Fix the tainted json of a Flask request for Flask 1.0.0. `get_data` method - Refactor FastAPI sources to use "taint_structure" instead of LazyTaintDict ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/appsec/_handlers.py | 85 ++-- ddtrace/appsec/_iast/_patch.py | 13 +- ddtrace/appsec/_iast/_taint_utils.py | 6 +- .../integrations/test_flask_telemetry.py | 2 +- tests/contrib/flask/test_flask_appsec_iast.py | 433 +++++++++++++++++- 5 files changed, 468 insertions(+), 71 deletions(-) diff --git a/ddtrace/appsec/_handlers.py b/ddtrace/appsec/_handlers.py index 495f96cfea5..4caae6efcf8 100644 --- a/ddtrace/appsec/_handlers.py +++ b/ddtrace/appsec/_handlers.py @@ -216,53 +216,58 @@ def _on_request_init(wrapped, instance, args, kwargs): def _on_flask_patch(flask_version): if _is_iast_enabled(): - try: - from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source - from ddtrace.appsec._iast._taint_tracking import OriginType + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source + from ddtrace.appsec._iast._patch import _patched_dictionary + from ddtrace.appsec._iast._patch import try_wrap_function_wrapper + from ddtrace.appsec._iast._taint_tracking import OriginType + + try_wrap_function_wrapper( + "werkzeug.datastructures", + "Headers.items", + functools.partial(if_iast_taint_yield_tuple_for, (OriginType.HEADER_NAME, OriginType.HEADER)), + ) + _set_metric_iast_instrumented_source(OriginType.HEADER_NAME) + _set_metric_iast_instrumented_source(OriginType.HEADER) - _w( - "werkzeug.datastructures", - "Headers.items", - functools.partial(if_iast_taint_yield_tuple_for, (OriginType.HEADER_NAME, OriginType.HEADER)), - ) - _set_metric_iast_instrumented_source(OriginType.HEADER_NAME) - _set_metric_iast_instrumented_source(OriginType.HEADER) + try_wrap_function_wrapper( + "werkzeug.datastructures", + "ImmutableMultiDict.__getitem__", + functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER), + ) + _set_metric_iast_instrumented_source(OriginType.PARAMETER) - _w( - "werkzeug.datastructures", - "ImmutableMultiDict.__getitem__", - functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER), - ) - _set_metric_iast_instrumented_source(OriginType.PARAMETER) + try_wrap_function_wrapper( + "werkzeug.datastructures", + "EnvironHeaders.__getitem__", + functools.partial(if_iast_taint_returned_object_for, OriginType.HEADER), + ) + _set_metric_iast_instrumented_source(OriginType.HEADER) - _w( - "werkzeug.datastructures", - "EnvironHeaders.__getitem__", - functools.partial(if_iast_taint_returned_object_for, OriginType.HEADER), - ) - _set_metric_iast_instrumented_source(OriginType.HEADER) + try_wrap_function_wrapper("werkzeug.wrappers.request", "Request.__init__", _on_request_init) - _w("werkzeug.wrappers.request", "Request.__init__", _on_request_init) + _set_metric_iast_instrumented_source(OriginType.PATH) + _set_metric_iast_instrumented_source(OriginType.QUERY) - _set_metric_iast_instrumented_source(OriginType.PATH) - _set_metric_iast_instrumented_source(OriginType.QUERY) + try_wrap_function_wrapper( + "werkzeug.wrappers.request", + "Request.get_data", + functools.partial(_patched_dictionary, OriginType.BODY, OriginType.BODY), + ) + try_wrap_function_wrapper( + "werkzeug.wrappers.request", + "Request.get_json", + functools.partial(_patched_dictionary, OriginType.BODY, OriginType.BODY), + ) + + _set_metric_iast_instrumented_source(OriginType.BODY) + if flask_version < (2, 0, 0): _w( - "werkzeug.wrappers.request", - "Request.get_data", - functools.partial(if_iast_taint_returned_object_for, OriginType.BODY), + "werkzeug._internal", + "_DictAccessorProperty.__get__", + functools.partial(if_iast_taint_returned_object_for, OriginType.QUERY), ) - _set_metric_iast_instrumented_source(OriginType.BODY) - - if flask_version < (2, 0, 0): - _w( - "werkzeug._internal", - "_DictAccessorProperty.__get__", - functools.partial(if_iast_taint_returned_object_for, OriginType.QUERY), - ) - _set_metric_iast_instrumented_source(OriginType.QUERY) - except Exception: - log.debug("Unexpected exception while patch IAST functions", exc_info=True) + _set_metric_iast_instrumented_source(OriginType.QUERY) def _on_flask_blocked_request(_): @@ -345,9 +350,9 @@ def _on_django_patch(): from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source from ddtrace.appsec._iast._taint_tracking import OriginType + # we instrument those sources on _on_django_func_wrapped _set_metric_iast_instrumented_source(OriginType.HEADER_NAME) _set_metric_iast_instrumented_source(OriginType.HEADER) - # we instrument those sources on _on_django_func_wrapped _set_metric_iast_instrumented_source(OriginType.PATH_PARAMETER) _set_metric_iast_instrumented_source(OriginType.PATH) _set_metric_iast_instrumented_source(OriginType.COOKIE) diff --git a/ddtrace/appsec/_iast/_patch.py b/ddtrace/appsec/_iast/_patch.py index d77259c2c4d..e0812cccfe2 100644 --- a/ddtrace/appsec/_iast/_patch.py +++ b/ddtrace/appsec/_iast/_patch.py @@ -9,6 +9,7 @@ from ddtrace.internal.logger import get_logger from ._metrics import _set_metric_iast_instrumented_source +from ._taint_utils import taint_structure from ._utils import _is_iast_enabled @@ -88,16 +89,10 @@ def if_iast_taint_yield_tuple_for(origins, wrapped, instance, args, kwargs): yield key, value -def _patched_fastapi_request_cookies(original_func, instance, args, kwargs): - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_utils import LazyTaintDict - +def _patched_dictionary(origin_key, origin_value, original_func, instance, args, kwargs): result = original_func(*args, **kwargs) - if isinstance(result, (LazyTaintDict)): - return result - - return LazyTaintDict(result, origins=(OriginType.COOKIE_NAME, OriginType.COOKIE), override_pyobject_tainted=True) + return taint_structure(result, origin_key, origin_value, override_pyobject_tainted=True) def _patched_fastapi_function(origin, original_func, instance, args, kwargs): @@ -130,7 +125,7 @@ def _on_iast_fastapi_patch(): try_wrap_function_wrapper( "starlette.requests", "cookie_parser", - _patched_fastapi_request_cookies, + functools.partial(_patched_dictionary, OriginType.COOKIE_NAME, OriginType.COOKIE), ) try_wrap_function_wrapper( "fastapi", diff --git a/ddtrace/appsec/_iast/_taint_utils.py b/ddtrace/appsec/_iast/_taint_utils.py index c4b31aad665..3927349e176 100644 --- a/ddtrace/appsec/_iast/_taint_utils.py +++ b/ddtrace/appsec/_iast/_taint_utils.py @@ -539,8 +539,8 @@ def check_tainted_dbapi_args(args, kwargs, tracer, integration_name, method): if asm_config._iast_lazy_taint: # redefining taint_structure to use lazy object if required - def taint_structure(main_obj, source_key, source_value, override_pyobject_tainted=False): # noqa: F811 + def taint_structure(main_obj, origin_key, origin_value, override_pyobject_tainted=False): # noqa: F811 if isinstance(main_obj, abc.Mapping): - return LazyTaintDict(main_obj, source_key, source_value, override_pyobject_tainted) + return LazyTaintDict(main_obj, (origin_key, origin_value), override_pyobject_tainted) elif isinstance(main_obj, abc.Sequence): - return LazyTaintList(main_obj, source_key, source_value, override_pyobject_tainted) + return LazyTaintList(main_obj, (origin_key, origin_value), override_pyobject_tainted) diff --git a/tests/appsec/integrations/test_flask_telemetry.py b/tests/appsec/integrations/test_flask_telemetry.py index 40b8db095ef..0539aaf6618 100644 --- a/tests/appsec/integrations/test_flask_telemetry.py +++ b/tests/appsec/integrations/test_flask_telemetry.py @@ -19,7 +19,7 @@ def test_flask_instrumented_metrics(telemetry_writer): from ddtrace.appsec._iast._taint_tracking import origin_to_str with override_global_config(dict(_iast_enabled=True)): - _on_flask_patch("2.0.0") + _on_flask_patch((2, 0, 0)) metrics_result = telemetry_writer._namespace._metrics_data metrics_source_tags_result = [metric._tags[0][1] for metric in metrics_result["generate-metrics"]["iast"].values()] diff --git a/tests/contrib/flask/test_flask_appsec_iast.py b/tests/contrib/flask/test_flask_appsec_iast.py index d197d639fd5..4835947b079 100644 --- a/tests/contrib/flask/test_flask_appsec_iast.py +++ b/tests/contrib/flask/test_flask_appsec_iast.py @@ -1,4 +1,5 @@ import json +import sys from flask import request from importlib_metadata import version @@ -6,6 +7,7 @@ from ddtrace.appsec._constants import IAST from ddtrace.appsec._iast import oce +from ddtrace.appsec._iast._patches.json_tainting import patch as patch_json from ddtrace.appsec._iast._utils import _is_python_version_supported as python_supported_by_iast from ddtrace.appsec._iast.constants import VULN_HEADER_INJECTION from ddtrace.appsec._iast.constants import VULN_INSECURE_COOKIE @@ -25,6 +27,7 @@ IAST_ENV_SAMPLING_0 = {"DD_IAST_REQUEST_SAMPLING": "0"} werkzeug_version = version("werkzeug") +flask_version = tuple([int(v) for v in version("flask").split(".")]) @pytest.fixture(autouse=True) @@ -53,6 +56,7 @@ def setUp(self): super(FlaskAppSecIASTEnabledTestCase, self).setUp() patch_sqlite_sqli() patch_header_injection() + patch_json() oce.reconfigure() self.tracer._iast_enabled = True @@ -272,7 +276,7 @@ def sqli_4(param_str): assert vulnerability["location"]["path"] == TEST_FILE_PATH assert vulnerability["hash"] == hash_value - @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") + @pytest.mark.skipif(sys.version_info < (3, 8), reason="some requests params fail in Python 3.7 or lower") def test_flask_simple_iast_path_header_and_querystring_tainted(self): @self.app.route("/sqli///", methods=["GET", "POST"]) def sqli_5(param_str, param_int): @@ -287,26 +291,20 @@ def sqli_5(param_str, param_int): assert header_ranges[0].source.name.lower() == "user-agent" assert header_ranges[0].source.origin == OriginType.HEADER - _ = get_tainted_ranges(request.query_string) - # TODO: this test fails in 3.7 - # assert query_string_ranges - # assert query_string_ranges[0].source.name == "http.request.query" - # assert query_string_ranges[0].source.origin == OriginType.QUERY + if flask_version > (2, 0): + query_string_ranges = get_tainted_ranges(request.query_string) + assert query_string_ranges + assert query_string_ranges[0].source.name == "http.request.query" + assert query_string_ranges[0].source.origin == OriginType.QUERY - _ = get_tainted_ranges(param_str) - # TODO: this test fails in 3.7 - # assert param_str_ranges - # assert param_str_ranges[0].source.name == "param_str" - # assert param_str_ranges[0].source.origin == OriginType.PATH_PARAMETER + request_path_ranges = get_tainted_ranges(request.path) + assert request_path_ranges + assert request_path_ranges[0].source.name == "http.request.path" + assert request_path_ranges[0].source.origin == OriginType.PATH + _ = get_tainted_ranges(param_str) assert not is_pyobject_tainted(param_int) - _ = get_tainted_ranges(request.path) - # TODO: this test fails in 3.7 - # assert request_path_ranges - # assert request_path_ranges[0].source.name == "http.request.path" - # assert request_path_ranges[0].source.origin == OriginType.PATH - request_form_name_ranges = get_tainted_ranges(request.form.get("name")) assert request_form_name_ranges assert request_form_name_ranges[0].source.name == "name" @@ -537,10 +535,409 @@ def sqli_9(): assert vulnerability["location"]["path"] == TEST_FILE_PATH assert vulnerability["hash"] == hash_value + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") + def test_flask_request_body(self): + @self.app.route("/sqli/body/", methods=("POST",)) + def sqli_10(): + import json + import sqlite3 + + from flask import request + + from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect + + con = sqlite3.connect(":memory:") + cur = con.cursor() + if flask_version > (2, 0): + json_data = request.json + else: + json_data = json.loads(request.data) + value = json_data.get("body") + assert value == "master" + assert is_pyobject_tainted(value) + query = add_aspect(add_aspect("SELECT tbl_name FROM sqlite_", value), " WHERE tbl_name LIKE 'password'") + # label test_flask_request_body + cur.execute(query) + + return "OK", 200 + + with override_global_config( + dict( + _iast_enabled=True, + _deduplication_enabled=False, + ) + ): + resp = self.client.post( + "/sqli/body/", data=json.dumps(dict(body="master")), content_type="application/json" + ) + assert resp.status_code == 200 + + root_span = self.pop_spans()[0] + assert root_span.get_metric(IAST.ENABLED) == 1.0 + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + assert loaded["sources"] == [{"name": "body", "origin": "http.request.body", "value": "master"}] + + line, hash_value = get_line_and_hash( + "test_flask_request_body", + VULN_SQL_INJECTION, + filename=TEST_FILE_PATH, + ) + vulnerability = loaded["vulnerabilities"][0] + assert vulnerability["type"] == VULN_SQL_INJECTION + assert vulnerability["evidence"] == { + "valueParts": [ + {"value": "SELECT tbl_name FROM sqlite_"}, + {"value": "master", "source": 0}, + {"value": " WHERE tbl_name LIKE '"}, + {"redacted": True}, + {"value": "'"}, + ] + } + assert vulnerability["location"]["line"] == line + assert vulnerability["location"]["path"] == TEST_FILE_PATH + assert vulnerability["hash"] == hash_value + + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") + def test_flask_request_body_complex_3_lvls(self): + @self.app.route("/sqli/body/", methods=("POST",)) + def sqli_11(): + import sqlite3 + + from flask import request + + from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect + + con = sqlite3.connect(":memory:") + cur = con.cursor() + + if flask_version > (2, 0): + json_data = request.json + else: + json_data = json.loads(request.data) + value = json_data.get("body").get("body2").get("body3") + assert value == "master" + assert is_pyobject_tainted(value) + query = add_aspect(add_aspect("SELECT tbl_name FROM sqlite_", value), " WHERE tbl_name LIKE 'password'") + # label test_flask_request_body_complex_3_lvls + cur.execute(query) + + return "OK", 200 + + with override_global_config( + dict( + _iast_enabled=True, + ) + ): + resp = self.client.post( + "/sqli/body/", + data=json.dumps(dict(body=dict(body2=dict(body3="master")))), + content_type="application/json", + ) + assert resp.status_code == 200 + + root_span = self.pop_spans()[0] + assert root_span.get_metric(IAST.ENABLED) == 1.0 + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + assert loaded["sources"] == [{"name": "body3", "origin": "http.request.body", "value": "master"}] + + line, hash_value = get_line_and_hash( + "test_flask_request_body_complex_3_lvls", + VULN_SQL_INJECTION, + filename=TEST_FILE_PATH, + ) + vulnerability = loaded["vulnerabilities"][0] + assert vulnerability["type"] == VULN_SQL_INJECTION + assert vulnerability["evidence"] == { + "valueParts": [ + {"value": "SELECT tbl_name FROM sqlite_"}, + {"value": "master", "source": 0}, + {"value": " WHERE tbl_name LIKE '"}, + {"redacted": True}, + {"value": "'"}, + ] + } + assert vulnerability["location"]["line"] == line + assert vulnerability["location"]["path"] == TEST_FILE_PATH + assert vulnerability["hash"] == hash_value + + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") + def test_flask_request_body_complex_3_lvls_and_list(self): + @self.app.route("/sqli/body/", methods=("POST",)) + def sqli_11(): + import sqlite3 + + from flask import request + + from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect + + con = sqlite3.connect(":memory:") + cur = con.cursor() + + if flask_version > (2, 0): + json_data = request.json + else: + json_data = json.loads(request.data) + value = json_data.get("body").get("body2").get("body3")[3] + assert value == "master" + assert is_pyobject_tainted(value) + query = add_aspect(add_aspect("SELECT tbl_name FROM sqlite_", value), " WHERE tbl_name LIKE 'password'") + # label test_flask_request_body_complex_3_lvls_and_list + cur.execute(query) + + return "OK", 200 + + with override_global_config( + dict( + _iast_enabled=True, + ) + ): + resp = self.client.post( + "/sqli/body/", + data=json.dumps(dict(body=dict(body2=dict(body3=["master3", "master2", "master1", "master"])))), + content_type="application/json", + ) + assert resp.status_code == 200 + + root_span = self.pop_spans()[0] + assert root_span.get_metric(IAST.ENABLED) == 1.0 + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + assert loaded["sources"] == [{"name": "body3", "origin": "http.request.body", "value": "master"}] + + line, hash_value = get_line_and_hash( + "test_flask_request_body_complex_3_lvls_and_list", + VULN_SQL_INJECTION, + filename=TEST_FILE_PATH, + ) + vulnerability = loaded["vulnerabilities"][0] + assert vulnerability["type"] == VULN_SQL_INJECTION + assert vulnerability["evidence"] == { + "valueParts": [ + {"value": "SELECT tbl_name FROM sqlite_"}, + {"value": "master", "source": 0}, + {"value": " WHERE tbl_name LIKE '"}, + {"redacted": True}, + {"value": "'"}, + ] + } + assert vulnerability["location"]["line"] == line + assert vulnerability["location"]["path"] == TEST_FILE_PATH + assert vulnerability["hash"] == hash_value + + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") + def test_flask_request_body_complex_3_lvls_list_dict(self): + @self.app.route("/sqli/body/", methods=("POST",)) + def sqli_11(): + import sqlite3 + + from flask import request + + from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect + + con = sqlite3.connect(":memory:") + cur = con.cursor() + + if flask_version > (2, 0): + json_data = request.json + else: + json_data = json.loads(request.data) + value = json_data.get("body").get("body2").get("body3")[3].get("body4") + assert value == "master" + assert is_pyobject_tainted(value) + query = add_aspect(add_aspect("SELECT tbl_name FROM sqlite_", value), " WHERE tbl_name LIKE 'password'") + # label test_flask_request_body_complex_3_lvls_list_dict + cur.execute(query) + + return "OK", 200 + + with override_global_config( + dict( + _iast_enabled=True, + ) + ): + resp = self.client.post( + "/sqli/body/", + data=json.dumps( + dict(body=dict(body2=dict(body3=["master3", "master2", "master1", {"body4": "master"}]))) + ), + content_type="application/json", + ) + assert resp.status_code == 200 + + root_span = self.pop_spans()[0] + assert root_span.get_metric(IAST.ENABLED) == 1.0 + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + assert loaded["sources"] == [{"name": "body4", "origin": "http.request.body", "value": "master"}] + + line, hash_value = get_line_and_hash( + "test_flask_request_body_complex_3_lvls_list_dict", + VULN_SQL_INJECTION, + filename=TEST_FILE_PATH, + ) + vulnerability = loaded["vulnerabilities"][0] + assert vulnerability["type"] == VULN_SQL_INJECTION + assert vulnerability["evidence"] == { + "valueParts": [ + {"value": "SELECT tbl_name FROM sqlite_"}, + {"value": "master", "source": 0}, + {"value": " WHERE tbl_name LIKE '"}, + {"redacted": True}, + {"value": "'"}, + ] + } + assert vulnerability["location"]["line"] == line + assert vulnerability["location"]["path"] == TEST_FILE_PATH + assert vulnerability["hash"] == hash_value + + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") + def test_flask_request_body_complex_json_all_types_of_values(self): + @self.app.route("/sqli/body/", methods=("POST",)) + def sqli_11(): + import sqlite3 + + from flask import request + + from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect + + def iterate_json(data, parent_key=""): + if isinstance(data, dict): + for key, value in data.items(): + iterate_json(value, key) + elif isinstance(data, list): + for index, item in enumerate(data): + iterate_json(item, parent_key) + else: + assert is_pyobject_tainted(parent_key), f"{parent_key} taint error" + if isinstance(data, str): + assert is_pyobject_tainted(data), f"{parent_key}.{data} taint error" + else: + assert not is_pyobject_tainted(data), f"{parent_key}.{data} taint error" + + if flask_version > (2, 0): + request_json = request.json + else: + request_json = json.loads(request.data) + + iterate_json(request_json) + + con = sqlite3.connect(":memory:") + cur = con.cursor() + + value = request_json.get("user").get("profile").get("preferences").get("extra") + assert value == "master" + assert is_pyobject_tainted(value) + query = add_aspect(add_aspect("SELECT tbl_name FROM sqlite_", value), " WHERE tbl_name LIKE 'password'") + # label test_flask_request_body_complex_json_all_types_of_values + cur.execute(query) + + return "OK", 200 + + with override_global_config( + dict( + _iast_enabled=True, + ) + ): + # random json with all kind of types + json_data = { + "user": { + "id": 12345, + "name": "John Doe", + "email": "johndoe@example.com", + "profile": { + "age": 30, + "gender": "male", + "preferences": { + "language": "English", + "timezone": "GMT+0", + "notifications": True, + "theme": "dark", + "extra": "master", + }, + "social_links": ["https://twitter.com/johndoe", "https://github.com/johndoe"], + }, + }, + "settings": { + "volume": 80, + "brightness": 50, + "wifi": { + "enabled": True, + "networks": [ + {"ssid": "HomeNetwork", "signal_strength": -40, "secured": True}, + {"ssid": "WorkNetwork", "signal_strength": -60, "secured": False}, + ], + }, + }, + "tasks": [ + {"task_id": 1, "title": "Finish project report", "due_date": "2024-08-25", "completed": False}, + { + "task_id": 2, + "title": "Buy groceries", + "due_date": "2024-08-23", + "completed": True, + "items": ["milk", "bread", "eggs"], + }, + ], + "random_values": [ + 42, + "randomString", + True, + None, + [3.14, 2.71, 1.618], + {"nested_key": "nestedValue", "nested_number": 999, "nested_array": [1, "two", None]}, + ], + "system": { + "os": "Linux", + "version": "5.10", + "uptime": 1234567, + "processes": {"running": 345, "sleeping": 56, "stopped": 2}, + }, + } + + resp = self.client.post( + "/sqli/body/", + data=json.dumps(json_data), + content_type="application/json", + ) + assert resp.status_code == 200 + + root_span = self.pop_spans()[0] + assert root_span.get_metric(IAST.ENABLED) == 1.0 + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + assert loaded["sources"] == [{"name": "extra", "origin": "http.request.body", "value": "master"}] + + line, hash_value = get_line_and_hash( + "test_flask_request_body_complex_json_all_types_of_values", + VULN_SQL_INJECTION, + filename=TEST_FILE_PATH, + ) + vulnerability = loaded["vulnerabilities"][0] + assert vulnerability["type"] == VULN_SQL_INJECTION + assert vulnerability["evidence"] == { + "valueParts": [ + {"value": "SELECT tbl_name FROM sqlite_"}, + {"value": "master", "source": 0}, + {"value": " WHERE tbl_name LIKE '"}, + {"redacted": True}, + {"value": "'"}, + ] + } + assert vulnerability["location"]["line"] == line + assert vulnerability["location"]["path"] == TEST_FILE_PATH + assert vulnerability["hash"] == hash_value + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") def test_flask_full_sqli_iast_enabled_http_request_header_values_scrubbed(self): @self.app.route("/sqli//", methods=["GET", "POST"]) - def sqli_10(param_str): + def sqli_12(param_str): import sqlite3 from flask import request From 32824935195eafbe2067ccd07047458e9fc77bfd Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Fri, 30 Aug 2024 08:41:33 -0700 Subject: [PATCH 11/17] chore(integrations): deprecate public patch.py modules (#10236) - Deprecates all modules that match the following format: `ddtrace/contrib/{integration_name}/patch.py` - Ensures deprecation warnings are not logged when integrations are patched - ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Rachel Yang --- ddtrace/contrib/aiobotocore/__init__.py | 8 ++++++-- ddtrace/contrib/aiobotocore/patch.py | 13 ++++++++++++- ddtrace/contrib/aiohttp/__init__.py | 6 +++++- ddtrace/contrib/aiohttp/patch.py | 13 ++++++++++++- ddtrace/contrib/aiohttp_jinja2/__init__.py | 6 +++++- ddtrace/contrib/aiohttp_jinja2/patch.py | 13 ++++++++++++- ddtrace/contrib/aiomysql/__init__.py | 6 +++++- ddtrace/contrib/aiomysql/patch.py | 13 ++++++++++++- ddtrace/contrib/aiopg/__init__.py | 6 +++++- ddtrace/contrib/aiopg/patch.py | 13 ++++++++++++- ddtrace/contrib/aioredis/__init__.py | 6 +++++- ddtrace/contrib/aioredis/patch.py | 14 +++++++++++++- ddtrace/contrib/algoliasearch/__init__.py | 6 +++++- ddtrace/contrib/algoliasearch/patch.py | 13 ++++++++++++- ddtrace/contrib/anthropic/__init__.py | 8 ++++++-- ddtrace/contrib/anthropic/patch.py | 13 ++++++++++++- ddtrace/contrib/aredis/__init__.py | 6 +++++- ddtrace/contrib/aredis/patch.py | 13 ++++++++++++- ddtrace/contrib/asyncio/__init__.py | 6 +++++- ddtrace/contrib/asyncio/patch.py | 13 ++++++++++++- ddtrace/contrib/asyncpg/__init__.py | 6 +++++- ddtrace/contrib/asyncpg/patch.py | 13 ++++++++++++- ddtrace/contrib/aws_lambda/patch.py | 13 ++++++++++++- ddtrace/contrib/boto/__init__.py | 6 +++++- ddtrace/contrib/boto/patch.py | 13 ++++++++++++- ddtrace/contrib/botocore/__init__.py | 6 +++++- ddtrace/contrib/botocore/patch.py | 13 ++++++++++++- ddtrace/contrib/bottle/__init__.py | 6 +++++- ddtrace/contrib/bottle/patch.py | 13 ++++++++++++- ddtrace/contrib/django/__init__.py | 8 ++++++-- ddtrace/contrib/django/patch.py | 13 ++++++++++++- ddtrace/contrib/dogpile_cache/__init__.py | 6 +++++- ddtrace/contrib/dogpile_cache/patch.py | 13 ++++++++++++- ddtrace/contrib/dramatiq/__init__.py | 6 +++++- ddtrace/contrib/dramatiq/patch.py | 13 ++++++++++++- ddtrace/contrib/elasticsearch/patch.py | 13 ++++++++++++- ddtrace/contrib/falcon/__init__.py | 6 +++++- ddtrace/contrib/falcon/patch.py | 13 ++++++++++++- ddtrace/contrib/fastapi/__init__.py | 6 +++++- ddtrace/contrib/fastapi/patch.py | 13 ++++++++++++- ddtrace/contrib/flask/__init__.py | 8 ++++++-- ddtrace/contrib/flask/patch.py | 13 ++++++++++++- ddtrace/contrib/futures/__init__.py | 6 +++++- ddtrace/contrib/futures/patch.py | 13 ++++++++++++- ddtrace/contrib/gevent/__init__.py | 5 ++++- ddtrace/contrib/gevent/patch.py | 13 ++++++++++++- ddtrace/contrib/graphql/__init__.py | 6 +++++- ddtrace/contrib/graphql/patch.py | 13 ++++++++++++- ddtrace/contrib/grpc/__init__.py | 6 +++++- ddtrace/contrib/grpc/patch.py | 13 ++++++++++++- ddtrace/contrib/httplib/__init__.py | 6 +++++- ddtrace/contrib/httplib/patch.py | 13 ++++++++++++- ddtrace/contrib/httpx/__init__.py | 6 +++++- ddtrace/contrib/httpx/patch.py | 13 ++++++++++++- ddtrace/contrib/jinja2/__init__.py | 6 +++++- ddtrace/contrib/jinja2/patch.py | 13 ++++++++++++- ddtrace/contrib/kafka/__init__.py | 6 +++++- ddtrace/contrib/kafka/patch.py | 13 ++++++++++++- ddtrace/contrib/kombu/__init__.py | 6 +++++- ddtrace/contrib/kombu/patch.py | 13 ++++++++++++- ddtrace/contrib/langchain/__init__.py | 6 +++++- ddtrace/contrib/langchain/patch.py | 13 ++++++++++++- ddtrace/contrib/logbook/__init__.py | 6 +++++- ddtrace/contrib/logbook/patch.py | 13 ++++++++++++- ddtrace/contrib/logging/__init__.py | 6 +++++- ddtrace/contrib/logging/patch.py | 13 ++++++++++++- ddtrace/contrib/loguru/__init__.py | 6 +++++- ddtrace/contrib/loguru/patch.py | 13 ++++++++++++- ddtrace/contrib/mako/__init__.py | 6 +++++- ddtrace/contrib/mako/patch.py | 13 ++++++++++++- ddtrace/contrib/mariadb/__init__.py | 6 +++++- ddtrace/contrib/mariadb/patch.py | 13 ++++++++++++- ddtrace/contrib/molten/__init__.py | 8 ++++++-- ddtrace/contrib/molten/patch.py | 13 ++++++++++++- ddtrace/contrib/mongoengine/__init__.py | 6 +++++- ddtrace/contrib/mongoengine/patch.py | 13 ++++++++++++- ddtrace/contrib/mysql/__init__.py | 6 +++++- ddtrace/contrib/mysql/patch.py | 13 ++++++++++++- ddtrace/contrib/mysqldb/__init__.py | 5 ++++- ddtrace/contrib/mysqldb/patch.py | 13 ++++++++++++- ddtrace/contrib/openai/__init__.py | 8 ++++++-- ddtrace/contrib/openai/patch.py | 13 ++++++++++++- ddtrace/contrib/psycopg/__init__.py | 8 ++++++-- ddtrace/contrib/psycopg/patch.py | 13 ++++++++++++- ddtrace/contrib/pylibmc/__init__.py | 5 ++++- ddtrace/contrib/pylibmc/patch.py | 13 ++++++++++++- ddtrace/contrib/pymemcache/__init__.py | 5 ++++- ddtrace/contrib/pymemcache/patch.py | 13 ++++++++++++- ddtrace/contrib/pymongo/__init__.py | 6 +++++- ddtrace/contrib/pymongo/patch.py | 13 ++++++++++++- ddtrace/contrib/pymysql/__init__.py | 6 +++++- ddtrace/contrib/pymysql/patch.py | 13 ++++++++++++- ddtrace/contrib/pynamodb/__init__.py | 6 +++++- ddtrace/contrib/pynamodb/patch.py | 13 ++++++++++++- ddtrace/contrib/pyodbc/__init__.py | 6 +++++- ddtrace/contrib/pyodbc/patch.py | 13 ++++++++++++- ddtrace/contrib/pyramid/__init__.py | 7 +++++-- ddtrace/contrib/pyramid/patch.py | 13 ++++++++++++- ddtrace/contrib/redis/__init__.py | 6 +++++- ddtrace/contrib/redis/patch.py | 13 ++++++++++++- ddtrace/contrib/rediscluster/__init__.py | 6 +++++- ddtrace/contrib/rediscluster/patch.py | 13 ++++++++++++- ddtrace/contrib/requests/__init__.py | 6 +++++- ddtrace/contrib/requests/patch.py | 13 ++++++++++++- ddtrace/contrib/sanic/__init__.py | 6 +++++- ddtrace/contrib/sanic/patch.py | 13 ++++++++++++- ddtrace/contrib/snowflake/__init__.py | 6 +++++- ddtrace/contrib/snowflake/patch.py | 13 ++++++++++++- ddtrace/contrib/sqlalchemy/__init__.py | 6 +++++- ddtrace/contrib/sqlalchemy/patch.py | 13 ++++++++++++- ddtrace/contrib/sqlite3/__init__.py | 6 +++++- ddtrace/contrib/sqlite3/patch.py | 13 ++++++++++++- ddtrace/contrib/starlette/__init__.py | 6 +++++- ddtrace/contrib/starlette/patch.py | 13 ++++++++++++- ddtrace/contrib/structlog/__init__.py | 6 +++++- ddtrace/contrib/structlog/patch.py | 13 ++++++++++++- ddtrace/contrib/subprocess/__init__.py | 6 +++++- ddtrace/contrib/subprocess/patch.py | 13 ++++++++++++- ddtrace/contrib/tornado/__init__.py | 14 ++++++++------ ddtrace/contrib/tornado/patch.py | 13 ++++++++++++- ddtrace/contrib/urllib/__init__.py | 6 +++++- ddtrace/contrib/urllib/patch.py | 13 ++++++++++++- ddtrace/contrib/urllib3/__init__.py | 6 +++++- ddtrace/contrib/urllib3/patch.py | 13 ++++++++++++- ddtrace/contrib/vertica/__init__.py | 6 +++++- ddtrace/contrib/vertica/patch.py | 13 ++++++++++++- ddtrace/contrib/webbrowser/__init__.py | 6 +++++- ddtrace/contrib/webbrowser/patch.py | 13 ++++++++++++- ddtrace/contrib/wsgi/wsgi.py | 13 ++++++++++++- ddtrace/contrib/yaaredis/__init__.py | 6 +++++- ddtrace/contrib/yaaredis/patch.py | 13 ++++++++++++- ...cate-public-patch-modules-108e06143646a735.yaml | 4 ++++ 132 files changed, 1135 insertions(+), 144 deletions(-) create mode 100644 releasenotes/notes/deprecate-public-patch-modules-108e06143646a735.yaml diff --git a/ddtrace/contrib/aiobotocore/__init__.py b/ddtrace/contrib/aiobotocore/__init__.py index ae26adacad9..c9568e34a14 100644 --- a/ddtrace/contrib/aiobotocore/__init__.py +++ b/ddtrace/contrib/aiobotocore/__init__.py @@ -32,8 +32,12 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: - # Required to allow users to import from `ddtrace.contrib.aiohttp.patch` directly - from . import patch as _ # noqa: F401, I001 + # Required to allow users to import from `ddtrace.contrib.aiobotocore.patch` directly + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.aiobotocore.patch import get_version diff --git a/ddtrace/contrib/aiobotocore/patch.py b/ddtrace/contrib/aiobotocore/patch.py index fee3f057ccd..ca0cf946acb 100644 --- a/ddtrace/contrib/aiobotocore/patch.py +++ b/ddtrace/contrib/aiobotocore/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.aiobotocore.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/aiohttp/__init__.py b/ddtrace/contrib/aiohttp/__init__.py index e6edf1d8ebd..0d7b5c75672 100644 --- a/ddtrace/contrib/aiohttp/__init__.py +++ b/ddtrace/contrib/aiohttp/__init__.py @@ -92,7 +92,11 @@ async def home_handler(request): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.aiohttp.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.aiohttp.middlewares import trace_app from ..internal.aiohttp.patch import get_version diff --git a/ddtrace/contrib/aiohttp/patch.py b/ddtrace/contrib/aiohttp/patch.py index 862d1d10ff2..26b3f433aac 100644 --- a/ddtrace/contrib/aiohttp/patch.py +++ b/ddtrace/contrib/aiohttp/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.aiohttp.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/aiohttp_jinja2/__init__.py b/ddtrace/contrib/aiohttp_jinja2/__init__.py index c00c55acc36..b6fa72b11a9 100644 --- a/ddtrace/contrib/aiohttp_jinja2/__init__.py +++ b/ddtrace/contrib/aiohttp_jinja2/__init__.py @@ -21,7 +21,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.aiohttp.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.aiohttp_jinja2.patch import get_version from ..internal.aiohttp_jinja2.patch import patch diff --git a/ddtrace/contrib/aiohttp_jinja2/patch.py b/ddtrace/contrib/aiohttp_jinja2/patch.py index 23bd01c4bed..29a0721b724 100644 --- a/ddtrace/contrib/aiohttp_jinja2/patch.py +++ b/ddtrace/contrib/aiohttp_jinja2/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.aiohttp_jinja2.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/aiomysql/__init__.py b/ddtrace/contrib/aiomysql/__init__.py index 6e68c2c72d6..3e938a74e3e 100644 --- a/ddtrace/contrib/aiomysql/__init__.py +++ b/ddtrace/contrib/aiomysql/__init__.py @@ -44,7 +44,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.aiohttp.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.aiomysql.patch import get_version from ..internal.aiomysql.patch import patch diff --git a/ddtrace/contrib/aiomysql/patch.py b/ddtrace/contrib/aiomysql/patch.py index 6871852b8f6..161263f2e5f 100644 --- a/ddtrace/contrib/aiomysql/patch.py +++ b/ddtrace/contrib/aiomysql/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.aiomysql.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/aiopg/__init__.py b/ddtrace/contrib/aiopg/__init__.py index 0b3875d7016..859d507f029 100644 --- a/ddtrace/contrib/aiopg/__init__.py +++ b/ddtrace/contrib/aiopg/__init__.py @@ -23,7 +23,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.aiohttp.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.aiopg.patch import get_version from ..internal.aiopg.patch import patch diff --git a/ddtrace/contrib/aiopg/patch.py b/ddtrace/contrib/aiopg/patch.py index c1302be21c1..05a6784604c 100644 --- a/ddtrace/contrib/aiopg/patch.py +++ b/ddtrace/contrib/aiopg/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.aiopg.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/aioredis/__init__.py b/ddtrace/contrib/aioredis/__init__.py index ca5238c3ad4..f23536e8cb2 100644 --- a/ddtrace/contrib/aioredis/__init__.py +++ b/ddtrace/contrib/aioredis/__init__.py @@ -77,7 +77,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.aioredis.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.aioredis.patch import get_version from ..internal.aioredis.patch import patch diff --git a/ddtrace/contrib/aioredis/patch.py b/ddtrace/contrib/aioredis/patch.py index ea824d2b8e0..1f9c6256571 100644 --- a/ddtrace/contrib/aioredis/patch.py +++ b/ddtrace/contrib/aioredis/patch.py @@ -1,3 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.aioredis.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module + +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/algoliasearch/__init__.py b/ddtrace/contrib/algoliasearch/__init__.py index 3d5abac0248..1ca59f4800c 100644 --- a/ddtrace/contrib/algoliasearch/__init__.py +++ b/ddtrace/contrib/algoliasearch/__init__.py @@ -30,7 +30,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.algoliasearch.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.algoliasearch.patch import get_version from ..internal.algoliasearch.patch import patch diff --git a/ddtrace/contrib/algoliasearch/patch.py b/ddtrace/contrib/algoliasearch/patch.py index f6ddddcffdf..715de26e60e 100644 --- a/ddtrace/contrib/algoliasearch/patch.py +++ b/ddtrace/contrib/algoliasearch/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.algoliasearch.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/anthropic/__init__.py b/ddtrace/contrib/anthropic/__init__.py index 9303f74a5ba..53a33482a24 100644 --- a/ddtrace/contrib/anthropic/__init__.py +++ b/ddtrace/contrib/anthropic/__init__.py @@ -88,10 +88,14 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.anthropic.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 + + from ..internal.anthropic.patch import get_version from ..internal.anthropic.patch import patch from ..internal.anthropic.patch import unpatch - from ..internal.anthropic.patch import get_version __all__ = ["patch", "unpatch", "get_version"] diff --git a/ddtrace/contrib/anthropic/patch.py b/ddtrace/contrib/anthropic/patch.py index 6944820f065..69a57222e1e 100644 --- a/ddtrace/contrib/anthropic/patch.py +++ b/ddtrace/contrib/anthropic/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.anthropic.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/aredis/__init__.py b/ddtrace/contrib/aredis/__init__.py index e9b3765c5df..850f056a2ba 100644 --- a/ddtrace/contrib/aredis/__init__.py +++ b/ddtrace/contrib/aredis/__init__.py @@ -74,7 +74,11 @@ async def example(): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.aredis.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.aredis.patch import get_version from ..internal.aredis.patch import patch diff --git a/ddtrace/contrib/aredis/patch.py b/ddtrace/contrib/aredis/patch.py index 98fe97ef393..19dc1bce0e8 100644 --- a/ddtrace/contrib/aredis/patch.py +++ b/ddtrace/contrib/aredis/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.aredis.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/asyncio/__init__.py b/ddtrace/contrib/asyncio/__init__.py index 6568e23f7a9..8cc9b47b536 100644 --- a/ddtrace/contrib/asyncio/__init__.py +++ b/ddtrace/contrib/asyncio/__init__.py @@ -15,7 +15,11 @@ # Required to allow users to import from `ddtrace.contrib.asyncio.patch` directly # Expose public methods - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.asyncio.helpers import ensure_future from ..internal.asyncio.helpers import run_in_executor from ..internal.asyncio.helpers import set_call_context diff --git a/ddtrace/contrib/asyncio/patch.py b/ddtrace/contrib/asyncio/patch.py index 26bfa5aa050..e6860d9604a 100644 --- a/ddtrace/contrib/asyncio/patch.py +++ b/ddtrace/contrib/asyncio/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.asyncio.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/asyncpg/__init__.py b/ddtrace/contrib/asyncpg/__init__.py index e29bb2bb8ff..13eb0046840 100644 --- a/ddtrace/contrib/asyncpg/__init__.py +++ b/ddtrace/contrib/asyncpg/__init__.py @@ -51,7 +51,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.asyncpg.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.asyncpg.patch import get_version from ..internal.asyncpg.patch import patch diff --git a/ddtrace/contrib/asyncpg/patch.py b/ddtrace/contrib/asyncpg/patch.py index c76d4d00aeb..16d1a2acf9c 100644 --- a/ddtrace/contrib/asyncpg/patch.py +++ b/ddtrace/contrib/asyncpg/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.asyncpg.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/aws_lambda/patch.py b/ddtrace/contrib/aws_lambda/patch.py index fe5128e6265..d527b777cd4 100644 --- a/ddtrace/contrib/aws_lambda/patch.py +++ b/ddtrace/contrib/aws_lambda/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.aws_lambda.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/boto/__init__.py b/ddtrace/contrib/boto/__init__.py index 7cb04345346..6b853c27853 100644 --- a/ddtrace/contrib/boto/__init__.py +++ b/ddtrace/contrib/boto/__init__.py @@ -36,7 +36,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.boto.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.boto.patch import get_version from ..internal.boto.patch import patch diff --git a/ddtrace/contrib/boto/patch.py b/ddtrace/contrib/boto/patch.py index 83711f238f3..b750103f155 100644 --- a/ddtrace/contrib/boto/patch.py +++ b/ddtrace/contrib/boto/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.boto.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/botocore/__init__.py b/ddtrace/contrib/botocore/__init__.py index c58ae524e89..94804407b4f 100644 --- a/ddtrace/contrib/botocore/__init__.py +++ b/ddtrace/contrib/botocore/__init__.py @@ -119,7 +119,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.botocore.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.botocore.patch import get_version from ..internal.botocore.patch import patch diff --git a/ddtrace/contrib/botocore/patch.py b/ddtrace/contrib/botocore/patch.py index f5527eb45be..d0ae736b683 100644 --- a/ddtrace/contrib/botocore/patch.py +++ b/ddtrace/contrib/botocore/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.botocore.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/bottle/__init__.py b/ddtrace/contrib/bottle/__init__.py index 2e40b75d41f..477ac9741ff 100644 --- a/ddtrace/contrib/bottle/__init__.py +++ b/ddtrace/contrib/bottle/__init__.py @@ -41,7 +41,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.bottle.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.bottle.patch import get_version from ..internal.bottle.patch import patch diff --git a/ddtrace/contrib/bottle/patch.py b/ddtrace/contrib/bottle/patch.py index 32f89abf54f..69428d3f2c5 100644 --- a/ddtrace/contrib/bottle/patch.py +++ b/ddtrace/contrib/bottle/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.bottle.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/django/__init__.py b/ddtrace/contrib/django/__init__.py index e206e4f62cc..b561babce17 100644 --- a/ddtrace/contrib/django/__init__.py +++ b/ddtrace/contrib/django/__init__.py @@ -185,12 +185,16 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.django.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods - from ..internal.django.patch import patch as _patch from ..internal.django.patch import get_version from ..internal.django.patch import patch + from ..internal.django.patch import patch as _patch from ..internal.django.patch import unpatch __all__ = ["patch", "unpatch", "_patch", "get_version"] diff --git a/ddtrace/contrib/django/patch.py b/ddtrace/contrib/django/patch.py index f98d22e82e4..9ac6fc6ce48 100644 --- a/ddtrace/contrib/django/patch.py +++ b/ddtrace/contrib/django/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.django.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/dogpile_cache/__init__.py b/ddtrace/contrib/dogpile_cache/__init__.py index c55e86d1e20..be09ac1de6c 100644 --- a/ddtrace/contrib/dogpile_cache/__init__.py +++ b/ddtrace/contrib/dogpile_cache/__init__.py @@ -44,7 +44,11 @@ def hello(name): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.dogpile_cache.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.dogpile_cache.patch import get_version diff --git a/ddtrace/contrib/dogpile_cache/patch.py b/ddtrace/contrib/dogpile_cache/patch.py index 3b318db1125..efe974592c2 100644 --- a/ddtrace/contrib/dogpile_cache/patch.py +++ b/ddtrace/contrib/dogpile_cache/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.dogpile_cache.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/dramatiq/__init__.py b/ddtrace/contrib/dramatiq/__init__.py index bb48ef86eb5..91f70275659 100644 --- a/ddtrace/contrib/dramatiq/__init__.py +++ b/ddtrace/contrib/dramatiq/__init__.py @@ -36,7 +36,11 @@ def my_other_task(content): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.dramatiq.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.dramatiq.patch import get_version diff --git a/ddtrace/contrib/dramatiq/patch.py b/ddtrace/contrib/dramatiq/patch.py index 97af21978d1..850726770b9 100644 --- a/ddtrace/contrib/dramatiq/patch.py +++ b/ddtrace/contrib/dramatiq/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.dramatiq.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/elasticsearch/patch.py b/ddtrace/contrib/elasticsearch/patch.py index 021518fc655..4957d2580e1 100644 --- a/ddtrace/contrib/elasticsearch/patch.py +++ b/ddtrace/contrib/elasticsearch/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.elasticsearch.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/falcon/__init__.py b/ddtrace/contrib/falcon/__init__.py index 363acf9edd2..653b9afa1f7 100644 --- a/ddtrace/contrib/falcon/__init__.py +++ b/ddtrace/contrib/falcon/__init__.py @@ -52,7 +52,11 @@ def on_falcon_request(span, request, response): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.falcon.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.falcon.middleware import TraceMiddleware from ..internal.falcon.patch import get_version from ..internal.falcon.patch import patch diff --git a/ddtrace/contrib/falcon/patch.py b/ddtrace/contrib/falcon/patch.py index 13ec3f93779..c07df81068e 100644 --- a/ddtrace/contrib/falcon/patch.py +++ b/ddtrace/contrib/falcon/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.falcon.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/fastapi/__init__.py b/ddtrace/contrib/fastapi/__init__.py index 12e27ccd5cb..e4bfe791401 100644 --- a/ddtrace/contrib/fastapi/__init__.py +++ b/ddtrace/contrib/fastapi/__init__.py @@ -58,7 +58,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.fastapi.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.fastapi.patch import get_version diff --git a/ddtrace/contrib/fastapi/patch.py b/ddtrace/contrib/fastapi/patch.py index 06c723f0939..3b645e3558e 100644 --- a/ddtrace/contrib/fastapi/patch.py +++ b/ddtrace/contrib/fastapi/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.fastapi.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/flask/__init__.py b/ddtrace/contrib/flask/__init__.py index e161bac99b7..cc8188c125d 100644 --- a/ddtrace/contrib/flask/__init__.py +++ b/ddtrace/contrib/flask/__init__.py @@ -104,9 +104,13 @@ def index(): with require_modules(required_modules) as missing_modules: if not missing_modules: # DEV: We do this so we can `@mock.patch('ddtrace.contrib.flask._patch.')` in tests - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 + from ..internal.flask.patch import get_version from ..internal.flask.patch import patch from ..internal.flask.patch import unpatch - from ..internal.flask.patch import get_version __all__ = ["patch", "unpatch", "get_version"] diff --git a/ddtrace/contrib/flask/patch.py b/ddtrace/contrib/flask/patch.py index 98bf00b2be1..7ffc3edd2ee 100644 --- a/ddtrace/contrib/flask/patch.py +++ b/ddtrace/contrib/flask/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.flask.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/futures/__init__.py b/ddtrace/contrib/futures/__init__.py index 686b400c49f..33f2f922187 100644 --- a/ddtrace/contrib/futures/__init__.py +++ b/ddtrace/contrib/futures/__init__.py @@ -24,7 +24,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.futures.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.futures.patch import get_version diff --git a/ddtrace/contrib/futures/patch.py b/ddtrace/contrib/futures/patch.py index 5bae0b6b648..27e97215842 100644 --- a/ddtrace/contrib/futures/patch.py +++ b/ddtrace/contrib/futures/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.futures.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/gevent/__init__.py b/ddtrace/contrib/gevent/__init__.py index 47b8a74d7a3..01a3cc0c0a2 100644 --- a/ddtrace/contrib/gevent/__init__.py +++ b/ddtrace/contrib/gevent/__init__.py @@ -44,7 +44,10 @@ def worker_function(): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.gevent.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ...provider import DefaultContextProvider as _DefaultContextProvider diff --git a/ddtrace/contrib/gevent/patch.py b/ddtrace/contrib/gevent/patch.py index 15ed2220893..488ea978f85 100644 --- a/ddtrace/contrib/gevent/patch.py +++ b/ddtrace/contrib/gevent/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.gevent.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/graphql/__init__.py b/ddtrace/contrib/graphql/__init__.py index 4fe4a6f1bf9..336e8a1520a 100644 --- a/ddtrace/contrib/graphql/__init__.py +++ b/ddtrace/contrib/graphql/__init__.py @@ -52,7 +52,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.graphql.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.graphql.patch import get_version diff --git a/ddtrace/contrib/graphql/patch.py b/ddtrace/contrib/graphql/patch.py index b9e35638706..d97814868a8 100644 --- a/ddtrace/contrib/graphql/patch.py +++ b/ddtrace/contrib/graphql/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.graphql.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/grpc/__init__.py b/ddtrace/contrib/grpc/__init__.py index 8b61b7b456f..8edfbf59846 100644 --- a/ddtrace/contrib/grpc/__init__.py +++ b/ddtrace/contrib/grpc/__init__.py @@ -83,7 +83,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.grpc.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.grpc.patch import get_version diff --git a/ddtrace/contrib/grpc/patch.py b/ddtrace/contrib/grpc/patch.py index 1886b947b6f..68286bd33be 100644 --- a/ddtrace/contrib/grpc/patch.py +++ b/ddtrace/contrib/grpc/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.grpc.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/httplib/__init__.py b/ddtrace/contrib/httplib/__init__.py index f935befd7b3..a79791785e5 100644 --- a/ddtrace/contrib/httplib/__init__.py +++ b/ddtrace/contrib/httplib/__init__.py @@ -65,7 +65,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 from ..internal.httplib.patch import get_version from ..internal.httplib.patch import patch diff --git a/ddtrace/contrib/httplib/patch.py b/ddtrace/contrib/httplib/patch.py index a86043c05f4..ef1f9c8d7f0 100644 --- a/ddtrace/contrib/httplib/patch.py +++ b/ddtrace/contrib/httplib/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.httplib.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/httpx/__init__.py b/ddtrace/contrib/httpx/__init__.py index b9116afb861..5acf7d9a3b6 100644 --- a/ddtrace/contrib/httpx/__init__.py +++ b/ddtrace/contrib/httpx/__init__.py @@ -85,7 +85,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.httpx.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.httpx.patch import get_version diff --git a/ddtrace/contrib/httpx/patch.py b/ddtrace/contrib/httpx/patch.py index 3fd6483a186..59122d66148 100644 --- a/ddtrace/contrib/httpx/patch.py +++ b/ddtrace/contrib/httpx/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.httpx.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/jinja2/__init__.py b/ddtrace/contrib/jinja2/__init__.py index 1b0959b29ef..512dbff81c0 100644 --- a/ddtrace/contrib/jinja2/__init__.py +++ b/ddtrace/contrib/jinja2/__init__.py @@ -35,7 +35,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.jinja2.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.jinja2.patch import get_version diff --git a/ddtrace/contrib/jinja2/patch.py b/ddtrace/contrib/jinja2/patch.py index d81b3d67778..ccedc3ee229 100644 --- a/ddtrace/contrib/jinja2/patch.py +++ b/ddtrace/contrib/jinja2/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.jinja2.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/kafka/__init__.py b/ddtrace/contrib/kafka/__init__.py index 05359f5c11d..1485226cddd 100644 --- a/ddtrace/contrib/kafka/__init__.py +++ b/ddtrace/contrib/kafka/__init__.py @@ -49,7 +49,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.kafka.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.kafka.patch import get_version diff --git a/ddtrace/contrib/kafka/patch.py b/ddtrace/contrib/kafka/patch.py index 6470048b748..8f43f6e0aba 100644 --- a/ddtrace/contrib/kafka/patch.py +++ b/ddtrace/contrib/kafka/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.kafka.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/kombu/__init__.py b/ddtrace/contrib/kombu/__init__.py index c93edc54141..de7ad71c76d 100644 --- a/ddtrace/contrib/kombu/__init__.py +++ b/ddtrace/contrib/kombu/__init__.py @@ -40,7 +40,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.kombu.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.kombu.patch import get_version diff --git a/ddtrace/contrib/kombu/patch.py b/ddtrace/contrib/kombu/patch.py index 2324e765666..faf5e07a54b 100644 --- a/ddtrace/contrib/kombu/patch.py +++ b/ddtrace/contrib/kombu/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.kombu.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/langchain/__init__.py b/ddtrace/contrib/langchain/__init__.py index 5fd0e7b2345..a336a12f677 100644 --- a/ddtrace/contrib/langchain/__init__.py +++ b/ddtrace/contrib/langchain/__init__.py @@ -215,7 +215,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.langchain.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.langchain.patch import get_version diff --git a/ddtrace/contrib/langchain/patch.py b/ddtrace/contrib/langchain/patch.py index 6cb2ab60916..f4c01c7f749 100644 --- a/ddtrace/contrib/langchain/patch.py +++ b/ddtrace/contrib/langchain/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.langchain.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/logbook/__init__.py b/ddtrace/contrib/logbook/__init__.py index 01133b441ca..76ff598a48a 100644 --- a/ddtrace/contrib/logbook/__init__.py +++ b/ddtrace/contrib/logbook/__init__.py @@ -56,7 +56,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.logbook.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.logbook.patch import get_version diff --git a/ddtrace/contrib/logbook/patch.py b/ddtrace/contrib/logbook/patch.py index e8c5e0557d1..2edfe480df9 100644 --- a/ddtrace/contrib/logbook/patch.py +++ b/ddtrace/contrib/logbook/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.logbook.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/logging/__init__.py b/ddtrace/contrib/logging/__init__.py index c95adc8c260..452b77f419d 100644 --- a/ddtrace/contrib/logging/__init__.py +++ b/ddtrace/contrib/logging/__init__.py @@ -69,7 +69,11 @@ def hello(): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.logging.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.logging.patch import get_version diff --git a/ddtrace/contrib/logging/patch.py b/ddtrace/contrib/logging/patch.py index a4dcbf5a462..578af8238e1 100644 --- a/ddtrace/contrib/logging/patch.py +++ b/ddtrace/contrib/logging/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.logging.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/loguru/__init__.py b/ddtrace/contrib/loguru/__init__.py index c39672c1bc9..12b3f014612 100644 --- a/ddtrace/contrib/loguru/__init__.py +++ b/ddtrace/contrib/loguru/__init__.py @@ -71,7 +71,11 @@ def log_format(record): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.loguru.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.loguru.patch import get_version diff --git a/ddtrace/contrib/loguru/patch.py b/ddtrace/contrib/loguru/patch.py index 6e8aac5d956..54b7bad1948 100644 --- a/ddtrace/contrib/loguru/patch.py +++ b/ddtrace/contrib/loguru/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.loguru.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/mako/__init__.py b/ddtrace/contrib/mako/__init__.py index 8ebc3f8a40b..9a2b24d3ede 100644 --- a/ddtrace/contrib/mako/__init__.py +++ b/ddtrace/contrib/mako/__init__.py @@ -18,7 +18,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.mako.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.mako.patch import get_version diff --git a/ddtrace/contrib/mako/patch.py b/ddtrace/contrib/mako/patch.py index 45c6fde4b94..56e3cc421cc 100644 --- a/ddtrace/contrib/mako/patch.py +++ b/ddtrace/contrib/mako/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.mako.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/mariadb/__init__.py b/ddtrace/contrib/mariadb/__init__.py index bb0c0607a7c..da5fe3d94cc 100644 --- a/ddtrace/contrib/mariadb/__init__.py +++ b/ddtrace/contrib/mariadb/__init__.py @@ -60,7 +60,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.mariadb.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.mariadb.patch import get_version diff --git a/ddtrace/contrib/mariadb/patch.py b/ddtrace/contrib/mariadb/patch.py index 2d47e09ddcd..7f7952df965 100644 --- a/ddtrace/contrib/mariadb/patch.py +++ b/ddtrace/contrib/mariadb/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.mariadb.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/molten/__init__.py b/ddtrace/contrib/molten/__init__.py index 0e346e9ca09..82bb573f7dd 100644 --- a/ddtrace/contrib/molten/__init__.py +++ b/ddtrace/contrib/molten/__init__.py @@ -42,11 +42,15 @@ def hello(name: str, age: int) -> str: with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.molten.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods + from ..internal.molten.patch import get_version from ..internal.molten.patch import patch from ..internal.molten.patch import unpatch - from ..internal.molten.patch import get_version __all__ = ["patch", "unpatch", "get_version"] diff --git a/ddtrace/contrib/molten/patch.py b/ddtrace/contrib/molten/patch.py index 0f9e6b94084..f888fd6875b 100644 --- a/ddtrace/contrib/molten/patch.py +++ b/ddtrace/contrib/molten/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.molten.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/mongoengine/__init__.py b/ddtrace/contrib/mongoengine/__init__.py index b82066c7e7b..0a1317dff5b 100644 --- a/ddtrace/contrib/mongoengine/__init__.py +++ b/ddtrace/contrib/mongoengine/__init__.py @@ -25,7 +25,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.mongoengine.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.mongoengine.patch import get_version diff --git a/ddtrace/contrib/mongoengine/patch.py b/ddtrace/contrib/mongoengine/patch.py index 88352c5d654..e277253a52c 100644 --- a/ddtrace/contrib/mongoengine/patch.py +++ b/ddtrace/contrib/mongoengine/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.mongoengine.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/mysql/__init__.py b/ddtrace/contrib/mysql/__init__.py index 3841187bcb3..db2c9f2f92d 100644 --- a/ddtrace/contrib/mysql/__init__.py +++ b/ddtrace/contrib/mysql/__init__.py @@ -71,7 +71,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.mysql.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.mysql.patch import get_version diff --git a/ddtrace/contrib/mysql/patch.py b/ddtrace/contrib/mysql/patch.py index 7216e58bd5c..2412c3c1dcd 100644 --- a/ddtrace/contrib/mysql/patch.py +++ b/ddtrace/contrib/mysql/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.mysql.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/mysqldb/__init__.py b/ddtrace/contrib/mysqldb/__init__.py index b9bddf38f55..cc361b99d3d 100644 --- a/ddtrace/contrib/mysqldb/__init__.py +++ b/ddtrace/contrib/mysqldb/__init__.py @@ -83,8 +83,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.mysqldb.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.mysqldb.patch import get_version from ..internal.mysqldb.patch import patch diff --git a/ddtrace/contrib/mysqldb/patch.py b/ddtrace/contrib/mysqldb/patch.py index fedac7fc738..f4f23c3751f 100644 --- a/ddtrace/contrib/mysqldb/patch.py +++ b/ddtrace/contrib/mysqldb/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.mysqldb.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/openai/__init__.py b/ddtrace/contrib/openai/__init__.py index c0435ec81fd..5b44c38b0fc 100644 --- a/ddtrace/contrib/openai/__init__.py +++ b/ddtrace/contrib/openai/__init__.py @@ -254,11 +254,15 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.openai.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods + from ..internal.openai.patch import get_version from ..internal.openai.patch import patch from ..internal.openai.patch import unpatch - from ..internal.openai.patch import get_version __all__ = ["patch", "unpatch", "get_version"] diff --git a/ddtrace/contrib/openai/patch.py b/ddtrace/contrib/openai/patch.py index 89ea9d21adf..24b180ffbca 100644 --- a/ddtrace/contrib/openai/patch.py +++ b/ddtrace/contrib/openai/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.openai.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/psycopg/__init__.py b/ddtrace/contrib/psycopg/__init__.py index df1e1177bc3..c32d7e7324f 100644 --- a/ddtrace/contrib/psycopg/__init__.py +++ b/ddtrace/contrib/psycopg/__init__.py @@ -67,8 +67,12 @@ with require_modules(required_modules) as missing_modules: # If psycopg and/or psycopg2 is available, patch these modules if len(missing_modules) < len(required_modules): - # Required to allow users to import from `ddtrace.contrib.openai.patch` directly - from . import patch as _ # noqa: F401, I001 + # Required to allow users to import from `ddtrace.contrib.psycopg.patch` directly + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.psycopg.patch import get_version diff --git a/ddtrace/contrib/psycopg/patch.py b/ddtrace/contrib/psycopg/patch.py index 91676a9aff5..41996806387 100644 --- a/ddtrace/contrib/psycopg/patch.py +++ b/ddtrace/contrib/psycopg/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.psycopg.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pylibmc/__init__.py b/ddtrace/contrib/pylibmc/__init__.py index c54e8c0a690..245a72fc549 100644 --- a/ddtrace/contrib/pylibmc/__init__.py +++ b/ddtrace/contrib/pylibmc/__init__.py @@ -27,8 +27,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.pylibmc.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.pylibmc.client import TracedClient from ..internal.pylibmc.patch import get_version diff --git a/ddtrace/contrib/pylibmc/patch.py b/ddtrace/contrib/pylibmc/patch.py index b1934884365..6292e0a209d 100644 --- a/ddtrace/contrib/pylibmc/patch.py +++ b/ddtrace/contrib/pylibmc/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.pylibmc.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pymemcache/__init__.py b/ddtrace/contrib/pymemcache/__init__.py index 647fb092e63..8f479d06fd0 100644 --- a/ddtrace/contrib/pymemcache/__init__.py +++ b/ddtrace/contrib/pymemcache/__init__.py @@ -38,8 +38,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.pymemcache.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.pymemcache.patch import get_version from ..internal.pymemcache.patch import patch diff --git a/ddtrace/contrib/pymemcache/patch.py b/ddtrace/contrib/pymemcache/patch.py index 00fcf02a3c1..cedf0eb7926 100644 --- a/ddtrace/contrib/pymemcache/patch.py +++ b/ddtrace/contrib/pymemcache/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.pymemcache.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pymongo/__init__.py b/ddtrace/contrib/pymongo/__init__.py index 04f206a6e1b..11f1a457187 100644 --- a/ddtrace/contrib/pymongo/__init__.py +++ b/ddtrace/contrib/pymongo/__init__.py @@ -43,7 +43,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.pymongo.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.pymongo.patch import get_version diff --git a/ddtrace/contrib/pymongo/patch.py b/ddtrace/contrib/pymongo/patch.py index e0cf8589dd4..348b8b31cf5 100644 --- a/ddtrace/contrib/pymongo/patch.py +++ b/ddtrace/contrib/pymongo/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.pymongo.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pymysql/__init__.py b/ddtrace/contrib/pymysql/__init__.py index b1524937194..cab64f08243 100644 --- a/ddtrace/contrib/pymysql/__init__.py +++ b/ddtrace/contrib/pymysql/__init__.py @@ -63,7 +63,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.pymysql.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.pymysql.patch import get_version diff --git a/ddtrace/contrib/pymysql/patch.py b/ddtrace/contrib/pymysql/patch.py index 415d70e010e..67c2bfd7772 100644 --- a/ddtrace/contrib/pymysql/patch.py +++ b/ddtrace/contrib/pymysql/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.pymysql.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pynamodb/__init__.py b/ddtrace/contrib/pynamodb/__init__.py index a23a3791cda..2b3ee76357f 100644 --- a/ddtrace/contrib/pynamodb/__init__.py +++ b/ddtrace/contrib/pynamodb/__init__.py @@ -37,7 +37,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.pynamodb.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.pynamodb.patch import get_version diff --git a/ddtrace/contrib/pynamodb/patch.py b/ddtrace/contrib/pynamodb/patch.py index 0e2a0495a41..0adcb852074 100644 --- a/ddtrace/contrib/pynamodb/patch.py +++ b/ddtrace/contrib/pynamodb/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.pynamodb.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pyodbc/__init__.py b/ddtrace/contrib/pyodbc/__init__.py index 7d6ce31687c..802209e89ec 100644 --- a/ddtrace/contrib/pyodbc/__init__.py +++ b/ddtrace/contrib/pyodbc/__init__.py @@ -61,7 +61,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.pyodbc.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.pyodbc.patch import get_version diff --git a/ddtrace/contrib/pyodbc/patch.py b/ddtrace/contrib/pyodbc/patch.py index 65edf406d7e..bdb59cc3907 100644 --- a/ddtrace/contrib/pyodbc/patch.py +++ b/ddtrace/contrib/pyodbc/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.pyodbc.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pyramid/__init__.py b/ddtrace/contrib/pyramid/__init__.py index 19fe86ff0fa..7e5679a402e 100644 --- a/ddtrace/contrib/pyramid/__init__.py +++ b/ddtrace/contrib/pyramid/__init__.py @@ -48,12 +48,15 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.pyramid.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.pyramid.patch import get_version from ..internal.pyramid.patch import patch - from ..internal.pyramid.trace import includeme from ..internal.pyramid.trace import trace_pyramid from ..internal.pyramid.trace import trace_tween_factory diff --git a/ddtrace/contrib/pyramid/patch.py b/ddtrace/contrib/pyramid/patch.py index ce945065ceb..7bda9baade0 100644 --- a/ddtrace/contrib/pyramid/patch.py +++ b/ddtrace/contrib/pyramid/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.pyramid.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/redis/__init__.py b/ddtrace/contrib/redis/__init__.py index 2b84b577f09..ae1db4f55e3 100644 --- a/ddtrace/contrib/redis/__init__.py +++ b/ddtrace/contrib/redis/__init__.py @@ -75,7 +75,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.redis.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.redis.patch import get_version diff --git a/ddtrace/contrib/redis/patch.py b/ddtrace/contrib/redis/patch.py index cbb973e9b1a..842d7b51092 100644 --- a/ddtrace/contrib/redis/patch.py +++ b/ddtrace/contrib/redis/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.redis.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/rediscluster/__init__.py b/ddtrace/contrib/rediscluster/__init__.py index 440deaebf52..7e0ea38165b 100644 --- a/ddtrace/contrib/rediscluster/__init__.py +++ b/ddtrace/contrib/rediscluster/__init__.py @@ -56,7 +56,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.rediscluster.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.rediscluster.patch import get_version diff --git a/ddtrace/contrib/rediscluster/patch.py b/ddtrace/contrib/rediscluster/patch.py index ccb23deec77..687cbcc3d77 100644 --- a/ddtrace/contrib/rediscluster/patch.py +++ b/ddtrace/contrib/rediscluster/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.rediscluster.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/requests/__init__.py b/ddtrace/contrib/requests/__init__.py index 66d0228c490..e1fcabc74d6 100644 --- a/ddtrace/contrib/requests/__init__.py +++ b/ddtrace/contrib/requests/__init__.py @@ -80,7 +80,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.requests.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.requests.patch import get_version diff --git a/ddtrace/contrib/requests/patch.py b/ddtrace/contrib/requests/patch.py index a9609f3efa9..1dedabfd59b 100644 --- a/ddtrace/contrib/requests/patch.py +++ b/ddtrace/contrib/requests/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.requests.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/sanic/__init__.py b/ddtrace/contrib/sanic/__init__.py index a13c0f505b5..e2b363a7de3 100644 --- a/ddtrace/contrib/sanic/__init__.py +++ b/ddtrace/contrib/sanic/__init__.py @@ -63,7 +63,11 @@ def index(request): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.sanic.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.sanic.patch import get_version diff --git a/ddtrace/contrib/sanic/patch.py b/ddtrace/contrib/sanic/patch.py index c5f6727b30e..53fa1631e37 100644 --- a/ddtrace/contrib/sanic/patch.py +++ b/ddtrace/contrib/sanic/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.sanic.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/snowflake/__init__.py b/ddtrace/contrib/snowflake/__init__.py index 98f749e1669..cd99af6cb8b 100644 --- a/ddtrace/contrib/snowflake/__init__.py +++ b/ddtrace/contrib/snowflake/__init__.py @@ -66,7 +66,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.snowflake.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.snowflake.patch import get_version diff --git a/ddtrace/contrib/snowflake/patch.py b/ddtrace/contrib/snowflake/patch.py index 5e01f498601..1fc46913482 100644 --- a/ddtrace/contrib/snowflake/patch.py +++ b/ddtrace/contrib/snowflake/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.snowflake.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/sqlalchemy/__init__.py b/ddtrace/contrib/sqlalchemy/__init__.py index b89db6ebc3a..0a8fd499a70 100644 --- a/ddtrace/contrib/sqlalchemy/__init__.py +++ b/ddtrace/contrib/sqlalchemy/__init__.py @@ -27,7 +27,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.sqlalchemy.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.sqlalchemy.engine import trace_engine diff --git a/ddtrace/contrib/sqlalchemy/patch.py b/ddtrace/contrib/sqlalchemy/patch.py index 8cd9ad0afab..f0a4002742a 100644 --- a/ddtrace/contrib/sqlalchemy/patch.py +++ b/ddtrace/contrib/sqlalchemy/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.sqlalchemy.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/sqlite3/__init__.py b/ddtrace/contrib/sqlite3/__init__.py index 670ba3d4615..b6cdb3e6dff 100644 --- a/ddtrace/contrib/sqlite3/__init__.py +++ b/ddtrace/contrib/sqlite3/__init__.py @@ -61,7 +61,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.sqlite3.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.sqlite3.patch import get_version diff --git a/ddtrace/contrib/sqlite3/patch.py b/ddtrace/contrib/sqlite3/patch.py index a3d93e88442..50d664e0ed3 100644 --- a/ddtrace/contrib/sqlite3/patch.py +++ b/ddtrace/contrib/sqlite3/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.sqlite3.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/starlette/__init__.py b/ddtrace/contrib/starlette/__init__.py index ddacc814925..5cb0f752c83 100644 --- a/ddtrace/contrib/starlette/__init__.py +++ b/ddtrace/contrib/starlette/__init__.py @@ -73,7 +73,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.starlette.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.starlette.patch import get_version diff --git a/ddtrace/contrib/starlette/patch.py b/ddtrace/contrib/starlette/patch.py index 42604cba8ce..2b227c4842c 100644 --- a/ddtrace/contrib/starlette/patch.py +++ b/ddtrace/contrib/starlette/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.starlette.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/structlog/__init__.py b/ddtrace/contrib/structlog/__init__.py index 6ba234ce062..684d677f22f 100644 --- a/ddtrace/contrib/structlog/__init__.py +++ b/ddtrace/contrib/structlog/__init__.py @@ -46,7 +46,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.structlog.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.structlog.patch import get_version diff --git a/ddtrace/contrib/structlog/patch.py b/ddtrace/contrib/structlog/patch.py index 4dd4ef7c455..3e25f60e00e 100644 --- a/ddtrace/contrib/structlog/patch.py +++ b/ddtrace/contrib/structlog/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.structlog.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/subprocess/__init__.py b/ddtrace/contrib/subprocess/__init__.py index 193584711df..98cd17f9642 100644 --- a/ddtrace/contrib/subprocess/__init__.py +++ b/ddtrace/contrib/subprocess/__init__.py @@ -27,7 +27,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.subprocess.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.subprocess.patch import get_version diff --git a/ddtrace/contrib/subprocess/patch.py b/ddtrace/contrib/subprocess/patch.py index a73f89a48f5..8c19326891c 100644 --- a/ddtrace/contrib/subprocess/patch.py +++ b/ddtrace/contrib/subprocess/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.subprocess.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/tornado/__init__.py b/ddtrace/contrib/tornado/__init__.py index 30d792ccb30..becc295ac40 100644 --- a/ddtrace/contrib/tornado/__init__.py +++ b/ddtrace/contrib/tornado/__init__.py @@ -112,17 +112,19 @@ def log_exception(self, typ, value, tb): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.tornado.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w - # Expose public methods - from ..internal.tornado.stack_context import TracerStackContext - from ..internal.tornado.stack_context import run_with_trace_context - - from ..internal.tornado.stack_context import context_provider + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 + # Expose public methods from ..internal.tornado.patch import get_version from ..internal.tornado.patch import patch from ..internal.tornado.patch import unpatch + from ..internal.tornado.stack_context import TracerStackContext + from ..internal.tornado.stack_context import context_provider + from ..internal.tornado.stack_context import run_with_trace_context __all__ = [ "patch", diff --git a/ddtrace/contrib/tornado/patch.py b/ddtrace/contrib/tornado/patch.py index c1261424ab8..92bd97321af 100644 --- a/ddtrace/contrib/tornado/patch.py +++ b/ddtrace/contrib/tornado/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.tornado.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/urllib/__init__.py b/ddtrace/contrib/urllib/__init__.py index 0d222937d45..3463eb7f31c 100644 --- a/ddtrace/contrib/urllib/__init__.py +++ b/ddtrace/contrib/urllib/__init__.py @@ -12,7 +12,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.urllib.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.urllib.patch import get_version diff --git a/ddtrace/contrib/urllib/patch.py b/ddtrace/contrib/urllib/patch.py index a4ce20e28a4..2176af4b2c3 100644 --- a/ddtrace/contrib/urllib/patch.py +++ b/ddtrace/contrib/urllib/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.urllib.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/urllib3/__init__.py b/ddtrace/contrib/urllib3/__init__.py index ad4eeb66985..21a592ed172 100644 --- a/ddtrace/contrib/urllib3/__init__.py +++ b/ddtrace/contrib/urllib3/__init__.py @@ -58,7 +58,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.urllib3.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.urllib3.patch import get_version diff --git a/ddtrace/contrib/urllib3/patch.py b/ddtrace/contrib/urllib3/patch.py index 26ec682f955..df0f7e37880 100644 --- a/ddtrace/contrib/urllib3/patch.py +++ b/ddtrace/contrib/urllib3/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.urllib3.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/vertica/__init__.py b/ddtrace/contrib/vertica/__init__.py index 597cd5fdb9b..e10c3fdab45 100644 --- a/ddtrace/contrib/vertica/__init__.py +++ b/ddtrace/contrib/vertica/__init__.py @@ -47,7 +47,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.vertica.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.vertica.patch import get_version diff --git a/ddtrace/contrib/vertica/patch.py b/ddtrace/contrib/vertica/patch.py index 499cc62034f..7e946f4df66 100644 --- a/ddtrace/contrib/vertica/patch.py +++ b/ddtrace/contrib/vertica/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.vertica.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/webbrowser/__init__.py b/ddtrace/contrib/webbrowser/__init__.py index 7e770f0b6cd..2eda99bd657 100644 --- a/ddtrace/contrib/webbrowser/__init__.py +++ b/ddtrace/contrib/webbrowser/__init__.py @@ -12,7 +12,11 @@ with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.webbrowser.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.webbrowser.patch import get_version diff --git a/ddtrace/contrib/webbrowser/patch.py b/ddtrace/contrib/webbrowser/patch.py index 1783e7b7567..a7aae36e607 100644 --- a/ddtrace/contrib/webbrowser/patch.py +++ b/ddtrace/contrib/webbrowser/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.webbrowser.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/wsgi/wsgi.py b/ddtrace/contrib/wsgi/wsgi.py index 808e1573dc3..7cbd291d61a 100644 --- a/ddtrace/contrib/wsgi/wsgi.py +++ b/ddtrace/contrib/wsgi/wsgi.py @@ -1,5 +1,16 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.wsgi.wsgi import * # noqa: F401,F403 from ..internal.wsgi.wsgi import _DDWSGIMiddlewareBase # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/yaaredis/__init__.py b/ddtrace/contrib/yaaredis/__init__.py index b4e224ec5cc..681d478cba6 100644 --- a/ddtrace/contrib/yaaredis/__init__.py +++ b/ddtrace/contrib/yaaredis/__init__.py @@ -74,7 +74,11 @@ async def example(): with require_modules(required_modules) as missing_modules: if not missing_modules: # Required to allow users to import from `ddtrace.contrib.yaaredis.patch` directly - from . import patch as _ # noqa: F401, I001 + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 # Expose public methods from ..internal.yaaredis.patch import get_version diff --git a/ddtrace/contrib/yaaredis/patch.py b/ddtrace/contrib/yaaredis/patch.py index d3bceddd498..4d27c351890 100644 --- a/ddtrace/contrib/yaaredis/patch.py +++ b/ddtrace/contrib/yaaredis/patch.py @@ -1,4 +1,15 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + from ..internal.yaaredis.patch import * # noqa: F401,F403 -# TODO: deprecate and remove this module +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/releasenotes/notes/deprecate-public-patch-modules-108e06143646a735.yaml b/releasenotes/notes/deprecate-public-patch-modules-108e06143646a735.yaml new file mode 100644 index 00000000000..a5dd99668f7 --- /dev/null +++ b/releasenotes/notes/deprecate-public-patch-modules-108e06143646a735.yaml @@ -0,0 +1,4 @@ +--- +deprecations: + - | + All public patch modules are deprecated. The non-deprecated methods are included in the ``__all__`` attribute. \ No newline at end of file From 0a49f527b7b1db4a2faa350ed6cbe31254ebb7bd Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Fri, 30 Aug 2024 18:08:08 +0100 Subject: [PATCH 12/17] feat(er): capture manual exceptions (#10430) We add a `span.exception` event listener when Exception Replay is enabled to capture any exception information that is manually attached to a span. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/bootstrap/preload.py | 4 +- ddtrace/debugging/_debugger.py | 5 -- ddtrace/debugging/_exception/replay.py | 54 +++++++++++++------ ...apture-exception-api-86a0a00e0b412567.yaml | 5 ++ tests/debugging/exception/test_replay.py | 32 ++++++++++- tests/debugging/mocking.py | 13 ++--- 6 files changed, 81 insertions(+), 32 deletions(-) create mode 100644 releasenotes/notes/chore-er-capture-exception-api-86a0a00e0b412567.yaml diff --git a/ddtrace/bootstrap/preload.py b/ddtrace/bootstrap/preload.py index 24160402174..e16821dc756 100644 --- a/ddtrace/bootstrap/preload.py +++ b/ddtrace/bootstrap/preload.py @@ -72,9 +72,9 @@ def register_post_preload(func: t.Callable) -> None: DynamicInstrumentation.enable() if er_config.enabled: # Exception Replay - from ddtrace.debugging._exception.replay import SpanExceptionProcessor + from ddtrace.debugging._exception.replay import SpanExceptionHandler - SpanExceptionProcessor().register() + SpanExceptionHandler.enable() if config._runtime_metrics_enabled: RuntimeWorker.enable() diff --git a/ddtrace/debugging/_debugger.py b/ddtrace/debugging/_debugger.py index a93d2077cae..ba24bda900e 100644 --- a/ddtrace/debugging/_debugger.py +++ b/ddtrace/debugging/_debugger.py @@ -23,7 +23,6 @@ from ddtrace import config as ddconfig from ddtrace._trace.tracer import Tracer from ddtrace.debugging._config import di_config -from ddtrace.debugging._exception.replay import SpanExceptionProcessor from ddtrace.debugging._function.discovery import FunctionDiscovery from ddtrace.debugging._function.store import FullyNamedWrappedFunction from ddtrace.debugging._function.store import FunctionStore @@ -274,7 +273,6 @@ def __exit__( class Debugger(Service): _instance: Optional["Debugger"] = None _probe_meter = _probe_metrics.get_meter("probe") - _span_processor: Optional[SpanExceptionProcessor] = None __rc_adapter__ = ProbeRCAdapter __uploader__ = LogsIntakeUploaderV1 @@ -328,9 +326,6 @@ def disable(cls, join: bool = True) -> None: atexit.unregister(cls.disable) unregister_post_run_module_hook(cls._on_run_module) - if cls._instance._span_processor: - cls._instance._span_processor.unregister() - cls._instance.stop(join=join) cls._instance = None diff --git a/ddtrace/debugging/_exception/replay.py b/ddtrace/debugging/_exception/replay.py index b01a94f6c36..29536e166f9 100644 --- a/ddtrace/debugging/_exception/replay.py +++ b/ddtrace/debugging/_exception/replay.py @@ -2,14 +2,12 @@ from dataclasses import dataclass from itertools import count from pathlib import Path -import sys from threading import current_thread from types import FrameType from types import TracebackType import typing as t import uuid -from ddtrace._trace.processor import SpanProcessor from ddtrace._trace.span import Span from ddtrace.debugging._probe.model import LiteralTemplateSegment from ddtrace.debugging._probe.model import LogLineProbe @@ -17,11 +15,15 @@ from ddtrace.debugging._signal.snapshot import Snapshot from ddtrace.debugging._uploader import LogsIntakeUploaderV1 from ddtrace.debugging._uploader import UploaderProduct +from ddtrace.internal import core +from ddtrace.internal.logger import get_logger from ddtrace.internal.packages import is_user_code from ddtrace.internal.rate_limiter import BudgetRateLimiterWithJitter as RateLimiter from ddtrace.internal.rate_limiter import RateLimitExceeded +log = get_logger(__name__) + GLOBAL_RATE_LIMITER = RateLimiter( limit_rate=1, # one trace per second raise_on_exceed=False, @@ -141,20 +143,18 @@ def can_capture(span: Span) -> bool: raise ValueError(msg) -@dataclass -class SpanExceptionProcessor(SpanProcessor): +class SpanExceptionHandler: __uploader__ = LogsIntakeUploaderV1 - def on_span_start(self, span: Span) -> None: - pass + _instance: t.Optional["SpanExceptionHandler"] = None - def on_span_finish(self, span: Span) -> None: - if not (span.error and can_capture(span)): - # No error or budget to capture + def on_span_exception( + self, span: Span, _exc_type: t.Type[BaseException], exc: BaseException, _tb: t.Optional[TracebackType] + ) -> None: + if span.get_tag(DEBUG_INFO_TAG) == "true" or not can_capture(span): + # Debug info for span already captured or no budget to capture return - _, exc, _tb = sys.exc_info() - chain, exc_id = unwind_exception_chain(exc, _tb) if not chain or exc_id is None: # No exceptions to capture @@ -208,12 +208,32 @@ def on_span_finish(self, span: Span) -> None: span.set_tag_str(DEBUG_INFO_TAG, "true") span.set_tag_str(EXCEPTION_ID_TAG, str(exc_id)) - def register(self) -> None: - super().register() + @classmethod + def enable(cls) -> None: + if cls._instance is not None: + log.debug("SpanExceptionHandler already enabled") + return + + log.debug("Enabling SpanExceptionHandler") + + instance = cls() + + instance.__uploader__.register(UploaderProduct.EXCEPTION_REPLAY) + core.on("span.exception", instance.on_span_exception, name=__name__) + + cls._instance = instance + + @classmethod + def disable(cls) -> None: + if cls._instance is None: + log.debug("SpanExceptionHandler already disabled") + return + + log.debug("Disabling SpanExceptionHandler") - self.__uploader__.register(UploaderProduct.EXCEPTION_REPLAY) + instance = cls._instance - def unregister(self) -> None: - self.__uploader__.unregister(UploaderProduct.EXCEPTION_REPLAY) + core.reset_listeners("span.exception", instance.on_span_exception) + instance.__uploader__.unregister(UploaderProduct.EXCEPTION_REPLAY) - return super().unregister() + cls._instance = None diff --git a/releasenotes/notes/chore-er-capture-exception-api-86a0a00e0b412567.yaml b/releasenotes/notes/chore-er-capture-exception-api-86a0a00e0b412567.yaml new file mode 100644 index 00000000000..fddb863d4bc --- /dev/null +++ b/releasenotes/notes/chore-er-capture-exception-api-86a0a00e0b412567.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Exception Replay will capture any exceptions that are manually attached to + a span with a call to ``set_exc_info``. diff --git a/tests/debugging/exception/test_replay.py b/tests/debugging/exception/test_replay.py index c87670bafc6..e2c75ff051e 100644 --- a/tests/debugging/exception/test_replay.py +++ b/tests/debugging/exception/test_replay.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +import sys import pytest @@ -79,7 +80,7 @@ def c(foo=42): snapshots = {str(s.uuid): s for s in uploader.collector.queue} for n, span in enumerate(self.spans): - assert span.get_tag("error.debug_info_captured") == "true" + assert span.get_tag(replay.DEBUG_INFO_TAG) == "true" exc_id = span.get_tag("_dd.debug.error.exception_id") @@ -146,7 +147,7 @@ def c(foo=42): number_of_exc_ids = 1 for n, span in enumerate(self.spans): - assert span.get_tag("error.debug_info_captured") == "true" + assert span.get_tag(replay.DEBUG_INFO_TAG) == "true" exc_id = span.get_tag("_dd.debug.error.exception_id") @@ -184,3 +185,30 @@ def c(foo=42): self.assert_span_count(6) # no new snapshots assert len(uploader.collector.queue) == 3 + + def test_debugger_capture_exception(self): + def a(v): + with self.trace("a") as span: + try: + raise ValueError("hello", v) + except Exception: + span.set_exc_info(*sys.exc_info()) + # Check that we don't capture multiple times + span.set_exc_info(*sys.exc_info()) + + def b(): + with self.trace("b"): + a(42) + + with exception_replay() as uploader: + with with_rate_limiter(RateLimiter(limit_rate=1, raise_on_exceed=False)): + b() + + self.assert_span_count(2) + assert len(uploader.collector.queue) == 1 + + span_b, span_a = self.spans + + assert span_a.name == "a" + assert span_a.get_tag(replay.DEBUG_INFO_TAG) == "true" + assert span_b.get_tag(replay.DEBUG_INFO_TAG) is None diff --git a/tests/debugging/mocking.py b/tests/debugging/mocking.py index 249b5794307..dee76125ba3 100644 --- a/tests/debugging/mocking.py +++ b/tests/debugging/mocking.py @@ -10,7 +10,7 @@ from ddtrace.debugging._config import di_config from ddtrace.debugging._debugger import Debugger -from ddtrace.debugging._exception.replay import SpanExceptionProcessor +from ddtrace.debugging._exception.replay import SpanExceptionHandler from ddtrace.debugging._probe.model import Probe from ddtrace.debugging._probe.remoteconfig import ProbePollerEvent from ddtrace.debugging._probe.remoteconfig import _filter_by_env_and_version @@ -196,15 +196,16 @@ def debugger(**config_overrides: Any) -> Generator[TestDebugger, None, None]: yield debugger -class MockSpanExceptionProcessor(SpanExceptionProcessor): +class MockSpanExceptionHandler(SpanExceptionHandler): __uploader__ = MockLogsIntakeUploaderV1 @contextmanager def exception_replay(**config_overrides: Any) -> Generator[MockLogsIntakeUploaderV1, None, None]: - processor = MockSpanExceptionProcessor() - processor.register() + MockSpanExceptionHandler.enable() + + handler = MockSpanExceptionHandler._instance try: - yield processor.__uploader__._instance + yield handler.__uploader__._instance finally: - processor.unregister() + MockSpanExceptionHandler.disable() From 487cd9e0cf5ed6bee9518c1f1d68049daae37acb Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:48:18 -0700 Subject: [PATCH 13/17] ci: move telemetry suite to gitlab (#10412) This change moves the `telemetry` test suite from circleci to gitlab for billing reasons. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Brett Langdon --- .circleci/config.templ.yml | 8 -------- .gitlab-ci.yml | 2 +- .gitlab/tests.yml | 13 +++++++++++-- .gitlab/tests/core.yml | 6 ++++++ tests/telemetry/test_writer.py | 8 ++++---- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.circleci/config.templ.yml b/.circleci/config.templ.yml index bda263b1f2a..4306decb37c 100644 --- a/.circleci/config.templ.yml +++ b/.circleci/config.templ.yml @@ -603,14 +603,6 @@ jobs: - run_test: pattern: "sourcecode" - telemetry: - parallelism: 6 - <<: *machine_executor - steps: - - run_test: - pattern: "telemetry" - snapshot: true - debugger: <<: *contrib_job steps: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f47ea38c3ad..414cecf2ae2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ stages: variables: REPO_LANG: python # "python" is used everywhere rather than "py" TESTRUNNER_IMAGE: ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b - CI_DEBUG_SERVICES: "true" + # CI_DEBUG_SERVICES: "true" .testrunner: image: $TESTRUNNER_IMAGE diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 3a69dd2dcaa..796067ca6b6 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -20,6 +20,7 @@ variables: alias: ddagent variables: DD_HOSTNAME: ddagent + LOG_LEVEL: ERROR DD_REMOTE_CONFIGURATION_ENABLED: true DD_SITE: datadoghq.com DD_API_KEY: invalid_but_this_is_fine @@ -27,9 +28,10 @@ variables: DD_REMOTE_CONFIGURATION_REFRESH_INTERVAL: 5s DD_DOGSTATSD_NON_LOCAL_TRAFFIC: true - name: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.17.0 + alias: testagent variables: - LOG_LEVEL: DEBUG - SNAPSHOT_DIR: /snapshots + LOG_LEVEL: INFO + SNAPSHOT_DIR: ${CI_PROJECT_DIR}/tests/snapshots SNAPSHOT_CI: 1 PORT: 9126 DD_POOL_TRACE_CHECK_FAILURES: true @@ -50,6 +52,13 @@ variables: ${RIOT_RUN_CMD} "${hash}" done +.test_base_riot_snapshot: + extends: .test_base_riot + before_script: + - !reference [.testrunner, before_script] + # DEV: Export here instead of via variables to avoid forwarding testagent requests to ddagent + - export DD_TRACE_AGENT_URL="http://testagent:9126" + include: - local: ".gitlab/tests/appsec.yml" - local: ".gitlab/tests/core.yml" diff --git a/.gitlab/tests/core.yml b/.gitlab/tests/core.yml index 0d42c275760..54856fe4a26 100644 --- a/.gitlab/tests/core.yml +++ b/.gitlab/tests/core.yml @@ -2,3 +2,9 @@ internal: extends: .test_base_riot variables: SUITE_NAME: "internal" + +telemetry: + extends: .test_base_riot_snapshot + parallel: 4 + variables: + SUITE_NAME: "telemetry" diff --git a/tests/telemetry/test_writer.py b/tests/telemetry/test_writer.py index a622af30e96..c4adc97d120 100644 --- a/tests/telemetry/test_writer.py +++ b/tests/telemetry/test_writer.py @@ -90,7 +90,6 @@ def test_app_started_event(telemetry_writer, test_agent_session, mock_time): {"name": "DD_SPAN_SAMPLING_RULES_FILE", "origin": "unknown", "value": None}, {"name": "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "origin": "unknown", "value": True}, {"name": "DD_TRACE_AGENT_TIMEOUT_SECONDS", "origin": "unknown", "value": 2.0}, - {"name": "DD_TRACE_AGENT_URL", "origin": "unknown", "value": "http://localhost:9126"}, {"name": "DD_TRACE_ANALYTICS_ENABLED", "origin": "unknown", "value": False}, {"name": "DD_TRACE_API_VERSION", "origin": "unknown", "value": None}, {"name": "DD_TRACE_CLIENT_IP_ENABLED", "origin": "unknown", "value": None}, @@ -165,9 +164,10 @@ def test_app_started_event(telemetry_writer, test_agent_session, mock_time): } requests[0]["body"]["payload"]["configuration"].sort(key=lambda c: c["name"]) result = _get_request_body(payload, "app-started") - result = {k: v for k, v in result.items() if k != "DD_TRACE_AGENT_URL"} - expected = {k: v for k, v in requests[0]["body"].items() if k != "DD_TRACE_AGENT_URL"} - assert expected == result + result["payload"]["configuration"] = [ + a for a in result["payload"]["configuration"] if a["name"] != "DD_TRACE_AGENT_URL" + ] + assert payload == result["payload"] @pytest.mark.parametrize( From 46377fef7f337e6b6e72e82870677df3e4164aad Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Fri, 30 Aug 2024 14:28:51 -0400 Subject: [PATCH 14/17] ci: move FastAPI tests from CircleCI to GitLab (#10463) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .circleci/config.templ.yml | 8 -------- .gitlab/tests.yml | 4 +++- .gitlab/tests/contrib.yml | 4 ++++ 3 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 .gitlab/tests/contrib.yml diff --git a/.circleci/config.templ.yml b/.circleci/config.templ.yml index 4306decb37c..95edbead77b 100644 --- a/.circleci/config.templ.yml +++ b/.circleci/config.templ.yml @@ -869,14 +869,6 @@ jobs: snapshot: true docker_services: "redis" - fastapi: - <<: *machine_executor - parallelism: 2 - steps: - - run_test: - pattern: "fastapi" - snapshot: true - flask: <<: *machine_executor parallelism: 10 diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 796067ca6b6..c40123ee260 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -56,11 +56,13 @@ variables: extends: .test_base_riot before_script: - !reference [.testrunner, before_script] - # DEV: Export here instead of via variables to avoid forwarding testagent requests to ddagent + # DEV: All job variables get shared with services, setting `DD_TRACE_AGENT_URL` on the testagent will tell it to forward all requests to the + # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent - export DD_TRACE_AGENT_URL="http://testagent:9126" include: - local: ".gitlab/tests/appsec.yml" + - local: ".gitlab/tests/contrib.yml" - local: ".gitlab/tests/core.yml" - local: ".gitlab/tests/tracer.yml" - local: ".gitlab/tests/profiling.yml" diff --git a/.gitlab/tests/contrib.yml b/.gitlab/tests/contrib.yml new file mode 100644 index 00000000000..db5bd8303d8 --- /dev/null +++ b/.gitlab/tests/contrib.yml @@ -0,0 +1,4 @@ +fastapi: + extends: .test_base_riot_snapshot + variables: + SUITE_NAME: "fastapi" From 79262a670e5579f010b9b394d388fc96c66f8106 Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Sat, 31 Aug 2024 07:40:56 -0700 Subject: [PATCH 15/17] ci: move two test suites to gitlab from circleci (#10466) This change moves the `ddtracerun` and `integration-snapshot` test suites from circleci to gitlab for billing purposes. It was necessary to adjust one of the ddtracerun tests to look for an optional environment variable specifying the location of the redis server, which had previously been `localhost` in both local and CI runs. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Brett Langdon --- .circleci/config.templ.yml | 19 ------------------- .gitlab/tests/core.yml | 15 +++++++++++++++ tests/commands/ddtrace_run_integration.py | 4 ++-- tests/contrib/config.py | 1 + 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/.circleci/config.templ.yml b/.circleci/config.templ.yml index 95edbead77b..356f0d68433 100644 --- a/.circleci/config.templ.yml +++ b/.circleci/config.templ.yml @@ -657,14 +657,6 @@ jobs: name: "Store core dumps" path: /tmp/core_dumps - integration_testagent: - <<: *machine_executor - steps: - - run_test: - snapshot: true - pattern: 'integration-snapshot*' - trace_agent_url: "" - vendor: <<: *contrib_job_small docker: @@ -682,17 +674,6 @@ jobs: snapshot: true docker_services: "localstack" - ddtracerun: - <<: *contrib_job - parallelism: 8 - docker: - - image: *ddtrace_dev_image - - image: *redis_image - steps: - - run_test: - pattern: 'ddtracerun' - trace_agent_url: "" - test_logging: <<: *contrib_job docker: diff --git a/.gitlab/tests/core.yml b/.gitlab/tests/core.yml index 54856fe4a26..51e727f3bc2 100644 --- a/.gitlab/tests/core.yml +++ b/.gitlab/tests/core.yml @@ -8,3 +8,18 @@ telemetry: parallel: 4 variables: SUITE_NAME: "telemetry" + +integration-testagent: + extends: .test_base_riot_snapshot + variables: + SUITE_NAME: "integration-snapshot*" + +ddtracerun: + extends: .test_base_riot + services: + - !reference [.test_base_riot, services] + - name: registry.ddbuild.io/redis:7.0.7 + alias: redis + variables: + SUITE_NAME: "ddtracerun" + TEST_REDIS_HOST: "redis" diff --git a/tests/commands/ddtrace_run_integration.py b/tests/commands/ddtrace_run_integration.py index 8a489afd26e..e52a0c9b8b0 100644 --- a/tests/commands/ddtrace_run_integration.py +++ b/tests/commands/ddtrace_run_integration.py @@ -11,7 +11,7 @@ if __name__ == "__main__": - r = redis.Redis(port=REDIS_CONFIG["port"]) + r = redis.Redis(host=REDIS_CONFIG["host"], port=REDIS_CONFIG["port"]) pin = Pin.get_from(r) assert pin @@ -36,7 +36,7 @@ assert span.error == 0 assert span.get_metric("network.destination.port") == REDIS_CONFIG["port"] assert span.get_metric("out.redis_db") == 0 - assert span.get_tag("out.host") == "localhost" + assert span.get_tag("out.host") == REDIS_CONFIG["host"] assert span.get_tag("redis.raw_command").startswith("mget 0 1 2 3") assert span.get_tag("redis.raw_command").endswith("...") diff --git a/tests/contrib/config.py b/tests/contrib/config.py index 7e75358f1f5..e99d2fba573 100644 --- a/tests/contrib/config.py +++ b/tests/contrib/config.py @@ -53,6 +53,7 @@ } REDIS_CONFIG = { + "host": os.getenv("TEST_REDIS_HOST", "localhost"), "port": int(os.getenv("TEST_REDIS_PORT", 6379)), } From 225dbdea156b4754cf85c8cb160e1554dd34f32e Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Sat, 31 Aug 2024 08:21:24 -0700 Subject: [PATCH 16/17] ci: move two test suites to gitlab (#10467) This change moves the `integration-agent` and `vendor` test suites from circleci to gitlab for billing purposes, while removing some unused logic from the `integration-agent` job. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Brett Langdon --- .circleci/config.templ.yml | 34 ---------------------------------- .gitlab/tests/core.yml | 10 ++++++++++ 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/.circleci/config.templ.yml b/.circleci/config.templ.yml index 356f0d68433..58b0a9c92ed 100644 --- a/.circleci/config.templ.yml +++ b/.circleci/config.templ.yml @@ -631,40 +631,6 @@ jobs: pattern: 'opentelemetry' snapshot: true - integration_agent: - <<: *machine_executor - parallelism: 2 - steps: - - attach_workspace: - at: . - - checkout - - setup_riot - - start_docker_services: - services: ddagent - - run: - environment: - RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>" - command: | - ulimit -c unlimited - ./scripts/run-test-suite 'integration-latest*' <> 1 - - run: - command: | - mkdir -p /tmp/core_dumps - cp core.* /tmp/core_dumps || true - ./scripts/bt - when: on_fail - - store_artifacts: - name: "Store core dumps" - path: /tmp/core_dumps - - vendor: - <<: *contrib_job_small - docker: - - image: *ddtrace_dev_image - steps: - - run_test: - pattern: 'vendor' - botocore: <<: *machine_executor parallelism: 6 diff --git a/.gitlab/tests/core.yml b/.gitlab/tests/core.yml index 51e727f3bc2..dc5db8185b8 100644 --- a/.gitlab/tests/core.yml +++ b/.gitlab/tests/core.yml @@ -14,6 +14,16 @@ integration-testagent: variables: SUITE_NAME: "integration-snapshot*" +integration-agent: + extends: .test_base_riot + variables: + SUITE_NAME: "integration-latest*" + +vendor: + extends: .test_base_riot + variables: + SUITE_NAME: "vendor" + ddtracerun: extends: .test_base_riot services: From 666f6ecd39a1937f18643bf05ab44c4a10003899 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Mon, 2 Sep 2024 10:54:07 +0200 Subject: [PATCH 17/17] chore(iast): testing fastapi and sqli (#10474) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .../fastapi/test_fastapi_appsec_iast.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/contrib/fastapi/test_fastapi_appsec_iast.py b/tests/contrib/fastapi/test_fastapi_appsec_iast.py index b732cebd1cb..405c5c1b89a 100644 --- a/tests/contrib/fastapi/test_fastapi_appsec_iast.py +++ b/tests/contrib/fastapi/test_fastapi_appsec_iast.py @@ -9,21 +9,30 @@ from fastapi.responses import JSONResponse import pytest +from ddtrace.appsec._constants import IAST from ddtrace.appsec._handlers import _on_asgi_request_parse_body from ddtrace.appsec._iast import oce from ddtrace.appsec._iast._patch import _on_iast_fastapi_patch +from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION +from ddtrace.appsec._iast.taint_sinks.header_injection import patch as patch_header_injection +from ddtrace.contrib.sqlite3.patch import patch as patch_sqlite_sqli from ddtrace.internal import core +from tests.appsec.iast.iast_utils import get_line_and_hash from tests.utils import override_env from tests.utils import override_global_config IAST_ENV = {"DD_IAST_REQUEST_SAMPLING": "100"} +TEST_FILE_PATH = "tests/contrib/fastapi/test_fastapi_appsec_iast.py" + fastapi_version = tuple([int(v) for v in _fastapi_version.split(".")]) def _aux_appsec_prepare_tracer(tracer): _on_iast_fastapi_patch() + patch_sqlite_sqli() + patch_header_injection() oce.reconfigure() tracer._iast_enabled = True @@ -276,3 +285,56 @@ async def test_route(item_id): assert result["ranges_start"] == 0 assert result["ranges_length"] == 8 assert result["ranges_origin"] == "http.request.path.parameter" + + +def test_fastapi_sqli_path_param(fastapi_application, client, tracer, test_spans): + @fastapi_application.get("/index.html/{param_str}") + async def test_route(param_str): + import sqlite3 + + from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect + + assert is_pyobject_tainted(param_str) + + con = sqlite3.connect(":memory:") + cur = con.cursor() + # label test_fastapi_sqli_path_parameter + cur.execute(add_aspect("SELECT 1 FROM ", param_str)) + + # test if asgi middleware is ok without any callback registered + core.reset_listeners(event_id="asgi.request.parse.body") + + with override_global_config(dict(_iast_enabled=True, _deduplication_enabled=False)), override_env(IAST_ENV): + # disable callback + _aux_appsec_prepare_tracer(tracer) + resp = client.get( + "/index.html/sqlite_master/", + ) + assert resp.status_code == 200 + + span = test_spans.pop_traces()[1][0] + assert span.get_metric(IAST.ENABLED) == 1.0 + + loaded = json.loads(span.get_tag(IAST.JSON)) + + assert loaded["sources"] == [ + {"origin": "http.request.path.parameter", "name": "param_str", "value": "sqlite_master"} + ] + + line, hash_value = get_line_and_hash( + "test_fastapi_sqli_path_parameter", VULN_SQL_INJECTION, filename=TEST_FILE_PATH + ) + vulnerability = loaded["vulnerabilities"][0] + assert vulnerability["type"] == VULN_SQL_INJECTION + assert vulnerability["evidence"] == { + "valueParts": [ + {"value": "SELECT "}, + {"redacted": True}, + {"value": " FROM "}, + {"value": "sqlite_master", "source": 0}, + ] + } + assert vulnerability["location"]["line"] == line + assert vulnerability["location"]["path"] == TEST_FILE_PATH + assert vulnerability["hash"] == hash_value