diff --git a/.github/workflows/django-overhead-profile.yml b/.github/workflows/django-overhead-profile.yml index f581749fe87..01a1626e0d8 100644 --- a/.github/workflows/django-overhead-profile.yml +++ b/.github/workflows/django-overhead-profile.yml @@ -15,6 +15,7 @@ jobs: runs-on: ubuntu-latest env: PREFIX: ${{ github.workspace }}/prefix + DD_CODE_ORIGIN_FOR_SPANS_ENABLED: "1" defaults: run: working-directory: ddtrace diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b3d1981a5ba..4abce66d343 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -14,26 +14,15 @@ jobs: strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - archs: x86_64 i686 - #- os: arm-4core-linux - # archs: aarch64 - - os: windows-latest - archs: AMD64 x86 - - os: macos-latest - archs: arm64 + os: [ubuntu-latest, windows-latest, macos-latest] + # Keep this in sync with hatch.toml + python-version: ["3.7", "3.10", "3.12"] steps: - uses: actions/checkout@v4 # Include all history and tags with: fetch-depth: 0 - - uses: actions/setup-python@v5 - name: Install Python - with: - python-version: '3.12' - - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install latest stable toolchain and rustfmt run: rustup update stable && rustup default stable && rustup component add rustfmt clippy @@ -44,4 +33,4 @@ jobs: version: "1.12.0" - name: Run tests - run: hatch run ddtrace_unit_tests:test + run: hatch run +py=${{ matrix.python-version }} ddtrace_unit_tests:test diff --git a/.gitignore b/.gitignore index ce0f64a9433..f6b1906f30c 100644 --- a/.gitignore +++ b/.gitignore @@ -141,10 +141,21 @@ ddtrace/_version.py artifacts/ # IAST cpp +ddtrace/appsec/_iast/_taint_tracking/cmake_install.cmake +ddtrace/appsec/_iast/_taint_tracking/CMakeCache.txt +ddtrace/appsec/_iast/_taint_tracking/Makefile ddtrace/appsec/_iast/_taint_tracking/cmake-build-debug/* ddtrace/appsec/_iast/_taint_tracking/_deps/* ddtrace/appsec/_iast/_taint_tracking/CMakeFiles/* - +ddtrace/appsec/_iast/_taint_tracking/tests/CMakeFiles/* +ddtrace/appsec/_iast/_taint_tracking/tests/cmake_install.cmake +ddtrace/appsec/_iast/_taint_tracking/tests/CMakeCache.txt +ddtrace/appsec/_iast/_taint_tracking/tests/Makefile +ddtrace/appsec/_iast/_taint_tracking/tests/CTestTestfile.cmake +ddtrace/appsec/_iast/_taint_tracking/tests/native_tests* +ddtrace/appsec/_iast/_taint_tracking/_vendor/pybind11/Makefile +ddtrace/appsec/_iast/_taint_tracking/_vendor/pybind11/CMakeFiles/* +ddtrace/appsec/_iast/_taint_tracking/_vendor/pybind11/cmake_install.cmake # CircleCI generated config .circleci/config.gen.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 643a2ec990e..a352f7f72ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ package-oci: onboarding_tests_installer: parallel: matrix: - - ONBOARDING_FILTER_WEBLOG: [test-app-python,test-app-python-container,test-app-python-alpine-libgcc] + - ONBOARDING_FILTER_WEBLOG: [test-app-python,test-app-python-container,test-app-python-alpine] onboarding_tests_k8s_injection: diff --git a/.riot/requirements/1280196.txt b/.riot/requirements/1280196.txt index ea2303daf1b..55d4fd22d35 100644 --- a/.riot/requirements/1280196.txt +++ b/.riot/requirements/1280196.txt @@ -2,28 +2,28 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --no-annotate .riot/requirements/1280196.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1280196.in # -attrs==23.1.0 -beautifulsoup4==4.12.2 -bottle==0.12.25 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 +attrs==24.2.0 +beautifulsoup4==4.12.3 +bottle==0.13.1 +coverage[toml]==7.6.1 +exceptiongroup==1.2.2 hypothesis==6.45.0 -importlib-metadata==7.0.0 +importlib-metadata==8.5.0 iniconfig==2.0.0 mock==5.1.0 opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 pytest-randomly==3.15.0 sortedcontainers==2.4.0 -soupsieve==2.5 -tomli==2.0.1 -waitress==2.1.2 -webob==1.8.7 -webtest==3.0.0 -zipp==3.17.0 +soupsieve==2.6 +tomli==2.0.2 +waitress==3.0.0 +webob==1.8.8 +webtest==3.0.1 +zipp==3.20.2 diff --git a/.riot/requirements/15dee3b.txt b/.riot/requirements/15dee3b.txt index 748fe265b41..2b8691e8510 100644 --- a/.riot/requirements/15dee3b.txt +++ b/.riot/requirements/15dee3b.txt @@ -2,28 +2,28 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --no-annotate .riot/requirements/15dee3b.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/15dee3b.in # -attrs==23.1.0 -beautifulsoup4==4.12.2 +attrs==24.2.0 +beautifulsoup4==4.12.3 bottle==0.12.25 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 +coverage[toml]==7.6.1 +exceptiongroup==1.2.2 hypothesis==6.45.0 -importlib-metadata==7.0.0 +importlib-metadata==8.5.0 iniconfig==2.0.0 mock==5.1.0 opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 pytest-randomly==3.15.0 sortedcontainers==2.4.0 -soupsieve==2.5 -tomli==2.0.1 -waitress==2.1.2 -webob==1.8.7 -webtest==3.0.0 -zipp==3.17.0 +soupsieve==2.6 +tomli==2.0.2 +waitress==3.0.0 +webob==1.8.8 +webtest==3.0.1 +zipp==3.20.2 diff --git a/.riot/requirements/18b1b66.txt b/.riot/requirements/18b1b66.txt index 026db1fa372..222f1e2b517 100644 --- a/.riot/requirements/18b1b66.txt +++ b/.riot/requirements/18b1b66.txt @@ -2,21 +2,21 @@ # This file is autogenerated by pip-compile with Python 3.7 # by the following command: # -# pip-compile --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/18b1b66.in +# pip-compile --allow-unsafe --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/18b1b66.in # -attrs==23.1.0 -beautifulsoup4==4.12.2 +attrs==24.2.0 +beautifulsoup4==4.12.3 bottle==0.12.25 coverage[toml]==7.2.7 -exceptiongroup==1.2.0 +exceptiongroup==1.2.2 hypothesis==6.45.0 importlib-metadata==6.7.0 iniconfig==2.0.0 mock==5.1.0 opentracing==2.4.0 -packaging==23.2 +packaging==24.0 pluggy==1.2.0 -pytest==7.4.3 +pytest==7.4.4 pytest-cov==4.1.0 pytest-mock==3.11.1 pytest-randomly==3.12.0 @@ -25,6 +25,6 @@ soupsieve==2.4.1 tomli==2.0.1 typing-extensions==4.7.1 waitress==2.1.2 -webob==1.8.7 -webtest==3.0.0 +webob==1.8.8 +webtest==3.0.1 zipp==3.15.0 diff --git a/.riot/requirements/573fdbf.txt b/.riot/requirements/573fdbf.txt index effce0978ae..d6aff21e2ac 100644 --- a/.riot/requirements/573fdbf.txt +++ b/.riot/requirements/573fdbf.txt @@ -2,28 +2,28 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --no-annotate .riot/requirements/573fdbf.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/573fdbf.in # -attrs==23.1.0 -beautifulsoup4==4.12.2 -bottle==0.12.25 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 +attrs==24.2.0 +beautifulsoup4==4.12.3 +bottle==0.13.1 +coverage[toml]==7.6.1 +exceptiongroup==1.2.2 hypothesis==6.45.0 -importlib-metadata==7.0.0 +importlib-metadata==8.5.0 iniconfig==2.0.0 mock==5.1.0 opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 pytest-randomly==3.15.0 sortedcontainers==2.4.0 -soupsieve==2.5 -tomli==2.0.1 -waitress==2.1.2 -webob==1.8.7 -webtest==3.0.0 -zipp==3.17.0 +soupsieve==2.6 +tomli==2.0.2 +waitress==3.0.0 +webob==1.8.8 +webtest==3.0.1 +zipp==3.20.2 diff --git a/.riot/requirements/760d56e.txt b/.riot/requirements/760d56e.txt index 53727148b51..d3e80a7a669 100644 --- a/.riot/requirements/760d56e.txt +++ b/.riot/requirements/760d56e.txt @@ -2,21 +2,21 @@ # This file is autogenerated by pip-compile with Python 3.7 # by the following command: # -# pip-compile --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/760d56e.in +# pip-compile --allow-unsafe --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/760d56e.in # -attrs==23.1.0 -beautifulsoup4==4.12.2 -bottle==0.12.25 +attrs==24.2.0 +beautifulsoup4==4.12.3 +bottle==0.13.1 coverage[toml]==7.2.7 -exceptiongroup==1.2.0 +exceptiongroup==1.2.2 hypothesis==6.45.0 importlib-metadata==6.7.0 iniconfig==2.0.0 mock==5.1.0 opentracing==2.4.0 -packaging==23.2 +packaging==24.0 pluggy==1.2.0 -pytest==7.4.3 +pytest==7.4.4 pytest-cov==4.1.0 pytest-mock==3.11.1 pytest-randomly==3.12.0 @@ -25,6 +25,6 @@ soupsieve==2.4.1 tomli==2.0.1 typing-extensions==4.7.1 waitress==2.1.2 -webob==1.8.7 -webtest==3.0.0 +webob==1.8.8 +webtest==3.0.1 zipp==3.15.0 diff --git a/.riot/requirements/8c110bf.txt b/.riot/requirements/8c110bf.txt index fed864f8b54..0a57b2a151b 100644 --- a/.riot/requirements/8c110bf.txt +++ b/.riot/requirements/8c110bf.txt @@ -2,28 +2,28 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --no-annotate .riot/requirements/8c110bf.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/8c110bf.in # -attrs==23.1.0 -beautifulsoup4==4.12.2 +attrs==24.2.0 +beautifulsoup4==4.12.3 bottle==0.12.25 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 +coverage[toml]==7.6.1 +exceptiongroup==1.2.2 hypothesis==6.45.0 -importlib-metadata==7.0.0 +importlib-metadata==8.5.0 iniconfig==2.0.0 mock==5.1.0 opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 pytest-randomly==3.15.0 sortedcontainers==2.4.0 -soupsieve==2.5 -tomli==2.0.1 -waitress==2.1.2 -webob==1.8.7 -webtest==3.0.0 -zipp==3.17.0 +soupsieve==2.6 +tomli==2.0.2 +waitress==3.0.0 +webob==1.8.8 +webtest==3.0.1 +zipp==3.20.2 diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectJoin.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectJoin.cpp index 60d4aa3dea1..476246e2a84 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectJoin.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectJoin.cpp @@ -55,7 +55,7 @@ aspect_join_str(PyObject* sep, PyObject* new_result{ new_pyobject_id(result) }; set_tainted_object(new_result, result_to, tx_taint_map); - Py_DecRef(result); + Py_DECREF(result); return new_result; } @@ -134,7 +134,7 @@ aspect_join(PyObject* sep, PyObject* result, PyObject* iterable_elements, const PyObject* new_result{ new_pyobject_id(result) }; set_tainted_object(new_result, result_to, tx_taint_map); - Py_DecRef(result); + Py_DECREF(result); return new_result; } @@ -158,9 +158,10 @@ api_join_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs) PyObject* list_aux = PyList_New(0); while ((item = PyIter_Next(iterator))) { PyList_Append(list_aux, item); + Py_DECREF(item); } arg0 = list_aux; - Py_DecRef(iterator); + Py_DECREF(iterator); decref_arg0 = true; } } @@ -181,7 +182,7 @@ api_join_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs) if (has_pyerr()) { if (decref_arg0) { - Py_DecRef(arg0); + Py_DECREF(arg0); } return nullptr; } @@ -190,13 +191,13 @@ api_join_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs) if (not ctx_map or ctx_map->empty() or get_pyobject_size(result) == 0) { // Empty result cannot have taint ranges if (decref_arg0) { - Py_DecRef(arg0); + Py_DECREF(arg0); } return result; } auto res = aspect_join(sep, result, arg0, ctx_map); if (decref_arg0) { - Py_DecRef(arg0); + Py_DECREF(arg0); } return res; }); diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectOperatorAdd.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectOperatorAdd.cpp index 0d613375aca..a92a1436f95 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectOperatorAdd.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectOperatorAdd.cpp @@ -21,17 +21,16 @@ add_aspect(PyObject* result_o, const size_t len_candidate_text{ get_pyobject_size(candidate_text) }; const size_t len_text_to_add{ get_pyobject_size(text_to_add) }; - if (len_text_to_add == 0 and len_candidate_text > 0) { - return candidate_text; - } - if (len_text_to_add > 0 and len_candidate_text == 0 and text_to_add == result_o) { - return text_to_add; + // Appended from or with nothing, ranges didn't change + if ((len_text_to_add == 0 and len_candidate_text > 0) || + (len_text_to_add > 0 and len_candidate_text == 0 and text_to_add == result_o)) { + return result_o; } const auto& to_candidate_text = get_tainted_object(candidate_text, tx_taint_map); if (to_candidate_text and to_candidate_text->get_ranges().size() >= TaintedObject::TAINT_RANGE_LIMIT) { const auto& res_new_id = new_pyobject_id(result_o); - Py_DecRef(result_o); + Py_DECREF(result_o); // If left side is already at the maximum taint ranges, we just reuse its // ranges, we don't need to look at left side. set_tainted_object(res_new_id, to_candidate_text, tx_taint_map); @@ -44,12 +43,20 @@ add_aspect(PyObject* result_o, } if (!to_text_to_add) { const auto& res_new_id = new_pyobject_id(result_o); - Py_DecRef(result_o); + Py_DECREF(result_o); set_tainted_object(res_new_id, to_candidate_text, tx_taint_map); return res_new_id; } - auto tainted = initializer->allocate_tainted_object_copy(to_candidate_text); + if (!to_candidate_text) { + const auto tainted = initializer->allocate_tainted_object(); + tainted->add_ranges_shifted(to_text_to_add, static_cast(len_candidate_text)); + set_tainted_object(result_o, tainted, tx_taint_map); + } + + // At this point we have both to_candidate_text and to_text_to_add to we add the + // ranges from both to result_o + const auto tainted = initializer->allocate_tainted_object_copy(to_candidate_text); tainted->add_ranges_shifted(to_text_to_add, static_cast(len_candidate_text)); set_tainted_object(result_o, tainted, tx_taint_map); @@ -84,6 +91,9 @@ api_add_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs) // PyNumber_Add actually works for any type! result_o = PyNumber_Add(candidate_text, text_to_add); + if (result_o == nullptr) { + return nullptr; + } TRY_CATCH_ASPECT("add_aspect", return result_o, , { const auto tx_map = Initializer::get_tainting_map(); @@ -116,8 +126,6 @@ api_add_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs) PyObject* api_add_inplace_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs) { - PyObject* result_o = nullptr; - if (nargs != 2) { py::set_error(PyExc_ValueError, MSG_ERROR_N_PARAMS); return nullptr; @@ -125,7 +133,10 @@ api_add_inplace_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs) PyObject* candidate_text = args[0]; PyObject* text_to_add = args[1]; - result_o = PyNumber_InPlaceAdd(candidate_text, text_to_add); + PyObject* result_o = PyNumber_InPlaceAdd(candidate_text, text_to_add); + if (result_o == nullptr) { + return nullptr; + } TRY_CATCH_ASPECT("add_inplace_aspect", return result_o, , { const auto tx_map = Initializer::get_tainting_map(); @@ -152,7 +163,6 @@ api_add_inplace_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs) is_notinterned_notfasttainted_unicode(text_to_add)) { return result_o; } - candidate_text = add_aspect(result_o, candidate_text, text_to_add, tx_map); - return candidate_text; + return add_aspect(result_o, candidate_text, text_to_add, tx_map); }); } \ No newline at end of file diff --git a/ddtrace/appsec/_iast/_taint_tracking/Initializer/Initializer.cpp b/ddtrace/appsec/_iast/_taint_tracking/Initializer/Initializer.cpp index 73b934366e2..657d8a92e10 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Initializer/Initializer.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Initializer/Initializer.cpp @@ -14,7 +14,7 @@ Initializer::Initializer() { // Fill the taintedobjects stack for (int i = 0; i < TAINTEDOBJECTS_STACK_SIZE; i++) { - available_taintedobjects_stack.push(new TaintedObject()); + available_taintedobjects_stack.push(make_shared()); } // Fill the ranges stack @@ -41,9 +41,6 @@ Initializer::clear_tainting_map(const TaintRangeMapTypePtr& tx_map) return; } std::lock_guard lock(active_map_addreses_mutex); - for (const auto& [fst, snd] : *tx_map) { - snd.second->decref(); - } tx_map->clear(); active_map_addreses.erase(tx_map.get()); } @@ -114,12 +111,12 @@ TaintedObjectPtr Initializer::allocate_tainted_object() { if (!available_taintedobjects_stack.empty()) { - const auto& toptr = available_taintedobjects_stack.top(); + const auto toptr = available_taintedobjects_stack.top(); available_taintedobjects_stack.pop(); return toptr; } // Stack is empty, create new object - return new TaintedObject(); + return make_shared(); } TaintedObjectPtr @@ -147,24 +144,6 @@ Initializer::allocate_tainted_object_copy(const TaintedObjectPtr& from) return allocate_ranges_into_taint_object_copy(from->ranges_); } -void -Initializer::release_tainted_object(TaintedObjectPtr tobj) -{ - if (!tobj) { - return; - } - - tobj->reset(); - if (available_taintedobjects_stack.size() < TAINTEDOBJECTS_STACK_SIZE) { - available_taintedobjects_stack.push(tobj); - return; - } - - // Stack full, just delete the object (but to a reset before so ranges are - // reused or freed) - delete tobj; -} - TaintRangePtr Initializer::allocate_taint_range(const RANGE_START start, const RANGE_LENGTH length, const Source& origin) { diff --git a/ddtrace/appsec/_iast/_taint_tracking/Initializer/Initializer.h b/ddtrace/appsec/_iast/_taint_tracking/Initializer/Initializer.h index d453ce5527f..6d7fbf36499 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Initializer/Initializer.h +++ b/ddtrace/appsec/_iast/_taint_tracking/Initializer/Initializer.h @@ -140,8 +140,6 @@ class Initializer */ TaintedObjectPtr allocate_tainted_object_copy(const TaintedObjectPtr& from); - void release_tainted_object(TaintedObjectPtr tobj); - // FIXME: these should be static functions of TaintRange // IMPORTANT: if the returned object is not assigned to the map, you have // responsibility of calling release_taint_range on it or you'll have a leak. diff --git a/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.cpp b/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.cpp index a15274cd5e0..ed50046cecf 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.cpp @@ -181,9 +181,8 @@ set_ranges(PyObject* str, const TaintRangeRefs& ranges, const TaintRangeMapTypeP auto new_tainted_object = initializer->allocate_ranges_into_taint_object(ranges); set_fast_tainted_if_notinterned_unicode(str); - new_tainted_object->incref(); if (it != tx_map->end()) { - it->second.second->decref(); + it->second.second.reset(); it->second = std::make_pair(get_internal_hash(str), new_tainted_object); return true; } @@ -312,7 +311,6 @@ get_tainted_object(PyObject* str, const TaintRangeMapTypePtr& tx_map) } if (get_internal_hash(str) != it->second.first) { - it->second.second->decref(); tx_map->erase(it); return nullptr; } @@ -368,15 +366,13 @@ set_tainted_object(PyObject* str, TaintedObjectPtr tainted_object, const TaintRa // If the tainted object is different, we need to decref the previous one // and incref the new one. But if it's the same object, we can avoid both // operations, since they would be redundant. - it->second.second->decref(); - tainted_object->incref(); + it->second.second.reset(); it->second = std::make_pair(get_internal_hash(str), tainted_object); } // Update the hash, because for bytearrays it could have changed after the extend operation it->second.first = get_internal_hash(str); return; } - tainted_object->incref(); tx_map->insert({ obj_id, std::make_pair(get_internal_hash(str), tainted_object) }); } diff --git a/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.h b/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.h index 6d6e1a9a279..efb817a3359 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.h +++ b/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.h @@ -17,7 +17,7 @@ namespace py = pybind11; class TaintedObject; // Alias -using TaintedObjectPtr = TaintedObject*; +using TaintedObjectPtr = shared_ptr; #ifdef NDEBUG // Decide wether to use abseil @@ -134,7 +134,7 @@ api_is_unicode_and_not_fast_tainted(const py::handle& str) return is_notinterned_notfasttainted_unicode(str.ptr()); } -TaintedObject* +TaintedObjectPtr get_tainted_object(PyObject* str, const TaintRangeMapTypePtr& tx_taint_map); Py_hash_t diff --git a/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintedObject.cpp b/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintedObject.cpp index 3a5aa2aa277..3b00a5d28e8 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintedObject.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintedObject.cpp @@ -140,34 +140,11 @@ void TaintedObject::reset() { move_ranges_to_stack(); - rc_ = 0; if (initializer) { ranges_.reserve(RANGES_INITIAL_RESERVE); } } -void -TaintedObject::incref() -{ - rc_++; -} - -void -TaintedObject::decref() -{ - if (--rc_ <= 0) { - release(); - } -} - -void -TaintedObject::release() -{ - // If rc_ is negative, there is a bug. - // assert(rc_ == 0); - initializer->release_tainted_object(this); -} - void pyexport_taintedobject(const py::module& m) { diff --git a/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintedObject.h b/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintedObject.h index a4198424a7c..88198fabcea 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintedObject.h +++ b/ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintedObject.h @@ -8,7 +8,6 @@ class TaintedObject private: TaintRangeRefs ranges_; - size_t rc_{}; public: constexpr static int TAINT_RANGE_LIMIT = 100; @@ -36,7 +35,7 @@ class TaintedObject [[nodiscard]] TaintRangeRefs get_ranges_copy() const { return ranges_; } - void add_ranges_shifted(TaintedObject* tainted_object, + void add_ranges_shifted(TaintedObjectPtr tainted_object, RANGE_START offset, RANGE_LENGTH max_length = -1, RANGE_START orig_offset = -1); @@ -53,12 +52,6 @@ class TaintedObject void move_ranges_to_stack(); void reset(); - - void incref(); - - void decref(); - - void release(); }; void diff --git a/ddtrace/appsec/_iast/_taint_tracking/Utils/StringUtils.cpp b/ddtrace/appsec/_iast/_taint_tracking/Utils/StringUtils.cpp index 0fd274925be..2df5ffa5e28 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Utils/StringUtils.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Utils/StringUtils.cpp @@ -112,6 +112,9 @@ new_pyobject_id(PyObject* tainted_object) const auto res = PyObject_CallFunctionObjArgs(bytes_join_ptr.ptr(), val, NULL); Py_XDECREF(val); Py_XDECREF(empty_bytes); + if (res == nullptr) { + return tainted_object; + } return res; } @@ -138,6 +141,9 @@ new_pyobject_id(PyObject* tainted_object) Py_XDECREF(val); Py_XDECREF(empty_bytes); Py_XDECREF(empty_bytearray); + if (res == nullptr) { + return tainted_object; + } return res; } return tainted_object; diff --git a/ddtrace/appsec/_iast/_taint_tracking/Utils/StringUtils.h b/ddtrace/appsec/_iast/_taint_tracking/Utils/StringUtils.h index d0dbf1ddf80..d2da5dcd821 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Utils/StringUtils.h +++ b/ddtrace/appsec/_iast/_taint_tracking/Utils/StringUtils.h @@ -81,12 +81,22 @@ template bool args_are_text_and_same_type(PyObject* first, PyObject* second, Args... args) { + if (first == nullptr || second == nullptr) { + return false; + } + + const auto type_first = PyObject_Type(first); + const auto type_second = PyObject_Type(second); + // Check if both first and second are valid text types and of the same type - if (first == nullptr || second == nullptr || !is_text(first) || !is_text(second) || - PyObject_Type(first) != PyObject_Type(second)) { + if (!is_text(first) || !is_text(second) || type_first != type_second) { + Py_XDECREF(type_first); + Py_XDECREF(type_second); return false; } + Py_XDECREF(type_first); + Py_XDECREF(type_second); // Recursively check the rest of the arguments return args_are_text_and_same_type(second, args...); } diff --git a/ddtrace/appsec/_iast/_taint_tracking/tests/test_add_aspect.cpp b/ddtrace/appsec/_iast/_taint_tracking/tests/test_add_aspect.cpp new file mode 100644 index 00000000000..85ac7dc3d95 --- /dev/null +++ b/ddtrace/appsec/_iast/_taint_tracking/tests/test_add_aspect.cpp @@ -0,0 +1,79 @@ +#include "Aspects/Helpers.h" + +#include +#include +#include + +using AspectAddCheck = PyEnvWithContext; + +TEST_F(AspectAddCheck, check_api_add_aspect_strings_candidate_text_empty) +{ + PyObject* candidate_text = this->StringToPyObjectStr(""); + PyObject* text_to_add = this->StringToPyObjectStr("def"); + PyObject* args_array[2]; + args_array[0] = candidate_text; + args_array[1] = text_to_add; + auto result = api_add_aspect(nullptr, args_array, 2); + EXPECT_FALSE(has_pyerr()); + + std::string result_string = this->PyObjectStrToString(result); + + EXPECT_EQ(result_string, "def"); + Py_DecRef(candidate_text); + Py_DecRef(text_to_add); + Py_DecRef(result); +} + +TEST_F(AspectAddCheck, check_api_add_aspect_strings_text_to_add_empty) +{ + PyObject* candidate_text = this->StringToPyObjectStr("abc"); + PyObject* text_to_add = this->StringToPyObjectStr(""); + PyObject* args_array[2]; + args_array[0] = candidate_text; + args_array[1] = text_to_add; + auto result = api_add_aspect(nullptr, args_array, 2); + EXPECT_FALSE(has_pyerr()); + + std::string result_string = this->PyObjectStrToString(result); + + EXPECT_EQ(result_string, "abc"); + Py_DecRef(candidate_text); + Py_DecRef(text_to_add); + Py_DecRef(result); +} + +TEST_F(AspectAddCheck, check_api_add_aspect_strings) +{ + PyObject* candidate_text = this->StringToPyObjectStr("abc"); + PyObject* text_to_add = this->StringToPyObjectStr("def"); + PyObject* args_array[2]; + args_array[0] = candidate_text; + args_array[1] = text_to_add; + auto result = api_add_aspect(nullptr, args_array, 2); + EXPECT_FALSE(has_pyerr()); + + std::string result_string = this->PyObjectStrToString(result); + + EXPECT_EQ(result_string, "abcdef"); + Py_DecRef(candidate_text); + Py_DecRef(text_to_add); + Py_DecRef(result); +} + +TEST_F(AspectAddCheck, check_api_add_aspect_bytes) +{ + PyObject* candidate_text = this->StringToPyObjectBytes("abc"); + PyObject* text_to_add = this->StringToPyObjectBytes("def"); + PyObject* args_array[2]; + args_array[0] = candidate_text; + args_array[1] = text_to_add; + auto result = api_add_aspect(nullptr, args_array, 2); + EXPECT_FALSE(has_pyerr()); + + std::string result_string = this->PyObjectBytesToString(result); + + EXPECT_EQ(result_string, "abcdef"); + Py_DecRef(candidate_text); + Py_DecRef(text_to_add); + Py_DecRef(result); +} diff --git a/ddtrace/appsec/_iast/_taint_tracking/tests/test_common.hpp b/ddtrace/appsec/_iast/_taint_tracking/tests/test_common.hpp index 81af3566b5a..b7338e3d683 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/tests/test_common.hpp +++ b/ddtrace/appsec/_iast/_taint_tracking/tests/test_common.hpp @@ -29,4 +29,22 @@ class PyEnvWithContext : public ::testing::Test initializer.reset(); py::finalize_interpreter(); } + + public: + PyObject* StringToPyObjectStr(const string& ob) { return PyUnicode_FromString(ob.c_str()); } + string PyObjectStrToString(PyObject* ob) + { + PyObject* utf8_str = PyUnicode_AsEncodedString(ob, "utf-8", "strict"); + const char* res_data = PyBytes_AsString(utf8_str); + std::string res_string(res_data); + Py_DecRef(utf8_str); + return res_string; + } + PyObject* StringToPyObjectBytes(const string& ob) { return PyBytes_FromString(ob.c_str()); } + string PyObjectBytesToString(PyObject* ob) + { + const char* res_data = PyBytes_AsString(ob); + std::string res_string(res_data); + return res_string; + } }; diff --git a/ddtrace/appsec/_iast/_taint_tracking/tests/test_helpers.cpp b/ddtrace/appsec/_iast/_taint_tracking/tests/test_helpers.cpp index 1078e23a1e0..825a03ba787 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/tests/test_helpers.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/tests/test_helpers.cpp @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/ddtrace/appsec/_iast/_taint_tracking/tests/test_index_aspect.cpp b/ddtrace/appsec/_iast/_taint_tracking/tests/test_index_aspect.cpp index 444325d8fd4..2961cc80ea8 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/tests/test_index_aspect.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/tests/test_index_aspect.cpp @@ -1,7 +1,6 @@ #include "Aspects/Helpers.h" #include -#include #include using AspectIndexCheck = PyEnvWithContext; diff --git a/ddtrace/appsec/_iast/_taint_tracking/tests/test_other.cpp b/ddtrace/appsec/_iast/_taint_tracking/tests/test_other.cpp index 940e52222b9..0a3ccf063bb 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/tests/test_other.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/tests/test_other.cpp @@ -1,6 +1,5 @@ #include #include -#include // jjj #include using PyByteArray_Check = PyEnvCheck; diff --git a/ddtrace/contrib/internal/botocore/services/bedrock.py b/ddtrace/contrib/internal/botocore/services/bedrock.py index fca33630f57..df547e8f095 100644 --- a/ddtrace/contrib/internal/botocore/services/bedrock.py +++ b/ddtrace/contrib/internal/botocore/services/bedrock.py @@ -318,7 +318,11 @@ def handle_bedrock_response( def patched_bedrock_api_call(original_func, instance, args, kwargs, function_vars): params = function_vars.get("params") pin = function_vars.get("pin") - model_provider, model_name = params.get("modelId").split(".") + model_meta = params.get("modelId").split(".") + if len(model_meta) == 2: + model_provider, model_name = model_meta + else: + _, model_provider, model_name = model_meta # cross-region inference integration = function_vars.get("integration") submit_to_llmobs = integration.llmobs_enabled and "embed" not in model_name with core.context_with_data( diff --git a/ddtrace/contrib/internal/pymongo/client.py b/ddtrace/contrib/internal/pymongo/client.py index f8441f3091b..cb7466ac7a8 100644 --- a/ddtrace/contrib/internal/pymongo/client.py +++ b/ddtrace/contrib/internal/pymongo/client.py @@ -10,10 +10,12 @@ # project import ddtrace +from ddtrace import Pin from ddtrace import config from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY from ddtrace.constants import SPAN_KIND from ddtrace.constants import SPAN_MEASURED_KEY +from ddtrace.contrib import trace_utils from ddtrace.ext import SpanKind from ddtrace.ext import SpanTypes from ddtrace.ext import db @@ -67,7 +69,7 @@ def __getddpin__(client): client.__getddpin__ = functools.partial(__getddpin__, client) # Set a pin on the traced mongo client - ddtrace.Pin(service=_DEFAULT_SERVICE).onto(client) + Pin(service=None).onto(client) # The function is exposed in the public API, but it is not used in the codebase. @@ -128,7 +130,7 @@ def _datadog_trace_operation(operation, wrapped): span = pin.tracer.trace( schematize_database_operation("pymongo.cmd", database_provider="mongodb"), span_type=SpanTypes.MONGODB, - service=pin.service, + service=trace_utils.ext_service(pin, config.pymongo), ) span.set_tag_str(COMPONENT, config.pymongo.integration_name) @@ -251,7 +253,7 @@ def _trace_cmd(cmd, socket_instance, address): s = pin.tracer.trace( schematize_database_operation("pymongo.cmd", database_provider="mongodb"), span_type=SpanTypes.MONGODB, - service=pin.service, + service=trace_utils.ext_service(pin, config.pymongo), ) s.set_tag_str(COMPONENT, config.pymongo.integration_name) diff --git a/ddtrace/contrib/internal/pymongo/patch.py b/ddtrace/contrib/internal/pymongo/patch.py index 5032ce3b48a..0c0927ffea1 100644 --- a/ddtrace/contrib/internal/pymongo/patch.py +++ b/ddtrace/contrib/internal/pymongo/patch.py @@ -16,6 +16,8 @@ from ddtrace.internal.wrapping import unwrap as _u from ddtrace.internal.wrapping import wrap as _w +from ....internal.schema import schematize_service_name + # keep TracedMongoClient import to maintain backwards compatibility from .client import TracedMongoClient # noqa: F401 from .client import _trace_mongo_client_init @@ -46,10 +48,7 @@ _CHECKOUT_FN_NAME = "get_socket" if pymongo.version_tuple < (4, 5) else "checkout" -config._add( - "pymongo", - dict(_default_service="pymongo"), -) +config._add("pymongo", dict(_default_service=schematize_service_name("pymongo"))) def get_version(): @@ -119,7 +118,7 @@ def traced_get_socket(func, args, kwargs): with pin.tracer.trace( "pymongo.%s" % _CHECKOUT_FN_NAME, - service=trace_utils.int_service(pin, config.pymongo), + service=trace_utils.ext_service(pin, config.pymongo), span_type=SpanTypes.MONGODB, ) as span: span.set_tag_str(COMPONENT, config.pymongo.integration_name) diff --git a/ddtrace/contrib/pytest/_plugin_v2.py b/ddtrace/contrib/pytest/_plugin_v2.py index 903f7b9f311..79818992601 100644 --- a/ddtrace/contrib/pytest/_plugin_v2.py +++ b/ddtrace/contrib/pytest/_plugin_v2.py @@ -29,6 +29,7 @@ from ddtrace.ext import test from ddtrace.ext.test_visibility import ITR_SKIPPING_LEVEL from ddtrace.ext.test_visibility.api import TestExcInfo +from ddtrace.ext.test_visibility.api import TestStatus from ddtrace.ext.test_visibility.api import disable_test_visibility from ddtrace.ext.test_visibility.api import enable_test_visibility from ddtrace.ext.test_visibility.api import is_test_visibility_enabled @@ -54,6 +55,12 @@ _NODEID_REGEX = re.compile("^((?P.*)/(?P[^/]*?))::(?P.*?)$") +class _TestOutcome(t.NamedTuple): + status: t.Optional[TestStatus] = None + skip_reason: t.Optional[str] = None + exc_info: t.Optional[TestExcInfo] = None + + def _handle_itr_should_skip(item, test_id) -> bool: """Checks whether a test should be skipped @@ -71,11 +78,11 @@ def _handle_itr_should_skip(item, test_id) -> bool: # Marking the test as forced run also applies to its hierarchy InternalTest.mark_itr_forced_run(test_id) return False - else: - InternalTest.mark_itr_skipped(test_id) - # Marking the test as skipped by ITR so that it appears in pytest's output - item.add_marker(pytest.mark.skip(reason=SKIPPED_BY_ITR_REASON)) # TODO don't rely on internal for reason - return True + + InternalTest.mark_itr_skipped(test_id) + # Marking the test as skipped by ITR so that it appears in pytest's output + item.add_marker(pytest.mark.skip(reason=SKIPPED_BY_ITR_REASON)) # TODO don't rely on internal for reason + return True return False @@ -311,7 +318,7 @@ def pytest_runtest_protocol(item, nextitem) -> None: return -def _pytest_runtest_makereport(item, call, outcome): +def _process_outcome(item, call, outcome) -> _TestOutcome: result = outcome.get_result() test_id = _get_test_id_from_item(item) @@ -322,7 +329,7 @@ def _pytest_runtest_makereport(item, call, outcome): # - it was skipped by ITR # - it was marked with skipif if InternalTest.is_finished(test_id): - return + return _TestOutcome() # In cases where a test was marked as XFAIL, the reason is only available during when call.when == "call", so we # add it as a tag immediately: @@ -339,7 +346,7 @@ def _pytest_runtest_makereport(item, call, outcome): # DEV NOTE: some skip scenarios (eg: skipif) have an exception during setup if call.when != "teardown" and not (has_exception or result.failed): - return + return _TestOutcome() xfail = hasattr(result, "wasxfail") or "xfail" in result.keywords xfail_reason_tag = InternalTest.get_tag(test_id, XFAIL_REASON) if xfail else None @@ -349,8 +356,8 @@ def _pytest_runtest_makereport(item, call, outcome): # that's why no XFAIL_REASON or test.RESULT tags will be added. if result.skipped: if InternalTest.was_skipped_by_itr(test_id): - # Items that were skipped by ITR already have their status set - return + # Items that were skipped by ITR already have their status and reason set + return _TestOutcome() if xfail and not has_skip_keyword: # XFail tests that fail are recorded skipped by pytest, should be passed instead @@ -358,11 +365,9 @@ def _pytest_runtest_makereport(item, call, outcome): InternalTest.set_tag(test_id, test.RESULT, test.Status.XFAIL.value) if xfail_reason_tag is None: InternalTest.set_tag(test_id, XFAIL_REASON, getattr(result, "wasxfail", "XFail")) - InternalTest.mark_pass(test_id) - return + return _TestOutcome(TestStatus.PASS) - InternalTest.mark_skip(test_id, _extract_reason(call)) - return + return _TestOutcome(TestStatus.SKIP, _extract_reason(call)) if result.passed: if xfail and not has_skip_keyword and not item.config.option.runxfail: @@ -371,24 +376,33 @@ def _pytest_runtest_makereport(item, call, outcome): InternalTest.set_tag(test_id, XFAIL_REASON, "XFail") InternalTest.set_tag(test_id, test.RESULT, test.Status.XPASS.value) - InternalTest.mark_pass(test_id) - return + return _TestOutcome(TestStatus.PASS) if xfail and not has_skip_keyword and not item.config.option.runxfail: # XPass (strict=True) are recorded failed by pytest, longrepr contains reason if xfail_reason_tag is None: InternalTest.set_tag(test_id, XFAIL_REASON, getattr(result, "longrepr", "XFail")) InternalTest.set_tag(test_id, test.RESULT, test.Status.XPASS.value) - InternalTest.mark_fail(test_id) - return + return _TestOutcome(TestStatus.FAIL) exc_info = TestExcInfo(call.excinfo.type, call.excinfo.value, call.excinfo.tb) if call.excinfo else None - InternalTest.mark_fail(test_id, exc_info) + return _TestOutcome(status=TestStatus.FAIL, exc_info=exc_info) + + +def _pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo, outcome: pytest.TestReport) -> None: + test_id = _get_test_id_from_item(item) + + test_outcome = _process_outcome(item, call, outcome) + + if test_outcome.status is None and call.when != "teardown": + return + + InternalTest.finish(test_id, test_outcome.status, test_outcome.skip_reason, test_outcome.exc_info) @pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call) -> None: +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> None: """Store outcome for tracing.""" outcome: pytest.TestReport outcome = yield diff --git a/ddtrace/contrib/trace_utils.py b/ddtrace/contrib/trace_utils.py index 3cc0cbdac4e..9fba9c0a7b5 100644 --- a/ddtrace/contrib/trace_utils.py +++ b/ddtrace/contrib/trace_utils.py @@ -404,14 +404,18 @@ def ext_service(pin, int_config, default=None): def _set_url_tag(integration_config, span, url, query): # type: (IntegrationConfig, Span, str, str) -> None - - if integration_config.http_tag_query_string: # Tagging query string in http.url - if config.global_query_string_obfuscation_disabled: # No redacting of query strings - span.set_tag_str(http.URL, url) - else: # Redact query strings - span.set_tag_str(http.URL, redact_url(url, config._obfuscation_query_string_pattern, query)) - else: # Not tagging query string in http.url + if not integration_config.http_tag_query_string: + span.set_tag_str(http.URL, strip_query_string(url)) + elif config.global_query_string_obfuscation_disabled: + # TODO(munir): This case exists for backwards compatibility. To remove query strings from URLs, + # users should set ``DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING=False``. This case should be + # removed when config.global_query_string_obfuscation_disabled is removed (v3.0). + span.set_tag_str(http.URL, url) + elif getattr(config._obfuscation_query_string_pattern, "pattern", None) == b"": + # obfuscation is disabled when DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP="" span.set_tag_str(http.URL, strip_query_string(url)) + else: + span.set_tag_str(http.URL, redact_url(url, config._obfuscation_query_string_pattern, query)) def set_http_meta( diff --git a/ddtrace/internal/ci_visibility/api/_test.py b/ddtrace/internal/ci_visibility/api/_test.py index b48de333641..7f9316c5ca4 100644 --- a/ddtrace/internal/ci_visibility/api/_test.py +++ b/ddtrace/internal/ci_visibility/api/_test.py @@ -126,13 +126,14 @@ def _telemetry_record_event_finished(self): def finish_test( self, - status: TestStatus, + status: Optional[TestStatus] = None, reason: Optional[str] = None, exc_info: Optional[TestExcInfo] = None, override_finish_time: Optional[float] = None, ) -> None: log.debug("Test Visibility: finishing %s, with status: %s, reason: %s", self, status, reason) - self.set_status(status) + if status is not None: + self.set_status(status) if reason is not None: self.set_tag(test.SKIP_REASON, reason) if exc_info is not None: diff --git a/ddtrace/internal/datadog/profiling/build_standalone.sh b/ddtrace/internal/datadog/profiling/build_standalone.sh index 238399e7d03..62ca35ba19f 100755 --- a/ddtrace/internal/datadog/profiling/build_standalone.sh +++ b/ddtrace/internal/datadog/profiling/build_standalone.sh @@ -73,9 +73,6 @@ cmake_args=( -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DLIB_INSTALL_DIR=$(realpath $MY_DIR)/lib - -DPython3_INCLUDE_DIRS=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") - -DPython3_EXECUTABLE=$(which python3) - -DPython3_LIBRARY=$(python3 -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))") ) # Initial build targets; no matter what, dd_wrapper is the base dependency, so it's always built diff --git a/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt b/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt index e9fdd8f391b..678ef83291c 100644 --- a/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt @@ -17,31 +17,27 @@ include(FindLibdatadog) add_subdirectory(../dd_wrapper ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build) +find_package(Python3 COMPONENTS Interpreter Development) # Make sure we have necessary Python variables if(NOT Python3_INCLUDE_DIRS) message(FATAL_ERROR "Python3_INCLUDE_DIRS not found") endif() -# This sets some parameters for the target build, which can only be defined by setup.py -set(ENV{PY_MAJOR_VERSION} ${PY_MAJOR_VERSION}) -set(ENV{PY_MINOR_VERSION} ${PY_MINOR_VERSION}) -set(ENV{PY_MICRO_VERSION} ${PY_MICRO_VERSION}) - -# if PYTHON_EXECUTABLE is unset or empty, but Python3_EXECUTABLE is set, use that -if(NOT PYTHON_EXECUTABLE AND Python3_EXECUTABLE) - set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE}) -endif() - # If we still don't have a Python executable, we can't continue -if(NOT PYTHON_EXECUTABLE) +if(NOT Python3_EXECUTABLE) message(FATAL_ERROR "Python executable not found") endif() +# This sets some parameters for the target build +set(ENV{PY_MAJOR_VERSION} ${Python3_VERSION_MAJOR}) +set(ENV{PY_MINOR_VERSION} ${Python3_VERSION_MINOR}) +set(ENV{PY_MICRO_VERSION} ${Python3_VERSION_PATCH}) + # Cythonize the .pyx file set(CRASHTRACKER_CPP_SRC ${CMAKE_CURRENT_BINARY_DIR}/_crashtracker.cpp) add_custom_command( OUTPUT ${CRASHTRACKER_CPP_SRC} - COMMAND ${PYTHON_EXECUTABLE} -m cython ${CMAKE_CURRENT_LIST_DIR}/_crashtracker.pyx -o ${CRASHTRACKER_CPP_SRC} + COMMAND ${Python3_EXECUTABLE} -m cython ${CMAKE_CURRENT_LIST_DIR}/_crashtracker.pyx -o ${CRASHTRACKER_CPP_SRC} DEPENDS ${CMAKE_CURRENT_LIST_DIR}/_crashtracker.pyx) # Specify the target C-extension that we want to build diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index fcb7e25d200..e0e71bad113 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -22,31 +22,28 @@ include(FindLibdatadog) # things may yet be factored differently. add_subdirectory(../dd_wrapper ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build) +find_package(Python3 COMPONENTS Interpreter Development) + # Make sure we have necessary Python variables if(NOT Python3_INCLUDE_DIRS) message(FATAL_ERROR "Python3_INCLUDE_DIRS not found") endif() -# This sets some parameters for the target build, which can only be defined by setup.py -set(ENV{PY_MAJOR_VERSION} ${PY_MAJOR_VERSION}) -set(ENV{PY_MINOR_VERSION} ${PY_MINOR_VERSION}) -set(ENV{PY_MICRO_VERSION} ${PY_MICRO_VERSION}) - -# if PYTHON_EXECUTABLE is unset or empty, but Python3_EXECUTABLE is set, use that -if(NOT PYTHON_EXECUTABLE AND Python3_EXECUTABLE) - set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE}) -endif() - # If we still don't have a Python executable, we can't continue -if(NOT PYTHON_EXECUTABLE) +if(NOT Python3_EXECUTABLE) message(FATAL_ERROR "Python executable not found") endif() +# This sets some parameters for the target build +set(ENV{PY_MAJOR_VERSION} ${Python3_VERSION_MAJOR}) +set(ENV{PY_MINOR_VERSION} ${Python3_VERSION_MINOR}) +set(ENV{PY_MICRO_VERSION} ${Python3_VERSION_PATCH}) + # Cythonize the .pyx file set(DDUP_CPP_SRC ${CMAKE_CURRENT_BINARY_DIR}/_ddup.cpp) add_custom_command( OUTPUT ${DDUP_CPP_SRC} - COMMAND ${PYTHON_EXECUTABLE} -m cython ${CMAKE_CURRENT_LIST_DIR}/_ddup.pyx -o ${DDUP_CPP_SRC} + COMMAND ${Python3_EXECUTABLE} -m cython ${CMAKE_CURRENT_LIST_DIR}/_ddup.pyx -o ${DDUP_CPP_SRC} DEPENDS ${CMAKE_CURRENT_LIST_DIR}/_ddup.pyx) # Specify the target C-extension that we want to build diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 8bcc96276d3..bc0caae58d8 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -21,6 +21,7 @@ include(FindCppcheck) # design is unknown. Hack it for now. add_subdirectory(../dd_wrapper ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build) +find_package(Python3 COMPONENTS Interpreter Development) # Make sure we have necessary Python variables if(NOT Python3_INCLUDE_DIRS) message(FATAL_ERROR "Python3_INCLUDE_DIRS not found") diff --git a/ddtrace/internal/test_visibility/api.py b/ddtrace/internal/test_visibility/api.py index cba827646f3..618dcf2499b 100644 --- a/ddtrace/internal/test_visibility/api.py +++ b/ddtrace/internal/test_visibility/api.py @@ -115,7 +115,7 @@ class FinishArgs(NamedTuple): """InternalTest allows finishing with an overridden finish time (for EFD and other retry purposes)""" test_id: InternalTestId - status: TestStatus + status: t.Optional[TestStatus] = None skip_reason: t.Optional[str] = None exc_info: t.Optional[TestExcInfo] = None override_finish_time: t.Optional[float] = None @@ -124,7 +124,7 @@ class FinishArgs(NamedTuple): @_catch_and_log_exceptions def finish( item_id: InternalTestId, - status: ext_api.TestStatus, + status: t.Optional[ext_api.TestStatus] = None, reason: t.Optional[str] = None, exc_info: t.Optional[ext_api.TestExcInfo] = None, override_finish_time: t.Optional[float] = None, diff --git a/ddtrace/internal/utils/http.py b/ddtrace/internal/utils/http.py index 48867cf9689..cba605a1527 100644 --- a/ddtrace/internal/utils/http.py +++ b/ddtrace/internal/utils/http.py @@ -74,21 +74,13 @@ def strip_query_string(url): def redact_query_string(query_string, query_string_obfuscation_pattern): - # type: (str, Optional[re.Pattern]) -> Union[bytes, str] - if query_string_obfuscation_pattern is None: - return query_string - + # type: (str, re.Pattern) -> Union[bytes, str] bytes_query = query_string if isinstance(query_string, bytes) else query_string.encode("utf-8") return query_string_obfuscation_pattern.sub(b"", bytes_query) def redact_url(url, query_string_obfuscation_pattern, query_string=None): # type: (str, re.Pattern, Optional[str]) -> Union[str,bytes] - - # Avoid further processing if obfuscation is disabled - if query_string_obfuscation_pattern is None: - return url - parts = compat.parse.urlparse(url) redacted_query = None diff --git a/ddtrace/settings/config.py b/ddtrace/settings/config.py index ab9ba671253..b92f87e6298 100644 --- a/ddtrace/settings/config.py +++ b/ddtrace/settings/config.py @@ -557,18 +557,16 @@ def __init__(self): dd_trace_obfuscation_query_string_regexp = _get_config( "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT ) - self.global_query_string_obfuscation_disabled = True # If empty obfuscation pattern + self.global_query_string_obfuscation_disabled = dd_trace_obfuscation_query_string_regexp == "" self._obfuscation_query_string_pattern = None self.http_tag_query_string = True # Default behaviour of query string tagging in http.url - if dd_trace_obfuscation_query_string_regexp != "": - self.global_query_string_obfuscation_disabled = False # Not empty obfuscation pattern - try: - self._obfuscation_query_string_pattern = re.compile( - dd_trace_obfuscation_query_string_regexp.encode("ascii") - ) - except Exception: - log.warning("Invalid obfuscation pattern, disabling query string tracing", exc_info=True) - self.http_tag_query_string = False # Disable query string tagging if malformed obfuscation pattern + try: + self._obfuscation_query_string_pattern = re.compile( + dd_trace_obfuscation_query_string_regexp.encode("ascii") + ) + except Exception: + log.warning("Invalid obfuscation pattern, disabling query string tracing", exc_info=True) + self.http_tag_query_string = False # Disable query string tagging if malformed obfuscation pattern self._ci_visibility_agentless_enabled = _get_config("DD_CIVISIBILITY_AGENTLESS_ENABLED", False, asbool) self._ci_visibility_agentless_url = _get_config("DD_CIVISIBILITY_AGENTLESS_URL", "") diff --git a/hatch.toml b/hatch.toml index 0c619d273f5..c582f8f6a9d 100644 --- a/hatch.toml +++ b/hatch.toml @@ -369,6 +369,7 @@ dependencies = [ [envs.ddtrace_unit_tests.env-vars] DD_IAST_ENABLED = "false" DD_REMOTE_CONFIGURATION_ENABLED = "false" +CMAKE_BUILD_PARALLEL_LEVEL="6" [envs.ddtrace_unit_tests.scripts] test = [ diff --git a/releasenotes/notes/botocore-bedrock-cross-region-inference-model-fix-179b7f1ddd6e8e02.yaml b/releasenotes/notes/botocore-bedrock-cross-region-inference-model-fix-179b7f1ddd6e8e02.yaml new file mode 100644 index 00000000000..be6f6d0dc27 --- /dev/null +++ b/releasenotes/notes/botocore-bedrock-cross-region-inference-model-fix-179b7f1ddd6e8e02.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + botocore: fixes bedrock model and model provider interpretation from ``modelId`` when using cross-region inference. diff --git a/releasenotes/notes/make-obfuscation-consistent-8a2361bf3298e1be.yaml b/releasenotes/notes/make-obfuscation-consistent-8a2361bf3298e1be.yaml new file mode 100644 index 00000000000..fd4e51ec995 --- /dev/null +++ b/releasenotes/notes/make-obfuscation-consistent-8a2361bf3298e1be.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + integrations: Ensures that ``http.url`` span tag contains the full query string when ``DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP `` is set to an empty string. diff --git a/scripts/iast/mod_leak_functions.py b/scripts/iast/mod_leak_functions.py index e43db154dbf..e55480a1d36 100644 --- a/scripts/iast/mod_leak_functions.py +++ b/scripts/iast/mod_leak_functions.py @@ -24,7 +24,9 @@ def test_doit(): string3 = string1 + string2 # 2 propagation ranges: hiroot1234 string4 = "-".join([string3, string3, string3]) # 6 propagation ranges: hiroot1234-hiroot1234-hiroot1234 - string5 = string4[0:20] # 1 propagation range: hiroot1234-hiroot123 + string4_2 = string1 + string4_2 += " " + " ".join(string_ for string_ in [string4, string4, string4]) + string5 = string4_2[0:20] # 1 propagation range: hiroot1234-hiroot123 string6 = string5.title() # 1 propagation range: Hiroot1234-Hiroot123 string7 = string6.upper() # 1 propagation range: HIROOT1234-HIROOT123 string8 = "%s_notainted" % string7 # 1 propagation range: HIROOT1234-HIROOT123_notainted diff --git a/setup.py b/setup.py index 196a2c452ca..f45e9b086c2 100644 --- a/setup.py +++ b/setup.py @@ -337,18 +337,12 @@ def build_extension_cmake(self, ext): cmake_build_dir = Path(self.build_lib.replace("lib.", "cmake."), ext.name).resolve() cmake_build_dir.mkdir(parents=True, exist_ok=True) - # Get development paths - python_include = sysconfig.get_paths()["include"] - python_lib = sysconfig.get_config_var("LIBDIR") - # Which commands are passed to _every_ cmake invocation cmake_args = ext.cmake_args or [] cmake_args += [ "-S{}".format(ext.source_dir), # cmake>=3.13 "-B{}".format(cmake_build_dir), # cmake>=3.13 - "-DPython3_INCLUDE_DIRS={}".format(python_include), - "-DPython3_LIBRARIES={}".format(python_lib), - "-DPYTHON_EXECUTABLE={}".format(sys.executable), + "-DPython3_ROOT_DIR={}".format(sysconfig.get_config_var("prefix")), "-DCMAKE_BUILD_TYPE={}".format(ext.build_type), "-DLIB_INSTALL_DIR={}".format(output_dir), "-DEXTENSION_NAME={}".format(extension_basename), @@ -536,11 +530,6 @@ def get_exts_for(name): CMakeExtension( "ddtrace.internal.datadog.profiling.ddup._ddup", source_dir=DDUP_DIR, - cmake_args=[ - "-DPY_MAJOR_VERSION={}".format(sys.version_info.major), - "-DPY_MINOR_VERSION={}".format(sys.version_info.minor), - "-DPY_MICRO_VERSION={}".format(sys.version_info.micro), - ], optional=False, ) ) @@ -549,11 +538,6 @@ def get_exts_for(name): CMakeExtension( "ddtrace.internal.datadog.profiling.crashtracker._crashtracker", source_dir=CRASHTRACKER_DIR, - cmake_args=[ - "-DPY_MAJOR_VERSION={}".format(sys.version_info.major), - "-DPY_MINOR_VERSION={}".format(sys.version_info.minor), - "-DPY_MICRO_VERSION={}".format(sys.version_info.micro), - ], optional=False, ) ) diff --git a/static-analysis.datadog.yml b/static-analysis.datadog.yml index 05e121907f8..156f3713b47 100644 --- a/static-analysis.datadog.yml +++ b/static-analysis.datadog.yml @@ -1,9 +1,12 @@ rulesets: - python-best-practices: rules: - nested-blocks: + comment-fixme-todo-ownership: ignore: - "**" max-function-lines: - ignore: + ignore: - "tests/ci_visibility/api/fake_runner_*.py" + nested-blocks: + ignore: + - "**" diff --git a/tests/contrib/botocore/bedrock_cassettes/anthropic_message_invoke.yaml b/tests/contrib/botocore/bedrock_cassettes/anthropic_message_invoke.yaml index a4abf1ac291..dfc0235d6cf 100644 --- a/tests/contrib/botocore/bedrock_cassettes/anthropic_message_invoke.yaml +++ b/tests/contrib/botocore/bedrock_cassettes/anthropic_message_invoke.yaml @@ -21,7 +21,7 @@ interactions: - !!binary | YXR0ZW1wdD0x method: POST - uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet-20240229-v1%3A0/invoke + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-3-sonnet-20240229-v1%3A0/invoke response: body: string: '{"id":"msg_01E6sPP1ksqicYCaBrkvzna8","type":"message","role":"assistant","content":[{"type":"text","text":"Hobbits diff --git a/tests/contrib/botocore/test_bedrock.py b/tests/contrib/botocore/test_bedrock.py index 90c7669cba5..3c91b147a97 100644 --- a/tests/contrib/botocore/test_bedrock.py +++ b/tests/contrib/botocore/test_bedrock.py @@ -173,6 +173,7 @@ def test_anthropic_invoke(bedrock_client, request_vcr): @pytest.mark.snapshot def test_anthropic_message_invoke(bedrock_client, request_vcr): body, model = json.dumps(_REQUEST_BODIES["anthropic_message"]), _MODELS["anthropic_message"] + model = "us." + model with request_vcr.use_cassette("anthropic_message_invoke.yaml"): response = bedrock_client.invoke_model(body=body, modelId=model) json.loads(response.get("body").read()) diff --git a/tests/contrib/botocore/test_bedrock_llmobs.py b/tests/contrib/botocore/test_bedrock_llmobs.py index ae2fb7894c6..b7e80651f35 100644 --- a/tests/contrib/botocore/test_bedrock_llmobs.py +++ b/tests/contrib/botocore/test_bedrock_llmobs.py @@ -128,6 +128,10 @@ def _test_llmobs_invoke(cls, provider, bedrock_client, mock_llmobs_span_writer, } with get_request_vcr().use_cassette(cassette_name): body, model = json.dumps(body), _MODELS[provider] + if provider == "anthropic_message": + # we do this to re-use a cassette which tests + # cross-region inference + model = "us." + model response = bedrock_client.invoke_model(body=body, modelId=model) json.loads(response.get("body").read()) span = mock_tracer.pop_traces()[0][0] diff --git a/tests/contrib/pymongo/test.py b/tests/contrib/pymongo/test.py index 69633b13e79..f439036fcc1 100644 --- a/tests/contrib/pymongo/test.py +++ b/tests/contrib/pymongo/test.py @@ -544,6 +544,27 @@ def test_user_specified_service_v0(self): assert len(spans) == 2 assert spans[1].service != "mysvc" + @TracerTestCase.run_in_subprocess(env_overrides=dict()) + def test_user_specified_service_default_override(self): + """ + Override service name using config + """ + from ddtrace import config + from ddtrace import patch + + cfg = config.pymongo + cfg["service"] = "new-mongo" + patch(pymongo=True) + assert cfg.service == "new-mongo", f"service name is {cfg.service}" + tracer = DummyTracer() + client = pymongo.MongoClient(port=MONGO_CONFIG["port"]) + Pin.get_from(client).clone(tracer=tracer).onto(client) + + client["testdb"].drop_collection("whatever") + spans = tracer.pop() + assert spans + assert spans[0].service == "new-mongo", f"service name is {spans[0].service}" + @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_SERVICE="mysvc", DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v1")) def test_user_specified_service_v1(self): """ @@ -563,6 +584,20 @@ def test_user_specified_service_v1(self): assert len(spans) == 2 assert spans[1].service == "mysvc" + @TracerTestCase.run_in_subprocess(env_overrides=dict()) + def test_unspecified_service_v0(self): + """ + v0: When a user does not specify a service for the app + The pymongo integration should use pymongo. + """ + tracer = DummyTracer() + client = pymongo.MongoClient(port=MONGO_CONFIG["port"]) + Pin.get_from(client).clone(tracer=tracer).onto(client) + client["testdb"].drop_collection("whatever") + spans = tracer.pop() + assert len(spans) == 2 + assert spans[1].service == "pymongo" + @TracerTestCase.run_in_subprocess( env_overrides=dict(DD_PYMONGO_SERVICE="mypymongo", DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v0") ) diff --git a/tests/tracer/test_env_vars.py b/tests/tracer/test_env_vars.py index 0191ece9e5f..272dd751196 100644 --- a/tests/tracer/test_env_vars.py +++ b/tests/tracer/test_env_vars.py @@ -8,7 +8,7 @@ "env_var_name,env_var_value,expected_obfuscation_config,expected_global_query_string_obfuscation_disabled," "expected_http_tag_query_string", [ - ("DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", "", None, True, True), + ("DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", "", 're.compile(b"")', True, True), ( "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", "(?i)(?:p(?:ass)?w(?:or))", @@ -64,7 +64,7 @@ def test_obfuscation_querystring_pattern_env_var( ], env=env, ) - assert b"AssertionError" not in out + assert b"AssertionError" not in out, out @pytest.mark.parametrize( diff --git a/tests/tracer/test_http.py b/tests/tracer/test_http.py index 03b003b1de5..5a8a02ebf51 100644 --- a/tests/tracer/test_http.py +++ b/tests/tracer/test_http.py @@ -45,16 +45,6 @@ def test_strip_query_string(url): ) -@pytest.mark.parametrize("url", _url_fixtures()) -def test_redact_url_obfuscation_disabled_without_param(url): - assert redact_url(url, None, None) == url - - -@pytest.mark.parametrize("url", _url_fixtures()) -def test_redact_url_obfuscation_disabled_with_param(url): - assert redact_url(url, None, "query_string") == url - - @pytest.mark.parametrize("url", _url_fixtures()) def test_redact_url_not_redacts_without_param(url): res = redact_url(url, re.compile(b"\\@"), None) diff --git a/tests/tracer/test_trace_utils.py b/tests/tracer/test_trace_utils.py index 07195e29009..d9c5508df93 100644 --- a/tests/tracer/test_trace_utils.py +++ b/tests/tracer/test_trace_utils.py @@ -1021,6 +1021,57 @@ def test_sanitized_url_in_http_meta(span, int_config): assert span.get_tag(http.URL) == FULL_URL +@pytest.mark.subprocess(env={"DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP": ""}) +def test_url_in_http_with_empty_obfuscation_regex(): + from ddtrace import config + from ddtrace import tracer + from ddtrace.contrib.trace_utils import set_http_meta + from ddtrace.ext import http + + assert config._obfuscation_query_string_pattern.pattern == b"", config._obfuscation_query_string_pattern + + SENSITIVE_URL = "http://weblog:7777/?application_key=123" + config._add("myint", dict()) + with tracer.trace("s") as span: + set_http_meta( + span, + config.myint, + method="GET", + url=SENSITIVE_URL, + status_code=200, + ) + assert span.get_tag(http.URL) == SENSITIVE_URL + + +# TODO(munir): Remove this test when global_query_string_obfuscation_disabled is removed +@pytest.mark.subprocess(env={"DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP": ""}) +def test_url_in_http_with_obfuscation_enabled_and_empty_regex(): + # Test that query strings are not added to urls when the obfuscation regex is an empty string + # and obfuscation is enabled (not disabled xD) + from ddtrace import config + from ddtrace import tracer + from ddtrace.contrib.trace_utils import set_http_meta + from ddtrace.ext import http + + # assert obfuscation is disabled when the regex is an empty string + assert config.global_query_string_obfuscation_disabled is True + assert config._obfuscation_query_string_pattern is not None + + # Enable obfucation with an empty regex + config.global_query_string_obfuscation_disabled = False + + config._add("myint", dict()) + with tracer.trace("s") as span: + set_http_meta( + span, + config.myint, + method="GET", + url="http://weblog:7777/?application_key=123", + status_code=200, + ) + assert span.get_tag(http.URL) == "http://weblog:7777/", span._meta + + def test_url_in_http_meta(span, int_config): SENSITIVE_QS_URL = "http://example.com/search?token=03cb9f67dbbc4cb8b963629951e10934&q=query#frag?ment" REDACTED_URL = "http://example.com/search?&q=query#frag?ment"