diff --git a/CHANGELOG.md b/CHANGELOG.md index a938a2b5..2686c215 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a _Nothing yet_ +## [0.4.4] - 2024-07-30 +### Fixed +- Fixed compatibility with NumPy 2. + ## [0.4.3] - 2022-12-24 ### Fixed - Fixed source distribution failing to build, due to `scikit-build` & `setuptools` version mismatch in `pyproject.toml`. @@ -119,7 +123,8 @@ _Nothing yet_ - Main functionality of `Thing`, `Data`, `Matrix`, `Vector`, `Sound`, `Spectrum`, `Spectrogram`, and `Intensity` classes. Preliminary implementations of `Pitch`, `Harmonicity`, `Formant`, and `MFCC`. - Basic Travis CI configuration. -[Unreleased]: https://github.com/YannickJadoul/Parselmouth/compare/v0.4.3...HEAD +[Unreleased]: https://github.com/YannickJadoul/Parselmouth/compare/v0.4.4...HEAD +[0.4.4]: https://github.com/YannickJadoul/Parselmouth/compare/v0.4.3...v0.4.4 [0.4.3]: https://github.com/YannickJadoul/Parselmouth/compare/v0.4.2...v0.4.3 [0.4.2]: https://github.com/YannickJadoul/Parselmouth/compare/v0.4.1...v0.4.2 [0.4.1]: https://github.com/YannickJadoul/Parselmouth/compare/v0.4.0...v0.4.1 diff --git a/docs/examples/pitch_manipulation.ipynb b/docs/examples/pitch_manipulation.ipynb index 7e6b8f29..6a0206ec 100644 --- a/docs/examples/pitch_manipulation.ipynb +++ b/docs/examples/pitch_manipulation.ipynb @@ -257,7 +257,7 @@ "version": "3.8.7" }, "nbsphinx": { - "timeout": 60 + "timeout": 60 } }, "nbformat": 4, diff --git a/pybind11/CMakeLists.txt b/pybind11/CMakeLists.txt index ded4dad9..00451d11 100644 --- a/pybind11/CMakeLists.txt +++ b/pybind11/CMakeLists.txt @@ -116,6 +116,7 @@ set(PYBIND11_HEADERS include/pybind11/eigen.h include/pybind11/embed.h include/pybind11/eval.h + include/pybind11/gil_safe_call_once.h include/pybind11/iostream.h include/pybind11/functional.h include/pybind11/numpy.h diff --git a/pybind11/include/pybind11/cast.h b/pybind11/include/pybind11/cast.h index ffa30975..0b1970b3 100644 --- a/pybind11/include/pybind11/cast.h +++ b/pybind11/include/pybind11/cast.h @@ -1202,7 +1202,7 @@ template <> class type_caster { if (!src) return false; else if (src.ptr() == Py_True) { value = true; return true; } else if (src.ptr() == Py_False) { value = false; return true; } - else if (convert || !strcmp("numpy.bool_", Py_TYPE(src.ptr())->tp_name)) { + else if (convert || is_numpy_bool(src)) { // (allow non-implicit conversion for numpy booleans) Py_ssize_t res = -1; @@ -1236,6 +1236,15 @@ template <> class type_caster { return handle(src ? Py_True : Py_False).inc_ref(); } PYBIND11_TYPE_CASTER(bool, _("bool")); + +private: + // Test if an object is a NumPy boolean (without fetching the type). + static inline bool is_numpy_bool(handle object) { + const char *type_name = Py_TYPE(object.ptr())->tp_name; + // Name changed to `numpy.bool` in NumPy 2, `numpy.bool_` is needed for 1.x support + return std::strcmp("numpy.bool", type_name) == 0 + || std::strcmp("numpy.bool_", type_name) == 0; + } }; // Helper class for UTF-{8,16,32} C++ stl strings: diff --git a/pybind11/include/pybind11/detail/common.h b/pybind11/include/pybind11/detail/common.h index 6c90e82b..e66022c4 100644 --- a/pybind11/include/pybind11/detail/common.h +++ b/pybind11/include/pybind11/detail/common.h @@ -45,6 +45,14 @@ # endif #endif +#if defined(PYBIND11_CPP20) +# define PYBIND11_CONSTINIT constinit +# define PYBIND11_DTOR_CONSTEXPR constexpr +#else +# define PYBIND11_CONSTINIT +# define PYBIND11_DTOR_CONSTEXPR +#endif + // Compiler version assertions #if defined(__INTEL_COMPILER) # if __INTEL_COMPILER < 1800 @@ -150,6 +158,10 @@ # undef copysign #endif +#if defined(PYBIND11_NUMPY_1_ONLY) +# define PYBIND11_INTERNAL_NUMPY_1_ONLY_DETECTED +#endif + #if defined(_MSC_VER) # if defined(PYBIND11_DEBUG_MARKER) # define _DEBUG diff --git a/pybind11/include/pybind11/gil_safe_call_once.h b/pybind11/include/pybind11/gil_safe_call_once.h new file mode 100644 index 00000000..a0deee1d --- /dev/null +++ b/pybind11/include/pybind11/gil_safe_call_once.h @@ -0,0 +1,85 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +//#include "gil.h" + +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +// Use the `gil_safe_call_once_and_store` class below instead of the naive +// +// static auto imported_obj = py::module_::import("module_name"); // BAD, DO NOT USE! +// +// which has two serious issues: +// +// 1. Py_DECREF() calls potentially after the Python interpreter was finalized already, and +// 2. deadlocks in multi-threaded processes (because of missing lock ordering). +// +// The following alternative avoids both problems: +// +// PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; +// auto &imported_obj = storage // Do NOT make this `static`! +// .call_once_and_store_result([]() { +// return py::module_::import("module_name"); +// }) +// .get_stored(); +// +// The parameter of `call_once_and_store_result()` must be callable. It can make +// CPython API calls, and in particular, it can temporarily release the GIL. +// +// `T` can be any C++ type, it does not have to involve CPython API types. +// +// The behavior with regard to signals, e.g. `SIGINT` (`KeyboardInterrupt`), +// is not ideal. If the main thread is the one to actually run the `Callable`, +// then a `KeyboardInterrupt` will interrupt it if it is running normal Python +// code. The situation is different if a non-main thread runs the +// `Callable`, and then the main thread starts waiting for it to complete: +// a `KeyboardInterrupt` will not interrupt the non-main thread, but it will +// get processed only when it is the main thread's turn again and it is running +// normal Python code. However, this will be unnoticeable for quick call-once +// functions, which is usually the case. +template +class gil_safe_call_once_and_store { +public: + // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called. + template + gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn) { + if (!is_initialized_) { // This read is guarded by the GIL. + // Multiple threads may enter here, because the GIL is released in the next line and + // CPython API calls in the `fn()` call below may release and reacquire the GIL. + gil_scoped_release gil_rel; // Needed to establish lock ordering. + std::call_once(once_flag_, [&] { + // Only one thread will ever enter here. + gil_scoped_acquire gil_acq; + ::new (storage_) T(fn()); // fn may release, but will reacquire, the GIL. + is_initialized_ = true; // This write is guarded by the GIL. + }); + // All threads will observe `is_initialized_` as true here. + } + // Intentionally not returning `T &` to ensure the calling code is self-documenting. + return *this; + } + + // This must only be called after `call_once_and_store_result()` was called. + T &get_stored() { + assert(is_initialized_); + return *reinterpret_cast(storage_); + } + + constexpr gil_safe_call_once_and_store() = default; + PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default; + +private: + alignas(T) char storage_[sizeof(T)] = {}; + std::once_flag once_flag_ = {}; + bool is_initialized_ = false; + // The `is_initialized_`-`storage_` pair is very similar to `std::optional`, + // but the latter does not have the triviality properties of former, + // therefore `std::optional` is not a viable alternative here. +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/pybind11/include/pybind11/numpy.h b/pybind11/include/pybind11/numpy.h index ec0a1d47..0608e831 100644 --- a/pybind11/include/pybind11/numpy.h +++ b/pybind11/include/pybind11/numpy.h @@ -10,7 +10,11 @@ #pragma once #include "pybind11.h" +#include "detail/common.h" #include "complex.h" +#include "gil_safe_call_once.h" +#include "pytypes.h" + #include #include #include @@ -25,6 +29,14 @@ #include #include +#if defined(PYBIND11_NUMPY_1_ONLY) && !defined(PYBIND11_INTERNAL_NUMPY_1_ONLY_DETECTED) +# error PYBIND11_NUMPY_1_ONLY must be defined before any pybind11 header is included. +#endif + +#if !defined(PYBIND11_NUMPY_1_ONLY) & PY_VERSION_HEX < 0x03090000 +# define PYBIND11_NUMPY_1_ONLY +#endif + #if defined(_MSC_VER) # pragma warning(push) # pragma warning(disable: 4127) // warning C4127: Conditional expression is constant @@ -33,7 +45,8 @@ /* This will be true on all flat address space platforms and allows us to reduce the whole npy_intp / ssize_t / Py_intptr_t business down to just ssize_t for all size and dimension types (e.g. shape, strides, indexing), instead of inflicting this - upon the library user. */ + upon the library user. + Note that NumPy 2 now uses ssize_t for `npy_intp` to simplify this. */ static_assert(sizeof(::pybind11::ssize_t) == sizeof(Py_intptr_t), "ssize_t != Py_intptr_t"); static_assert(std::is_signed::value, "Py_intptr_t must be signed"); // We now can reinterpret_cast between py::ssize_t and Py_intptr_t (MSVC + PyPy cares) @@ -48,7 +61,8 @@ template <> struct handle_type_name { static constexpr auto name = _("num template struct npy_format_descriptor; -struct PyArrayDescr_Proxy { +/* NumPy 1 proxy (always includes legacy fields) */ +struct PyArrayDescr1_Proxy { PyObject_HEAD PyObject *typeobj; char kind; @@ -63,6 +77,45 @@ struct PyArrayDescr_Proxy { PyObject *names; }; +#ifndef PYBIND11_NUMPY_1_ONLY +struct PyArrayDescr_Proxy { + PyObject_HEAD + PyObject *typeobj; + char kind; + char type; + char byteorder; + char _former_flags; + int type_num; + /* Additional fields are NumPy version specific. */ +}; +#else +/* NumPy 1.x only, we can expose all fields */ +using PyArrayDescr_Proxy = PyArrayDescr1_Proxy; +#endif + +#ifndef PYBIND11_NUMPY_1_ONLY +/* NumPy 2 proxy, including legacy fields */ +struct PyArrayDescr2_Proxy { + PyObject_HEAD + PyObject *typeobj; + char kind; + char type; + char byteorder; + char _former_flags; + int type_num; + std::uint64_t flags; + ssize_t elsize; + ssize_t alignment; + PyObject *metadata; + Py_hash_t hash; + void *reserved_null[2]; + /* The following fields only exist if 0 <= type_num < 2056 */ + char *subarray; + PyObject *fields; + PyObject *names; +}; +#endif + struct PyArray_Proxy { PyObject_HEAD char *data; @@ -115,6 +168,29 @@ inline numpy_internals& get_numpy_internals() { return *ptr; } +inline PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { + module_ numpy = module_::import("numpy"); + str version_string = numpy.attr("__version__"); + + module_ numpy_lib = module_::import("numpy.lib"); + object numpy_version = numpy_lib.attr("NumpyVersion")(version_string); + int major_version = numpy_version.attr("major").cast(); + + #ifdef PYBIND11_NUMPY_1_ONLY + if (major_version >= 2) { + throw std::runtime_error( + "This extension was built with PYBIND11_NUMPY_1_ONLY defined, " + "but NumPy 2 is used in this process. For NumPy2 compatibility, " + "this extension needs to be rebuilt without the PYBIND11_NUMPY_1_ONLY define."); + } + #endif + + /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially + became a private module. */ + std::string numpy_core_path = major_version >= 2 ? "numpy._core" : "numpy.core"; + return module_::import((numpy_core_path + "." + submodule_name).c_str()); +} + template struct same_size { template using as = bool_constant; }; @@ -164,14 +240,16 @@ struct npy_api { NPY_ULONG_, NPY_ULONGLONG_, NPY_UINT_), }; + unsigned int PyArray_RUNTIME_VERSION_; + typedef struct { Py_intptr_t *ptr; int len; } PyArray_Dims; static npy_api& get() { - static npy_api api = lookup(); - return api; + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage.call_once_and_store_result(lookup).get_stored(); } bool PyArray_Check_(PyObject *obj) const { @@ -197,8 +275,10 @@ struct npy_api { PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); int (*PyArray_DescrConverter_) (PyObject *, PyObject **); bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); +#ifdef PYBIND11_NUMPY_1_ONLY int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, unsigned char, PyObject **, int *, Py_intptr_t *, PyObject **, PyObject *); +#endif PyObject *(*PyArray_Squeeze_)(PyObject *); // Unused. Not removed because that affects ABI of the class. int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); @@ -213,30 +293,39 @@ struct npy_api { API_PyArray_DescrFromScalar = 57, API_PyArray_FromAny = 69, API_PyArray_Resize = 80, - API_PyArray_CopyInto = 82, + // CopyInto was slot 82 and 50 was effectively an alias. NumPy 2 removed 82. + API_PyArray_CopyInto = 50, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, API_PyArray_DescrNewFromType = 96, API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, +#ifdef PYBIND11_NUMPY_1_ONLY API_PyArray_GetArrayParamsFromObject = 278, +#endif API_PyArray_Squeeze = 136, API_PyArray_SetBaseObject = 282 }; static npy_api lookup() { - module_ m = module_::import("numpy.core.multiarray"); + module_ m = detail::import_numpy_core_submodule("multiarray"); auto c = m.attr("_ARRAY_API"); #if PY_MAJOR_VERSION >= 3 void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), NULL); #else void **api_ptr = (void **) PyCObject_AsVoidPtr(c.ptr()); #endif + if (api_ptr == nullptr) { + //raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer."); + throw error_already_set(); + } npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion); - if (api.PyArray_GetNDArrayCFeatureVersion_() < 0x7) + api.PyArray_RUNTIME_VERSION_ = api.PyArray_GetNDArrayCFeatureVersion_(); + if (api.PyArray_RUNTIME_VERSION_ < 0x7) { pybind11_fail("pybind11 numpy support requires numpy >= 1.7.0"); + } DECL_NPY_API(PyArray_Type); DECL_NPY_API(PyVoidArrType_Type); DECL_NPY_API(PyArrayDescr_Type); @@ -250,7 +339,9 @@ struct npy_api { DECL_NPY_API(PyArray_DescrNewFromType); DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_EquivTypes); +#ifdef PYBIND11_NUMPY_1_ONLY DECL_NPY_API(PyArray_GetArrayParamsFromObject); +#endif DECL_NPY_API(PyArray_Squeeze); DECL_NPY_API(PyArray_SetBaseObject); #undef DECL_NPY_API @@ -274,6 +365,16 @@ inline const PyArrayDescr_Proxy* array_descriptor_proxy(const PyObject* ptr) { return reinterpret_cast(ptr); } +inline const PyArrayDescr1_Proxy *array_descriptor1_proxy(const PyObject *ptr) { + return reinterpret_cast(ptr); +} + +#ifndef PYBIND11_NUMPY_1_ONLY +inline const PyArrayDescr2_Proxy *array_descriptor2_proxy(const PyObject *ptr) { + return reinterpret_cast(ptr); +} +#endif + inline bool check_flags(const void* ptr, int flag) { return (flag == (array_proxy(ptr)->flags & flag)); } @@ -498,14 +599,32 @@ class dtype : public object { } /// Size of the data type in bytes. +#ifdef PYBIND11_NUMPY_1_ONLY + ssize_t itemsize() const { return detail::array_descriptor_proxy(m_ptr)->elsize; } +#else ssize_t itemsize() const { - return detail::array_descriptor_proxy(m_ptr)->elsize; + if (detail::npy_api::get().PyArray_RUNTIME_VERSION_ < 0x12) { + return detail::array_descriptor1_proxy(m_ptr)->elsize; + } + return detail::array_descriptor2_proxy(m_ptr)->elsize; } +#endif /// Returns true for structured data types. +#ifdef PYBIND11_NUMPY_1_ONLY + bool has_fields() const { return detail::array_descriptor_proxy(m_ptr)->names != nullptr; } +#else bool has_fields() const { - return detail::array_descriptor_proxy(m_ptr)->names != nullptr; + if (detail::npy_api::get().PyArray_RUNTIME_VERSION_ < 0x12) { + return detail::array_descriptor1_proxy(m_ptr)->names != nullptr; + } + const auto *proxy = detail::array_descriptor2_proxy(m_ptr); + if (proxy->type_num < 0 || proxy->type_num >= 2056) { + return false; + } + return proxy->names != nullptr; } +#endif /// Single-character type code. char kind() const { @@ -513,10 +632,14 @@ class dtype : public object { } private: - static object _dtype_from_pep3118() { - static PyObject *obj = module_::import("numpy.core._internal") - .attr("_dtype_from_pep3118").cast().release().ptr(); - return reinterpret_borrow(obj); + static object &_dtype_from_pep3118() { + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result([]() { + return detail::import_numpy_core_submodule("_internal") + .attr("_dtype_from_pep3118"); + }) + .get_stored(); } dtype strip_padding(ssize_t itemsize) { @@ -641,9 +764,7 @@ class array : public buffer { } /// Byte size of a single element - ssize_t itemsize() const { - return detail::array_descriptor_proxy(detail::array_proxy(m_ptr)->descr)->elsize; - } + ssize_t itemsize() const { return dtype().itemsize(); } /// Total number of bytes ssize_t nbytes() const { diff --git a/pybind11/include/pybind11/pybind11.h b/pybind11/include/pybind11/pybind11.h index 53503c4a..553d0289 100644 --- a/pybind11/include/pybind11/pybind11.h +++ b/pybind11/include/pybind11/pybind11.h @@ -47,6 +47,7 @@ #include "detail/class.h" #include "detail/init.h" +#include #include #include #include @@ -2187,7 +2188,9 @@ class gil_scoped_acquire { class gil_scoped_release { public: + // PRECONDITION: The GIL must be held when this constructor is called. explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { + assert(PyGILState_Check()); // `get_internals()` must be called here unconditionally in order to initialize // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an // initialization race could occur as multiple threads try `gil_scoped_acquire`. @@ -2236,7 +2239,11 @@ class gil_scoped_acquire { class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() { state = PyEval_SaveThread(); } + // PRECONDITION: The GIL must be held when this constructor is called. + gil_scoped_release() { + assert(PyGILState_Check()); + state = PyEval_SaveThread(); + } ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; diff --git a/pybind11/tools/pybind11NewTools.cmake b/pybind11/tools/pybind11NewTools.cmake index 18da8be1..5b516b3e 100644 --- a/pybind11/tools/pybind11NewTools.cmake +++ b/pybind11/tools/pybind11NewTools.cmake @@ -99,7 +99,7 @@ if(NOT DEFINED PYTHON_MODULE_EXTENSION) execute_process( COMMAND "${${_Python}_EXECUTABLE}" "-c" - "from distutils import sysconfig as s;print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" + "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" OUTPUT_VARIABLE _PYTHON_MODULE_EXTENSION ERROR_VARIABLE _PYTHON_MODULE_EXTENSION_ERR OUTPUT_STRIP_TRAILING_WHITESPACE)