Support numpy>=2 and remove numpy<2 pin (#3040)

* Build wheels with latest numpy and test them with oldest supported numpy

* Update oldest supported Python and Numpy versions

* Pin `numpy<2` for documentation only

The documentation requires an old version of bokeh which isn't compatible with `numpy>=2`.

* Cast to `c_long` in RxD on Python side before passing to C++

With Numpy2 there was an inconsistency in the integer types, i.e. the C++ assumed a smaller
type than the Python side, on Windows only. Causing SEGFAULT. This fixes the issue by using
consistent integer types.

.github/workflows/coverage.yml

@@ -58,7 +58,7 @@ jobs:
         run: |
           python3 -m venv music-venv
           source music-venv/bin/activate
-          python3 -m pip install 'mpi4py<4' "cython" 'numpy<2' setuptools
+          python3 -m pip install 'mpi4py<4' cython numpy setuptools
           sudo mkdir -p $MUSIC_INSTALL_DIR
           sudo chown -R $USER $MUSIC_INSTALL_DIR
           curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/refs/tags/${MUSIC_VERSION}.zip

.github/workflows/neuron-ci.yml

@@ -207,7 +207,7 @@ jobs:
         run: |
           python3 -m venv music-venv
           source music-venv/bin/activate
-          python3 -m pip install 'mpi4py<4' "cython" 'numpy<2' setuptools
+          python3 -m pip install 'mpi4py<4' cython numpy setuptools
           sudo mkdir -p $MUSIC_INSTALL_DIR
           sudo chown -R $USER $MUSIC_INSTALL_DIR
           curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/refs/tags/${MUSIC_VERSION}.zip

ci/win_build_cmake.sh

@@ -30,7 +30,7 @@ cd $BUILD_SOURCESDIRECTORY/build
 	-DNRN_ENABLE_RX3D=ON  \
 	-DNRN_RX3D_OPT_LEVEL=2 \
 	-DNRN_BINARY_DIST_BUILD=ON \
-	-DPYTHON_EXECUTABLE=/c/Python38/python.exe \
+	-DPYTHON_EXECUTABLE=/c/Python39/python.exe \
 	-DNRN_ENABLE_PYTHON_DYNAMIC=ON  \
 	-DNRN_PYTHON_DYNAMIC='c:/Python38/python.exe;c:/Python39/python.exe;c:/Python310/python.exe;c:/Python311/python.exe;c:/Python312/python.exe'  \
 	-DCMAKE_INSTALL_PREFIX='/c/nrn-install' \

ci/win_install_deps.cmd

@@ -23,11 +23,11 @@ pwsh -command "(Get-Content C:\Python310\Lib\distutils\cygwinccompiler.py) -repl
 pwsh -command "(Get-Content C:\Python311\Lib\distutils\cygwinccompiler.py) -replace 'msvcr100', 'msvcrt' | Out-File C:\Python311\Lib\distutils\cygwinccompiler.py"
 
 :: install numpy
-C:\Python38\python.exe -m pip install  numpy==1.17.5 "cython" || goto :error
-C:\Python39\python.exe -m pip install  numpy==1.19.3 "cython" || goto :error
-C:\Python310\python.exe -m pip install numpy==1.21.3 "cython" || goto :error
-C:\Python311\python.exe -m pip install numpy==1.23.5 "cython" || goto :error
-C:\Python312\python.exe -m pip install numpy==1.26.3 "cython" || goto :error
+C:\Python38\python.exe -m pip install  numpy cython || goto :error
+C:\Python39\python.exe -m pip install  numpy cython || goto :error
+C:\Python310\python.exe -m pip install numpy cython || goto :error
+C:\Python311\python.exe -m pip install numpy cython || goto :error
+C:\Python312\python.exe -m pip install numpy cython || goto :error
 :: setuptools 70.2 leads to an error
 C:\Python312\python.exe -m pip install setuptools==70.1.1 || goto :error
 

ci/win_test_installer.cmd

@@ -21,9 +21,24 @@ C:\Python38\python -c "import neuron; neuron.test(); quit()" || set "errorfound=
 C:\Python39\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
 C:\Python310\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
 C:\Python311\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
-:: install numpy dependency
-python -m pip install "numpy<2"
+C:\Python312\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
+
+:: install oldest supported numpy
+C:\Python38\python.exe -m pip install -r packaging/python/oldest_numpy_requirements.txt || goto :error
+C:\Python39\python.exe -m pip install -r packaging/python/oldest_numpy_requirements.txt || goto :error
+C:\Python310\python.exe -m pip install -r packaging/python/oldest_numpy_requirements.txt || goto :error
+C:\Python311\python.exe -m pip install -r packaging/python/oldest_numpy_requirements.txt || goto :error
+C:\Python312\python.exe -m pip install -r packaging/python/oldest_numpy_requirements.txt || goto :error
+
+:: test all pythons again
+C:\Python38\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
+C:\Python39\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
+C:\Python310\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
+C:\Python311\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
+C:\Python312\python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
+
 :: run also using whatever is system python
+python -m pip install numpy
 python --version
 python -c "import neuron; neuron.test(); quit()" || set "errorfound=y"
 

docs/install/install_instructions.md

@@ -482,7 +482,7 @@ share/lib/python/neuron/rxd/geometry3d/surfaces.cpp:14605:41: error: no member n
 ```
 often there's something related to NumPy nearby, e.g. `npy`.
 
-The issue is that certain versions of NEURON (9.0 and earlier) are not
+The issue is that certain versions of NEURON (below 9.0) are not
 compatible with `numpy>=2`. Check the numpy version, e.g.,
 ```
 python -c "import numpy; print(numpy.__version__)"

nrn_requirements.txt

@@ -11,5 +11,5 @@ packaging
 pytest<=8.1.1 # potential bug from 8.2.0 due to parallelism?
 pytest-cov
 mpi4py<4 # MUSIC not compatible with MPI 4
-numpy<2
+numpy
 find_libpython

packaging/python/build_requirements.txt

@@ -1,8 +1,3 @@
 cython
 packaging
-numpy==1.17.5;python_version=='3.8'
-numpy==1.19.3;python_version=='3.9' and platform_machine!='arm64'
-numpy==1.21.3;python_version=='3.9' and platform_machine=='arm64'
-numpy==1.21.4;python_version=='3.10'
-numpy==1.23.5;python_version=='3.11'
-numpy==1.26.0;python_version=='3.12'
+numpy

packaging/python/oldest_numpy_requirements.txt

@@ -0,0 +1,5 @@
+numpy==1.20.3;python_version=='3.9' and platform_machine!='arm64'
+numpy==1.21.6;python_version=='3.9' and platform_machine=='arm64'
+numpy==1.21.6;python_version=='3.10'
+numpy==1.23.5;python_version=='3.11'
+numpy==1.26.4;python_version=='3.12'

packaging/python/test_requirements.txt

@@ -0,0 +1,2 @@
+pytest
+setuptools;python_version>='3.12' # From 3.12, no longer installed by default

packaging/python/test_wheels.sh

@@ -237,6 +237,12 @@ test_wheel () {
 }
 
 
+test_wheel_basic_python () {
+    echo "=========== BASIC PYTHON TESTS ==========="
+    $python_exe -c "import neuron; neuron.test(); neuron.test_rxd()"
+}
+
+
 echo "== Testing $python_wheel using $python_exe ($python_ver) =="
 
 
@@ -257,10 +263,8 @@ fi
 $python_exe -m pip install --upgrade pip
 
 
-# install numpy, pytest and neuron
-# we install setuptools because since python 3.12 it is no more installed
-# by default
-$python_exe -m pip install "numpy<2" pytest setuptools
+# install test requirements
+$python_exe -m pip install -r packaging/python/test_requirements.txt
 $python_exe -m pip install $python_wheel
 $python_exe -m pip show neuron || $python_exe -m pip show neuron-nightly
 
@@ -271,8 +275,14 @@ if echo $compile_options | grep "NRN_ENABLE_CORENEURON=ON" > /dev/null ; then
   has_coreneuron=true
 fi
 
-# run tests
-test_wheel "${python_exe}"
+# run tests with latest NumPy
+echo " == Running tests with latest NumPy == "
+test_wheel
+
+# run basic python tests with oldest supported NumPy
+echo " == Running basic python tests with oldest supported NumPy == "
+$python_exe -m pip install -r packaging/python/oldest_numpy_requirements.txt
+test_wheel_basic_python
 
 # cleanup
 if [[ "$use_venv" != "false" ]]; then

setup.py

@@ -354,7 +354,7 @@ def setup_package():
     NRN_COLLECT_DIRS = ["bin", "lib", "include", "share"]
 
     docs_require = []  # sphinx, themes, etc
-    maybe_rxd_reqs = ["numpy<2", "Cython"] if Components.RX3D else []
+    maybe_rxd_reqs = ["numpy", "Cython"] if Components.RX3D else []
     maybe_docs = docs_require if "docs" in sys.argv else []
     maybe_test_runner = ["pytest-runner"] if "test" in sys.argv else []
 
@@ -509,7 +509,7 @@ def setup_package():
         },
         cmdclass=dict(build_ext=CMakeAugmentedBuilder, docs=Docs),
         install_requires=[
-            "numpy>=1.9.3,<2",
+            "numpy>=1.9.3",
             "packaging",
             "find_libpython",
             "setuptools<=70.3.0",

share/lib/python/neuron/rxd/rxd.py

@@ -61,7 +61,7 @@
 setup_solver.argtypes = [
     ndpointer(ctypes.c_double),
     ctypes.c_int,
-    numpy.ctypeslib.ndpointer(numpy.int_, flags="contiguous"),
+    numpy.ctypeslib.ndpointer(ctypes.c_long, flags="contiguous"),
     ctypes.c_int,
 ]
 
@@ -630,8 +630,8 @@ def _matrix_to_rxd_sparse(m):
     return (
         n,
         len(nonzero_i),
-        numpy.ascontiguousarray(nonzero_i, dtype=numpy.int_),
-        numpy.ascontiguousarray(nonzero_j, dtype=numpy.int_),
+        numpy.ascontiguousarray(nonzero_i, dtype=ctypes.c_long),
+        numpy.ascontiguousarray(nonzero_j, dtype=ctypes.c_long),
         nonzero_values,
     )
 
@@ -709,7 +709,7 @@ def _setup_matrices():
         n = len(_node_get_states())
 
         volumes = node._get_data()[0]
-        zero_volume_indices = (numpy.where(volumes == 0)[0]).astype(numpy.int_)
+        zero_volume_indices = (numpy.where(volumes == 0)[0]).astype(ctypes.c_long)
         if species._has_1d:
             # TODO: initialization is slow. track down why
             for sr in _species_get_all_species():
@@ -1896,7 +1896,7 @@ def _init():
     _setup_matrices()
     # if species._has_1d and species._1d_submatrix_n():
     # volumes = node._get_data()[0]
-    # zero_volume_indices = (numpy.where(volumes == 0)[0]).astype(numpy.int_)
+    # zero_volume_indices = (numpy.where(volumes == 0)[0]).astype(ctypes.c_long)
     # setup_solver(_node_get_states(), len(_node_get_states()), zero_volume_indices, len(zero_volume_indices), h._ref_t, h._ref_dt)
     clear_rates()
     _setup_memb_currents()

share/lib/python/neuron/rxd/species.py

@@ -60,12 +60,12 @@
     ctypes.c_int,
     ctypes.py_object,
     ctypes.c_long,
-    numpy.ctypeslib.ndpointer(dtype=int),
-    numpy.ctypeslib.ndpointer(dtype=int),
+    numpy.ctypeslib.ndpointer(dtype=ctypes.c_long),
+    numpy.ctypeslib.ndpointer(dtype=ctypes.c_long),
     ctypes.c_long,
-    numpy.ctypeslib.ndpointer(dtype=int),
+    numpy.ctypeslib.ndpointer(dtype=ctypes.c_long),
     ctypes.c_long,
-    numpy.ctypeslib.ndpointer(dtype=int),
+    numpy.ctypeslib.ndpointer(dtype=ctypes.c_long),
     ctypes.c_long,
     numpy.ctypeslib.ndpointer(dtype=float),
     ctypes.c_double,
@@ -842,7 +842,7 @@ def line_defs(self, nodes, direction, nodes_length):
 
         # sort list for parallelization
         line_defs.sort(key=lambda x: x[1], reverse=True)
-        line_defs = numpy.asarray(line_defs, dtype=int)
+        line_defs = numpy.asarray(line_defs, dtype=ctypes.c_long)
         line_defs = line_defs.reshape(2 * len(line_defs))
         return line_defs
 
@@ -862,7 +862,7 @@ def ordered_nodes(self, p_line_defs, direction, neighbors):
 
     def create_neighbors_array(self, nodes, nodes_length):
         self._isalive()
-        my_array = numpy.zeros((nodes_length, 3), dtype=int)
+        my_array = numpy.zeros((nodes_length, 3), dtype=ctypes.c_long)
         for n in nodes:
             for i, ele in enumerate(n.neighbors[::2]):
                 my_array[n._index, i] = ele if ele is not None else -1

test/rxd/conftest.py

@@ -1,5 +1,6 @@
 import os.path as osp
 import numpy
+import ctypes
 import pytest
 import gc
 
@@ -81,7 +82,7 @@ def neuron_nosave_instance(neuron_import):
     rxd.rxd.rxd_include_node_flux1D(0, None, None, None)
     rxd.species._has_1d = False
     rxd.species._has_3d = False
-    rxd.rxd._zero_volume_indices = numpy.ndarray(0, dtype=numpy.int_)
+    rxd.rxd._zero_volume_indices = numpy.ndarray(0, dtype=ctypes.c_long)
     rxd.set_solve_type(dimension=1)