build: Raise Python minimum to 3.7, pybind11 to 2.7 (#1855)

Drop Python 2.7 support. Upgrade from very old cmake FindPython to findPython3, update idioms. Simplify a lot of cruft that was supporting python2, as well as supporting build features we don't use (long ago, I needed some special library naming conventions at SPI, etc., that we no longer use). Change the module filename from "OSL" to "oslquery" to reflect that it's only covering oslquery and make room for other python bindings in the future. Signed-off-by: Larry Gritz <[email protected]>
AcademySoftwareFoundation · Sep 5, 2024 · b472adc · b472adc
1 parent 7738e23
commit b472adc
Show file tree

Hide file tree

Showing 27 changed files with 95 additions and 125 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -349,8 +349,8 @@ jobs:
             cxx_std: 17
             openexr_ver: v3.1.11
             openimageio_ver: v2.4.13.0
-            pybind11_ver: v2.6.2
-            python_ver: 2.7
+            pybind11_ver: v2.7.0
+            python_ver: 3.8
             simd: sse4.2
             setenvs: export CMAKE_BUILD_TYPE=Debug
                             LLVM_VERSION=9.0.0
@@ -365,7 +365,7 @@ jobs:
             openexr_ver: v3.1.11
             openimageio_ver: release
             pybind11_ver: v2.8.1
-            # python_ver: 2.7   FIXME
+            python_ver: 3.8
             simd: avx2,f16c
             setenvs: export LLVM_VERSION=10.0.0
                             OPENIMAGEIO_CMAKE_FLAGS="-DBUILD_FMT_VERSION=7.0.1"

diff --git a/CHANGES.md b/CHANGES.md
@@ -5,8 +5,9 @@ Release 1.14 -- in progress (compared to 1.13)
 -------------------------------------------------------------
 ### New minimum dependencies and compatibility changes:
 
-* Note: it is likely that by the time it is released, OSL 1.14 will require
-  Imath >= 3.1, Python >= 3.7, C++ >= 17, gcc >= 9.3.
+* *C++*: Move to C++17 standard minimum (from 14), which also implies a minimum gcc 9.3 (raised from 6.3)
+* *Python*: 3.7 minimum.
+* *Imath*: minimum raised to 3.1 (from 2.4)
 
 
 ### OSL Language, standard library, and oslc compiler (for shader writers):

diff --git a/INSTALL.md b/INSTALL.md
@@ -55,7 +55,7 @@ NEW or CHANGED dependencies since the last major release are **bold**.
     * [Cuda](https://developer.nvidia.com/cuda-downloads) 9.0 or higher. It is
       recommended that you use 11.0 or higher.
 
-* [Imath](https://github.com/AcademySoftwareFoundation/Imath) 3.1 or newer.
+* [Imath](https://github.com/AcademySoftwareFoundation/Imath) **3.1 or newer**.
 * [Flex](https://github.com/westes/flex) 2.5.35 or newer and
   [GNU Bison](https://www.gnu.org/software/bison/) 2.7 or newer.
   Note that on some MacOS/xcode releases, the system-installed Bison is too
@@ -67,13 +67,10 @@ NEW or CHANGED dependencies since the last major release are **bold**.
   be operative.
 * (optional) Python: If you are building the Python bindings or running the
   testsuite:
-    * Python >= 2.7 (tested against 2.7, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12)
-      NOTE: It is likely that 1.13 is the last release that will support
-      Python 2.7.
-    * pybind11 >= 2.4.2 (Tested through 2.11. Note that pybind11 v2.10+ does
-      not support Python < 3.6.)
+    * **Python >= 3.7** (tested through 3.12)
+    * **pybind11 >= 2.7** (tested through 2.13)
     * NumPy
-* (optional) Qt5 >= 5.6 or Qt6 (tested Qt5 through 5.15 and Qt6 through 6.6).
+* (optional) Qt5 >= 5.6 or Qt6 (tested Qt5 through 5.15 and Qt6 through 6.7).
   If not found at build time, the `osltoy` application will be disabled.
 
 

diff --git a/Makefile b/Makefile
@@ -210,14 +210,6 @@ ifneq (${PYTHON_VERSION},)
 MY_CMAKE_FLAGS += -DPYTHON_VERSION:STRING=${PYTHON_VERSION}
 endif
 
-ifneq (${PYLIB_LIB_PREFIX},)
-MY_CMAKE_FLAGS += -DPYLIB_LIB_PREFIX:BOOL=${PYLIB_LIB_PREFIX}
-endif
-
-ifneq (${PYLIB_INCLUDE_SONAME},)
-MY_CMAKE_FLAGS += -DPYLIB_INCLUDE_SONAME:BOOL=${PYLIB_INCLUDE_SONAME}
-endif
-
 
 #$(info MY_CMAKE_FLAGS = ${MY_CMAKE_FLAGS})
 #$(info MY_MAKE_FLAGS = ${MY_MAKE_FLAGS})

diff --git a/doc/build_install/windows/build_osl.py b/doc/build_install/windows/build_osl.py
@@ -34,8 +34,6 @@
 #
 #
 
-from __future__ import print_function
-
 from distutils.spawn import find_executable
 
 import argparse

diff --git a/src/build-scripts/ci-startup.bash b/src/build-scripts/ci-startup.bash
@@ -27,7 +27,7 @@ export LSAN_OPTIONS=suppressions=$PWD/src/build-scripts/nosanitize.txt
 export ASAN_OPTIONS=print_suppressions=0:detect_odr_violation=1
 
 export USE_PYTHON=${USE_PYTHON:=1}
-export PYTHON_VERSION=${PYTHON_VERSION:="2.7"}
+export PYTHON_VERSION=${PYTHON_VERSION:="3.7"}
 export PYTHONPATH=/usr/local/lib/python${PYTHON_VERSION}/site-packages:$PYTHONPATH
 export PYTHONPATH=/usr/local/lib64/python${PYTHON_VERSION}/site-packages:$PYTHONPATH
 export PYTHONPATH=$OSL_ROOT/lib/python${PYTHON_VERSION}/site-packages:$PYTHONPATH

diff --git a/src/build-scripts/docdeep.py b/src/build-scripts/docdeep.py
@@ -1,6 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function, absolute_import
-
 # Copyright Contributors to the Open Shading Language project.
 # SPDX-License-Identifier: BSD-3-Clause
 # https://github.com/AcademySoftwareFoundation/OpenShadingLanguage

diff --git a/src/build-scripts/gh-installdeps.bash b/src/build-scripts/gh-installdeps.bash
@@ -107,12 +107,8 @@ else
     if [[ "${PYTHON_VERSION}" == "3.9" ]] ; then
         time sudo apt-get -q install -y python3.9-dev python3-numpy
         pip3 --version
-        pip3 install numpy
-    elif [[ "$PYTHON_VERSION" == "2.7" ]] ; then
-        time sudo apt-get -q install -y python-dev python-numpy
-    else
-        pip3 install numpy
     fi
+    pip3 install numpy
 
     if [[ "$CXX" == "g++-9" ]] ; then
         time sudo apt-get install -y g++-9

diff --git a/src/build-scripts/gh-win-installdeps.bash b/src/build-scripts/gh-win-installdeps.bash
@@ -29,7 +29,7 @@ if [[ "$PYTHON_VERSION" == "3.6" ]] ; then
     export CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH;/c/hostedtoolcache/windows/Python/3.6.8/x64"
 else
     export CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH;/c/hostedtoolcache/windows/Python/3.7.9/x64"
-    export Python_EXECUTABLE="/c/hostedtoolcache/windows/Python/3.7.9/x64/python.exe"
+    export Python3_EXECUTABLE="/c/hostedtoolcache/windows/Python/3.7.9/x64/python.exe"
     export PYTHONPATH=$OpenImageIO_ROOT/lib/python${PYTHON_VERSION}/site-packages
 fi
 pip install numpy

diff --git a/src/build-scripts/hidesymbols.map b/src/build-scripts/hidesymbols.map
@@ -1,4 +1,4 @@
 {
-    global: *OSL*; test_shade; *osl*imageio*; *osl_input_extensions;
+    global: *OSL*; test_shade; *osl*imageio*; *osl_input_extensions; PyInit*;
     local: osl_*; *pvt*; *;
 };
diff --git a/src/build-scripts/process-ptx.py b/src/build-scripts/process-ptx.py
@@ -7,8 +7,6 @@
 #  * add the `.visible` directive to all functions
 #  * add the `.visible` directive to global/constant variables
 
-from __future__ import print_function, absolute_import
-
 import re
 import sys
 

diff --git a/src/build-scripts/serialize-bc.py b/src/build-scripts/serialize-bc.py
@@ -6,8 +6,6 @@
 # Turn an LLVM-compiled bitfile into a C++ source file where the compiled
 # bitcode is in a huge array.
 
-from __future__ import print_function, absolute_import
-
 import sys
 
 in_name = sys.argv[1]

diff --git a/src/cmake/cuda_macros.cmake b/src/cmake/cuda_macros.cmake
@@ -185,15 +185,15 @@ function ( LLVM_COMPILE_CUDA llvm_src headers prefix llvm_bc_cpp_generated extra
     MAKE_CUDA_BITCODE (${llvm_src} "" llvm_bc "${extra_clang_args}")
 
     add_custom_command (OUTPUT ${llvm_bc_cpp}
-        COMMAND ${Python_EXECUTABLE} "${CMAKE_SOURCE_DIR}/src/build-scripts/serialize-bc.py" ${llvm_bc} ${llvm_bc_cpp} ${prefix}
+        COMMAND ${Python3_EXECUTABLE} "${CMAKE_SOURCE_DIR}/src/build-scripts/serialize-bc.py" ${llvm_bc} ${llvm_bc_cpp} ${prefix}
         MAIN_DEPENDENCY ${llvm_src}
         DEPENDS "${CMAKE_SOURCE_DIR}/src/build-scripts/serialize-bc.py" ${llvm_src} ${headers} ${llvm_bc}
         WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" )
 endfunction ()
 
 function ( MAKE_EMBEDDED_CPP symbol_name output_cpp input_file )
     add_custom_command ( OUTPUT ${output_cpp}
-        COMMAND ${Python_EXECUTABLE} "${CMAKE_SOURCE_DIR}/src/build-scripts/serialize-bc.py"
+        COMMAND ${Python3_EXECUTABLE} "${CMAKE_SOURCE_DIR}/src/build-scripts/serialize-bc.py"
             ${input_file} ${output_cpp} "${symbol_name}"
 
         DEPENDS "${CMAKE_SOURCE_DIR}/src/build-scripts/serialize-bc.py" ${linked_bc}
@@ -227,7 +227,7 @@ function ( CUDA_SHADEOPS_COMPILE prefix output_bc output_ptx input_srcs headers
         COMMAND ${LLVM_OPT_TOOL} ${opt_tool_flags} ${linked_bc} -o ${linked_bc}
         COMMAND ${LLVM_LLC_TOOL} --march=nvptx64 -mcpu=${CUDA_TARGET_ARCH} ${linked_bc} -o ${linked_ptx}
         # This script converts all of the .weak functions defined in the PTX into .visible functions.
-        COMMAND ${Python_EXECUTABLE} "${CMAKE_SOURCE_DIR}/src/build-scripts/process-ptx.py"
+        COMMAND ${Python3_EXECUTABLE} "${CMAKE_SOURCE_DIR}/src/build-scripts/process-ptx.py"
             ${linked_ptx} ${linked_ptx}
             DEPENDS ${shadeops_bc_list} ${exec_headers} ${PROJECT_PUBLIC_HEADERS} ${input_srcs} ${headers}
                 "${CMAKE_SOURCE_DIR}/src/build-scripts/process-ptx.py"

diff --git a/src/cmake/externalpackages.cmake b/src/cmake/externalpackages.cmake
@@ -123,6 +123,9 @@ checked_find_package (partio)
 
 # From pythonutils.cmake
 find_python ()
+if (USE_PYTHON)
+    checked_find_package (pybind11 REQUIRED VERSION_MIN 2.7)
+endif ()
 
 
 # Qt -- used for osltoy

diff --git a/src/cmake/llvm_macros.cmake b/src/cmake/llvm_macros.cmake
@@ -112,7 +112,7 @@ function ( EMBED_LLVM_BITCODE_IN_CPP src_list suffix output_name list_to_append_
      # Serialize the linked bitcode into a CPP file 
     set ( src_bc_cpp "${CMAKE_CURRENT_BINARY_DIR}/${output_name}.bc.cpp" )
     add_custom_command ( OUTPUT ${src_bc_cpp}
-        COMMAND ${Python_EXECUTABLE} "${_THIS_MODULE_BASE_DIR}/../build-scripts/serialize-bc.py"
+        COMMAND ${Python3_EXECUTABLE} "${_THIS_MODULE_BASE_DIR}/../build-scripts/serialize-bc.py"
             ${linked_src_bc} ${src_bc_cpp} ${output_name}
         DEPENDS "${_THIS_MODULE_BASE_DIR}/../build-scripts/serialize-bc.py" ${linked_src_bc}
         ${exec_headers} ${PROJECT_PUBLIC_HEADERS}

diff --git a/src/cmake/pythonutils.cmake b/src/cmake/pythonutils.cmake
@@ -3,58 +3,63 @@
 # https://github.com/AcademySoftwareFoundation/OpenShadingLanguage
 
 # Python-related options.
-option (USE_PYTHON "Build the Python bindings" ON)
+set_option (USE_PYTHON "Build the Python bindings" ON)
 set (PYTHON_VERSION "" CACHE STRING "Target version of python to try to find")
-option (PYLIB_INCLUDE_SONAME "If ON, soname/soversion will be set for Python module library" OFF)
-option (PYLIB_LIB_PREFIX "If ON, prefix the Python module with 'lib'" OFF)
-set (PYMODULE_SUFFIX "" CACHE STRING "Suffix to add to Python module init namespace")
+if (WIN32)
+    set (PYLIB_LIB_TYPE SHARED CACHE STRING "Type of library to build for python module (MODULE or SHARED)")
+else ()
+    set (PYLIB_LIB_TYPE MODULE CACHE STRING "Type of library to build for python module (MODULE or SHARED)")
+endif ()
 
 
 # Find Python. This macro should only be called if python is required. If
 # Python cannot be found, it will be a fatal error.
 macro (find_python)
     if (NOT VERBOSE)
-        set (PythonInterp_FIND_QUIETLY true)
-        set (PythonLibs_FIND_QUIETLY true)
+        set (PythonInterp3_FIND_QUIETLY true)
+        set (PythonLibs3_FIND_QUIETLY true)
     endif ()
 
     # Attempt to find the desired version, but fall back to other
     # additional versions.
-    set (_req REQUIRED)
-    if (PYTHON_VERSION)
-        list (APPEND _req EXACT)
+    unset (_req)
+    if (USE_PYTHON)
+        set (_req REQUIRED)
+        if (PYTHON_VERSION)
+            list (APPEND _req EXACT)
+        endif ()
     endif ()
-    checked_find_package (Python ${PYTHON_VERSION}
+    checked_find_package (Python3 ${PYTHON_VERSION}
                           ${_req}
+                          VERSION_MIN 3.7
                           COMPONENTS Interpreter Development
-                          PRINT Python_Development_FOUND Python_VERSION
-                                Python_EXECUTABLE Python_LIBRARIES
-                                Python_Development_FOUND Python_Interpreter_FOUND)
+                          PRINT Python3_VERSION Python3_EXECUTABLE
+                                Python3_LIBRARIES
+                                Python3_Development_FOUND
+                                Python3_Interpreter_FOUND )
 
     # The version that was found may not be the default or user
     # defined one.
-    set (PYTHON_VERSION_FOUND ${Python_VERSION_MAJOR}.${Python_VERSION_MINOR})
+    set (PYTHON_VERSION_FOUND ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR})
 
     # Give hints to subsequent pybind11 searching to ensure that it finds
     # exactly the same version that we found.
-    set (PythonInterp_FIND_VERSION PYTHON_VERSION_FOUND)
-    set (PythonInterp_FIND_VERSION_MAJOR ${Python_VERSION_MAJOR})
+    set (PythonInterp3_FIND_VERSION PYTHON_VERSION_FOUND)
+    set (PythonInterp3_FIND_VERSION_MAJOR ${Python3_VERSION_MAJOR})
 
     if (NOT DEFINED PYTHON_SITE_DIR)
         set (PYTHON_SITE_DIR "${CMAKE_INSTALL_LIBDIR}/python${PYTHON_VERSION_FOUND}/site-packages")
     endif ()
+    message (VERBOSE "    Python version found ${PYTHON_VERSION_FOUND}")
     message (VERBOSE "    Python site packages dir ${PYTHON_SITE_DIR}")
-    message (VERBOSE "    Python to include 'lib' prefix: ${PYLIB_LIB_PREFIX}")
-    message (VERBOSE "    Python to include SO version: ${PYLIB_INCLUDE_SONAME}")
 endmacro()
 
 
-
 ###########################################################################
 # pybind11
 
 macro (setup_python_module)
-    cmake_parse_arguments (lib "" "TARGET;MODULE;LIBS" "SOURCES" ${ARGN})
+    cmake_parse_arguments (lib "" "TARGET;MODULE" "SOURCES;LIBS" ${ARGN})
     # Arguments: <prefix> <options> <one_value_keywords> <multi_value_keywords> args...
 
     set (target_name ${lib_TARGET})
@@ -64,46 +69,35 @@ macro (setup_python_module)
         set_property (SOURCE ${lib_SOURCES} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-macro-redefined ")
     endif ()
 
-    # Add the library itself
-    add_library (${target_name} MODULE ${lib_SOURCES})
+    pybind11_add_module(${target_name} ${PYLIB_LIB_TYPE} ${lib_SOURCES})
 
+#    # Add the library itself
+#    add_library (${target_name} MODULE ${lib_SOURCES})
+#
     # Declare the libraries it should link against
     target_link_libraries (${target_name}
-                           PRIVATE
-                                pybind11::pybind11
-                                Python::Python
-                                ${lib_LIBS})
-    if (APPLE)
-        set_target_properties (${target_name} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-    endif ()
+                           PRIVATE ${lib_LIBS})
 
-    # Exclude the 'lib' prefix from the name
-    if (NOT PYLIB_LIB_PREFIX)
-        target_compile_definitions(${target_name}
-                                   PRIVATE "PYMODULE_NAME=${lib_MODULE}")
-        set_target_properties (${target_name} PROPERTIES
-                               OUTPUT_NAME ${lib_MODULE}
-                               PREFIX "")
-    else ()
-        target_compile_definitions(${target_name}
-                                   PRIVATE "PYMODULE_NAME=Py${lib_MODULE}")
-        set_target_properties (${target_name} PROPERTIES
-                               OUTPUT_NAME Py${lib_MODULE}
-                               PREFIX lib)
+    set (_module_LINK_FLAGS "${VISIBILITY_MAP_COMMAND} ${EXTRA_DSO_LINK_ARGS}")
+    if (UNIX AND NOT APPLE)
+        # Hide symbols from any static dependent libraries embedded here.
+        set (_module_LINK_FLAGS "${_module_LINK_FLAGS} -Wl,--exclude-libs,ALL")
     endif ()
+    set_target_properties (${target_name} PROPERTIES LINK_FLAGS ${_module_LINK_FLAGS})
 
-    if (PYLIB_INCLUDE_SONAME)
-        message(VERBOSE "Setting Py${lib_MODULE} SOVERSION to: ${SOVERSION}")
-        set_target_properties(${target_name} PROPERTIES
-            VERSION ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}
-            SOVERSION ${SOVERSION} )
-    endif()
+    # Exclude the 'lib' prefix from the name
+    target_compile_definitions(${target_name}
+                               PRIVATE "PYMODULE_NAME=${lib_MODULE}")
+    set_target_properties (${target_name} PROPERTIES
+                           OUTPUT_NAME ${lib_MODULE}
+                           # PREFIX ""
+                           )
 
-    if (WIN32)
-        set_target_properties (${target_name} PROPERTIES
-                               DEBUG_POSTFIX "_d"
-                               SUFFIX ".pyd")
-    endif()
+#    if (WIN32)
+#        set_target_properties (${target_name} PROPERTIES
+#                               DEBUG_POSTFIX "_d"
+#                               SUFFIX ".pyd")
+#    endif()
 
     # In the build area, put it in lib/python so it doesn't clash with the
     # non-python libraries of the same name (which aren't prefixed by "lib"
@@ -114,8 +108,10 @@ macro (setup_python_module)
             )
 
     install (TARGETS ${target_name}
-             RUNTIME DESTINATION ${PYTHON_SITE_DIR} COMPONENT user
-             LIBRARY DESTINATION ${PYTHON_SITE_DIR} COMPONENT user)
+             RUNTIME DESTINATION ${PYTHON_SITE_DIR}/${lib_MODULE} COMPONENT user
+             LIBRARY DESTINATION ${PYTHON_SITE_DIR}/${lib_MODULE} COMPONENT user)
+
+    install(FILES __init__.py DESTINATION ${PYTHON_SITE_DIR}/${lib_MODULE})
 
 endmacro ()
 
diff --git a/src/cmake/testing.cmake b/src/cmake/testing.cmake
@@ -37,7 +37,7 @@ macro (add_one_testsuite testname testsrcdir)
     set (testdir "${CMAKE_BINARY_DIR}/testsuite/${testname}")
     file (MAKE_DIRECTORY "${testdir}")
     if (NOT _tst_COMMAND)
-        set (_tst_COMMAND ${Python_EXECUTABLE} "${testsuite}/runtest.py" ${testdir})
+        set (_tst_COMMAND ${Python3_EXECUTABLE} "${testsuite}/runtest.py" ${testdir})
         if (MSVC_IDE)
             list (APPEND _tst_COMMAND --devenv-config $<CONFIGURATION>
                                       --solution-path "${CMAKE_BINARY_DIR}" )
@@ -410,7 +410,7 @@ macro (osl_add_all_tests)
     # We also exclude these tests if this is a sanitizer build, because the
     # Python interpreter itself won't be linked with the right asan
     # libraries to run correctly.
-    if (USE_PYTHON AND NOT SANITIZE)
+    if (USE_PYTHON AND Python3_Development_FOUND AND NOT SANITIZE)
         TESTSUITE ( python-oslquery )
     endif ()
 

diff --git a/src/doc/CMakeLists.txt b/src/doc/CMakeLists.txt
@@ -41,7 +41,7 @@ macro (docdeep_generate)
     message (VERBOSE "docdeep will make '${mdfile}'")
     set (docdeep_program ${CMAKE_SOURCE_DIR}/src/build-scripts/docdeep.py)
     add_custom_command (OUTPUT ${mdfile}
-        COMMAND ${Python_EXECUTABLE} ${docdeep_program} -d ${_docdeep_NAME} -s docs.css
+        COMMAND ${Python3_EXECUTABLE} ${docdeep_program} -d ${_docdeep_NAME} -s docs.css
                 ${_docdeep_SOURCE} > "${CMAKE_CURRENT_BINARY_DIR}/${mdfile}"
         MAIN_DEPENDENCY ${docdeep_program}
         DEPENDS ${_docdeep_SOURCE}

diff --git a/src/doc/conf.py b/src/doc/conf.py
@@ -16,7 +16,6 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
-from __future__ import absolute_import
 import sys
 import os
 import shlex