Make DALI tests compatible with Python 3.12 (#5452)

- adds additional replacements for functions used by nose
  and removed in python 3.12
- remove numpy version pinning as the <1.26 is not compatible
  with python 3.12

Signed-off-by: Janusz Lisiecki <[email protected]>
Signed-off-by: Kamil Tokarski <[email protected]>
Co-authored-by: Kamil Tokarski <[email protected]>

Acknowledgements.txt

@@ -4285,4 +4285,128 @@ JAX
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.
+
+==============================================================================
+Python
+
+A. HISTORY OF THE SOFTWARE
+==========================
+
+Python was created in the early 1990s by Guido van Rossum at Stichting
+Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands
+as a successor of a language called ABC.  Guido remains Python's
+principal author, although it includes many contributions from others.
+
+In 1995, Guido continued his work on Python at the Corporation for
+National Research Initiatives (CNRI, see https://www.cnri.reston.va.us)
+in Reston, Virginia where he released several versions of the
+software.
+
+In May 2000, Guido and the Python core development team moved to
+BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
+year, the PythonLabs team moved to Digital Creations, which became
+Zope Corporation.  In 2001, the Python Software Foundation (PSF, see
+https://www.python.org/psf/) was formed, a non-profit organization
+created specifically to own Python-related Intellectual Property.
+Zope Corporation was a sponsoring member of the PSF.
+
+All Python releases are Open Source (see https://opensource.org for
+the Open Source Definition).  Historically, most, but not all, Python
+releases have also been GPL-compatible; the table below summarizes
+the various releases.
+
+    Release         Derived     Year        Owner       GPL-
+                    from                                compatible? (1)
+
+    0.9.0 thru 1.2              1991-1995   CWI         yes
+    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
+    1.6             1.5.2       2000        CNRI        no
+    2.0             1.6         2000        BeOpen.com  no
+    1.6.1           1.6         2001        CNRI        yes (2)
+    2.1             2.0+1.6.1   2001        PSF         no
+    2.0.1           2.0+1.6.1   2001        PSF         yes
+    2.1.1           2.1+2.0.1   2001        PSF         yes
+    2.1.2           2.1.1       2002        PSF         yes
+    2.1.3           2.1.2       2002        PSF         yes
+    2.2 and above   2.1.1       2001-now    PSF         yes
+
+Footnotes:
+
+(1) GPL-compatible doesn't mean that we're distributing Python under
+    the GPL.  All Python licenses, unlike the GPL, let you distribute
+    a modified version without making your changes open source.  The
+    GPL-compatible licenses make it possible to combine Python with
+    other software that is released under the GPL; the others don't.
+
+(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
+    because its license has a choice of law clause.  According to
+    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
+    is "not incompatible" with the GPL.
+
+Thanks to the many outside volunteers who have worked under Guido's
+direction to make these releases possible.
+
+
+B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
+===============================================================
+
+Python software and documentation are licensed under the
+Python Software Foundation License Version 2.
+
+Starting with Python 3.8.6, examples, recipes, and other code in
+the documentation are dual licensed under the PSF License Version 2
+and the Zero-Clause BSD license.
+
+Some software incorporated into Python is under different licenses.
+The licenses are listed with code falling under that license.
+
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation;
+All Rights Reserved" are retained in Python alone or in any derivative version
+prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.

dali/python/nvidia/dali/_autograph/core/converter_testing.py

@@ -15,7 +15,7 @@
 """Base class for tests in this module."""
 
 import contextlib
-import imp
+import types
 import inspect
 import sys
 
@@ -32,7 +32,7 @@
 def allowlist(f):
     """Helper that marks a callable as allowlisted."""
     if "allowlisted_module_for_testing" not in sys.modules:
-        allowlisted_mod = imp.new_module("allowlisted_module_for_testing")
+        allowlisted_mod = types.ModuleType("allowlisted_module_for_testing")
         sys.modules["allowlisted_module_for_testing"] = allowlisted_mod
         config.CONVERSION_RULES = (
             config.DoNotConvert("allowlisted_module_for_testing"),
@@ -76,7 +76,7 @@ def __init__(self, converters, ag_overrides, operator_overload=hooks.OperatorBas
     def get_extra_locals(self):
         retval = super(TestingTranspiler, self).get_extra_locals()
         if self._ag_overrides:
-            modified_ag = imp.new_module("fake_autograph")
+            modified_ag = types.ModuleType("fake_autograph")
             modified_ag.__dict__.update(retval["ag__"].__dict__)
             modified_ag.__dict__.update(self._ag_overrides)
             retval["ag__"] = modified_ag

dali/python/nvidia/dali/backend.py

@@ -41,14 +41,6 @@ def deprecation_warning(what):
     Init(OpSpec("CPUAllocator"), OpSpec("PinnedCPUAllocator"), OpSpec("GPUAllocator"))
     initialized = True
 
-    # py3.12 warning
-    if sys.version_info[0] == 3 and sys.version_info[1] >= 12:
-        deprecation_warning(
-            "DALI support for Python {0}.{1} is experimental and some "
-            "functionalities may not work."
-            "".format(sys.version_info[0], sys.version_info[1])
-        )
-
     # py3.6 warning
     if sys.version_info[0] == 3 and sys.version_info[1] == 6:
         deprecation_warning(

dali/python/nvidia/dali/reducers.py

@@ -20,8 +20,10 @@
 import importlib
 
 
-def dummy_lambda():
-    pass
+# Don't allow any reformatters turning it into regular `def` function.
+# The whole point of this object is to have
+# properties (name) specific to lambda.
+dummy_lambda = lambda: 0  # noqa: E731
 
 
 # unfortunately inspect.getclosurevars does not yield global names referenced by
@@ -121,9 +123,13 @@ def reducer_override(self, obj):
             try:
                 pickle.dumps(obj)
             except AttributeError as e:
-                if "Can't pickle local object" in str(e):
+                str_e = str(e)
+                # For Python <3.12.5 and 3.12.5 respectively.
+                if "Can't pickle local object" in str_e or "Can't get local object" in str_e:
                     return function_by_value_reducer(obj)
             except pickle.PicklingError as e:
-                if "it's not the same object as" in str(e):
+                str_e = str(e)
+                # For jupyter notebook issues and Python 3.12.5+ respectively
+                if "it's not the same object as" in str_e or "Can't pickle local object" in str_e:
                     return function_by_value_reducer(obj)
         return NotImplemented

dali/python/setup.py.in

@@ -83,6 +83,9 @@ For more details please check the
           # Currently supported range of versions.
           'astunparse >= 1.6.0',
           'gast >= 0.3.3',
+          # the latest astunparse (1.6.3) doesn't work with any other six than
+          # 1.16 on python 3.12 due to import six.moves
+          'six >= 1.16',
           'dm-tree',
           @DALI_INSTALL_REQUIRES_NVIMGCODEC@
           ],

dali/test/python/autograph/converters/test_call_trees.py

@@ -14,7 +14,7 @@
 # ==============================================================================
 """Tests for call_trees module."""
 
-import imp
+import types
 
 from nvidia.dali._autograph.converters import call_trees
 from nvidia.dali._autograph.converters import functions
@@ -193,7 +193,7 @@ def f(h, g, a, *args):
     def test_debugger_set_trace(self):
         tracking_list = []
 
-        pdb = imp.new_module("fake_pdb")
+        pdb = types.ModuleType("fake_pdb")
         pdb.set_trace = lambda: tracking_list.append(1)
 
         def f():

dali/test/python/autograph/core/test_converter.py

@@ -14,7 +14,7 @@
 # ==============================================================================
 """Tests for converter module."""
 
-import imp
+import types
 
 from nvidia.dali._autograph.core import converter
 from nvidia.dali._autograph.core import converter_testing
@@ -40,7 +40,7 @@ def f():
         opts_packed = templates.replace(template, opts_ast=opts_ast)
 
         reparsed, _, _ = loader.load_ast(opts_packed)
-        fake_ag = imp.new_module("fake_ag")
+        fake_ag = types.ModuleType("fake_ag")
         fake_ag.ConversionOptions = converter.ConversionOptions
         fake_ag.Feature = converter.Feature
         reparsed.ag__ = fake_ag

dali/test/python/autograph/impl/test_api.py

@@ -19,7 +19,6 @@
 import contextlib
 import functools
 import gc
-import imp
 import inspect
 import os
 import re
@@ -593,7 +592,7 @@ def test_converted_call_tf_op_forced(self):
         self.assertAllEqual(self.evaluate(x), 2)
 
     def test_converted_call_exec_generated_code(self):
-        temp_mod = imp.new_module("test_module")
+        temp_mod = types.ModuleType("test_module")
         dynamic_code = """
       def foo(x):
         return x + 1

dali/test/python/autograph/impl/test_conversion.py

@@ -14,7 +14,7 @@
 # ==============================================================================
 """Tests for conversion module."""
 
-import imp
+import types
 import sys
 import unittest
 
@@ -40,7 +40,7 @@ def test_fn():
         self.assertFalse(conversion.is_allowlisted(test_fn))
 
     def test_is_allowlisted_callable_allowlisted_call(self):
-        allowlisted_mod = imp.new_module("test_allowlisted_call")
+        allowlisted_mod = types.ModuleType("test_allowlisted_call")
         sys.modules["test_allowlisted_call"] = allowlisted_mod
         config.CONVERSION_RULES = (
             config.DoNotConvert("test_allowlisted_call"),

dali/test/python/autograph/pyct/test_inspect_utils.py

@@ -17,7 +17,7 @@
 import abc
 import collections
 import functools
-import imp
+import types
 import textwrap
 import unittest
 
@@ -292,8 +292,8 @@ def local_fn():
 
     def test_getqualifiedname(self):
         foo = object()
-        qux = imp.new_module("quxmodule")
-        bar = imp.new_module("barmodule")
+        qux = types.ModuleType("quxmodule")
+        bar = types.ModuleType("barmodule")
         baz = object()
         bar.baz = baz
 
@@ -321,7 +321,7 @@ def test_getqualifiedname_efficiency(self):
             current_level = []
             for j in range(10):
                 mod_name = "mod_{}_{}".format(i, j)
-                mod = imp.new_module(mod_name)
+                mod = types.ModuleType(mod_name)
                 current_level.append(mod)
                 if i == 9 and j == 9:
                     mod.foo = foo
@@ -347,7 +347,7 @@ def test_getqualifiedname_cycles(self):
         ns = {}
         mods = []
         for i in range(10):
-            mod = imp.new_module("mod_{}".format(i))
+            mod = types.ModuleType("mod_{}".format(i))
             if i == 9:
                 mod.foo = foo
             # Module i refers to module i+1

dali/test/python/autograph/pyct/test_loader.py

@@ -21,8 +21,10 @@
 import unittest
 
 import gast
+from distutils.version import LooseVersion
 
 from nvidia.dali._autograph.pyct import ast_util
+from nvidia.dali._autograph.pyct import gast_util
 from nvidia.dali._autograph.pyct import loader
 from nvidia.dali._autograph.pyct import parser
 from nvidia.dali._autograph.pyct import pretty_printer
@@ -77,6 +79,7 @@ def test_load_ast(self):
             decorator_list=[],
             returns=None,
             type_comment=None,
+            **{"type_params": []} if gast_util.get_gast_version() >= LooseVersion("0.5.5") else {},
         )
 
         module, source, _ = loader.load_ast(node)

dali/test/python/autograph/pyct/test_templates.py

@@ -14,7 +14,7 @@
 # ==============================================================================
 """Tests for templates module."""
 
-import imp
+import types
 import unittest
 
 import gast
@@ -132,7 +132,7 @@ def test_fn(a):
 
         node = templates.replace(template, foo="b")[0]
         result, _, _ = loader.load_ast(node)
-        mod = imp.new_module("test")
+        mod = types.ModuleType("test")
         mod.b = 3
         self.assertEqual(3, result.test_fn(mod))
 

dali/test/python/checkpointing/test_dali_checkpointing.py

@@ -27,7 +27,7 @@
     get_dali_extra_path,
     module_functions,
 )
-from nose_utils import assert_warns
+from nose_utils import assert_warns, assert_raises
 from nose2.tools import params, cartesian_params
 from dataclasses import dataclass
 from nvidia.dali import tfrecord as tfrec
@@ -36,7 +36,6 @@
 from nvidia.dali.auto_aug import trivial_augment as ta
 from reader.test_numpy import is_gds_supported
 from nose.plugins.attrib import attr
-from nose_utils import assert_raises
 
 
 reader_signed_off = create_sign_off_decorator()