[python] fix compatibility for Python 3.12

doc/contributing/sw/adding_python_depedencies.md

@@ -4,5 +4,5 @@ To comply with software supply chain security requirements of various project pa
 We accomplish this with the help of the `uv pip compile` command, which is part of the `uv` package.
 
 If you need to add another Python package to the project, do so by:
-1. adding the package and version number to the `pyproject.toml` file, in the form of `<package>==<version>`, and
+1. adding the package and version number to the `pyproject.toml` file, in the form of `<package> ~= <version>` (or `<package> == <version>` in case a specific version is needed), and
 1. run the script `util/sh/scripts/gen-python-requirements.sh`, which will auto-generate the updated `python-requirements.txt` file.

pyproject.toml

@@ -12,20 +12,25 @@ dependencies = [
   # Dependency management
   "uv==0.4.10",
 
+  # Polyfill library
+  "importlib-resources ~= 5.12", # Need for Python<3.9
+  # This are condition dependency of importlib-resources
+  # Add it here otherwise it'll get dropped when uv/pip tries to lock dependency with Python>=3.10.
+  "zipp ~= 3.16", # Need for Python<3.10
+
   # Keep sorted
   "beautifulsoup4==4.12.2",
   "hjson==3.1.0",
-  "importlib_metadata==6.8.0",
   "libclang==16.0.0",
-  "libcst==0.4.1",
-  "lxml==4.9.2",
+  "libcst == 1.1.0", # 1.2.0+ needs Python 3.9+
+  "lxml ~= 5.0",
   "mako==1.1.6",
   "pluralizer==1.2.0",
   "pycryptodome==3.15.0",
   "pyelftools==0.29",
   "pytest-timeout==2.1.0",
   "pytest==7.0.1",
-  "pyyaml==6.0",
+  "pyyaml ~= 6.0.2",
   "rich==12.6.0", # maximum version compatible with Python 3.6.8 (used on CentOS7 nightly regression machines)
   "semantic_version==2.10.0",
   "tabulate==0.8.10",
@@ -35,7 +40,6 @@ dependencies = [
   "jsonschema==4.17.3",
   # These are conditional dependency of jsonschema when Python<3.9.
   # Add them here otherwise they'll get dropped when uv/pip tries to lock dependency with Python>=3.9.
-  "importlib-resources==1.4.0",
   "pkgutil_resolve_name==1.3.10",
 
   # Dependencies: gen-flash-img.py
@@ -55,13 +59,12 @@ dependencies = [
   "termcolor==1.1.0",
 
   # Linters
-  "flake8==5.0.4",
+  "flake8 ~= 7.1",
   "isort==5.10.1",
   "mypy==0.971",
   "yapf==0.32.0",
 
   # Type stubs for mypy checking.
-  "types-pkg_resources==0.1.3",
   "types-pyyaml==6.0.11",
   "types-tabulate==0.8.11",
 

util/reggen/BUILD

@@ -217,6 +217,7 @@ py_library(
         ":window",
         requirement("mako"),
         requirement("pyyaml"),
+        requirement("importlib_resources"),
     ],
 )
 
@@ -227,6 +228,7 @@ py_library(
         ":ip_block",
         requirement("mako"),
         requirement("pyyaml"),
+        requirement("importlib_resources"),
     ],
 )
 
@@ -260,6 +262,7 @@ py_library(
         ":reg_base",
         ":register",
         requirement("mako"),
+        requirement("importlib_resources"),
     ],
 )
 
@@ -298,6 +301,7 @@ py_library(
         ":ip_block",
         requirement("hjson"),
         requirement("mako"),
+        requirement("importlib_resources"),
     ],
 )
 

util/reggen/gen_dv.py

@@ -13,7 +13,7 @@
 
 from mako import exceptions  # type: ignore
 from mako.lookup import TemplateLookup  # type: ignore
-from pkg_resources import resource_filename
+import importlib_resources
 
 from reggen.ip_block import IpBlock
 from reggen.multi_register import MultiRegister
@@ -152,7 +152,7 @@ def get_block_base_name(dv_base_names_map: Dict[str, DvBaseNames], block: str) -
 def gen_dv(block: IpBlock, dv_base_names: List[str], outdir: str) -> int:
     '''Generate DV files for an IpBlock'''
 
-    lookup = TemplateLookup(directories=[resource_filename('reggen', '.')])
+    lookup = TemplateLookup(directories=[str(importlib_resources.files('reggen'))])
     uvm_reg_tpl = lookup.get_template('uvm_reg.sv.tpl')
 
     # Generate the RAL package(s). For a device interface with no name we

util/reggen/gen_fpv.py

@@ -12,15 +12,15 @@
 import yaml
 from mako import exceptions  # type: ignore
 from mako.template import Template  # type: ignore
-from pkg_resources import resource_filename
+import importlib_resources
 
 from reggen.ip_block import IpBlock
 
 
 def gen_fpv(block: IpBlock, outdir: str) -> int:
     # Read Register templates
     fpv_csr_tpl = Template(
-        filename=resource_filename('reggen', 'fpv_csr.sv.tpl'))
+        filename=str(importlib_resources.files('reggen') / "fpv_csr.sv.tpl"))
 
     device_hier_paths = block.bus_interfaces.device_hier_paths
 

util/reggen/gen_rtl.py

@@ -9,7 +9,7 @@
 
 from mako import exceptions  # type: ignore
 from mako.template import Template  # type: ignore
-from pkg_resources import resource_filename
+import importlib_resources
 
 from reggen.ip_block import IpBlock
 from reggen.lib import check_int
@@ -94,9 +94,9 @@ def get_reg_tx_type(block: IpBlock, reg: RegBase, hw2reg: bool) -> str:
 def gen_rtl(block: IpBlock, outdir: str) -> int:
     # Read Register templates
     reg_top_tpl = Template(
-        filename=resource_filename('reggen', 'reg_top.sv.tpl'))
+        filename=str(importlib_resources.files('reggen') / 'reg_top.sv.tpl'))
     reg_pkg_tpl = Template(
-        filename=resource_filename('reggen', 'reg_pkg.sv.tpl'))
+        filename=str(importlib_resources.files('reggen') / 'reg_pkg.sv.tpl'))
 
     # In case the generated package contains alias definitions, we add
     # the alias implementation identifier to the package name so that it

util/reggen/gen_sec_cm_testplan.py

@@ -9,7 +9,7 @@
 import hjson  # type: ignore
 from mako import exceptions  # type: ignore
 from mako.lookup import TemplateLookup  # type: ignore
-from pkg_resources import resource_filename
+import importlib_resources
 
 from reggen.ip_block import IpBlock
 
@@ -51,7 +51,7 @@ def gen_sec_cm_testplan(block: IpBlock, outdir: str) -> int:
 
         return 0
 
-    lookup = TemplateLookup(directories=[resource_filename('reggen', '.')])
+    lookup = TemplateLookup(directories=[str(importlib_resources.files('reggen'))])
     sec_cm_testplan_tpl = lookup.get_template('sec_cm_testplan.hjson.tpl')
     with open(outfile, 'w', encoding='UTF-8') as f:
         try:

util/reggen/version.py

@@ -8,7 +8,7 @@
 import sys
 from typing import List
 
-import pkg_resources  # part of setuptools
+from importlib.metadata import version
 
 
 def show_and_exit(clitool: str, packages: List[str]) -> None:
@@ -21,5 +21,5 @@ def show_and_exit(clitool: str, packages: List[str]) -> None:
         ver = 'not found (not in Git repository?)'
     sys.stderr.write(clitool + " Git version " + ver + '\n')
     for p in packages:
-        sys.stderr.write(p + ' ' + pkg_resources.require(p)[0].version + '\n')
+        sys.stderr.write(p + ' ' + version(p) + '\n')
     exit(0)

util/tlgen/BUILD

@@ -23,6 +23,7 @@ py_library(
     deps = [
         "//util/reggen:validate",
         requirement("mako"),
+        requirement("importlib_resources"),
     ],
 )
 

util/tlgen/generate.py

@@ -9,7 +9,7 @@
 
 from mako import exceptions  # type: ignore
 from mako.template import Template  # type: ignore
-from pkg_resources import resource_filename
+import importlib_resources
 
 from .xbar import Xbar
 
@@ -21,13 +21,13 @@ def generate(xbar: Xbar, library_name: str = "ip") -> List[Tuple[str, Any]]:
     pairs of files to write, each in the form (path, contents).
     """
     xbar_rtl_tpl = Template(
-        filename=resource_filename('tlgen', 'xbar.rtl.sv.tpl'))
+        filename=str(importlib_resources.files('tlgen') / 'xbar.rtl.sv.tpl'))
     xbar_pkg_tpl = Template(
-        filename=resource_filename('tlgen', 'xbar.pkg.sv.tpl'))
+        filename=str(importlib_resources.files('tlgen') / 'xbar.pkg.sv.tpl'))
     xbar_core_tpl = Template(
-        filename=resource_filename('tlgen', 'xbar.core.tpl'))
+        filename=str(importlib_resources.files('tlgen') / 'xbar.core.tpl'))
     xbar_hjson_tpl = Template(
-        filename=resource_filename('tlgen', 'xbar.hjson.tpl'))
+        filename=str(importlib_resources.files('tlgen') / 'xbar.hjson.tpl'))
     try:
         out_rtl = xbar_rtl_tpl.render(xbar=xbar)
         out_pkg = xbar_pkg_tpl.render(xbar=xbar)

util/tlgen/generate_tb.py

@@ -9,7 +9,7 @@
 
 from mako import exceptions  # type: ignore
 from mako.template import Template  # type: ignore
-from pkg_resources import resource_filename
+import importlib_resources
 
 from .xbar import Xbar
 
@@ -25,7 +25,7 @@ def generate_tb(xbar: Xbar,
     ]
 
     for fname in tb_files:
-        tpl = Template(filename=resource_filename('tlgen', fname + '.tpl'))
+        tpl = Template(filename=str(importlib_resources.files('tlgen') / (fname + '.tpl')))
 
         # some files need to be renamed
         if fname == "xbar.sim.core":

util/topgen/BUILD

@@ -42,6 +42,7 @@ py_library(
         "//util/reggen:params",
         "//util/reggen:window",
         requirement("mako"),
+        requirement("importlib_resources"),
     ],
 )
 

util/topgen/gen_dv.py

@@ -7,7 +7,7 @@
 
 from mako import exceptions  # type: ignore
 from mako.lookup import TemplateLookup  # type: ignore
-from pkg_resources import resource_filename
+import importlib_resources
 
 from reggen.gen_dv import gen_core_file
 
@@ -24,8 +24,8 @@ def gen_dv(top: Top,
            outdir: str) -> int:
     '''Generate DV RAL model for a Top'''
     # Read template
-    lookup = TemplateLookup(directories=[resource_filename('topgen', '.'),
-                                         resource_filename('reggen', '.')])
+    lookup = TemplateLookup(directories=[str(importlib_resources.files('topgen')),
+                                         str(importlib_resources.files('reggen'))])
     uvm_reg_tpl = lookup.get_template('top_uvm_reg.sv.tpl')
 
     # Expand template

util/uvmdvgen/gen_agent.py

@@ -8,7 +8,7 @@
 
 from mako import exceptions
 from mako.template import Template
-from pkg_resources import resource_filename
+import importlib_resources
 
 
 def gen_agent(name, has_separate_host_device_driver, root_dir, vendor):
@@ -50,7 +50,7 @@ def gen_agent(name, has_separate_host_device_driver, root_dir, vendor):
         fname = src_prefix + src + src_suffix
 
         # read template
-        tpl = Template(filename=resource_filename('uvmdvgen', ftpl))
+        tpl = Template(filename=str(importlib_resources.files('uvmdvgen') / ftpl))
 
         if not os.path.exists(path_dir): os.system("mkdir -p " + path_dir)
         with open(path_dir + "/" + fname, 'w') as fout:

util/uvmdvgen/gen_env.py

@@ -8,7 +8,7 @@
 import logging as log
 
 from mako.template import Template
-from pkg_resources import resource_filename
+import importlib_resources
 from uvmdvgen import VENDOR_DEFAULT
 
 
@@ -69,7 +69,7 @@ def gen_env(name, is_cip, has_ral, has_interrupts, has_alerts, num_edn,
             continue
 
         # read template
-        tpl = Template(filename=resource_filename('uvmdvgen', ftpl))
+        tpl = Template(filename=str(importlib_resources.files('uvmdvgen') / ftpl))
 
         # create rendered file
         with open(file_path, 'w') as fout: