From 4ccb9c42734a1c06cccb7bd3e0e381a70eabadfd Mon Sep 17 00:00:00 2001
From: CrazyBolillo <antonio@zoftko.com>
Date: Wed, 3 Apr 2024 21:32:51 -0600
Subject: [PATCH] Improve performance by caching find_spec

Certain checkers upstream on pylint like import-error heavily use
find_spec. This method is IO intensive as it looks for files
across several search paths to return a ModuleSpec.

Since imports across files may repeat themselves it makes sense to cache
this method in order to speed up the linting process.

Local testing shows that caching reduces the total amount of calls to
find_module methods (used by find_spec) by about 50%. Linting the test
repository in the related issue goes from 40 seconds to 37 seconds. This
was on a NVME disk and after warmup, so timing gains may be bigger on
slower file systems like the one mentioned in the referenced issue.

Closes pylint-dev/pylint#9310.
---
 astroid/interpreter/_import/spec.py | 14 ++++++++++++++
 tests/test_manager.py               |  1 +
 tests/test_modutils.py              |  1 +
 3 files changed, 16 insertions(+)

diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py
index 93096e54e6..618faa4456 100644
--- a/astroid/interpreter/_import/spec.py
+++ b/astroid/interpreter/_import/spec.py
@@ -24,6 +24,8 @@
 
 from . import util
 
+modpath_cache = {}
+
 
 # The MetaPathFinder protocol comes from typeshed, which says:
 # Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder`
@@ -423,6 +425,18 @@ def _find_spec_with_path(
     raise ImportError(f"No module named {'.'.join(module_parts)}")
 
 
+def cache_modpath(func):
+    def wrapper(*args):
+        key = ".".join(args[0])
+        if key not in modpath_cache:
+            modpath_cache[key] = func(*args)
+
+        return modpath_cache[key]
+
+    return wrapper
+
+
+@cache_modpath
 def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSpec:
     """Find a spec for the given module.
 
diff --git a/tests/test_manager.py b/tests/test_manager.py
index 7861927930..4db6e7053a 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -41,6 +41,7 @@ class AstroidManagerTest(
 ):
     def setUp(self) -> None:
         super().setUp()
+        astroid.interpreter._import.spec.modpath_cache.clear()
         self.manager = test_utils.brainless_manager()
 
     def test_ast_from_file(self) -> None:
diff --git a/tests/test_modutils.py b/tests/test_modutils.py
index 929c58992c..8f21064cb2 100644
--- a/tests/test_modutils.py
+++ b/tests/test_modutils.py
@@ -41,6 +41,7 @@ class ModuleFileTest(unittest.TestCase):
     package = "mypypa"
 
     def tearDown(self) -> None:
+        astroid.interpreter._import.spec.modpath_cache.clear()
         for k in list(sys.path_importer_cache):
             if "MyPyPa" in k:
                 del sys.path_importer_cache[k]