feat: Consider packages installed if the venv includes them from the …

…system-site (#2219)

news/2216.feature.md

@@ -0,0 +1 @@
+Consider packages as installed if the venv includes them from the system-site-packages.

src/pdm/environments/python.py

@@ -5,6 +5,7 @@
 
 from pdm.environments.base import BaseEnvironment
 from pdm.models.in_process import get_sys_config_paths
+from pdm.models.working_set import WorkingSet
 
 if TYPE_CHECKING:
     from pdm.project import Project
@@ -40,3 +41,12 @@ def process_env(self) -> dict[str, str]:
         if venv is not None and self.prefix is None:
             env.update(venv.env_vars())
         return env
+
+    def get_working_set(self) -> WorkingSet:
+        scheme = self.get_paths()
+        paths = [scheme["platlib"], scheme["purelib"]]
+        venv = self.interpreter.get_venv()
+        shared_paths = []
+        if venv is not None and venv.include_system_site_packages:
+            shared_paths.extend(venv.base_paths)
+        return WorkingSet(paths, shared_paths=list(dict.fromkeys(shared_paths)))

src/pdm/installers/synchronizers.py

@@ -215,6 +215,7 @@ def compare_with_working_set(self) -> tuple[list[str], list[str], list[str]]:
         candidates = self.candidates.copy()
         to_update: set[str] = set()
         to_remove: set[str] = set()
+        to_add: set[str] = set()
         locked_repository = self.environment.project.locked_repository
         all_candidate_keys = list(locked_repository.all_candidates)
 
@@ -224,23 +225,24 @@ def compare_with_working_set(self) -> tuple[list[str], list[str], list[str]]:
             if key in candidates:
                 can = candidates.pop(key)
                 if self._should_update(dist, can):
-                    to_update.add(key)
+                    if working_set.is_owned(key):
+                        to_update.add(key)
+                    else:
+                        to_add.add(key)
             elif (
-                self.only_keep or self.clean and key not in all_candidate_keys
-            ) and key not in self.SEQUENTIAL_PACKAGES:
+                (self.only_keep or self.clean and key not in all_candidate_keys)
+                and key not in self.SEQUENTIAL_PACKAGES
+                and working_set.is_owned(key)
+            ):
                 # Remove package only if it is not required by any group
                 # Packages for packaging will never be removed
                 to_remove.add(key)
-        to_add = {
+        to_add.update(
             strip_extras(name)[0]
             for name, _ in candidates.items()
             if name != self.self_key and strip_extras(name)[0] not in working_set
-        }
-        return (
-            sorted(to_add),
-            sorted(to_update),
-            sorted(to_remove),
         )
+        return (sorted(to_add), sorted(to_update), sorted(to_remove))
 
     def synchronize(self) -> None:
         """Synchronize the working set with pinned candidates."""

src/pdm/models/venv.py

@@ -4,7 +4,9 @@
 import sys
 from pathlib import Path
 
-from pdm.utils import get_venv_like_prefix
+from pdm.compat import cached_property
+from pdm.models.in_process import get_sys_config_paths
+from pdm.utils import find_python_in_path, get_venv_like_prefix
 
 IS_WIN = sys.platform == "win32"
 BIN_DIR = "Scripts" if IS_WIN else "bin"
@@ -46,3 +48,32 @@ def from_interpreter(cls, interpreter: Path) -> VirtualEnv | None:
     def env_vars(self) -> dict[str, str]:
         key = "CONDA_PREFIX" if self.is_conda else "VIRTUAL_ENV"
         return {key: str(self.root)}
+
+    @cached_property
+    def venv_config(self) -> dict[str, str]:
+        venv_cfg = self.root / "pyvenv.cfg"
+        if not venv_cfg.exists():
+            return {}
+        parsed: dict[str, str] = {}
+        with venv_cfg.open() as fp:
+            for line in fp:
+                if "=" in line:
+                    k, v = line.split("=", 1)
+                    k = k.strip().lower()
+                    v = v.strip()
+                    if k == "include-system-site-packages":
+                        v = v.lower()
+                    parsed[k] = v
+        return parsed
+
+    @property
+    def include_system_site_packages(self) -> bool:
+        return self.venv_config.get("include-system-site-packages") == "true"
+
+    @cached_property
+    def base_paths(self) -> list[str]:
+        home = Path(self.venv_config["home"])
+        base_executable = find_python_in_path(home) or find_python_in_path(home.parent)
+        assert base_executable is not None
+        paths = get_sys_config_paths(str(base_executable))
+        return [paths["purelib"], paths["platlib"]]

src/pdm/models/working_set.py

@@ -2,6 +2,7 @@
 
 import itertools
 import sys
+from collections import ChainMap
 from pathlib import Path
 from typing import Iterable, Iterator, Mapping
 
@@ -60,23 +61,34 @@ def distributions(path: list[str]) -> Iterable[im.Distribution]:
 class WorkingSet(Mapping[str, im.Distribution]):
     """A dictionary of currently installed distributions"""
 
-    def __init__(self, paths: list[str] | None = None):
+    def __init__(self, paths: list[str] | None = None, shared_paths: list[str] | None = None) -> None:
         if paths is None:
             paths = sys.path
+        if shared_paths is None:
+            shared_paths = []
         self._dist_map = {
             normalize_name(dist.metadata["Name"]): dist
             for dist in distributions(path=list(dict.fromkeys(paths)))
             if dist.metadata["Name"]
         }
+        self._shared_map = {
+            normalize_name(dist.metadata["Name"]): dist
+            for dist in distributions(path=list(dict.fromkeys(shared_paths)))
+            if dist.metadata["Name"]
+        }
+        self._iter_map = ChainMap(self._dist_map, self._shared_map)
 
     def __getitem__(self, key: str) -> im.Distribution:
-        return self._dist_map[key]
+        return self._iter_map[key]
+
+    def is_owned(self, key: str) -> bool:
+        return key in self._dist_map
 
     def __len__(self) -> int:
-        return len(self._dist_map)
+        return len(self._iter_map)
 
     def __iter__(self) -> Iterator[str]:
-        return iter(self._dist_map)
+        return iter(self._iter_map)
 
     def __repr__(self) -> str:
-        return repr(self._dist_map)
+        return repr(self._iter_map)

src/pdm/pytest.py

@@ -265,6 +265,9 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
     def add_distribution(self, dist: Distribution) -> None:
         self._data[dist.name] = dist
 
+    def is_owned(self, key: str) -> bool:
+        return key in self._data
+
     def __getitem__(self, key: str) -> Distribution:
         return self._data[key]