refactor(has_pkg): introduce strict flag (MODFLOW-USGS#106)

* add strict flag to has_pkg() toggling whether to try to import or only check metadata
* always invalidate/refresh the installed package cache if strict is on
* add pytest-virtualenv to test dependencies, test with/without strict
* use --dist loadfile with xdist in CI for compatibility
* use strict=True for requires_pkg marker

.github/workflows/ci.yml

@@ -167,7 +167,8 @@ jobs:
           BIN_PATH: ~/.local/bin/modflow
           REPOS_PATH: ${{ github.workspace }}
           GITHUB_TOKEN: ${{ github.token }}
-        run: pytest -v -n auto --durations 0 --ignore modflow_devtools/test/test_download.py
+        # use --dist loadfile to so tests requiring pytest-virtualenv run on the same worker
+        run: pytest -v -n auto --dist loadfile --durations 0 --ignore modflow_devtools/test/test_download.py
 
       - name: Run network-dependent tests
         # only invoke the GH API on one OS and Python version

conftest.py

@@ -1 +1,4 @@
+from pathlib import Path
+
 pytest_plugins = ["modflow_devtools.fixtures"]
+project_root_path = Path(__file__).parent

modflow_devtools/markers.py

@@ -22,7 +22,7 @@ def requires_exe(*exes):
 
 
 def requires_pkg(*pkgs):
-    missing = {pkg for pkg in pkgs if not has_pkg(pkg)}
+    missing = {pkg for pkg in pkgs if not has_pkg(pkg, strict=True)}
     return pytest.mark.skipif(
         missing,
         reason=f"missing package{'s' if len(missing) != 1 else ''}: "

modflow_devtools/misc.py

@@ -398,21 +398,47 @@ def has_exe(exe):
     return _has_exe_cache[exe]
 
 
-def has_pkg(pkg):
+def has_pkg(pkg: str, strict: bool = False) -> bool:
     """
     Determines if the given Python package is installed.
 
+    Parameters
+    ----------
+    pkg : str
+        Name of the package to check.
+    strict : bool
+        If False, only check if package metadata is available.
+        If True, try to import the package (all dependencies must be present).
+
+    Returns
+    -------
+    bool
+        True if the package is installed, otherwise False.
+
+    Notes
+    -----
     Originally written by Mike Toews ([email protected]) for FloPy.
     """
-    if pkg not in _has_pkg_cache:
-        found = True
+
+    def try_import():
+        try:  # import name, e.g. "import shapefile"
+            importlib.import_module(pkg)
+            return True
+        except ModuleNotFoundError:
+            return False
+
+    def try_metadata() -> bool:
         try:  # package name, e.g. pyshp
             metadata.distribution(pkg)
+            return True
         except metadata.PackageNotFoundError:
-            try:  # import name, e.g. "import shapefile"
-                importlib.import_module(pkg)
-            except ModuleNotFoundError:
-                found = False
-        _has_pkg_cache[pkg] = found
+            return False
+
+    found = False
+    if not strict:
+        found = pkg in _has_pkg_cache or try_metadata()
+    if not found:
+        found = try_import()
+    _has_pkg_cache[pkg] = found
 
     return _has_pkg_cache[pkg]

modflow_devtools/test/test_misc.py

@@ -2,14 +2,17 @@
 import shutil
 from os import environ
 from pathlib import Path
+from pprint import pprint
 from typing import List
 
 import pytest
+from conftest import project_root_path
 from modflow_devtools.misc import (
     get_model_paths,
     get_namefile_paths,
     get_packages,
     has_package,
+    has_pkg,
     set_dir,
     set_env,
 )
@@ -249,3 +252,34 @@ def test_get_namefile_paths_select_patterns():
 def test_get_namefile_paths_select_packages():
     paths = get_namefile_paths(_examples_path, packages=["wel"])
     assert len(paths) >= 43
+
+
+@pytest.mark.slow
+def test_has_pkg(virtualenv):
+    python = virtualenv.python
+    venv = Path(python).parent
+    pkg = "pytest"
+    dep = "pluggy"
+    print(
+        f"Using temp venv at {venv} with python {python} to test has_pkg('{pkg}') with and without '{dep}'"
+    )
+
+    # install a package and remove one of its dependencies
+    virtualenv.run(f"pip install {project_root_path}")
+    virtualenv.run(f"pip install {pkg}")
+    virtualenv.run(f"pip uninstall -y {dep}")
+
+    # check with/without strict mode
+    for strict in [False, True]:
+        cmd = (
+            f"from modflow_devtools.misc import has_pkg; print(has_pkg('{pkg}'"
+            + (", strict=True))" if strict else "))")
+        )
+        exp = "False" if strict else "True"
+        assert (
+            virtualenv.run(
+                f'{python} -c "{cmd}"',
+                capture=True,
+            ).strip()
+            == exp
+        )

pyproject.toml

@@ -58,6 +58,7 @@ test = [
     "pytest-cases",
     "pytest-cov",
     "pytest-dotenv",
+    "pytest-virtualenv",
     "pytest-xdist",
     "PyYaml"
 ]