Make pytest the only requirement for running test suite without errors (

#1088)

* Switch to find_packages() and add missing __init__.py files

* Add missing pyyaml dependency

* Handle requests as optional dependency

* Handle JSON schema dependencies as optional

* Handle mongomock dependency as optional

* Remove spurious undefined pytest marker

* Remove redundant notebook tests

* Add missing requests requirement for parameter_sweep

* Change CI job for noneditable install to only require pytest

* Address unused imports pylint failures in unchanged files

Probably they have been ignored until now because not in a package?

* Add yet another WET import guard for mongomock

* Add missing __init__.py files for package data to work correctly

* Address spurious Pylint failure

* Modify "Getting Started" docs to install pytest instead

* Replace most import guards with Pyomo's attempt_import()

* Address Pylint failure

.github/workflows/checks.yml

@@ -204,7 +204,7 @@ jobs:
     - name: Install watertap and testing dependencies
       run: |
         echo '::group::Output of "pip install" commands'
-        pip install --progress-bar off "watertap[testing] @ ${_install_url}"
+        pip install --progress-bar off "watertap @ ${_install_url}" pytest
         echo '::endgroup::'
         echo '::group::Display installed packages'
         conda list

docs/getting_started.rst

@@ -112,11 +112,11 @@ Next, we can obtain Ipopt and CBC from conda-forge:
 Running the test suite
 ----------------------
 
-To run the WaterTAP test suite, first install the optional testing dependencies using pip:
+To run the WaterTAP test suite, first install the ``pytest`` test framework:
 
 .. code-block:: shell
 
-   pip install "watertap[testing]"
+   pip install pytest
 
 Then, run the following command to run the complete WaterTAP test suite:
 

setup.py

@@ -14,7 +14,7 @@
 """
 
 # Always prefer setuptools over distutils
-from setuptools import setup, find_namespace_packages
+from setuptools import setup, find_packages
 from pathlib import Path
 
 cwd = Path(__file__).parent
@@ -72,7 +72,7 @@
     ],
     keywords="water systems, chemical engineering, process modeling, filtration, desalination, nawi",
     # just include watertap and everything under it
-    packages=find_namespace_packages(
+    packages=find_packages(
         include=("watertap*",),
     ),
     python_requires=">=3.7",
@@ -81,6 +81,7 @@
         # maintainers: switch to SPECIAL_DEPENDENCIES_FOR_RELEASE when cutting a release of watertap
         *SPECIAL_DEPENDENCIES_FOR_PRERELEASE,
         "pyomo>=6.6.1",  # (also needed for units in electrolyte database (edb))
+        "pyyaml",  # watertap.core.wt_database
         # the following requirements are for the electrolyte database (edb)
         "pymongo>3",  # database interface
         "fastjsonschema",  # schema validation
@@ -90,6 +91,7 @@
         "scipy",
         # for parameter_sweep
         "h5py",
+        "requests",
         # for watertap.ui.api_model (though may be generally useful)
         "pydantic < 2",
         "numpy",

watertap/edb/commands.py

@@ -22,9 +22,10 @@
 
 # third-party
 import click
-from json_schema_for_humans import generate as schema_gen
-from json_schema_for_humans.generation_configuration import GenerationConfiguration
 from pymongo.errors import ConnectionFailure
+from pyomo.common.dependencies import attempt_import
+
+_, json_schema_available = attempt_import("json_schema_for_humans")
 
 # package
 from .db_api import ElectrolyteDB
@@ -399,6 +400,15 @@ def drop_database(url, database, yes):
     "-d", "--database", help="Database name", default=ElectrolyteDB.DEFAULT_DB
 )
 def schema(output_file, output_format, data_type, url, database):
+    if not json_schema_available:
+        raise ImportError(
+            "json-schema-for-humans (EDB optional dependency) not installed"
+        )
+    from json_schema_for_humans import generate as schema_gen
+    from json_schema_for_humans.generation_configuration import (
+        GenerationConfiguration,
+    )
+
     print_messages = _log.isEnabledFor(logging.ERROR)
     if output_file:
         stream = output_file

watertap/edb/tests/test_commands.py

@@ -107,7 +107,9 @@ def __call__(self, url=None, **kwargs) -> EDBClient:
 
 @pytest.fixture(scope="function")
 def mock_edb(monkeypatch) -> EDBClientFactory:
-    import mongomock
+    mongomock = pytest.importorskip(
+        "mongomock", reason="mongomock (EDB optional dependency) not available"
+    )
 
     # NOTE since mongomock clients store data in memory,
     # the same MongoClient instance must be kept and used for the lifetime of the fixture

watertap/edb/tests/test_edb.py

@@ -15,15 +15,21 @@
 import json
 import os
 import pytest
-import mongomock
+
+from pyomo.common.dependencies import attempt_import
 
 from watertap.edb import commands
 from watertap.edb.db_api import ElectrolyteDB
 from watertap.edb.validate import validate
 
+mongomock, mongomock_available = attempt_import("mongomock")
+
 
 class MockDB(ElectrolyteDB):
     def __init__(self, db="foo", **kwargs):
+        if not mongomock_available:
+            pytest.skip(reason="mongomock (EDB optional dependency) not available")
+
         self._client = mongomock.MongoClient()
         self._db = getattr(self._client, db)
         # note: don't call superclass!

watertap/edb/tests/test_schemas.py

@@ -13,6 +13,10 @@
 Test schemas module
 """
 import pytest
+
+pytest.importorskip(
+    "watertap.edb.schemas", reason="Missing optional dependencies for EDB schemas"
+)
 from ..schemas import schemas
 from ..data_model import Reaction
 

watertap/edb/tests/util.py

@@ -13,12 +13,18 @@
 Utility functions for EDB tests
 """
 import pytest
-import mongomock
+from pyomo.common.dependencies import attempt_import
+
 from watertap.edb.db_api import ElectrolyteDB
 
+mongomock, mongomock_available = attempt_import("mongomock")
+
 
 class MockDB(ElectrolyteDB):
     def __init__(self, db="foo", **kwargs):
+        if not mongomock_available:
+            pytest.skip(reason="mongomock (EDB optional dependency) not available")
+
         self._client = mongomock.MongoClient()
         self._db = getattr(self._client, db)
         # note: don't call superclass!

watertap/examples/chemistry/tests/conftest.py

@@ -11,16 +11,20 @@
 #################################################################################
 import pytest
 from _pytest.config import Config
+from pyomo.common.dependencies import attempt_import
 
 from watertap.edb import ElectrolyteDB
 from watertap.edb.commands import _load_bootstrap
 
+mongomock, mongomock_available = attempt_import("mongomock")
+
 
 class MockDB(ElectrolyteDB):
     def __init__(self, db="foo", **kwargs):
-        from mongomock import MongoClient
+        if not mongomock_available:
+            pytest.skip(reason="mongomock (EDB optional dependency) not available")
 
-        self._client = MongoClient()
+        self._client = mongomock.MongoClient()
         self._db = getattr(self._client, db)
         # note: don't call superclass!
         self._database_name = db

watertap/examples/edb/tests/conftest.py

@@ -11,16 +11,20 @@
 #################################################################################
 import pytest
 from _pytest.config import Config
+from pyomo.common.dependencies import attempt_import
 
 from watertap.edb import ElectrolyteDB
 from watertap.edb.commands import _load_bootstrap
 
+mongomock, mongomock_available = attempt_import("mongomock")
+
 
 class MockDB(ElectrolyteDB):
     def __init__(self, db="foo", **kwargs):
-        from mongomock import MongoClient
+        if not mongomock_available:
+            pytest.skip(reason="mongomock (EDB optional dependency) not available")
 
-        self._client = MongoClient()
+        self._client = mongomock.MongoClient()
         self._db = getattr(self._client, db)
         # note: don't call superclass!
         self._database_name = db

watertap/tools/parameter_sweep/parameter_sweep.py

@@ -13,7 +13,6 @@
 import pyomo.environ as pyo
 import warnings
 import copy
-import requests
 import time
 
 from abc import abstractmethod, ABC
@@ -27,6 +26,9 @@
 from pyomo.common.modeling import unique_component_name
 from pyomo.core.base import _VarData, _ExpressionData
 from pyomo.core.base.param import _ParamData
+from pyomo.common.dependencies import attempt_import
+
+requests, requests_available = attempt_import("requests")
 
 from watertap.tools.parameter_sweep.parameter_sweep_writer import ParameterSweepWriter
 from watertap.tools.parameter_sweep.sampling_types import SamplingType, LinearSample
@@ -191,6 +193,11 @@ def assign_variable_names(model, outputs):
                 outputs[output_name] = exprs[output_name]
 
     def _publish_updates(self, iteration, solve_status, solve_time):
+        if not requests_available:
+            raise ImportError(
+                "requests (parameter_sweep optional dependency) not installed"
+            )
+
         if self.config.publish_progress:
             publish_dict = {
                 "worker_number": self.parallel_manager.get_rank(),