diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4738d35..dd5a40c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,15 +1,12 @@ name: CI on: push: - branches: - - main - - develop* paths-ignore: - '**.md' pull_request: branches: - main - - develop* + - develop paths-ignore: - '**.md' jobs: @@ -19,7 +16,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 @@ -83,24 +80,24 @@ jobs: fail-fast: false matrix: os: [ ubuntu-22.04, macos-12, windows-2022 ] - python: [ 3.8, 3.9, "3.10", "3.11" ] + python: [ 3.8, 3.9, "3.10", "3.11", "3.12" ] env: GCC_V: 11 steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: modflow-devtools - name: Checkout modflow6 - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: MODFLOW-USGS/modflow6 path: modflow6 - name: Checkout modflow6 examples - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: MODFLOW-USGS/modflow6-examples path: modflow6-examples @@ -116,7 +113,6 @@ jobs: with: repository: MODFLOW-USGS/modflow6-largetestmodels path: modflow6-largetestmodels - token: ${{ github.token }} - name: Install executables uses: modflowpy/install-modflow-action@v1 @@ -131,10 +127,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - cache: 'pip' - cache-dependency-path: | - modflow-devtools/pyproject.toml - modflow6-examples/etc/requirements*.txt - name: Install Python packages working-directory: modflow-devtools @@ -162,22 +154,19 @@ jobs: run: python ci_build_files.py - name: Run local tests - working-directory: modflow-devtools + working-directory: modflow-devtools/autotest env: - BIN_PATH: ~/.local/bin/modflow REPOS_PATH: ${{ github.workspace }} - GITHUB_TOKEN: ${{ github.token }} # use --dist loadfile to so tests requiring pytest-virtualenv run on the same worker - run: pytest -v -n auto --dist loadfile --durations 0 --ignore modflow_devtools/test/test_download.py + run: pytest -v -n auto --dist loadfile --durations 0 --ignore test_download.py - name: Run network-dependent tests # only invoke the GH API on one OS and Python version # to avoid rate limits (1000 rqs / hour / repository) # https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits if: runner.os == 'Linux' && matrix.python == '3.8' - working-directory: modflow-devtools + working-directory: modflow-devtools/autotest env: - BIN_PATH: ~/.local/bin/modflow REPOS_PATH: ${{ github.workspace }} GITHUB_TOKEN: ${{ github.token }} - run: pytest -v -n auto --durations 0 modflow_devtools/test/test_download.py \ No newline at end of file + run: pytest -v -n auto --durations 0 test_download.py \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad286b2..7a2bd3a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -164,6 +164,10 @@ jobs: permissions: contents: write pull-requests: write + id-token: write # mandatory for trusted publishing + environment: # requires a 'release' environment in repo settings + name: release + url: https://pypi.org/p/modflow-devtools steps: - name: Checkout main branch @@ -188,12 +192,14 @@ jobs: - name: Check package run: twine check --strict dist/* - - name: Publish package - if: ${{ env.TWINE_USERNAME != '' }} - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: twine upload dist/* + - name: Upload package + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 reset: name: Draft reset PR diff --git a/.gitignore b/.gitignore index 4ad02b9..20eb2ac 100644 --- a/.gitignore +++ b/.gitignore @@ -128,8 +128,9 @@ dmypy.json # Pyre type checker .pyre/ -# pycharme +# IDEs .idea/ +.vscode/ # downloaded exe modflow_devtools/bin/ diff --git a/DEVELOPER.md b/DEVELOPER.md index b074df5..499e1b8 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -38,25 +38,21 @@ This repository's tests use [`pytest`](https://docs.pytest.org/en/latest/) and s This repository's tests expect a few environment variables: -- `BIN_PATH`: path to MODFLOW 6 and related executables - `REPOS_PATH`: the path to MODFLOW 6 example model repositories - `GITHUB_TOKEN`: a GitHub authentication token These may be set manually, but the recommended approach is to configure environment variables in a `.env` file in the project root, for instance: ``` -BIN_PATH=/path/to/modflow/executables REPOS_PATH=/path/to/repos GITHUB_TOKEN=yourtoken... ``` The tests use [`pytest-dotenv`](https://github.com/quiqua/pytest-dotenv) to detect and load variables from this file. -**Note:** at minimum, the tests require that the `mf6` (or `mf6.exe` on Windows) executable is present in `BIN_PATH`. - ### Running the tests -Tests should be run from the project root. To run the tests in parallel with verbose output: +Tests should be run from the `autotest` directory. To run the tests in parallel with verbose output: ```shell pytest -v -n auto @@ -64,11 +60,11 @@ pytest -v -n auto ### Writing new tests -Tests should follow a few conventions for ease of use and maintenance. +Tests follow a few conventions for ease of use and maintenance. #### Temporary directories -Tests which must write to disk should use `pytest`'s built-in `temp_dir` fixture or one of this package's own scoped temporary directory fixtures. +Tests which must write to disk use `pytest`'s built-in `temp_dir` fixture or one of this package's own scoped temporary directory fixtures. ## Releasing diff --git a/HISTORY.md b/HISTORY.md index 0f7d8d9..de0e5cb 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,15 @@ +### Version 1.3.0 + +#### New features + +* [feat(fixtures)](https://github.com/MODFLOW-USGS/modflow-devtools/commit/0ce571411b6b35bc62d4f333d1a961bd2f202784): Add --tabular pytest CLI arg and corresponding fixture (#116). Committed by wpbonelli on 2023-09-12. +* [feat(timeit)](https://github.com/MODFLOW-USGS/modflow-devtools/commit/506a238f6f31d827015a6c6f5ba1867ee55948a7): Add function timing decorator (#118). Committed by wpbonelli on 2023-09-12. +* [feat(executables)](https://github.com/MODFLOW-USGS/modflow-devtools/commit/5b61a4b393b0bcd40aafeb87d1e80b3e557e0f05): Support .get(key, default) like dict (#125). Committed by wpbonelli on 2023-11-21. + +#### Refactoring + +* [refactor](https://github.com/MODFLOW-USGS/modflow-devtools/commit/cd644fa90885cde04f36f24e44cfe922b2a38897): Support python 3.12, various updates (#124). Committed by wpbonelli on 2023-11-11. + ### Version 1.2.0 #### New features diff --git a/modflow_devtools/test/__init__.py b/autotest/__init__.py similarity index 100% rename from modflow_devtools/test/__init__.py rename to autotest/__init__.py diff --git a/conftest.py b/autotest/conftest.py similarity index 100% rename from conftest.py rename to autotest/conftest.py diff --git a/pytest.ini b/autotest/pytest.ini similarity index 60% rename from pytest.ini rename to autotest/pytest.ini index 2e0fa86..73fd077 100644 --- a/pytest.ini +++ b/autotest/pytest.ini @@ -1,8 +1,8 @@ [pytest] -addopts = -ra +addopts = -ra --color=yes python_files = test_*.py *_test*.py markers = - slow: tests that don't complete in a few seconds + slow: tests not completing in a few seconds meta: run by other tests (e.g. testing fixtures) \ No newline at end of file diff --git a/modflow_devtools/test/test_build.py b/autotest/test_build.py similarity index 99% rename from modflow_devtools/test/test_build.py rename to autotest/test_build.py index 3966ec0..36697f7 100644 --- a/modflow_devtools/test/test_build.py +++ b/autotest/test_build.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest + from modflow_devtools.build import meson_build from modflow_devtools.markers import requires_pkg diff --git a/modflow_devtools/test/test_download.py b/autotest/test_download.py similarity index 99% rename from modflow_devtools/test/test_download.py rename to autotest/test_download.py index 9fbfff7..9d5173d 100644 --- a/modflow_devtools/test/test_download.py +++ b/autotest/test_download.py @@ -1,5 +1,6 @@ import pytest from flaky import flaky + from modflow_devtools.download import ( download_and_unzip, download_artifact, diff --git a/autotest/test_executables.py b/autotest/test_executables.py new file mode 100644 index 0000000..c06edc3 --- /dev/null +++ b/autotest/test_executables.py @@ -0,0 +1,28 @@ +import sys +from pathlib import Path +from shutil import which + +import pytest + +from modflow_devtools.executables import Executables +from modflow_devtools.misc import add_sys_path, get_suffixes + +ext, _ = get_suffixes(sys.platform) +exe_stem = "pytest" +exe_path = Path(which(exe_stem)) +bin_path = exe_path.parent +exe = f"{exe_stem}{ext}" + + +@pytest.fixture +def exes(): + with add_sys_path(bin_path): + yield Executables(**{exe_stem: bin_path / exe}) + + +def test_access(exes): + # support both attribute and dictionary style access + assert exes.pytest == exes["pytest"] == exe_path + # .get() works too + assert exes.get("not a target") is None + assert exes.get("not a target", exes["pytest"]) == exes["pytest"] diff --git a/modflow_devtools/test/test_fixtures.py b/autotest/test_fixtures.py similarity index 90% rename from modflow_devtools/test/test_fixtures.py rename to autotest/test_fixtures.py index c5364e6..f959d0c 100644 --- a/modflow_devtools/test/test_fixtures.py +++ b/autotest/test_fixtures.py @@ -139,7 +139,7 @@ def test_keep_class_scoped_tmpdir(tmp_path, arg): TestKeepClassScopedTmpdirInner.test_keep_class_scoped_tmpdir_inner.__name__, "-M", "test_keep", - "-K", + arg, tmp_path, ] assert pytest.main(args) == ExitCode.OK @@ -160,19 +160,14 @@ def test_keep_module_scoped_tmpdir(tmp_path, arg): test_keep_module_scoped_tmpdir_inner.__name__, "-M", "test_keep", - "-K", + arg, tmp_path, ] assert pytest.main(args) == ExitCode.OK this_path = Path(__file__) keep_path = ( - tmp_path - / f"{str(this_path.parent.parent.name)}.{str(this_path.parent.name)}.{str(this_path.stem)}0" + tmp_path / f"{str(this_path.parent.name)}.{str(this_path.stem)}0" ) - from pprint import pprint - - print(keep_path) - pprint(list(keep_path.glob("*"))) assert test_keep_fname in [f.name for f in keep_path.glob("*")] @@ -186,7 +181,7 @@ def test_keep_session_scoped_tmpdir(tmp_path, arg, request): test_keep_session_scoped_tmpdir_inner.__name__, "-M", "test_keep", - "-K", + arg, tmp_path, ] assert pytest.main(args) == ExitCode.OK @@ -316,3 +311,34 @@ def test_pandas(pandas, arg, function_tmpdir): assert "True" in res elif pandas == "no": assert "False" in res + + +test_tabular_fname = "tabular.txt" + + +@pytest.mark.meta("test_tabular") +def test_tabular_inner(function_tmpdir, tabular): + with open(function_tmpdir / test_tabular_fname, "w") as f: + f.write(str(tabular)) + + +@pytest.mark.parametrize("tabular", ["raw", "recarray", "dataframe"]) +@pytest.mark.parametrize("arg", ["--tabular", "-T"]) +def test_tabular(tabular, arg, function_tmpdir): + inner_fn = test_tabular_inner.__name__ + args = [ + __file__, + "-v", + "-s", + "-k", + inner_fn, + arg, + tabular, + "--keep", + function_tmpdir, + "-M", + "test_tabular", + ] + assert pytest.main(args) == ExitCode.OK + res = open(next(function_tmpdir.rglob(test_tabular_fname))).readlines()[0] + assert tabular == res diff --git a/autotest/test_markers.py b/autotest/test_markers.py new file mode 100644 index 0000000..cd105c4 --- /dev/null +++ b/autotest/test_markers.py @@ -0,0 +1,59 @@ +from os import environ +from platform import python_version, system +from shutil import which + +from packaging.version import Version + +from modflow_devtools.markers import * + +exe = "pytest" + + +@requires_exe(exe) +def test_require_exe(): + assert which(exe) + require_exe(exe) + require_program(exe) + + +exes = [exe, "python"] + + +@require_exe(*exes) +def test_require_exe_multiple(): + assert all(which(exe) for exe in exes) + + +@requires_pkg("pytest") +def test_requires_pkg(): + import numpy + + assert numpy is not None + + +@requires_pkg("pytest", "pluggy") +def test_requires_pkg_multiple(): + import pluggy + import pytest + + assert pluggy is not None and pytest is not None + + +@requires_platform("Windows") +def test_requires_platform(): + assert system() == "Windows" + + +@excludes_platform("Darwin", ci_only=True) +def test_requires_platform_ci_only(): + if "CI" in environ: + assert system() != "Darwin" + + +py_ver = python_version() + + +@pytest.mark.parametrize("version", ["3.12", "3.11"]) +def test_requires_python(version): + if Version(py_ver) >= Version(version): + assert requires_python(version) diff --git a/modflow_devtools/test/test_misc.py b/autotest/test_misc.py similarity index 89% rename from modflow_devtools/test/test_misc.py rename to autotest/test_misc.py index 79ea786..7b65641 100644 --- a/modflow_devtools/test/test_misc.py +++ b/autotest/test_misc.py @@ -1,12 +1,13 @@ import os +import re import shutil from os import environ from pathlib import Path -from pprint import pprint +from time import sleep from typing import List import pytest -from conftest import project_root_path + from modflow_devtools.misc import ( get_model_paths, get_namefile_paths, @@ -15,6 +16,7 @@ has_pkg, set_dir, set_env, + timed, ) @@ -25,7 +27,7 @@ def test_set_dir(tmp_path): assert Path(os.getcwd()) != tmp_path -def test_set_env(tmp_path): +def test_set_env(): # test adding a variable key = "TEST_ENV" val = "test" @@ -254,32 +256,27 @@ def test_get_namefile_paths_select_packages(): assert len(paths) >= 43 -@pytest.mark.slow -def test_has_pkg(virtualenv): - python = virtualenv.python - venv = Path(python).parent - pkg = "pytest" - dep = "pluggy" - print( - f"Using temp venv at {venv} with python {python} to test has_pkg('{pkg}') with and without '{dep}'" - ) - - # install a package and remove one of its dependencies - virtualenv.run(f"pip install {project_root_path}") - virtualenv.run(f"pip install {pkg}") - virtualenv.run(f"pip uninstall -y {dep}") - - # check with/without strict mode - for strict in [False, True]: - cmd = ( - f"from modflow_devtools.misc import has_pkg; print(has_pkg('{pkg}'" - + (", strict=True))" if strict else "))") - ) - exp = "False" if strict else "True" - assert ( - virtualenv.run( - f'{python} -c "{cmd}"', - capture=True, - ).strip() - == exp - ) +def test_has_pkg(): + assert has_pkg("pytest") + assert not has_pkg("notapkg") + + +def test_timed1(capfd): + def sleep1(): + sleep(0.001) + + timed(sleep1)() + cap = capfd.readouterr() + print(cap.out) + assert re.match(r"sleep1 took \d+\.\d+ ms", cap.out) + + +def test_timed2(capfd): + @timed + def sleep1dec(): + sleep(0.001) + + sleep1dec() + cap = capfd.readouterr() + print(cap.out) + assert re.match(r"sleep1dec took \d+\.\d+ ms", cap.out) diff --git a/modflow_devtools/test/test_ostags.py b/autotest/test_ostags.py similarity index 99% rename from modflow_devtools/test/test_ostags.py rename to autotest/test_ostags.py index 7ad3b91..22e93f1 100644 --- a/modflow_devtools/test/test_ostags.py +++ b/autotest/test_ostags.py @@ -1,6 +1,7 @@ from platform import system import pytest + from modflow_devtools.ostags import ( OSTag, get_binary_suffixes, diff --git a/modflow_devtools/test/test_zip.py b/autotest/test_zip.py similarity index 67% rename from modflow_devtools/test/test_zip.py rename to autotest/test_zip.py index b1aa28a..070b9c4 100644 --- a/modflow_devtools/test/test_zip.py +++ b/autotest/test_zip.py @@ -2,91 +2,73 @@ import shutil import sys import zipfile -from os import environ from pathlib import Path from pprint import pprint +from shutil import which from zipfile import ZipFile import pytest + from modflow_devtools.markers import excludes_platform from modflow_devtools.misc import get_suffixes, set_dir from modflow_devtools.zip import MFZipFile -_bin_path = Path(environ.get("BIN_PATH")).expanduser().absolute() -_ext, _ = get_suffixes(sys.platform) - - -@pytest.fixture(scope="module") -def empty_archive(module_tmpdir) -> Path: - # https://stackoverflow.com/a/25195628/6514033 - data = b"PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - path = module_tmpdir / "empty.zip" - - with open(path, "wb") as zip: - zip.write(data) - - return path - - -@pytest.fixture(scope="module") -def nonempty_archive(module_tmpdir) -> Path: - if not _bin_path.is_dir(): - pytest.skip(f"BIN_PATH ({_bin_path}) is not a directory") - - zip_path = module_tmpdir / "nonempty.zip" - txt_path = module_tmpdir / "hw.txt" - exe_path = _bin_path / f"mf6{_ext}" - - # create a zip file with a text file and an executable - shutil.copy(exe_path, module_tmpdir) - with open(txt_path, "w") as f: - f.write("hello world") - - with set_dir(module_tmpdir): - zip = MFZipFile(zip_path.name, "w") - zip.write(txt_path.name, compress_type=zipfile.ZIP_DEFLATED) - zip.write(exe_path.name, compress_type=zipfile.ZIP_DEFLATED) - zip.close() - - return zip_path +ext, _ = get_suffixes(sys.platform) +exe_stem = "pytest" +exe_path = Path(which(exe_stem)) +exe_name = f"{exe_stem}{ext}" def test_compressall(function_tmpdir): zip_file = function_tmpdir / "output.zip" input_dir = function_tmpdir / "input" input_dir.mkdir() - with open(input_dir / "data.txt", "w") as f: f.write("hello world") MFZipFile.compressall(str(zip_file), dir_pths=str(input_dir)) - pprint(list(function_tmpdir.iterdir())) assert zip_file.exists() output_dir = function_tmpdir / "output" output_dir.mkdir() - ZipFile(zip_file).extractall(path=str(output_dir)) - pprint(list(output_dir.iterdir())) assert (output_dir / "data.txt").is_file() +@pytest.fixture(scope="module") +def empty_archive(module_tmpdir) -> Path: + # https://stackoverflow.com/a/25195628/6514033 + data = b"PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + path = module_tmpdir / "empty.zip" + with open(path, "wb") as zip: + zip.write(data) + yield path + + def test_extractall_empty(empty_archive, function_tmpdir): zf = MFZipFile(empty_archive, "r") zf.extractall(str(function_tmpdir)) - assert not any(function_tmpdir.iterdir()) +@pytest.fixture(scope="module") +def archive(module_tmpdir) -> Path: + zip_path = module_tmpdir / "nonempty.zip" + shutil.copy(exe_path, module_tmpdir) + with set_dir(module_tmpdir): + zip = MFZipFile(zip_path.name, "w") + zip.write(exe_path.name, compress_type=zipfile.ZIP_DEFLATED) + zip.close() + yield zip_path + + @pytest.mark.parametrize("mf", [True, False]) @excludes_platform("Windows") -def test_preserves_execute_permission(function_tmpdir, nonempty_archive, mf): - zip = MFZipFile(nonempty_archive) if mf else ZipFile(nonempty_archive) +def test_extractall_preserves_execute_permission(function_tmpdir, archive, mf): + zip = MFZipFile(archive) if mf else ZipFile(archive) zip.extractall(path=str(function_tmpdir)) - - exe_path = function_tmpdir / f"mf6{_ext}" - - assert exe_path.is_file() - assert os.access(exe_path, os.X_OK) == mf + path = function_tmpdir / exe_name + assert path.is_file() + assert os.access(path, os.X_OK) == mf diff --git a/docs/conf.py b/docs/conf.py index ba8c65b..aa50e63 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ project = "modflow-devtools" author = "MODFLOW Team" -release = "1.2.0" +release = "1.3.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/index.rst b/docs/index.rst index cbeaa40..e71195d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ The `modflow-devtools` package provides a set of tools for developing and testin md/download.md md/ostags.md md/zip.md + md/timed.md .. toctree:: diff --git a/docs/md/download.md b/docs/md/download.md index 9a3643b..069f0af 100644 --- a/docs/md/download.md +++ b/docs/md/download.md @@ -20,7 +20,7 @@ print([asset["name"] for asset in assets]) This yields `['code.json', 'linux.zip', 'mac.zip', 'win64.zip']`. -Equivalently, using the `list_release_assets()` function to list the latest release assets directly: +Equivalently, using the `get_release_assets()` function to list the latest release assets directly: ```python from modflow_devtools.download import get_release_assets diff --git a/docs/md/executables.md b/docs/md/executables.md index d235cc5..ef1085f 100644 --- a/docs/md/executables.md +++ b/docs/md/executables.md @@ -28,15 +28,6 @@ The `targets` fixture can then be injected into test functions: def test_targets(targets): # attribute- and dictionary-style access is supported assert targets["mf6"] == targets.mf6 -``` - -There is also a convenience function for getting a program's version string. The function will automatically strip the program name from the output (assumed delimited with `:`). - -```python -import subprocess - -def test_executables_version(targets): - # returns e.g. '6.4.1 Release 12/09/2022' - assert targets.get_version(targets.mf6) == \ - subprocess.check_output([f"{targets.mf6}", "-v"]).decode('utf-8').strip().split(":")[1].strip() + # .get() works too + assert targets.get("not a target") is None ``` diff --git a/docs/md/markers.md b/docs/md/markers.md index bd9714c..a50aec3 100644 --- a/docs/md/markers.md +++ b/docs/md/markers.md @@ -78,4 +78,11 @@ Both these markers accept a `ci_only` flag, which indicates whether the policy s Markers are also provided to ping network resources and skip if unavailable: - `@requires_github`: skips if `github.com` is unreachable -- `@requires_spatial_reference`: skips if `spatialreference.org` is unreachable \ No newline at end of file +- `@requires_spatial_reference`: skips if `spatialreference.org` is unreachable + +## Aliases + +All markers are aliased to imperative mood, e.g. `require_github`. Some have other aliases as well: + +`requires_pkg` -> `require[s]_package` +`requires_exe` -> `require[s]_program` diff --git a/docs/md/ostags.md b/docs/md/ostags.md index ea9440c..6108dc7 100644 --- a/docs/md/ostags.md +++ b/docs/md/ostags.md @@ -46,11 +46,11 @@ The second argument specifies the mapping in format `2`, where ` ## Getting suffixes -A convenience function is available to get the appropriate binary file extensions for a given operating system, identified by tag (or the current operating system if no tag is provided). The return value is a 2-tuple containing the executable and library extensions, respectively. +A convenience function is available to get the appropriate binary file extensions for a given operating system, identified by any supported OS tag, or the current operating system if no tag is provided. The return value is a 2-tuple containing the executable and library extensions, respectively. ```python get_binary_suffixes() # get extensions for current OS get_binary_suffixes("linux") # returns ("", ".so") -get_binary_suffixes("linux") # returns ("", ".so") +get_binary_suffixes("mac") # returns ("", ".dylib") get_binary_suffixes("win64") # returns (".exe", ".dll") ``` diff --git a/docs/md/timed.md b/docs/md/timed.md new file mode 100644 index 0000000..d4de7ab --- /dev/null +++ b/docs/md/timed.md @@ -0,0 +1,21 @@ +# `timed` + +There is a `@timed` decorator function available in the `modflow_devtools.misc` module. Applying it to any function prints a (rough) benchmark to `stdout` when the function returns. For instance: + +```python +from modflow_devtools.misc import timed + +@timed +def sleep1(): + sleep(0.001) + +sleep1() # prints e.g. "sleep1 took 1.26 ms" +``` + +It can also wrap a function directly: + +```python +timed(sleep1)() +``` + +The [`timeit`](https://docs.python.org/3/library/timeit.html) built-in module is used internally, however the timed function is only called once, where by default, `timeit` averages multiple runs. \ No newline at end of file diff --git a/modflow_devtools/__init__.py b/modflow_devtools/__init__.py index a37151c..be89f8f 100644 --- a/modflow_devtools/__init__.py +++ b/modflow_devtools/__init__.py @@ -1,6 +1,6 @@ __author__ = "Joseph D. Hughes" -__date__ = "Sep 12, 2023" -__version__ = "1.2.0" +__date__ = "Nov 21, 2023" +__version__ = "1.3.0" __maintainer__ = "Joseph D. Hughes" __email__ = "jdhughes@usgs.gov" __status__ = "Production" diff --git a/modflow_devtools/download.py b/modflow_devtools/download.py index 3741f05..fcf045b 100644 --- a/modflow_devtools/download.py +++ b/modflow_devtools/download.py @@ -87,7 +87,7 @@ def get_response_json(): return json.loads(resp.read().decode()) except urllib.error.HTTPError as err: if err.code == 401 and os.environ.get("GITHUB_TOKEN"): - raise ValueError("GITHUB_TOKEN env is invalid") from err + raise ValueError("GITHUB_TOKEN is invalid") from err elif err.code == 403 and "rate limit exceeded" in err.reason: raise ValueError( f"use GITHUB_TOKEN env to bypass rate limit ({err})" diff --git a/modflow_devtools/executables.py b/modflow_devtools/executables.py index 28c966d..0c9705e 100644 --- a/modflow_devtools/executables.py +++ b/modflow_devtools/executables.py @@ -1,10 +1,6 @@ -import sys -from os import PathLike from pathlib import Path from types import SimpleNamespace -from typing import Dict, Optional - -from modflow_devtools.misc import get_suffixes, run_cmd +from typing import Dict class Executables(SimpleNamespace): @@ -21,20 +17,12 @@ def __setitem__(self, key, item): def __getitem__(self, key): return self.__dict__[key] + def get(self, key, default=None): + return self.as_dict().get(key, default) + def as_dict(self) -> Dict[str, Path]: """ Returns a dictionary mapping executable names to paths. """ return self.__dict__.copy() - - @staticmethod - def get_version(path: PathLike = None, flag: str = "-v") -> Optional[str]: - """Get an executable's version string.""" - - out, err, ret = run_cmd(str(path), flag) - if ret == 0: - out = "".join(out).strip() - return out.split(":")[1].strip() - else: - return None diff --git a/modflow_devtools/fixtures.py b/modflow_devtools/fixtures.py index 1fa695c..0700cf8 100644 --- a/modflow_devtools/fixtures.py +++ b/modflow_devtools/fixtures.py @@ -101,6 +101,14 @@ def use_pandas(request): raise ValueError(f"Unsupported value for --pandas: {pandas}") +@pytest.fixture +def tabular(request): + tab = request.config.option.TABULAR + if tab not in ["raw", "recarray", "dataframe"]: + raise ValueError(f"Unsupported value for --tabular: {tab}") + return tab + + # configuration hooks @@ -163,7 +171,16 @@ def pytest_addoption(parser): action="store", default="yes", dest="PANDAS", - help="Package input data can be provided as either pandas dataframes or numpy recarrays. By default, pandas dataframes are used. To test with numpy recarrays, use 'no'. To randomize selection (per test), use 'random'.", + help="Indicates whether to use pandas, where multiple approaches are available. Select 'yes', 'no', or 'random'.", + ) + + parser.addoption( + "-T", + "--tabular", + action="store", + default="raw", + dest="TABULAR", + help="Configure tabular data representation for model input. Select 'raw', 'recarray', or 'dataframe'.", ) diff --git a/modflow_devtools/markers.py b/modflow_devtools/markers.py index d40ca5c..1c21e43 100644 --- a/modflow_devtools/markers.py +++ b/modflow_devtools/markers.py @@ -1,4 +1,11 @@ -from platform import system +""" +Pytest markers to toggle tests based on environment conditions. +Occasionally useful to directly assert environment expectations. +""" + +from platform import python_version, system + +from packaging.version import Version from modflow_devtools.imports import import_optional_dependency from modflow_devtools.misc import ( @@ -10,6 +17,7 @@ ) pytest = import_optional_dependency("pytest") +py_ver = Version(python_version()) def requires_exe(*exes): @@ -21,6 +29,23 @@ def requires_exe(*exes): ) +def requires_python(version, bound="lower"): + if not isinstance(version, str): + raise ValueError(f"Version must a string") + + py_tgt = Version(version) + if bound == "lower": + return py_ver >= py_tgt + elif bound == "upper": + return py_ver <= py_tgt + elif bound == "exact": + return py_ver == py_tgt + else: + return ValueError( + f"Invalid bound type: {bound} (use 'upper', 'lower', or 'exact')" + ) + + def requires_pkg(*pkgs): missing = {pkg for pkg in pkgs if not has_pkg(pkg, strict=True)} return pytest.mark.skipif( @@ -69,3 +94,20 @@ def excludes_branch(branch): not is_connected("spatialreference.org"), reason="spatialreference.org is required.", ) + + +# imperative mood renaming, and some aliases + +require_exe = requires_exe +require_program = requires_exe +requires_program = requires_exe +require_python = requires_python +require_pkg = requires_pkg +require_package = requires_pkg +requires_package = requires_pkg +require_platform = requires_platform +exclude_platform = excludes_platform +require_branch = requires_branch +exclude_branch = excludes_branch +require_github = requires_github +require_spatial_reference = requires_spatial_reference diff --git a/modflow_devtools/misc.py b/modflow_devtools/misc.py index 7dd9210..4915d8c 100644 --- a/modflow_devtools/misc.py +++ b/modflow_devtools/misc.py @@ -3,12 +3,14 @@ import sys import traceback from contextlib import contextmanager +from functools import wraps from importlib import metadata from os import PathLike, chdir, environ, getcwd from os.path import basename, normpath from pathlib import Path from shutil import which from subprocess import PIPE, Popen +from timeit import timeit from typing import List, Optional, Tuple from urllib import request @@ -64,8 +66,9 @@ def set_env(*remove, **update): class add_sys_path: """ - Context manager for temporarily editing the system path - (https://stackoverflow.com/a/39855753/6514033) + Context manager to add temporarily to the system path. + + Adapted from https://stackoverflow.com/a/39855753/6514033. """ def __init__(self, path): @@ -442,3 +445,44 @@ def try_metadata() -> bool: _has_pkg_cache[pkg] = found return _has_pkg_cache[pkg] + + +def timed(f): + """ + Decorator for estimating runtime of any function. + Prints estimated time to stdout, in milliseconds. + + Parameters + ---------- + f : function + Function to time. + + Notes + ----- + Adapted from https://stackoverflow.com/a/27737385/6514033. + Uses the built-in timeit module internally. + + Returns + ------- + function + The decorated function. + """ + + @wraps(f) + def _timed(*args, **kw): + res = None + + def call(): + nonlocal res + res = f(*args, **kw) + + t = timeit(lambda: call(), number=1) + if "log_time" in kw: + name = kw.get("log_name", f.__name__.upper()) + kw["log_time"][name] = int(t * 1000) + else: + print(f"{f.__name__} took {t * 1000:.2f} ms") + + return res + + return _timed diff --git a/modflow_devtools/test/test_executables.py b/modflow_devtools/test/test_executables.py deleted file mode 100644 index 483103a..0000000 --- a/modflow_devtools/test/test_executables.py +++ /dev/null @@ -1,44 +0,0 @@ -import subprocess -import sys -from os import environ -from pathlib import Path - -import pytest -from modflow_devtools.executables import Executables -from modflow_devtools.misc import add_sys_path, get_suffixes - -_bin_path = Path(environ.get("BIN_PATH")).expanduser().absolute() -_ext, _ = get_suffixes(sys.platform) - - -@pytest.fixture -def exes(): - if not _bin_path.is_dir(): - pytest.skip(f"BIN_PATH ({_bin_path}) is not a directory") - - with add_sys_path(str(_bin_path)): - yield Executables( - **{ - "mf6": _bin_path / f"mf6{_ext}", - } - ) - - -def test_get_version(exes): - ver_str = Executables.get_version(exes.mf6) - version = ( - subprocess.check_output([f"{exes.mf6}", "-v"]) - .decode("utf-8") - .split(":")[1] - .strip() - ) - assert ver_str == version - assert int(ver_str[0].split(".")[0]) >= 6 - - -def test_mapping(exes): - print(exes.mf6) - assert ( - exes.mf6 == exes["mf6"] - ) # should support both attribute and dictionary access - assert exes.mf6 == _bin_path / f"mf6{_ext}" # should be the correct path diff --git a/modflow_devtools/test/test_markers.py b/modflow_devtools/test/test_markers.py deleted file mode 100644 index 95f497c..0000000 --- a/modflow_devtools/test/test_markers.py +++ /dev/null @@ -1,49 +0,0 @@ -from os import environ -from platform import system -from shutil import which - -from modflow_devtools.markers import ( - excludes_platform, - requires_exe, - requires_pkg, - requires_platform, -) - - -@requires_exe("mf6") -def test_requires_exe(): - assert which("mf6") - - -exes = ["mfusg", "mfnwt"] - - -@requires_exe(*exes) -def test_requires_exe_multiple(): - assert all(which(exe) for exe in exes) - - -@requires_pkg("numpy") -def test_requires_pkg(): - import numpy - - assert numpy is not None - - -@requires_pkg("numpy", "matplotlib") -def test_requires_pkg_multiple(): - import matplotlib - import numpy - - assert numpy is not None and matplotlib is not None - - -@requires_platform("Windows") -def test_requires_platform(): - assert system() == "Windows" - - -@excludes_platform("Darwin", ci_only=True) -def test_requires_platform_ci_only(): - if "CI" in environ: - assert system() != "Darwin" diff --git a/pyproject.toml b/pyproject.toml index c3630b7..4d2a701 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,9 @@ name = "modflow-devtools" description = "Python tools for MODFLOW development" authors = [ {name = "Joseph D. Hughes", email = "modflow@usgs.gov"}, + {name = "Michael Reno", email = "mreno@ucar.edu"}, + {name = "Mike Taves", email = "mwtoews@gmail.com"}, + {name = "Wes Bonelli", email = "wbonelli@ucar.edu"}, ] maintainers = [ {name = "Joseph D. Hughes", email = "modflow@usgs.gov"}, @@ -33,6 +36,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Hydrology" ] requires-python = ">=3.8" @@ -58,7 +62,6 @@ test = [ "pytest-cases", "pytest-cov", "pytest-dotenv", - "pytest-virtualenv", "pytest-xdist", "PyYaml" ] @@ -84,10 +87,11 @@ verbose = true [tool.isort] profile = "black" -src_paths = ["src/modflow_devtools"] +src_paths = ["modflow_devtools"] line_length = 79 [tool.setuptools] +packages = ["modflow_devtools"] include-package-data = true zip-safe = false diff --git a/version.txt b/version.txt index 867e524..589268e 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.2.0 \ No newline at end of file +1.3.0 \ No newline at end of file