From 14da16a7d3b423fb94e970f8623556f1b9b70e9c Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Tue, 30 Jan 2024 15:27:07 +0000 Subject: [PATCH] (#819) Add some unit tests --- .github/workflows/pin_versions.py | 34 ++- .github/workflows/test_data/pip_freeze.txt | 248 ++++++++++++++++++ .github/workflows/test_data/setup.cfg | 80 ++++++ .github/workflows/test_data/setup.cfg.pinned | 80 ++++++ .../workflows/test_data/setup.cfg.unpinned | 80 ++++++ .github/workflows/test_pin_versions.py | 78 ++++++ 6 files changed, 586 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/test_data/pip_freeze.txt create mode 100644 .github/workflows/test_data/setup.cfg create mode 100644 .github/workflows/test_data/setup.cfg.pinned create mode 100644 .github/workflows/test_data/setup.cfg.unpinned create mode 100644 .github/workflows/test_pin_versions.py diff --git a/.github/workflows/pin_versions.py b/.github/workflows/pin_versions.py index ffe2e84af..8cf5e3f48 100755 --- a/.github/workflows/pin_versions.py +++ b/.github/workflows/pin_versions.py @@ -7,7 +7,7 @@ from functools import partial from sys import stderr, stdout -SETUP_CFG_PATTERN = re.compile("(.*?)(@(.*))?\n") +SETUP_CFG_PATTERN = re.compile("(.*?)\\s*(@(.*))?\n") PIP = "pip" @@ -20,9 +20,7 @@ def normalize(package_name: str): def fetch_pin_versions() -> dict[str, str]: - process = subprocess.run( - [PIP, "freeze"], capture_output=True, encoding=locale.getpreferredencoding() - ) + process = run_pip_freeze() if process.returncode == 0: output = process.stdout lines = output.split("\n") @@ -40,21 +38,29 @@ def fetch_pin_versions() -> dict[str, str]: exit(1) +def run_pip_freeze(): + process = subprocess.run( + [PIP, "freeze"], capture_output=True, encoding=locale.getpreferredencoding() + ) + return process + + def process_setup_cfg(input_fname, output_fname, dependency_processor): with open(input_fname) as input_file: with open(output_fname, "w") as output_file: - while line := input_file.readline(): - output_file.write(line) - if line.startswith("install_requires"): - break - - while (line := input_file.readline()) and not line.isspace(): - dependency_processor(line, output_file) + process_files(input_file, output_file, dependency_processor) - output_file.write(line) - while line := input_file.readline(): - output_file.write(line) +def process_files(input_file, output_file, dependency_processor): + while line := input_file.readline(): + output_file.write(line) + if line.startswith("install_requires"): + break + while (line := input_file.readline()) and not line.isspace(): + dependency_processor(line, output_file) + output_file.write(line) + while line := input_file.readline(): + output_file.write(line) def strip_comment(line: str): diff --git a/.github/workflows/test_data/pip_freeze.txt b/.github/workflows/test_data/pip_freeze.txt new file mode 100644 index 000000000..e1acb1a2c --- /dev/null +++ b/.github/workflows/test_data/pip_freeze.txt @@ -0,0 +1,248 @@ +accessible-pygments==0.0.4 +aioca==1.7 +aiohttp==3.9.1 +aiosignal==1.3.1 +alabaster==0.7.16 +aniso8601==9.0.1 +anyio==4.2.0 +appdirs==1.4.4 +asciitree==0.3.3 +asttokens==2.4.1 +async-timeout==4.0.3 +attrs==23.2.0 +Babel==2.14.0 +beautifulsoup4==4.12.3 +bidict==0.22.1 +black==24.1.0 +blinker==1.7.0 +blueapi==0.3.15 +bluesky==1.12.0 +bluesky-kafka==0.10.0 +bluesky-live==0.0.8 +boltons==23.1.1 +build==1.0.3 +cachetools==5.3.2 +caproto==1.1.1 +certifi==2023.11.17 +cfgv==3.4.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.3 +cloudpickle==3.0.0 +colorama==0.4.6 +comm==0.2.1 +confluent-kafka==2.3.0 +contourpy==1.2.0 +coverage==7.4.0 +cycler==0.12.1 +dask==2024.1.0 +databroker==1.2.5 +dataclasses-json==0.6.3 +decorator==5.1.1 +Deprecated==1.2.14 +diff_cover==8.0.3 +distlib==0.3.8 +dls-bluesky-core==0.0.3 +dls-dodal==1.13.1 +dnspython==2.5.0 +docopt==0.6.2 +doct==1.1.0 +docutils==0.20.1 +email-validator==2.1.0.post1 +entrypoints==0.4 +epicscorelibs==7.0.7.99.0.2 +event-model==1.19.9 +exceptiongroup==1.2.0 +executing==2.0.1 +fastapi==0.98.0 +fasteners==0.19 +filelock==3.13.1 +Flask==3.0.1 +Flask-RESTful==0.3.10 +fonttools==4.47.2 +freephil==0.2.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gitdb==4.0.11 +GitPython==3.1.41 +googleapis-common-protos==1.59.1 +graypy==2.1.0 +greenlet==3.0.3 +grpcio==1.60.0 +h11==0.14.0 +h5py==3.10.0 +hdf5plugin==4.3.0 +HeapDict==1.0.1 +historydict==1.2.6 +httpcore==1.0.2 +httptools==0.6.1 +httpx==0.26.0 +humanize==4.9.0 +-e git+ssh://git@github.com/DiamondLightSource/hyperion.git@5b88ce8b69483397adb66f78da5970a5186fcae2#egg=hyperion +identify==2.5.33 +idna==3.6 +imageio==2.33.1 +imagesize==1.4.1 +importlib-metadata==6.11.0 +importlib-resources==6.1.1 +iniconfig==2.0.0 +intake==0.6.4 +ipython==8.20.0 +ipywidgets==8.1.1 +ispyb==10.0.0 +itsdangerous==2.1.2 +jedi==0.19.1 +Jinja2==3.1.3 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +jupyterlab-widgets==3.0.9 +kiwisolver==1.4.5 +livereload==2.6.3 +locket==1.0.0 +MarkupSafe==2.1.4 +marshmallow==3.20.2 +matplotlib==3.8.2 +matplotlib-inline==0.1.6 +mockito==1.4.0 +mongoquery==1.4.2 +msgpack==1.0.7 +msgpack-numpy==0.4.8 +multidict==6.0.4 +mypy==1.8.0 +mypy-extensions==1.0.0 +mysql-connector-python==8.3.0 +networkx==3.2.1 +nexgen==0.8.0 +nodeenv==1.8.0 +nose2==0.14.0 +nslsii==0.9.1 +numcodecs==0.12.1 +numpy==1.26.3 +opencv-python-headless==4.9.0.80 +opentelemetry-api==1.22.0 +opentelemetry-distro==0.43b0 +opentelemetry-exporter-jaeger==1.21.0 +opentelemetry-exporter-jaeger-proto-grpc==1.21.0 +opentelemetry-exporter-jaeger-thrift==1.21.0 +opentelemetry-instrumentation==0.43b0 +opentelemetry-sdk==1.22.0 +opentelemetry-semantic-conventions==0.43b0 +ophyd==1.9.0 +ophyd-async @ git+https://github.com/bluesky/ophyd-async@ec5729640041ee5b77b4614158793af3a34cf9d8 +orjson==3.9.12 +p4p==4.1.12 +packaging==23.2 +pandas==2.2.0 +parso==0.8.3 +partd==1.4.1 +pathlib2==2.3.7.post1 +pathspec==0.12.1 +pexpect==4.9.0 +pika==1.3.2 +pillow==10.2.0 +PIMS==0.6.1 +Pint==0.23 +pipdeptree==2.13.2 +platformdirs==4.1.0 +pluggy==1.4.0 +ply==3.11 +pre-commit==3.6.0 +prettytable==3.9.0 +prompt-toolkit==3.0.43 +protobuf==4.25.2 +psutil==5.9.8 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pvxslibs==1.3.1 +py==1.11.0 +pydantic==1.10.14 +pydata-sphinx-theme==0.15.2 +pyepics==3.5.2 +Pygments==2.17.2 +pymongo==4.6.1 +pyOlog==4.5.0 +pyparsing==3.1.1 +pyproject-api==1.6.1 +pyproject_hooks==1.0.0 +pyright==1.1.348 +pyright_diff_quality_plugin @ git+https://github.com/DiamondLightSource/pyright_diff_quality_plugin.git@77fc0819e708eb16ffdcaea06552a2804294b53e +pytest==7.4.4 +pytest-asyncio==0.23.3 +pytest-cov==4.1.0 +pytest-random-order==1.1.1 +python-dateutil==2.8.2 +python-dotenv==1.0.1 +python-multipart==0.0.6 +pytz==2023.3.post1 +PyYAML==6.0.1 +pyzmq==25.1.2 +redis==5.0.1 +referencing==0.32.1 +requests==2.31.0 +rpds-py==0.17.1 +ruff==0.1.14 +scanspec==0.6.5 +scipy==1.12.0 +semver==3.0.2 +setuptools-dso==2.10 +six==1.16.0 +slicerator==1.1.0 +smmap==5.0.1 +sniffio==1.3.0 +snowballstemmer==2.2.0 +soupsieve==2.5 +Sphinx==7.2.6 +sphinx-autobuild==2021.3.14 +sphinx-copybutton==0.5.2 +sphinx_design==0.5.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +SQLAlchemy==1.4.51 +stack-data==0.6.3 +starlette==0.27.0 +stomp.py==8.1.0 +suitcase-mongo==0.4.0 +suitcase-msgpack==0.3.0 +suitcase-utils==0.5.4 +super-state-machine==2.0.2 +tabulate==0.9.0 +thrift==0.16.0 +tifffile==2023.12.9 +tomli==2.0.1 +toolz==0.12.1 +tornado==6.4 +tox==3.28.0 +tox-direct==0.4 +tqdm==4.66.1 +traitlets==5.14.1 +types-mock==5.1.0.20240106 +types-PyYAML==6.0.12.12 +types-requests==2.31.0.20240125 +typing-inspect==0.9.0 +typing_extensions==4.5.0 +tzdata==2023.4 +tzlocal==5.2 +ujson==5.9.0 +urllib3==2.1.0 +uvicorn==0.27.0 +uvloop==0.19.0 +virtualenv==20.25.0 +watchfiles==0.21.0 +wcwidth==0.2.13 +websocket-client==1.7.0 +websockets==12.0 +Werkzeug==3.0.1 +widgetsnbextension==4.0.9 +workflows==2.26 +wrapt==1.16.0 +xarray==2024.1.1 +yarl==1.9.4 +zarr==2.16.1 +zict==2.2.0 +zipp==3.17.0 +zmq==0.0.0 +zocalo==0.30.2 diff --git a/.github/workflows/test_data/setup.cfg b/.github/workflows/test_data/setup.cfg new file mode 100644 index 000000000..aa07251c1 --- /dev/null +++ b/.github/workflows/test_data/setup.cfg @@ -0,0 +1,80 @@ +[metadata] +name = hyperion +description = Unattended MX data collection using BlueSky / Ophyd +url = https://github.com/DiamondLightSource/hyperion +license = BSD 3-Clause License +long_description = file: README.rst +long_description_content_type = text/x-rst +classifiers = + Development Status :: 3 - Alpha + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + +[options] +python_requires = >=3.9 +packages = find: +package_dir = + =src +install_requires = + bluesky + pyepics + blueapi + flask-restful + ispyb + scanspec + numpy + nexgen @ git+https://github.com/dials/nexgen.git@db4858f6d91a3d07c6c0f815ef752849c0bf79d4 + opentelemetry-distro + opentelemetry-exporter-jaeger + ophyd + semver + # For databroker + humanize + pandas + xarray + doct + databroker + dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@0aabdb389f65a30e629cec60a36c3b5298b7647f + pydantic<2.0 # See https://github.com/DiamondLightSource/hyperion/issues/774 + scipy + pyzmq<25 # See https://github.com/DiamondLightSource/hyperion/issues/1103 + +[options.entry_points] +console_scripts = + hyperion = hyperion.__main__:main + hyperion-callbacks = hyperion.external_interaction.callbacks.__main__:main + +[options.extras_require] +dev = + GitPython + black + pytest-cov + pytest-random-order + pytest-asyncio + ipython + mockito + pre-commit + mypy + matplotlib + tox + build + ruff + diff-cover + pyright + pyright_diff_quality_plugin @ git+https://github.com/DiamondLightSource/pyright_diff_quality_plugin.git + + +[options.packages.find] +where = src + +[options.package_data] +hyperion = *.txt + +[mypy] +# Ignore missing stubs for modules we use +ignore_missing_imports = True +#needed for opentelemetry +namespace_packages = true +[mypy-opentelemetry.sdk.*] +implicit_reexport = True diff --git a/.github/workflows/test_data/setup.cfg.pinned b/.github/workflows/test_data/setup.cfg.pinned new file mode 100644 index 000000000..604ed1339 --- /dev/null +++ b/.github/workflows/test_data/setup.cfg.pinned @@ -0,0 +1,80 @@ +[metadata] +name = hyperion +description = Unattended MX data collection using BlueSky / Ophyd +url = https://github.com/DiamondLightSource/hyperion +license = BSD 3-Clause License +long_description = file: README.rst +long_description_content_type = text/x-rst +classifiers = + Development Status :: 3 - Alpha + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + +[options] +python_requires = >=3.9 +packages = find: +package_dir = + =src +install_requires = + bluesky @ 1.12.0 + pyepics @ 3.5.2 + blueapi @ 0.3.15 + flask-restful @ 0.3.10 + ispyb @ 10.0.0 + scanspec @ 0.6.5 + numpy @ 1.26.3 + nexgen @ 0.8.0 + opentelemetry-distro @ 0.43b0 + opentelemetry-exporter-jaeger @ 1.21.0 + ophyd @ 1.9.0 + semver @ 3.0.2 + # For databroker + humanize @ 4.9.0 + pandas @ 2.2.0 + xarray @ 2024.1.1 + doct @ 1.1.0 + databroker @ 1.2.5 + dls-dodal @ 1.13.1 + pydantic<2.0 # See https://github.com/DiamondLightSource/hyperion/issues/774 + scipy @ 1.12.0 + pyzmq<25 # See https://github.com/DiamondLightSource/hyperion/issues/1103 + +[options.entry_points] +console_scripts = + hyperion = hyperion.__main__:main + hyperion-callbacks = hyperion.external_interaction.callbacks.__main__:main + +[options.extras_require] +dev = + GitPython + black + pytest-cov + pytest-random-order + pytest-asyncio + ipython + mockito + pre-commit + mypy + matplotlib + tox + build + ruff + diff-cover + pyright + pyright_diff_quality_plugin @ git+https://github.com/DiamondLightSource/pyright_diff_quality_plugin.git + + +[options.packages.find] +where = src + +[options.package_data] +hyperion = *.txt + +[mypy] +# Ignore missing stubs for modules we use +ignore_missing_imports = True +#needed for opentelemetry +namespace_packages = true +[mypy-opentelemetry.sdk.*] +implicit_reexport = True diff --git a/.github/workflows/test_data/setup.cfg.unpinned b/.github/workflows/test_data/setup.cfg.unpinned new file mode 100644 index 000000000..eed433078 --- /dev/null +++ b/.github/workflows/test_data/setup.cfg.unpinned @@ -0,0 +1,80 @@ +[metadata] +name = hyperion +description = Unattended MX data collection using BlueSky / Ophyd +url = https://github.com/DiamondLightSource/hyperion +license = BSD 3-Clause License +long_description = file: README.rst +long_description_content_type = text/x-rst +classifiers = + Development Status :: 3 - Alpha + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + +[options] +python_requires = >=3.9 +packages = find: +package_dir = + =src +install_requires = + bluesky + pyepics + blueapi + flask-restful + ispyb + scanspec + numpy + nexgen + opentelemetry-distro + opentelemetry-exporter-jaeger + ophyd + semver + # For databroker + humanize + pandas + xarray + doct + databroker + dls-dodal + pydantic<2.0 # See https://github.com/DiamondLightSource/hyperion/issues/774 + scipy + pyzmq<25 # See https://github.com/DiamondLightSource/hyperion/issues/1103 + +[options.entry_points] +console_scripts = + hyperion = hyperion.__main__:main + hyperion-callbacks = hyperion.external_interaction.callbacks.__main__:main + +[options.extras_require] +dev = + GitPython + black + pytest-cov + pytest-random-order + pytest-asyncio + ipython + mockito + pre-commit + mypy + matplotlib + tox + build + ruff + diff-cover + pyright + pyright_diff_quality_plugin @ git+https://github.com/DiamondLightSource/pyright_diff_quality_plugin.git + + +[options.packages.find] +where = src + +[options.package_data] +hyperion = *.txt + +[mypy] +# Ignore missing stubs for modules we use +ignore_missing_imports = True +#needed for opentelemetry +namespace_packages = true +[mypy-opentelemetry.sdk.*] +implicit_reexport = True diff --git a/.github/workflows/test_pin_versions.py b/.github/workflows/test_pin_versions.py new file mode 100644 index 000000000..64583603a --- /dev/null +++ b/.github/workflows/test_pin_versions.py @@ -0,0 +1,78 @@ +import io +from functools import partial +from unittest.mock import MagicMock, patch + +import pin_versions +import pytest + + +@pytest.fixture +def pinned_versions(): + with patch("pin_versions.run_pip_freeze") as run_pip_freeze: + with open("test_data/pip_freeze.txt") as freeze_output: + mock_process = MagicMock() + run_pip_freeze.return_value = mock_process + mock_process.stdout = freeze_output.read() + mock_process.returncode = 0 + + +@pytest.mark.parametrize( + "input, expected", + [ + ("", ("", None)), + ( + " pydantic<2.0 # See https://github.com/DiamondLightSource/hyperion/issues/774", + ( + " pydantic<2.0 ", + " See https://github.com/DiamondLightSource/hyperion/issues/774", + ), + ), + ], +) +def test_strip_comment(input, expected): + assert pin_versions.strip_comment(input) == expected + + +@pytest.mark.parametrize( + "input, expected", + [ + ("dls-dodal", "dls-dodal"), + ("dls_dodal", "dls-dodal"), + ("dls.dodal", "dls-dodal"), + ], +) +def test_normalize(input, expected): + assert pin_versions.normalize(input) == expected + + +def test_unpin(): + with io.StringIO() as output_file: + with open("test_data/setup.cfg") as input_file: + pin_versions.process_files( + input_file, output_file, pin_versions.unpin_versions + ) + with open("test_data/setup.cfg.unpinned") as expected_file: + assert output_file.getvalue() == expected_file.read() + + +@patch("pin_versions.stdout") +def test_write_commit_message(mock_stdout, pinned_versions): + installed_versions = pin_versions.fetch_pin_versions() + pin_versions.write_commit_message(installed_versions) + mock_stdout.write.assert_called_once_with( + "Pin dependencies prior to release. Dodal 1.13.1, nexgen 0.8.0" + ) + + +def test_pin(pinned_versions): + installed_versions = pin_versions.fetch_pin_versions() + with io.StringIO() as output_file: + with open("test_data/setup.cfg.unpinned") as input_file: + pin_versions.process_files( + input_file, + output_file, + partial(pin_versions.update_setup_cfg_line, installed_versions), + ) + + with open("test_data/setup.cfg.pinned") as expected_file: + assert output_file.getvalue() == expected_file.read()