From 9bc34d052c96d41d510d1b6d4315b5adf88f48f6 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Thu, 20 Apr 2023 12:49:13 +0100 Subject: [PATCH] Refactor dependency locks (#3879) Fixes: #3702 --- .config/requirements-docs.txt | 2 + .config/requirements-lock.txt | 42 ++++++++++ .config/requirements-test.txt | 14 ++++ .config/requirements.in | 12 +++ requirements.txt => .config/requirements.txt | 50 +++++------- .gitignore | 1 + .pre-commit-config.yaml | 27 ++++--- pyproject.toml | 80 ++++---------------- src/molecule/__init__.py | 10 ++- src/molecule/driver/base.py | 3 +- src/molecule/scenarios.py | 36 +++++---- src/molecule/test/functional/conftest.py | 2 +- tox.ini | 11 ++- 13 files changed, 153 insertions(+), 137 deletions(-) create mode 100644 .config/requirements-docs.txt create mode 100644 .config/requirements-lock.txt create mode 100644 .config/requirements-test.txt create mode 100644 .config/requirements.in rename requirements.txt => .config/requirements.txt (69%) diff --git a/.config/requirements-docs.txt b/.config/requirements-docs.txt new file mode 100644 index 000000000..72fea1f8f --- /dev/null +++ b/.config/requirements-docs.txt @@ -0,0 +1,2 @@ +mkdocs-ansible[lock]>=0.1.4 +pipdeptree>=2.4.0 diff --git a/.config/requirements-lock.txt b/.config/requirements-lock.txt new file mode 100644 index 000000000..5af77861a --- /dev/null +++ b/.config/requirements-lock.txt @@ -0,0 +1,42 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --no-annotate --output-file=.config/requirements-lock.txt --resolver=backtracking --strip-extras --unsafe-package=ruamel-yaml-clib pyproject.toml +# +ansible-compat==3.0.2 +ansible-core==2.14.4 +arrow==1.2.3 +attrs==23.1.0 +binaryornot==0.4.4 +certifi==2022.12.7 +cffi==1.15.1 +chardet==5.1.0 +charset-normalizer==3.1.0 +click==8.1.3 +click-help-colors==0.9.1 +cookiecutter==2.1.1 +cryptography==40.0.2 +enrich==1.2.7 +idna==3.4 +jinja2==3.1.2 +jinja2-time==0.2.0 +jsonschema==4.17.3 +markdown-it-py==2.2.0 +markupsafe==2.1.2 +mdurl==0.1.2 +packaging==23.1 +pluggy==1.0.0 +pycparser==2.21 +pygments==2.15.1 +pyrsistent==0.19.3 +python-dateutil==2.8.2 +python-slugify==8.0.1 +pyyaml==6.0 +requests==2.28.2 +resolvelib==0.8.1 +rich==13.3.4 +six==1.16.0 +subprocess-tee==0.4.1 +text-unidecode==1.3 +urllib3==1.26.15 diff --git a/.config/requirements-test.txt b/.config/requirements-test.txt new file mode 100644 index 000000000..e416b0a8c --- /dev/null +++ b/.config/requirements-test.txt @@ -0,0 +1,14 @@ +# Do not include ansible or any package that would drag ansible in here +# We want to assure test extra provides tools to test molecule and its +# related tools/plugins but w/o ansible, which can be installed separated. +ansible-lint >= 6.12.1 +ansi2html >= 1.8.0 +coverage >= 7.0.3 +filelock >= 3.9.0 +pexpect >= 4.8.0, < 5 +pytest-mock >= 3.10.0 +pytest-plus >= 0.4.0 +pytest-testinfra >= 7.0.0 +pytest-xdist >= 3.1.0 +pytest >= 7.2.0 +check-jsonschema diff --git a/.config/requirements.in b/.config/requirements.in new file mode 100644 index 000000000..bc63c6408 --- /dev/null +++ b/.config/requirements.in @@ -0,0 +1,12 @@ +ansible-compat >= 2.2.0 +ansible-core >= 2.12.10 +click >= 8.0, < 9 +click-help-colors >= 0.9 +cookiecutter >= 1.7.3 # dependency issues in older versions +enrich >= 1.2.7 +jsonschema >= 4.9.1 +Jinja2 >= 2.11.3 +packaging +pluggy >= 0.7.1, < 2.0 +PyYAML >= 5.1 +rich >= 9.5.1 diff --git a/requirements.txt b/.config/requirements.txt similarity index 69% rename from requirements.txt rename to .config/requirements.txt index c0c954a3e..2e1a36f17 100644 --- a/requirements.txt +++ b/.config/requirements.txt @@ -2,25 +2,22 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --extra=docs --extra=lint --extra=test --no-annotate --output-file=requirements.txt --resolver=backtracking --strip-extras pyproject.toml tools/opts.txt +# pip-compile --extra=docs --extra=test --no-annotate --output-file=.config/requirements.txt --resolver=backtracking --strip-extras pyproject.toml # ansi2html==1.8.0 ansible-compat==3.0.2 ansible-core==2.14.4 ansible-lint==6.14.6 -argparse-manpage==4.1 arrow==1.2.3 -asyncio==3.4.3 attrs==23.1.0 -babel==2.12.1 +beautifulsoup4==4.12.1 binaryornot==0.4.4 black==23.3.0 bracex==2.3.post1 -cairocffi==1.5.1 +cairocffi==1.5.0 cairosvg==2.7.0 certifi==2022.12.7 cffi==1.15.1 -cfgv==3.3.1 chardet==5.1.0 charset-normalizer==3.1.0 check-jsonschema==0.22.0 @@ -30,57 +27,53 @@ colorama==0.4.6 cookiecutter==2.1.1 coverage==7.2.3 cryptography==40.0.2 +csscompressor==0.9.5 cssselect2==0.7.0 -dacite==1.8.0 defusedxml==0.7.1 -distlib==0.3.6 -distro==1.8.0 enrich==1.2.7 exceptiongroup==1.1.1 execnet==1.9.0 filelock==3.12.0 -flake8==6.0.0 ghp-import==2.1.0 -gitdb==4.0.10 -gitpython==3.1.31 -griffe==0.27.1 -identify==2.5.22 +griffe==0.26.0 +htmlmin2==0.1.13 idna==3.4 -importlib-metadata==6.5.0 +importlib-metadata==6.1.0 iniconfig==2.0.0 jinja2==3.1.2 jinja2-time==0.2.0 +jsmin==3.0.1 jsonschema==4.17.3 markdown==3.3.7 +markdown-exec==1.4.0 markdown-include==0.8.1 markdown-it-py==2.2.0 markupsafe==2.1.2 -mccabe==0.7.0 mdurl==0.1.2 mergedeep==1.3.4 mkdocs==1.4.2 +mkdocs-ansible==0.1.4 mkdocs-autorefs==0.4.1 -mkdocs-git-revision-date-localized-plugin==1.2.0 -mkdocs-material==9.1.6 +mkdocs-gen-files==0.4.0 +mkdocs-htmlproofer-plugin==0.11.0 +mkdocs-material==9.1.5 mkdocs-material-extensions==1.1.1 -mkdocs-multirepo-plugin==0.6.1 +mkdocs-minify-plugin==0.6.4 +mkdocs-monorepo-plugin==1.0.4 mkdocstrings==0.21.2 mkdocstrings-python==0.9.0 mypy-extensions==1.0.0 -nodeenv==1.7.0 -packaging==23.1 +packaging==23.0 pathspec==0.11.1 pexpect==4.8.0 pillow==9.5.0 +pipdeptree==2.7.0 platformdirs==3.2.0 pluggy==1.0.0 -pre-commit==3.2.2 ptyprocess==0.7.0 -pycodestyle==2.10.0 pycparser==2.21 -pyflakes==3.0.1 -pygments==2.15.1 -pymdown-extensions==9.11 +pygments==2.14.0 +pymdown-extensions==9.10 pyrsistent==0.19.3 pytest==7.3.1 pytest-mock==3.10.0 @@ -89,7 +82,6 @@ pytest-testinfra==7.0.0 pytest-xdist==3.2.1 python-dateutil==2.8.2 python-slugify==8.0.1 -pytz==2023.3 pyyaml==6.0 pyyaml-env-tag==0.1 regex==2023.3.23 @@ -99,15 +91,13 @@ rich==13.3.4 ruamel-yaml==0.17.21 ruamel-yaml-clib==0.2.7 six==1.16.0 -smmap==5.0.0 +soupsieve==2.4 subprocess-tee==0.4.1 text-unidecode==1.3 tinycss2==1.2.1 tomli==2.0.1 typing-extensions==4.5.0 -typing-inspect==0.8.0 urllib3==1.26.15 -virtualenv==20.21.0 watchdog==3.0.0 wcmatch==8.4.1 webencodings==0.5.1 diff --git a/.gitignore b/.gitignore index cf21ff68d..a2ac138b4 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ molecule/** tools/test-schema/node_modules site .DS_Store +src/molecule/_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 91112832f..575cd9476 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,10 +20,6 @@ repos: additional_dependencies: - prettier - prettier-plugin-toml - - repo: https://github.com/PyCQA/doc8 - rev: "v1.1.1" - hooks: - - id: doc8 - repo: https://github.com/codespell-project/codespell rev: v2.2.4 hooks: @@ -57,7 +53,7 @@ repos: types: [file, yaml] entry: yamllint --strict - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: "v0.0.261" + rev: "v0.0.262" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -102,10 +98,23 @@ repos: rev: 6.13.0 hooks: - id: pip-compile - entry: pip-compile -q --resolver=backtracking --strip-extras --no-annotate --output-file=requirements.txt tools/opts.txt pyproject.toml --extra docs --extra test --extra lint - files: ^(pyproject\.toml|requirements.txt|constraints\.txt)$ + entry: pip-compile -q --resolver=backtracking --strip-extras --no-annotate --output-file=.config/requirements.txt pyproject.toml --extra docs --extra test + files: ^(pyproject\.toml|\.config\/.*)$ - id: pip-compile name: pip-compile-upgrade - entry: pip-compile -q --resolver=backtracking --strip-extras -q --upgrade --no-annotate --output-file=requirements.txt tools/opts.txt pyproject.toml --extra docs --extra test --extra lint - files: ^(pyproject\.toml|requirements.txt|constraints\.txt)$ + alias: up + entry: pip-compile -q --resolver=backtracking --strip-extras -q --upgrade --no-annotate --output-file=.config/requirements.txt pyproject.toml --extra docs --extra test + files: ^(pyproject\.toml|\.config\/.*)$ stages: [manual] + - id: pip-compile + name: lock + alias: lock + always_run: true + entry: pip-compile --upgrade --resolver=backtracking --no-annotate --output-file=.config/requirements-lock.txt pyproject.toml --strip-extras --unsafe-package ruamel-yaml-clib + files: ^.config\/requirements.*$ + language: python + language_version: "3.9" # minimal we support officially + pass_filenames: false + stages: [manual] + additional_dependencies: + - pip>=22.3.1 diff --git a/pyproject.toml b/pyproject.toml index 08c619b07..03a029a5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,15 @@ [build-system] requires = [ - "setuptools >= 61.0", # PEP-621 - "setuptools_scm[toml] >= 7.0.0", + "setuptools >= 63.0.0", # required by pyproject+setuptools_scm integration + "setuptools_scm[toml] >= 7.0.5", # required for "no-local-version" scheme + ] build-backend = "setuptools.build_meta" [project] # https://peps.python.org/pep-0621/#readme requires-python = ">=3.9" -dynamic = ["version"] +dynamic = ["version", "dependencies", "optional-dependencies"] name = "molecule" description = "Molecule aids in the development and testing of Ansible roles" readme = "README.md" @@ -22,7 +23,8 @@ classifiers = [ "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", + "Operating System :: MacOS", + "Operating System :: POSIX", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -35,20 +37,6 @@ classifiers = [ "Topic :: Utilities", ] keywords = ["ansible", "roles", "testing", "molecule"] -dependencies = [ - "ansible-compat >= 2.2.0", - "ansible-core >= 2.12.10", - "click >= 8.0, < 9", - "click-help-colors >= 0.9", - "cookiecutter >= 1.7.3", # dependency issues in older versions - "enrich >= 1.2.7", - "jsonschema >= 4.9.1", - "Jinja2 >= 2.11.3", - "packaging", - "pluggy >= 0.7.1, < 2.0", - "PyYAML >= 5.1", - "rich >= 9.5.1", -] [project.urls] homepage = "https://github.com/ansible-community/molecule" @@ -56,43 +44,6 @@ documentation = "https://molecule.readthedocs.io/" repository = "https://github.com/ansible-community/molecule" changelog = "https://github.com/ansible-community/molecule/releases" -[project.optional-dependencies] -docs = [ - "argparse-manpage", - "cairosvg", - "markdown-include", - "mkdocs", - "mkdocs-git-revision-date-localized-plugin", - "mkdocs-material", - "mkdocs-material-extensions", - "mkdocs-multirepo-plugin", - "mkdocstrings[python]", - "pillow", - "pymdown-extensions", -] -lint = [ - "check-jsonschema >= 0.20.0", - "flake8 >= 6.0.0", - "jsonschema >= 4.17.3", - "pre-commit >= 2.21.0", - "yamllint >= 1.28.0", -] -test = [ - # Do not include ansible or any package that would drag ansible in here - # We want to assure test extra provides tools to test molecule and its - # related tools/plugins but w/o ansible, which can be installed separated. - "ansible-lint >= 6.12.1", - "ansi2html >= 1.8.0", - "coverage >= 7.0.3", - "filelock >= 3.9.0", - "pexpect >= 4.8.0, < 5", - "pytest-mock >= 3.10.0", - "pytest-plus >= 0.4.0", - "pytest-testinfra >= 7.0.0", - "pytest-xdist >= 3.1.0", - "pytest >= 7.2.0", -] - [project.scripts] molecule = "molecule.__main__:main" @@ -117,18 +68,6 @@ omit = ["test/*"] fail_under = 88 skip_covered = true -# skip_empty = true -[tool.doc8] -# see https://github.com/PyCQA/doc8/issues/75 -ignore-path = [ - ".eggs", - ".tox", - "requirements.txt", - "build", - "molecule.egg-info", - "src/molecule.egg-info", -] - [tool.black] skip-string-normalization = false @@ -264,5 +203,12 @@ parametrize-values-type = "tuple" [tool.ruff.isort] known-first-party = ["molecule"] +[tool.setuptools.dynamic] +optional-dependencies.docs = { file = [".config/requirements-docs.txt"] } +optional-dependencies.test = { file = [".config/requirements-test.txt"] } +optional-dependencies.lock = { file = [".config/requirements-lock.txt"] } +dependencies = { file = [".config/requirements.in"] } + [tool.setuptools_scm] local_scheme = "no-local-version" +write_to = "src/molecule/_version.py" diff --git a/src/molecule/__init__.py b/src/molecule/__init__.py index 7d3ae0b90..2876824d8 100644 --- a/src/molecule/__init__.py +++ b/src/molecule/__init__.py @@ -19,8 +19,10 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. """Molecule version information.""" +try: + from ._version import __version__, version # type: ignore +except ImportError: # pragma: no cover + __version__ = "0.1.dev1" + version = __version__ - -from importlib.metadata import version - -__version__ = version("molecule") +__all__ = ("__version__", "version") diff --git a/src/molecule/driver/base.py b/src/molecule/driver/base.py index 187d84b87..e81cbf6c9 100644 --- a/src/molecule/driver/base.py +++ b/src/molecule/driver/base.py @@ -22,6 +22,7 @@ import inspect import os from abc import ABCMeta, abstractmethod +from importlib.metadata import version import molecule from molecule.status import Status @@ -41,7 +42,7 @@ def __init__(self, config=None) -> None: self._config = config self._path = os.path.abspath(os.path.dirname(inspect.getfile(self.__class__))) self.module = self.__module__.split(".", maxsplit=1)[0] - self.version = molecule.version(self.module) + self.version = version(self.module) @property @abstractmethod diff --git a/src/molecule/scenarios.py b/src/molecule/scenarios.py index 84622d3db..0afcbd42d 100644 --- a/src/molecule/scenarios.py +++ b/src/molecule/scenarios.py @@ -121,22 +121,20 @@ def _get_matrix(self): :returns: dict """ - return dict( - { - scenario.name: { - "check": scenario.check_sequence, - "cleanup": scenario.cleanup_sequence, - "converge": scenario.converge_sequence, - "create": scenario.create_sequence, - "dependency": scenario.dependency_sequence, - "destroy": scenario.destroy_sequence, - "idempotence": scenario.idempotence_sequence, - "prepare": scenario.prepare_sequence, - "side_effect": scenario.side_effect_sequence, - "syntax": scenario.syntax_sequence, - "test": scenario.test_sequence, - "verify": scenario.verify_sequence, - } - for scenario in self.all - }, - ) + return { + scenario.name: { + "check": scenario.check_sequence, + "cleanup": scenario.cleanup_sequence, + "converge": scenario.converge_sequence, + "create": scenario.create_sequence, + "dependency": scenario.dependency_sequence, + "destroy": scenario.destroy_sequence, + "idempotence": scenario.idempotence_sequence, + "prepare": scenario.prepare_sequence, + "side_effect": scenario.side_effect_sequence, + "syntax": scenario.syntax_sequence, + "test": scenario.test_sequence, + "verify": scenario.verify_sequence, + } + for scenario in self.all + } diff --git a/src/molecule/test/functional/conftest.py b/src/molecule/test/functional/conftest.py index 67cccab9a..2716a0d2a 100644 --- a/src/molecule/test/functional/conftest.py +++ b/src/molecule/test/functional/conftest.py @@ -48,7 +48,7 @@ @pytest.fixture(scope="session", autouse=True) def require_installed_package(): try: - molecule.version("molecule") + molecule.__version__ except Exception as e: pytest.fail(f"Functional tests require molecule package to be installed: {e}") diff --git a/tox.ini b/tox.ini index c61cebdaa..2b1beb02a 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ setenv = COVERAGE_FILE={env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} COVERAGE_PROCESS_START={toxinidir}/pyproject.toml MOLECULE_NO_LOG=0 - PIP_CONSTRAINT = {toxinidir}/requirements.txt + PIP_CONSTRAINT = {toxinidir}/.config/requirements.txt devel: PIP_CONSTRAINT=/dev/null PIP_DISABLE_PIP_VERSION_CHECK=1 PYTHONDONTWRITEBYTECODE=1 @@ -47,16 +47,11 @@ setenv = # Temporare remove "-n auto" as it seems to break coverage on this project. _EXTRAS=-l deps = - # py,py{39,310,311,312}: --editable .[docker,lint,podman,test,windows] devel: git+https://github.com/ansible/ansible#egg=ansible-core # pytest-molecule not used but we want to check that it does not conflict devel: git+https://github.com/ansible-community/pytest-molecule#egg=pytest-molecule extras = - docker - lint - podman test - windows commands = ansibledevel: ansible-galaxy install git+https://github.com/ansible-collections/community.general.git # failsafe as pip may install incompatible dependencies @@ -98,6 +93,10 @@ setenv = {[testenv:lint]setenv} commands = # manual hook calls the optional pip-compile-upgrade hook after pip-compile {[testenv:lint]commands} --hook-stage manual pip-compile + # Update pre-commit hooks + -pre-commit autoupdate + # We fail if files are modified at the end + git diff --exit-code [testenv:docs] description = Build documentation