From a1b5ba0094ebbd2879e471671cb7d7e1f93d636b Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Sat, 1 Jul 2023 12:35:16 +0200 Subject: [PATCH] FEAT: automatically update settings in `pyproject.toml` (#143) * FEAT: automatically update `black` configuration * FEAT: automatically update `nbqa.addopts` settings * MAINT: switch to `tomlkit` Just like `ruamel.yaml`, `tomlkit` perserves formatting of the TOML file when dumping it --- .cspell.json | 1 + pyproject.toml | 12 +-- setup.cfg | 2 +- src/repoma/check_dev_files/black.py | 143 ++++++++++++---------------- src/repoma/utilities/pyproject.py | 46 +++++++-- tests/check_dev_files/test_black.py | 128 ------------------------- 6 files changed, 109 insertions(+), 223 deletions(-) delete mode 100644 tests/check_dev_files/test_black.py diff --git a/.cspell.json b/.cspell.json index a02087f4..252914e4 100644 --- a/.cspell.json +++ b/.cspell.json @@ -111,6 +111,7 @@ "showcode", "showtags", "taplo", + "tomlkit", "unittests", "venv" ] diff --git a/pyproject.toml b/pyproject.toml index 0f1ccc87..9fd07de2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,12 +25,12 @@ exclude = ''' include = '\.pyi?$' preview = true target-version = [ - 'py310', - 'py311', - 'py36', - 'py37', - 'py38', - 'py39', + "py310", + "py311", + "py36", + "py37", + "py38", + "py39", ] [tool.isort] diff --git a/setup.cfg b/setup.cfg index ce28fa07..4542c55b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,7 +36,7 @@ install_requires = pip-tools PyYAML ruamel.yaml # better YAML dumping - toml + tomlkit packages = find: package_dir = =src diff --git a/src/repoma/check_dev_files/black.py b/src/repoma/check_dev_files/black.py index fe14ac8c..8636af63 100644 --- a/src/repoma/check_dev_files/black.py +++ b/src/repoma/check_dev_files/black.py @@ -1,11 +1,9 @@ -"""Check :file:`pyproject.toml` black config.""" -from textwrap import dedent -from typing import List, Optional +"""Update :file:`pyproject.toml` black configuration.""" from ruamel.yaml.comments import CommentedMap from repoma.errors import PrecommitError -from repoma.utilities import CONFIG_PATH, natural_sorting +from repoma.utilities import CONFIG_PATH from repoma.utilities.executor import Executor from repoma.utilities.precommit import ( find_repo, @@ -13,79 +11,66 @@ update_precommit_hook, update_single_hook_precommit_repo, ) -from repoma.utilities.pyproject import load_pyproject +from repoma.utilities.pyproject import ( + get_sub_table, + load_pyproject, + to_toml_array, + write_pyproject, +) from repoma.utilities.setup_cfg import get_supported_python_versions def main() -> None: if not CONFIG_PATH.pyproject.exists(): return - config = _load_black_config() executor = Executor() - executor(_check_line_length, config) - executor(_check_activate_preview, config) - executor(_check_option_ordering, config) - executor(_check_target_versions, config) - executor(_check_pyproject) + executor(_remove_outdated_settings) + executor(_update_black_settings) + executor(_update_nbqa_settings) executor(_update_precommit_repo) executor(_update_precommit_nbqa_hook) executor.finalize() -def _load_black_config(content: Optional[str] = None) -> dict: - config = load_pyproject(content) - return config.get("tool", {}).get("black", {}) - - -def _check_activate_preview(config: dict) -> None: - expected_option = "preview" - if config.get(expected_option) is not True: - raise PrecommitError(dedent(f""" - An option in pyproject.toml is wrong or missing. Should be: - - [tool.black] - {expected_option} = true - """).strip()) - - -def _check_line_length(config: dict) -> None: - if config.get("line-length") is not None: - raise PrecommitError( - "pyproject.toml should not specify a line-length (default to 88)." +def _remove_outdated_settings() -> None: + pyproject = load_pyproject() + settings = get_sub_table(pyproject, "tool.black", create=True) + forbidden_options = ("line-length",) + removed_options = set() + for option in forbidden_options: + if option in settings: + removed_options.add(option) + settings.remove(option) + if removed_options: + write_pyproject(pyproject) + msg = ( + f"Removed {', '.join(sorted(removed_options))} option from black" + f" configuration in {CONFIG_PATH.pyproject}" ) + raise PrecommitError(msg) -def _check_option_ordering(config: dict) -> None: - options = list(config) - sorted_options = sorted(config, key=natural_sorting) - if sorted_options != options: - error_message = dedent(""" - Options in pyproject.toml should be alphabetically sorted: - - [tool.black] - """).strip() - for option in sorted_options: - error_message += f"\n{option} = ..." - raise PrecommitError(error_message) +def _update_black_settings() -> None: + pyproject = load_pyproject() + settings = get_sub_table(pyproject, "tool.black", create=True) + versions = get_supported_python_versions() + target_version = to_toml_array(sorted("py" + v.replace(".", "") for v in versions)) + minimal_settings = { + "preview": True, + "target-version": target_version, + } + if not __complies(settings, minimal_settings): + settings.update(minimal_settings) + write_pyproject(pyproject) + msg = f"Updated black configuration in {CONFIG_PATH.pyproject}" + raise PrecommitError(msg) -def _check_target_versions(config: dict) -> None: - target_versions = config.get("target-version", []) - supported_python_versions = get_supported_python_versions() - expected_target_versions = sorted( - "py" + s.replace(".", "") for s in supported_python_versions - ) - if target_versions != expected_target_versions: - error_message = dedent(""" - Black target versions in pyproject.toml should be as follows: - - [tool.black] - target-version = [ - """).strip() - for version in expected_target_versions: - error_message += f"\n '{version}'," - error_message += "\n]" - raise PrecommitError(error_message) +def __complies(settings: dict, minimal_settings: dict) -> bool: + for key, value in minimal_settings.items(): + if settings.get(key) != value: + return False + return True def _update_precommit_repo() -> None: @@ -106,29 +91,25 @@ def _update_precommit_nbqa_hook() -> None: ) -def _check_pyproject() -> None: +def _update_nbqa_settings() -> None: + # cspell:ignore addopts if not CONFIG_PATH.precommit.exists(): return - config, _ = load_round_trip_precommit_config() - nbqa_repo = find_repo(config, "https://github.com/nbQA-dev/nbQA") - if nbqa_repo is None: + if not __has_nbqa_precommit_repo(): return - nbqa_config = _load_nbqa_black_config() - if nbqa_config != ["--line-length=85"]: - error_message = dedent(""" - Configuration of nbqa-black in pyproject.toml should be as follows: - - [tool.nbqa.addopts] - black = [ - "--line-length=85", - ] - - This is to ensure that code blocks render nicely in the sphinx-book-theme. - """).strip() - raise PrecommitError(error_message) + pyproject = load_pyproject() + nbqa_table = get_sub_table(pyproject, "tool.nbqa.addopts", create=True) + expected = ["--line-length=85"] + if nbqa_table.get("black") != expected: + nbqa_table["black"] = expected + write_pyproject(pyproject) + msg = "Added nbQA configuration for black" + raise PrecommitError(msg) -def _load_nbqa_black_config(content: Optional[str] = None) -> List[str]: - # cspell:ignore addopts - config = load_pyproject(content) - return config.get("tool", {}).get("nbqa", {}).get("addopts", {}).get("black", {}) +def __has_nbqa_precommit_repo() -> bool: + config, _ = load_round_trip_precommit_config() + nbqa_repo = find_repo(config, "https://github.com/nbQA-dev/nbQA") + if nbqa_repo is None: + return False + return True diff --git a/src/repoma/utilities/pyproject.py b/src/repoma/utilities/pyproject.py index 133658f8..c9e3014d 100644 --- a/src/repoma/utilities/pyproject.py +++ b/src/repoma/utilities/pyproject.py @@ -1,14 +1,46 @@ -"""Tools for loading and inspecting :code:`pyproject.toml`.""" -from collections import OrderedDict -from typing import Optional +"""Tools for loading, inspecting, and updating :code:`pyproject.toml`.""" +import os +from typing import Any, Iterable, Optional -import toml +import tomlkit +from tomlkit.container import Container +from tomlkit.items import Array, Table +from tomlkit.toml_document import TOMLDocument from repoma.utilities import CONFIG_PATH -def load_pyproject(content: Optional[str] = None) -> dict: +def load_pyproject(content: Optional[str] = None) -> TOMLDocument: + if not os.path.exists(CONFIG_PATH.pyproject): + return TOMLDocument() if content is None: with open(CONFIG_PATH.pyproject) as stream: - return toml.load(stream, _dict=OrderedDict) - return toml.loads(content, _dict=OrderedDict) + return tomlkit.loads(stream.read()) + return tomlkit.loads(content) + + +def get_sub_table(config: Container, dotted_header: str, create: bool = False) -> Table: + """Get a TOML sub-table through a dotted header key.""" + current_table: Any = config + for header in dotted_header.split("."): + if header not in current_table: + if create: + current_table[header] = tomlkit.table() + else: + raise KeyError(f"TOML data does not contain {dotted_header!r}") + current_table = current_table[header] + return current_table + + +def write_pyproject(config: TOMLDocument) -> None: + src = tomlkit.dumps(config, sort_keys=True) + with open(CONFIG_PATH.pyproject, "w") as stream: + stream.write(src) + + +def to_toml_array(items: Iterable[Any]) -> Array: + array = tomlkit.array() + array.extend(items) + if len(array) > 1: + array.multiline(True) + return array diff --git a/tests/check_dev_files/test_black.py b/tests/check_dev_files/test_black.py deleted file mode 100644 index 7009955e..00000000 --- a/tests/check_dev_files/test_black.py +++ /dev/null @@ -1,128 +0,0 @@ -from textwrap import dedent - -import pytest - -from repoma.check_dev_files.black import ( - _check_activate_preview, - _check_line_length, - _check_option_ordering, - _check_target_versions, - _load_black_config, - _load_nbqa_black_config, -) -from repoma.errors import PrecommitError - - -def test_check_line_length(): - toml_content = dedent(""" - [tool.black] - line-length = 79 - """).strip() - config = _load_black_config(toml_content) - with pytest.raises(PrecommitError) as error: - _check_line_length(config) - assert ( - error.value.args[0] - == "pyproject.toml should not specify a line-length (default to 88)." - ) - - -@pytest.mark.parametrize( - "toml_content", - [ - """[tool.black]""", - dedent(""" - [tool.config] - preview = false - """).strip(), - ], -) -def test_check_activate_preview(toml_content: str): - toml_content = """[tool.black]""" - config = _load_black_config(toml_content) - with pytest.raises(PrecommitError) as error: - _check_activate_preview(config) - assert error.value.args[0] == dedent(""" - An option in pyproject.toml is wrong or missing. Should be: - - [tool.black] - preview = true - """).strip() - - -def test_check_target_versions(): - toml_content = dedent(""" - [tool.black] - target-version = [ - 'py36', - 'py37', - 'py310', - ] - """).strip() - config = _load_black_config(toml_content) - with pytest.raises(PrecommitError) as error: - _check_target_versions(config) - assert error.value.args[0] == dedent(""" - Black target versions in pyproject.toml should be as follows: - - [tool.black] - target-version = [ - 'py310', - 'py311', - 'py36', - 'py37', - 'py38', - 'py39', - ] - """).strip() - - -def test_load_config_from_pyproject(): - config = _load_black_config() - assert config["preview"] is True - assert "exclude" in config - - -def test_load_config_from_string(): - toml_content = dedent(R""" - [tool.black] - preview = true - include = '\.pyi?$' - line-length = 88 - """).strip() - config = _load_black_config(toml_content) - assert config == { - "preview": True, - "include": R"\.pyi?$", - "line-length": 88, - } - - -def test_load_nbqa_black_config(): - # cspell:ignore addopts - toml_content = dedent(R""" - [tool.nbqa.addopts] - black = [ - "--line-length=85", - ] - """).strip() - config = _load_nbqa_black_config(toml_content) - assert config == ["--line-length=85"] - - -def test_check_option_ordering(): - toml_content = dedent(R""" - [tool.black] - preview = true - line-length = 88 - """).strip() - config = _load_black_config(toml_content) - with pytest.raises(PrecommitError) as error: - _check_option_ordering(config) - assert error.value.args[0] == dedent(""" - Options in pyproject.toml should be alphabetically sorted: - - [tool.black] - line-length = ... - preview = ... - """).strip()