diff --git a/.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml b/.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml index 141b646..f9c2502 100644 --- a/.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml +++ b/.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml @@ -31,7 +31,13 @@ on: required: false type: boolean default: false - pre-commit-hook-skip-list: + pre-commit-repo-update-skip-list: + description: A comma-separated list of pre-commit repo urls to skip updates + for (only applicable when `update-pre-commit=true`). + required: false + type: string + default: '' + pre-commit-hook-run-skip-list: description: A comma-separated list of pre-commit hooks to skip (only applicable when `run-pre-commit=true`). required: false @@ -89,7 +95,8 @@ jobs: dependency-dict: ${{ inputs.dependency-dict }} update-pre-commit: ${{ inputs.update-pre-commit }} run-pre-commit: ${{ inputs.run-pre-commit }} - pre-commit-hook-skip-list: ${{ inputs.pre-commit-hook-skip-list }} + pre-commit-repo-update-skip-list: ${{ inputs.pre-commit-repo-update-skip-list }} + pre-commit-hook-run-skip-list: ${{ inputs.pre-commit-hook-run-skip-list }} export-dependency-groups: ${{ inputs.export-dependency-groups }} - if: ${{ !endsWith(github.repository, '/python-package-ci-cd') }} # Run the public action when this is run outside the python-package-ci-cd repository uses: tektronix/python-package-ci-cd/actions/update_development_dependencies@v1.4.1 @@ -97,7 +104,7 @@ jobs: dependency-dict: ${{ inputs.dependency-dict }} update-pre-commit: ${{ inputs.update-pre-commit }} run-pre-commit: ${{ inputs.run-pre-commit }} - pre-commit-hook-skip-list: ${{ inputs.pre-commit-hook-skip-list }} + pre-commit-hook-run-skip-list: ${{ inputs.pre-commit-hook-run-skip-list }} export-dependency-groups: ${{ inputs.export-dependency-groups }} - uses: stefanzweifel/git-auto-commit-action@8621497c8c39c72f3e2a999a26b4ca1b5058a842 # v5.0.1 with: diff --git a/.github/workflows/test-actions.yml b/.github/workflows/test-actions.yml index d44d815..7cc92b0 100644 --- a/.github/workflows/test-actions.yml +++ b/.github/workflows/test-actions.yml @@ -150,7 +150,8 @@ jobs: update-pre-commit: true run-pre-commit: true dependency-dict: '{"dev": ["pyright"]}' - pre-commit-hook-skip-list: remove-tabs,forbid-tabs,check-readthedocs,check-dependabot,check-github-actions,check-github-workflows,commitizen,blacken-docs,yamlfix,hadolint,mdformat,markdown-link-check,check-poetry,toml-sort-fix,pyright,poetry-audit,ruff,ruff-format,docformatter,renovate-config-validator,actionlint + pre-commit-repo-update-skip-list: https://github.com/pre-commit/pre-commit-hooks,https://github.com/executablebooks/mdformat + pre-commit-hook-run-skip-list: remove-tabs,forbid-tabs,check-readthedocs,check-dependabot,check-github-actions,check-github-workflows,commitizen,blacken-docs,yamlfix,hadolint,mdformat,markdown-link-check,check-poetry,toml-sort-fix,pyright,poetry-audit,ruff,ruff-format,docformatter,renovate-config-validator,actionlint export-dependency-groups: | actions-update_development_dependencies:actions/update_development_dependencies, tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 740b97c..4d2ef03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,9 +18,14 @@ Valid subsections within a version are: Things to be included in the next release go here. +### Added + +- Added a new `pre-commit-repo-update-skip-list` input parameter to the `update_development_dependencies` action and the `_reusable-update-python-and-pre-commit-dependencies.yml` workflow to allow users to skip updating specific `pre-commit` hooks. + ### Changed - Bumped dependency versions. +- _**BREAKING CHANGE**_: Renamed the `pre-commit-hook-skip-list` input parameter to `pre-commit-hook-run-skip-list` in the `_reusable-update-python-and-pre-commit-dependencies.yml` workflow and `update_development_dependencies` action. --- diff --git a/actions/update_development_dependencies/action.yml b/actions/update_development_dependencies/action.yml index 8346218..8ed484f 100644 --- a/actions/update_development_dependencies/action.yml +++ b/actions/update_development_dependencies/action.yml @@ -29,7 +29,12 @@ inputs: the update-pre-commit input to `true`. required: false default: 'false' - pre-commit-hook-skip-list: + pre-commit-repo-update-skip-list: + description: A comma-separated list of pre-commit repo urls to skip updates for + (only applicable when `update-pre-commit=true`). + required: false + default: '' + pre-commit-hook-run-skip-list: description: A comma-separated list of pre-commit hooks to skip (only applicable when `run-pre-commit=true`). required: false diff --git a/actions/update_development_dependencies/main.py b/actions/update_development_dependencies/main.py index a31c06b..e5cd8df 100644 --- a/actions/update_development_dependencies/main.py +++ b/actions/update_development_dependencies/main.py @@ -20,6 +20,8 @@ from pathlib import Path +import yaml + from pypi_simple import PyPISimple from yamlfix import fix_files # pyright: ignore[reportUnknownVariableType] @@ -126,7 +128,31 @@ def update_poetry_dependencies( ) -def update_pre_commit_dependencies(python_executable: str, repository_root_directory: Path) -> None: +def get_pre_commit_repos(repository_root_directory: Path) -> list[str]: + """Get the list of repo urls from the .pre-commit-config.yaml file. + + Args: + repository_root_directory: The root directory of the repository. + + Returns: + The list of repo urls. + """ + pre_commit_file_data = yaml.safe_load( + (repository_root_directory / ".pre-commit-config.yaml").read_text() + ) + repo_list: list[str] = [] + for repo in pre_commit_file_data.get("repos", []): + if (repo_url := str(repo["repo"])) == "local": + continue # skip local repos, they don't need to be updated + repo_list.append(repo_url) + return repo_list + + +def update_pre_commit_dependencies( + python_executable: str, + repository_root_directory: Path, + pre_commit_repo_update_skip_list: list[str], +) -> None: """Update the pre-commit dependencies in the .pre-commit-config.yaml file. This function will also fix the formatting of the yaml file using the `yamlfix` package. @@ -134,13 +160,19 @@ def update_pre_commit_dependencies(python_executable: str, repository_root_direc Args: python_executable: The path to the python executable to use. repository_root_directory: The root directory of the repository. + pre_commit_repo_update_skip_list: A list of pre-commit repo urls to skip updating. """ run_cmd_in_subprocess( f"git config --global --add safe.directory " f'"{repository_root_directory.resolve().as_posix()}"' ) - # Update pre-commit config file - run_cmd_in_subprocess(f'"{python_executable}" -m pre_commit autoupdate --freeze') + # Update every hook in the pre-commit config file, skipping any hooks in the skip list + for repo in get_pre_commit_repos(repository_root_directory): + if repo in pre_commit_repo_update_skip_list: + continue + run_cmd_in_subprocess( + f'"{python_executable}" -m pre_commit autoupdate --freeze --repo {repo}' + ) # Fix the formatting of the pre-commit config file with warnings.catch_warnings(): @@ -194,7 +226,10 @@ def main() -> None: export_dependency_groups = [ x.strip() for x in os.environ["INPUT_EXPORT-DEPENDENCY-GROUPS"].split(",") if x ] - pre_commit_hook_skip_list = os.environ["INPUT_PRE-COMMIT-HOOK-SKIP-LIST"] + pre_commit_hook_run_skip_list = os.environ["INPUT_PRE-COMMIT-HOOK-RUN-SKIP-LIST"] + pre_commit_repo_update_skip_list = [ + x.strip() for x in os.environ["INPUT_PRE-COMMIT-REPO-UPDATE-SKIP-LIST"].split(",") if x + ] install_dependencies = os.environ["INPUT_INSTALL-DEPENDENCIES"].lower() in _ENV_VAR_TRUE_VALUES run_pre_commit = os.environ["INPUT_RUN-PRE-COMMIT"].lower() in _ENV_VAR_TRUE_VALUES update_pre_commit = os.environ["INPUT_UPDATE-PRE-COMMIT"].lower() in _ENV_VAR_TRUE_VALUES @@ -208,14 +243,16 @@ def main() -> None: python_executable, repo_root_path, dependency_dict, lock_only=not install_dependencies ) if update_pre_commit or run_pre_commit: - update_pre_commit_dependencies(python_executable, repo_root_path) + update_pre_commit_dependencies( + python_executable, repo_root_path, pre_commit_repo_update_skip_list + ) if export_dependency_groups: export_requirements_files(python_executable, export_dependency_groups) if run_pre_commit: # Run the pre-commit hooks, ignore any errors since they are # just being run to auto-fix files. with contextlib.suppress(subprocess.CalledProcessError): - os.environ["SKIP"] = pre_commit_hook_skip_list + os.environ["SKIP"] = pre_commit_hook_run_skip_list run_cmd_in_subprocess(f'"{python_executable}" -m pre_commit run --all-files') diff --git a/actions/update_development_dependencies/readme.md b/actions/update_development_dependencies/readme.md index 7f8fbef..f5f194e 100644 --- a/actions/update_development_dependencies/readme.md +++ b/actions/update_development_dependencies/readme.md @@ -20,15 +20,16 @@ This action enables updating Python development dependencies using the ## Inputs -| Input variable | Necessity | Description | Default | -| --------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | -| `repo-root` | optional | The root directory of the repository. | `.` | -| `install-dependencies` | optional | A boolean indicating if packages should be installed via poetry (this is not usually needed). | `false` | -| `dependency-dict` | optional | Specify a valid JSON dictionary of dependency groups to update, where each key is a dependency group name, and each value is a list of dependencies to update within that group, e.g. `'{"dev": ["pylint", "ruff"], "tests": ["ruff"]}'`. Use an empty string, e.g. `""`, for dependencies located in the default group' | `{}` | -| `update-pre-commit` | optional | A boolean indicating if the pre-commit hooks should be updated. | `false` | -| `run-pre-commit` | optional | A boolean indicating to run the pre-commit hooks to perform auto-fixing after updating the dependencies. Setting this input to `true` will also set the update-pre-commit input to `true`. | `false` | -| `pre-commit-hook-skip-list` | optional | A comma-separated list of pre-commit hooks to skip (only applicable when `run-pre-commit=true`). | `""` | -| `export-dependency-groups` | optional | A comma-separated list of dependency groups to export to a `requirements.txt` file. The format is `group1,group2:custom-path/to/test/folder`. | `""` | +| Input variable | Necessity | Description | Default | +| ---------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | +| `repo-root` | optional | The root directory of the repository. | `.` | +| `install-dependencies` | optional | A boolean indicating if packages should be installed via poetry (this is not usually needed). | `false` | +| `dependency-dict` | optional | Specify a valid JSON dictionary of dependency groups to update, where each key is a dependency group name, and each value is a list of dependencies to update within that group, e.g. `'{"dev": ["pylint", "ruff"], "tests": ["ruff"]}'`. Use an empty string, e.g. `""`, for dependencies located in the default group' | `{}` | +| `update-pre-commit` | optional | A boolean indicating if the pre-commit hooks should be updated. | `false` | +| `run-pre-commit` | optional | A boolean indicating to run the pre-commit hooks to perform auto-fixing after updating the dependencies. Setting this input to `true` will also set the update-pre-commit input to `true`. | `false` | +| `pre-commit-repo-update-skip-list` | optional | A comma-separated list of pre-commit repo urls to skip updates for (only applicable when `update-pre-commit=true`). | `""` | +| `pre-commit-hook-run-skip-list` | optional | A comma-separated list of pre-commit hooks to skip (only applicable when `run-pre-commit=true`). | `""` | +| `export-dependency-groups` | optional | A comma-separated list of dependency groups to export to a `requirements.txt` file. The format is `group1,group2:custom-path/to/test/folder`. | `""` | ## Example @@ -53,7 +54,8 @@ jobs: dependency-dict: '{"dev": ["pylint", "ruff"], "tests": ["ruff"]}' # optional, but without it nothing will get updated by Poetry update-pre-commit: true # optional run-pre-commit: true # optional - pre-commit-hook-skip-list: 'pylint' # optional, hooks that don't auto-fix things can (and probably should be) skipped + pre-commit-repo-update-skip-list: 'https://github.com/pre-commit/pre-commit-hooks' # optional + pre-commit-hook-run-skip-list: 'pylint' # optional, hooks that don't auto-fix things can (and probably should be) skipped export-dependency-groups: 'docs,tests:custom-path/to/test/folder' # optional - uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/tests/test_update_development_dependencies.py b/tests/test_update_development_dependencies.py index 3940276..d5f7327 100644 --- a/tests/test_update_development_dependencies.py +++ b/tests/test_update_development_dependencies.py @@ -18,6 +18,11 @@ ) PYTHON_EXECUTABLE = Path(sys.executable).as_posix() +PRE_COMMIT_REPO_UPDATE_SKIP_LIST = [ + "https://github.com/executablebooks/mdformat", + "https://github.com/Lucas-C/pre-commit-hooks", + "https://github.com/PyCQA/docformatter", +] @pytest.fixture(autouse=True) @@ -41,14 +46,105 @@ def fixture_repo_root_dir( repo_root_directory = tmp_path / "repo" repo_root_directory.mkdir() monkeypatch.chdir(repo_root_directory) - (repo_root_directory / ".pre-commit-config.yaml").touch() + (repo_root_directory / ".pre-commit-config.yaml").write_text( + """--- +default_install_hook_types: [pre-commit, commit-msg] +default_stages: [pre-commit] +ci: + autofix_prs: false + autoupdate_schedule: weekly + autoupdate_commit_msg: 'chore(pre-commit-deps): pre-commit autoupdate' + skip: + - check-poetry + - pylint +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # frozen: v5.0.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-toml + - id: check-json + - id: check-xml + - id: file-contents-sorter + files: ^(doc_config/known_words.txt)$ + args: [--unique, --ignore-case] + - id: requirements-txt-fixer + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-case-conflict + - id: check-merge-conflict + - id: check-added-large-files + args: [--maxkb=3000, --enforce-all] + - id: forbid-submodules + - id: pretty-format-json + args: [--autofix, --indent=4] + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: a30f0d816e5062a67d87c8de753cfe499672b959 # frozen: v1.5.5 + hooks: + - id: remove-tabs + - id: forbid-tabs + - repo: https://github.com/Mateusz-Grzelinski/actionlint-py + rev: 27445053da613c660ed5895d9616662059a53ca7 # frozen: v1.7.3.17 + hooks: + - id: actionlint + additional_dependencies: [pyflakes, shellcheck-py] + - repo: https://github.com/lyz-code/yamlfix + rev: 8072181c0f2eab9f2dd8db2eb3b9556d7cd0bd74 # frozen: 1.17.0 + hooks: + - id: yamlfix + - repo: https://github.com/AleksaC/hadolint-py + rev: e70baeefd566058716df2f29eae8fe8ffc213a9f # frozen: v2.12.1b3 + hooks: + - id: hadolint + args: [--ignore=DL3008, --ignore=DL3018] + - repo: https://github.com/executablebooks/mdformat + rev: 86542e37a3a40974eb812b16b076220fe9bb4278 # frozen: 0.7.18 + hooks: + - id: mdformat + args: [--number, --end-of-line, keep] + additional_dependencies: + - mdformat-admon + - mdformat-beautysh + - repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + pass_filenames: true + args: [-sn] + - id: pyright + name: pyright + entry: pyright + language: system + types: [python] + pass_filenames: false + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 8983acb92ee4b01924893632cf90af926fa608f0 # frozen: v0.7.0 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + # TODO: Re-enable this once https://github.com/PyCQA/docformatter/issues/293 is resolved + # - repo: https://github.com/PyCQA/docformatter + # rev: dfefe062799848234b4cd60b04aa633c0608025e # frozen: v1.7.5 + # hooks: + # - id: docformatter + # additional_dependencies: [tomli] +""" + ) (repo_root_directory / "dev").mkdir() (repo_root_directory / "dev" / "requirements.txt").touch() monkeypatch.setenv("INPUT_REPO-ROOT", str(repo_root_directory)) monkeypatch.setenv("INPUT_DEPENDENCY-DICT", '{"dev": ["pytest"]}') monkeypatch.setenv("INPUT_EXPORT-DEPENDENCY-GROUPS", "dev") - monkeypatch.setenv("INPUT_PRE-COMMIT-HOOK-SKIP-LIST", "") + monkeypatch.setenv( + "INPUT_PRE-COMMIT-REPO-UPDATE-SKIP-LIST", ",".join(PRE_COMMIT_REPO_UPDATE_SKIP_LIST) + ) + monkeypatch.setenv("INPUT_PRE-COMMIT-HOOK-RUN-SKIP-LIST", "") monkeypatch.setenv("INPUT_INSTALL-DEPENDENCIES", "true") monkeypatch.setenv("INPUT_RUN-PRE-COMMIT", "true") monkeypatch.setenv("INPUT_UPDATE-PRE-COMMIT", "true") @@ -101,7 +197,9 @@ def test_update_pre_commit_dependencies( ) -> None: """Test the update_pre_commit_dependencies function.""" with patch("subprocess.check_call") as mock_subproc_call: - update_pre_commit_dependencies(sys.executable, repo_root_dir) + update_pre_commit_dependencies( + sys.executable, repo_root_dir, PRE_COMMIT_REPO_UPDATE_SKIP_LIST + ) # Check the calls to subprocess.check_call expected_calls = [ @@ -115,9 +213,63 @@ def test_update_pre_commit_dependencies( f"{repo_root_dir.resolve().as_posix()}", ] ), - call([PYTHON_EXECUTABLE, "-m", "pre_commit", "autoupdate", "--freeze"]), + call( + [ + PYTHON_EXECUTABLE, + "-m", + "pre_commit", + "autoupdate", + "--freeze", + "--repo", + "https://github.com/pre-commit/pre-commit-hooks", + ] + ), + call( + [ + PYTHON_EXECUTABLE, + "-m", + "pre_commit", + "autoupdate", + "--freeze", + "--repo", + "https://github.com/Mateusz-Grzelinski/actionlint-py", + ] + ), + call( + [ + PYTHON_EXECUTABLE, + "-m", + "pre_commit", + "autoupdate", + "--freeze", + "--repo", + "https://github.com/lyz-code/yamlfix", + ] + ), + call( + [ + PYTHON_EXECUTABLE, + "-m", + "pre_commit", + "autoupdate", + "--freeze", + "--repo", + "https://github.com/AleksaC/hadolint-py", + ] + ), + call( + [ + PYTHON_EXECUTABLE, + "-m", + "pre_commit", + "autoupdate", + "--freeze", + "--repo", + "https://github.com/astral-sh/ruff-pre-commit", + ] + ), ] - assert mock_subproc_call.call_count == 2 + assert mock_subproc_call.call_count == 6 mock_subproc_call.assert_has_calls(expected_calls, any_order=True) @@ -163,7 +315,7 @@ def test_main( # Call the main function main() assert mock_subproc_call.called - assert mock_subproc_call.call_count == 9 + assert mock_subproc_call.call_count == 13 def test_main_no_install_dependencies( diff --git a/workflows/update-python-and-pre-commit-dependencies.md b/workflows/update-python-and-pre-commit-dependencies.md index a659feb..419b495 100644 --- a/workflows/update-python-and-pre-commit-dependencies.md +++ b/workflows/update-python-and-pre-commit-dependencies.md @@ -29,15 +29,16 @@ updates for the Python dependencies. ## Inputs -| Input variable | Necessity | Description | Default | -| --------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | -| `commit-user-name` | required | The name of the user to use when committing changes to the repository. | | -| `commit-user-email` | required | The email of the user to use when committing changes to the repository. | | -| `dependency-dict` | optional | Specify a valid JSON dictionary of dependency groups to update, where each key is a dependency group name, and each value is a list of dependencies to update within that group, e.g. `'{"dev": ["pylint", "ruff"], "tests": ["ruff"]}'`. Use an empty string, e.g. `""`, for dependencies located in the default group' | `{}` | -| `update-pre-commit` | optional | A boolean indicating if the pre-commit hooks should be updated. | `false` | -| `run-pre-commit` | optional | A boolean indicating to run the pre-commit hooks to perform auto-fixing after updating the dependencies. Setting this input to `true` will also set the update-pre-commit input to `true`. | `false` | -| `pre-commit-hook-skip-list` | optional | A comma-separated list of pre-commit hooks to skip (only applicable when `run-pre-commit=true`). | `""` | -| `export-dependency-groups` | optional | A comma-separated list of dependency groups to export to a `requirements.txt` file. The format is `group1,group2:custom-path/to/test/folder`. | `""` | +| Input variable | Necessity | Description | Default | +| ---------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | +| `commit-user-name` | required | The name of the user to use when committing changes to the repository. | | +| `commit-user-email` | required | The email of the user to use when committing changes to the repository. | | +| `dependency-dict` | optional | Specify a valid JSON dictionary of dependency groups to update, where each key is a dependency group name, and each value is a list of dependencies to update within that group, e.g. `'{"dev": ["pylint", "ruff"], "tests": ["ruff"]}'`. Use an empty string, e.g. `""`, for dependencies located in the default group' | `{}` | +| `update-pre-commit` | optional | A boolean indicating if the pre-commit hooks should be updated. | `false` | +| `run-pre-commit` | optional | A boolean indicating to run the pre-commit hooks to perform auto-fixing after updating the dependencies. Setting this input to `true` will also set the update-pre-commit input to `true`. | `false` | +| `pre-commit-repo-update-skip-list` | optional | A comma-separated list of pre-commit repo urls to skip updates for (only applicable when `update-pre-commit=true`). | `""` | +| `pre-commit-hook-run-skip-list` | optional | A comma-separated list of pre-commit hooks to skip (only applicable when `run-pre-commit=true`). | `""` | +| `export-dependency-groups` | optional | A comma-separated list of dependency groups to export to a `requirements.txt` file. The format is `group1,group2:custom-path/to/test/folder`. | `""` | ## Secrets @@ -64,7 +65,8 @@ jobs: dependency-dict: '{"dev": ["pylint", "ruff"], "tests": ["ruff"]}' # optional, but without it nothing will get updated by Poetry update-pre-commit: true # optional run-pre-commit: true # optional - pre-commit-hook-skip-list: pylint,pyright,pyroma,poetry-audit # optional, hooks that don't auto-fix things can (and probably should be) skipped + pre-commit-repo-update-skip-list: 'https://github.com/pre-commit/pre-commit-hooks' # optional + pre-commit-hook-run-skip-list: pylint,pyright,pyroma,poetry-audit # optional, hooks that don't auto-fix things can (and probably should be) skipped export-dependency-groups: 'docs,tests:custom-path/to/test/folder' # optional permissions: contents: write