Add cached GitHub Action workflow for pre-commit(minitest)

- Also exclude files with shared imports from autoflake
  (autoflake does not know about them and would remove them)
- Also add `additional_dependencies: [XenAPI]` for the unittest
- Also add `exit $EXIT_CODE` to minimaltest to propagate errors

Signed-off-by: Bernhard Kaindl <[email protected]>

.github/workflows/main.yml

@@ -0,0 +1,65 @@
+name: Run pre-commit checks and unit tests
+
+on:
+  push:
+  pull_request:
+
+concurrency:  # Cancel pending and in-progress workflows for the same PR, branch or tag:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+env:
+  # No warnings for pip and pytest themselves; pytest enables warnings in conftest.py
+  PYTHONWARNINGS: ignore
+  # Development Mode for stronger checks: https://docs.python.org/3/library/devmode.html
+  PYTHONDEVMODE: yes
+jobs:
+  python-checks:
+    name: minimaltest (pre-commit)
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          #: Install Python 2.7 from Ubuntu 20.04 using apt-get install
+          sudo apt-get update && sudo apt-get install -y python2
+          curl -sSL https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py
+          python2 get-pip.py
+          if [ -f requirements.txt ]; then pip2 install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip2 install -r requirements-dev.txt; fi
+          pip2 install pylint==1.9.4
+
+      - name: Run python2 -m unittest discover to execute all unit and integration tests
+        run: python2 -m unittest discover
+      - name: Run pylint --py3k to check which Python3 conversions are still to do
+        env:
+          disabled: no-absolute-import,print-statement
+
+          # FIXME/TODO: These checks by pylint --py3k are currently failing:
+          py3_todo: "\
+            unicode-builtin,\
+            comprehension-escape,\
+            dict-keys-not-iterating,\
+            old-division"
+
+        run: python2 -m pylint --py3k --rcfile /dev/null -f parseable -d $disabled,$py3_todo *.py */*.py
+      # Fast, cached pre-commit setup for GitHub CI as described here:
+      # https://www.python4data.science/en/latest/productive/git/advanced/hooks/ci.html
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+      - name: Setup the cache for installing pre-commit from the GitHub Action's cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          # Ensure that we don't load outdate cache if the python version or the pre-comming config changed:
+          # The cache key is the key for saving and loading the cached files:
+          # The cache key is pre-commit|<path of the Python interpreter>|<hash of .pre-commit-config.yaml>
+          key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+      - name: Run pre-commit checks
+        uses: pre-commit/[email protected]
+        env:
+          # Skip the no-commit-to-branch check inside of GitHub CI (for CI on merge to master)
+          SKIP: no-commit-to-branch

.pre-commit-config.yaml

@@ -83,7 +83,8 @@ repos:
         # For pytest, users need to install pytest. Then use this instead:
         # entry: env PYTHONDEVMODE=yes python3 -m pytest
         pass_filenames: false
-        language: system
+        additional_dependencies: [XenAPI]
+        language: python
         types: [python]
 -   repo: https://github.com/PyCQA/autoflake
     rev: v2.2.1
@@ -100,7 +101,20 @@ repos:
         files: \.py$
         exclude: |
             (?x)^(
+                # These modules import Python stdlib modules that they don't use themselfes
+                # but other XSConsole modules which wildcard-import * expect them.
+                #for other XSConsole modules:
+                # Until the nested wildcard imports are fixed, we can only exclude them:
                 simpleconfig.py|
+                XSConsoleBases.py|
+                XSConsoleCurses.py|
+                XSConsoleData.py|
+                XSConsoleHotData.py|
+                XSConsoleMenus.py|
+                XSConsoleRootDialogue.py|
+                XSConsoleState.py|
+                XSConsoleTerm.py|
+                XSConsoleUtils.py|
                 XSConsoleStandard.py
             )
 -   repo: https://github.com/akaihola/darker
@@ -140,7 +154,9 @@ repos:
     hooks:
     -   id: git-diff # For reference: https://github.com/pre-commit/pre-commit/issues/1712
         name: When pre-commit fixers made changes, run 'git diff' to show the changes
-        entry: git diff --exit-code  # When changes were made, the exit code is needed to show them
+        # --exit-code: If diffs were found, make pre-commit to show the output of "git diff"
+        # --ws-error-highlight=old: Highlight the old, removed trailing space filled in red:
+        entry: git diff --ws-error-highlight=old --exit-code
         language: system
         pass_filenames: false
         always_run: true

Makefile

@@ -173,11 +173,15 @@ pylint.txt: .pylintrc $(ALL_SCRIPTS)
 	.git/pre-commit.env/bin/pip install --quiet --upgrade pre-commit
 
 minimaltest: .git/pre-commit.env/bin/pre-commit
-	@.git/pre-commit.env/bin/pre-commit run --show-diff-on-failure || {\
-	 echo "-----------------------------------------------------------------------";\
-	 echo "If you see a diff above, it shows the changes which pre-commit made.";\
-	 echo "Run 'git add -p' to add those changes to the index and repeat the test.";\
-	 echo "-----------------------------------------------------------------------";}
+	@.git/pre-commit.env/bin/pre-commit run ||                                                        \
+	{                                                                                                 \
+	 EXIT_CODE=$$?;                                                                                   \
+	 echo "----------------------------------------------------------------------------------------"; \
+	 echo "If you see the output of 'git diff' above, it shows the changes which pre-commit made.";   \
+	 echo "Run 'git add -p' to add those changes to the index and then call make minimaltest again."; \
+	 echo "----------------------------------------------------------------------------------------"; \
+	 exit $$EXIT_CODE;                                                                                \
+	}
 	@[ -e .git/hooks/pre-commit ] || \
 	 echo "Please run make minimaltest-install"