Add ability to assert mypy errors

.github/workflows/ci.yml

@@ -22,9 +22,9 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install --upgrade --no-cache-dir -e '.[dev]'
-      - name: Type checking
+      - name: Test (mypy)
         run: |
-          mypy tests/typechecked
+          python run_mypy_assertions.py
       - name: Test
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run_mypy_assertions.py

@@ -0,0 +1,57 @@
+import subprocess
+from pathlib import Path
+
+if __name__ == "__main__":
+    source_dir = Path("tests/typechecked")  # Files that must be validated using `mypy`
+
+    # Run `mypy` and get the output.
+    p = subprocess.Popen(
+        ["mypy", source_dir],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = p.communicate()
+    assert stderr == b"", "`stderr` must be empty."
+    print("Output of `mypy`:")
+    print(stdout.decode())
+
+    unvalidated_errors = []  # Errors that were not marked as intended in the source
+
+    for line in stdout.decode().splitlines():
+        # Parse line in the output of `mypy`. If it cannot be parsed, just skip it.
+        try:
+            path, line_number, status, message = line.split(":", 3)
+        except ValueError:
+            continue
+
+        # We only need to validate errors.
+        if status.lower().strip() != "error":
+            continue
+
+        # Get the line in the source that caused the error.
+        with open(path, "r") as f:
+            path_lines = f.read().splitlines()
+        source_line = path_lines[int(line_number) - 1]
+
+        # See if the error was intended.
+        try:
+            code, match = source_line.split("# mypy: E: ", 1)
+            validated = match.lower() in message.lower()
+        except ValueError:
+            validated = False
+
+        # If it wasn't intended, record the error.
+        if not validated:
+            unvalidated_errors.append(line)
+
+    # Return failure if there are any unvalidated errors.
+    if unvalidated_errors:
+        print("These errors were not validated:")
+        for error in unvalidated_errors:
+            print(error)
+        exit(1)
+    else:
+        print("All errors were validated!")
+        exit(0)
+else:
+    print(__name__)

tests/typechecked/test_overload.py

@@ -23,4 +23,4 @@ def test_overload() -> None:
     assert f(1) == 1
     assert f("1") == "1"
     with pytest.raises(NotFoundLookupError):
-        f(1.0)  # type: ignore
+        f(1.0)  # mypy: E: [call-overload]