baseline allow

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Enable stub mode within `TYPE_CHECKING` branches (#702)
 - Infer from overloads - add default value in impl (#697)
 - Warn for missing returns with explicit `Any` return types (#715)
+- `--baseline-allow` and `--baseline-ban` for baseline management (#710)
 ### Fixes
 - positional arguments on overloads break super (#697)
 - positional arguments on overloads duplicate unions (#697)

mypy/build.py

@@ -66,6 +66,7 @@
     is_sub_path,
     is_typeshed_file,
     module_prefix,
+    plural_s,
     read_py_file,
     time_ref,
     time_spent_us,
@@ -1073,14 +1074,14 @@ def save_baseline(manager: BuildManager):
         # Indicate that writing was canceled
         manager.options.write_baseline = False
         return
-    new_baseline = manager.errors.prepare_baseline_errors()
+    new_baseline, rejected = manager.errors.prepare_baseline_errors()
     file = Path(manager.options.baseline_file)
     if not new_baseline:
         if file.exists():
             file.unlink()
-            print("No errors, baseline file removed")
+            print("No baselinable errors, baseline file removed")
         elif manager.options.write_baseline:
-            print("No errors, no baseline to write")
+            print("No baselinable errors, no baseline to write")
         # Indicate that writing was canceled
         manager.options.write_baseline = False
         return
@@ -1090,13 +1091,21 @@ def save_baseline(manager: BuildManager):
         file.parent.mkdir(parents=True)
     data: BaselineType = {
         "files": new_baseline,
-        "format": "1.7",
+        "format": "2.6",
         "targets": manager.options._targets,
     }
     with file.open("w") as f:
         json.dump(data, f, indent=2, sort_keys=True)
     if not manager.options.write_baseline and manager.options.auto_baseline:
-        manager.stdout.write(f"Baseline successfully updated at {file}\n")
+        removed = len(
+            [
+                error
+                for file in manager.errors.original_baseline.values()
+                for error in file
+                if error["code"].startswith("error:")
+            ]
+        ) - len(manager.errors.baseline_stats["total"])
+        manager.stdout.write(f"{removed} error{plural_s(removed)} removed from baseline {file}\n")
 
 
 def load_baseline(options: Options, errors: Errors, stdout: TextIO) -> None:
@@ -1150,7 +1159,7 @@ def error(msg: str | None = None) -> bool:
 
     if baseline_format is None and error():
         return
-    elif baseline_format != "1.7":
+    elif baseline_format != "2.6":
         if not options.write_baseline:
             error(
                 f"error: Baseline file '{file}' was generated with an old version of basedmypy.\n"

mypy/errors.py

@@ -1362,7 +1362,9 @@ def initialize_baseline(
         self.baseline = baseline_errors
         self.baseline_targets = targets
 
-    def prepare_baseline_errors(self) -> dict[str, list[StoredBaselineError]]:
+    def prepare_baseline_errors(
+        self,
+    ) -> (dict[str, list[StoredBaselineError]], list[StoredBaselineError]):
         """Create a dict representing the error portion of an error baseline file"""
 
         def remove_duplicates(errors: list[ErrorInfo]) -> list[ErrorInfo]:
@@ -1386,10 +1388,29 @@ def remove_duplicates(errors: list[ErrorInfo]) -> list[ErrorInfo]:
                 i += 1
             return unduplicated_result
 
+        allowed = self.options.baseline_allows
+        banned = self.options.baseline_bans
+        rejected = []
+        error_list = []
+
+        def gate_keep(error: ErrorInfo) -> bool:
+            """should an error be accepted into the baseline"""
+
+            def yes_no(condition: bool) -> bool:
+                if error.severity == "error":
+                    (error_list if condition else rejected).append(error)
+                return condition
+
+            if allowed:
+                return yes_no(error.code in allowed)
+            if banned:
+                return yes_no(error.code not in banned)
+            return yes_no(True)
+
         result = {
             self.common_path(file): [
                 {
-                    "code": error.code.code if error.code else None,
+                    "code": f"{error.severity}:{error.code.code if error.code else None}",
                     "column": error.column,
                     "line": error.line,
                     "message": error.message,
@@ -1399,18 +1420,21 @@ def remove_duplicates(errors: list[ErrorInfo]) -> list[ErrorInfo]:
                     and cast(List[str], self.read_source(file))[error.line - 1].strip(),
                 }
                 for error in remove_duplicates(errors)
+                if gate_keep(error)
                 # don't store reveal errors
                 if error.code != codes.REVEAL
             ]
             for file, errors in self.all_errors.items()
         }
+        result = {file: errors for file, errors in result.items() if errors}
         for file in result.values():
             previous = 0
             for error in file:
                 error["offset"] = cast(int, error["line"]) - previous
                 previous = cast(int, error["line"])
                 del error["line"]
-        return cast(Dict[str, List[StoredBaselineError]], result)
+        self.baseline_stats = {"total": error_list, "rejected": len(rejected)}
+        return cast(Dict[str, List[StoredBaselineError]], result), rejected
 
     def filter_baseline(
         self, errors: list[ErrorInfo], path: str, source_lines: list[str] | None

mypy/main.py

@@ -9,6 +9,7 @@
 import time
 from collections import defaultdict
 from gettext import gettext
+from operator import itemgetter
 from typing import IO, Any, Final, NoReturn, Sequence, TextIO
 
 import mypy.options
@@ -26,6 +27,7 @@
 from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path
 from mypy.options import COMPLETE_FEATURES, INCOMPLETE_FEATURES, BuildType, Options
 from mypy.split_namespace import SplitNamespace
+from mypy.util import plural_s
 from mypy.version import __based_version__, __version__
 
 orig_stat: Final = os.stat
@@ -122,41 +124,63 @@ def main(
     if messages and n_notes < len(messages):
         code = 2 if blockers else 1
     if options.error_summary:
+        if options.summary and res and n_errors:
+            stdout.write("\n")
+            stdout.write(formatter.style("error summary:\n", color="none", bold=True))
+            all_errors = defaultdict(lambda: 0)
+            for errors in res.manager.errors.error_info_map.values():
+                for error in errors:
+                    if error.severity != "error" or not error.code:
+                        continue
+                    all_errors[error.code.code] += 1
+            if all_errors:
+                max_code_name_length = max(len(code_name) for code_name in all_errors)
+                for error_code, count in sorted(all_errors.items(), key=itemgetter(1)):
+                    stdout.write(f"  {error_code:<{max_code_name_length}} {count:>5}\n")
         if options.write_baseline and res:
-            new_errors = n_errors
             n_files = len(res.manager.errors.all_errors)
-            total = []
-            # This is stupid, but it's just to remove the dupes from the unfiltered errors
-            for errors in res.manager.errors.all_errors.values():
-                temp = res.manager.errors.render_messages(errors)
-                total.extend(res.manager.errors.remove_duplicates(temp))
-            n_errors = len([error for error in total if error[5] == "error"])
-        else:
-            new_errors = -1
+            stats = res.manager.errors.baseline_stats
+            rejected = stats["rejected"]
+            total = len(stats["total"])
+            new_errors = n_errors - rejected
+            previous = res.manager.errors.original_baseline
+            stdout.write(formatter.style("baseline:\n", color="none", bold=True))
+            if new_errors >= 0:
+                stdout.write(f"  {new_errors} new error{plural_s(new_errors)}\n")
+            stdout.write(f"  {total} error{plural_s(total)} in baseline\n")
+            difference = (
+                len(
+                    [
+                        error
+                        for file in previous.values()
+                        for error in file
+                        if error["code"].startswith("error:")
+                    ]
+                )
+                - total
+            )
+            if difference > 0:
+                stdout.write(
+                    f"  {difference} error{plural_s(difference)} less than previous baseline\n"
+                )
+            if rejected:
+                stdout.write(f"  {rejected} error{plural_s(rejected)} rejected\n")
+            stdout.write(f"  baseline successfully written to {options.baseline_file}\n")
+            stdout.flush()
         if n_errors:
             summary = formatter.format_error(
-                n_errors,
-                n_files,
-                len(sources),
-                new_errors=new_errors,
-                blockers=blockers,
-                use_color=options.color_output,
+                n_errors, n_files, len(sources), blockers=blockers, use_color=options.color_output
             )
             stdout.write(summary + "\n")
+        if n_errors >= 100 and not options.write_baseline:
+            stdout.write(
+                "That's a lot of errors, perhaps you would want to write an error baseline (`--write-baseline`)\n"
+            )
         # Only notes should also output success
         elif not messages or n_notes == len(messages):
             stdout.write(formatter.format_success(len(sources), options.color_output) + "\n")
         stdout.flush()
 
-    if options.write_baseline:
-        stdout.write(
-            formatter.style(
-                f"Baseline successfully written to {options.baseline_file}\n", "green", bold=True
-            )
-        )
-        stdout.flush()
-        code = 0
-
     if options.install_types and not options.non_interactive:
         result = install_types(formatter, options, after_run=True, non_interactive=False)
         if result:
@@ -563,6 +587,20 @@ def add_invertible_flag(
         action="store",
         help=f"Use baseline info in the given file (defaults to '{defaults.BASELINE_FILE}')",
     )
+    based_group.add_argument(
+        "--baseline-allow",
+        metavar="NAME",
+        action="append",
+        default=[],
+        help="Allow error codes into the baseline",
+    )
+    based_group.add_argument(
+        "--baseline-ban",
+        metavar="NAME",
+        action="append",
+        default=[],
+        help="Prevent error codes from being written to the baseline",
+    )
     add_invertible_flag(
         "--no-auto-baseline",
         default=True,
@@ -619,6 +657,12 @@ def add_invertible_flag(
         "You probably want to set this on a module override",
         group=based_group,
     )
+    add_invertible_flag(
+        "--no-summary",
+        default=True,
+        help="don't show an error code summary at the end",
+        group=based_group,
+    )
     add_invertible_flag(
         "--ide", default=False, help="Best default for IDE integration.", group=based_group
     )
@@ -1460,6 +1504,8 @@ def set_ide_flags() -> None:
     if invalid_codes:
         parser.error(f"Invalid error code(s): {', '.join(sorted(invalid_codes))}")
 
+    options.baseline_bans |= {error_codes[code] for code in set(options.baseline_ban)}
+    options.baseline_allows |= {error_codes[code] for code in set(options.baseline_allow)}
     options.disabled_error_codes |= {error_codes[code] for code in disabled_codes}
     options.enabled_error_codes |= {error_codes[code] for code in enabled_codes}
 

mypy/options.py

@@ -165,6 +165,11 @@ def __init__(self) -> None:
         # Based options
         self.write_baseline = False
         self.baseline_file = defaults.BASELINE_FILE
+        self.baseline_allow: list[str] = []
+        self.baseline_allows: set[ErrorCode] = set()
+        self.baseline_ban: list[str] = []
+        self.baseline_bans: set[ErrorCode] = set()
+        self.summary = True
         self.auto_baseline = True
         self.default_return = True
         self.infer_function_types = True
@@ -609,7 +614,12 @@ def select_options_affecting_cache(self) -> Mapping[str, object]:
         result: dict[str, object] = {}
         for opt in OPTIONS_AFFECTING_CACHE:
             val = getattr(self, opt)
-            if opt in ("disabled_error_codes", "enabled_error_codes"):
+            if opt in (
+                "disabled_error_codes",
+                "enabled_error_codes",
+                "baseline_allows",
+                "baseline_bans",
+            ):
                 val = sorted([code.code for code in val])
             result[opt] = val
         return result

mypy/test/testcmdline.py

@@ -65,6 +65,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None:
     if "# dont-normalize-output:" in testcase.input:
         testcase.normalize_output = False
     args.append("--show-traceback")
+    args.append("--no-summary")
     based = "based" in testcase.parent.name
     if not based:
         args.append("--no-strict")
@@ -116,7 +117,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None:
     # Remove temp file.
     os.remove(program_path)
     # Compare actual output to expected.
-    if testcase.output_files:
+    if testcase.output_files and False:
         assert not testcase.output, "output not checked when outfile supplied"
         # Ignore stdout, but we insist on empty stderr and zero status.
         if err or result:
@@ -126,6 +127,8 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None:
             )
         check_test_output_files(testcase, step)
     else:
+        if testcase.output_files:
+            check_test_output_files(testcase, step)
         if testcase.normalize_output:
             out = normalize_error_messages(err + out)
         obvious_result = 1 if out else 0

mypy/util.py

@@ -824,12 +824,9 @@ def format_error(
         *,
         blockers: bool = False,
         use_color: bool = True,
-        new_errors: int = -1,
     ) -> str:
         """Format a short summary in case of errors."""
         msg = f"Found {n_errors} error{plural_s(n_errors)} "
-        if new_errors != -1:
-            msg += f"({new_errors} new error{plural_s(new_errors)}) "
         msg += f"in {n_files} file{plural_s(n_files)}"
         if blockers:
             msg += " (errors prevented further checking)"