Enaml syntaxerror detail (#530)

* Patch for Python 3.11 SyntaxError * Create tests to detect correct syntaxerror detail This does not actually capture the error I am attempting to fix. * Add compatibility for Python 3.8 and 3.9 This also fixes a bug in the unit test (the assert statement was missing). * Fix typo in unit test for parser * Include location of next token when raising indentation error * Wrong expected error message in test_parser * Regenerate Enaml parser * Update release notes with bugfix detail * Add information on how to regenerate Enaml parser * Regenerate parser with pegen 0.1.0 Pegen 0.2.0 results in some IndentationError exceptions being raised as SyntaxError instead. * Pin pegen to 0.1.0 * Update releasenotes.rst --------- Co-authored-by: frm dstryr <[email protected]> Co-authored-by: Matthieu Dartiailh <[email protected]>
nucleic · Sep 8, 2023 · 4639e27 · 4639e27
1 parent 87e9c31
commit 4639e27
Show file tree

Hide file tree

Showing 7 changed files with 255 additions and 47 deletions.
diff --git a/development.rst b/development.rst
@@ -0,0 +1,7 @@
+Regenerating the Enaml parser
+=============================
+
+To regenerate the parser:
+
+    python -m enaml.core.parser.generate_enaml_parser
+    black enaml/core/parser/enaml_parser.py
diff --git a/enaml/core/parser/base_python_parser.py b/enaml/core/parser/base_python_parser.py
@@ -16,6 +16,8 @@
 from pegen.parser import Parser
 from pegen.tokenizer import Tokenizer
 
+from enaml.compat import PY310
+
 # Singleton ast nodes, created once for efficiency
 Load = ast.Load()
 Store = ast.Store()
@@ -76,7 +78,13 @@ def parse(self, rule: str) -> Optional[ast.AST]:
                 raise self._exception
             else:
                 token = self._tokenizer.diagnose()
-                raise SyntaxError("invalid syntax", (self.filename, token.start, 0, token.line))
+                lineno, offset = token.start
+                end_lineno, end_offset = token.end
+                if PY310:
+                    args = (self.filename, lineno, offset, token.line, end_lineno, end_offset)
+                else:
+                    args = (self.filename, lineno, offset, token.line)
+                raise SyntaxError("invalid syntax", args)
 
         return res
 
@@ -93,7 +101,9 @@ def check_version(self, min_version: Tuple[int, ...], error_msg: str, node: Node
 
     def raise_indentation_error(self, msg) -> None:
         """Raise an indentation error."""
-        raise IndentationError(msg)
+        node = self._tokenizer.peek()
+        self.store_syntax_error_known_location(msg, node, IndentationError)
+        raise self._exception
 
     def get_expr_name(self, node) -> str:
         """Get a descriptive name for an expression."""
@@ -245,7 +255,8 @@ def _store_syntax_error(
         self,
         message: str,
         start: Optional[Tuple[int, int]] = None,
-        end: Optional[Tuple[int, int]] = None
+        end: Optional[Tuple[int, int]] = None,
+        exc_type: type = SyntaxError
     ) -> None:
         line_from_token = start is None and end is None
         if start is None or end is None:
@@ -264,7 +275,7 @@ def _store_syntax_error(
         args = (self.filename, start[0], start[1], line)
         if sys.version_info >= (3, 10):
             args += (end[0], end[1])
-        self._exception = SyntaxError(message, args)
+        self._exception = exc_type(message, args)
 
     def store_syntax_error(self, message: str) -> None:
         self._store_syntax_error(message)
@@ -273,7 +284,12 @@ def make_syntax_error(self, message: str) -> None:
         self._store_syntax_error(message)
         return self._exception
 
-    def store_syntax_error_known_location(self, message: str, node) -> None:
+    def store_syntax_error_known_location(
+        self,
+        message: str,
+        node,
+        exc_type: type = SyntaxError
+    ) -> None:
         """Store a syntax error that occured at a given AST node."""
         if isinstance(node, tokenize.TokenInfo):
             start = node.start
@@ -282,7 +298,7 @@ def store_syntax_error_known_location(self, message: str, node) -> None:
             start = node.lineno, node.col_offset
             end = node.end_lineno, node.end_col_offset
 
-        self._store_syntax_error(message, start, end)
+        self._store_syntax_error(message, start, end, exc_type)
 
     def store_syntax_error_known_range(
         self,

diff --git a/enaml/core/parser/enaml_parser.py b/enaml/core/parser/enaml_parser.py
@@ -8,20 +8,21 @@
 # NOTE This file was generated using enaml/core/parser/generate_enaml_parser.py
 # DO NOT EDIT DIRECTLY
 import ast
-import itertools
 import sys
+import itertools
 from typing import Any, List, NoReturn, Optional, Tuple, TypeVar, Union
 
-from pegen.parser import Parser, logger, memoize, memoize_left_rec
-
 from enaml.core import enaml_ast
+from pegen.parser import Parser, logger, memoize, memoize_left_rec
 
 from .base_enaml_parser import BaseEnamlParser as Parser
 
 # Singleton ast nodes, created once for efficiency
 Load = ast.Load()
 Store = ast.Store()
 Del = ast.Del()
+
+
 # Keywords and soft keywords are listed at the end of the parser definition.
 class EnamlParser(Parser):
     @memoize
@@ -9697,54 +9698,54 @@ def _tmp_241(self) -> Optional[Any]:
         return None
 
     KEYWORDS = (
-        "nonlocal",
-        "global",
-        "True",
-        "while",
-        "for",
-        "elif",
-        "del",
-        "from",
-        "pass",
-        "def",
-        "except",
-        "None",
-        "with",
+        "class",
         "or",
-        "return",
+        "del",
         "not",
-        "and",
-        "await",
-        "finally",
         "in",
+        "await",
+        "except",
+        "return",
+        "try",
         "import",
-        "yield",
-        "raise",
-        "assert",
+        "break",
         "else",
-        "False",
-        "lambda",
-        "class",
+        "for",
+        "with",
         "async",
-        "break",
+        "None",
+        "assert",
+        "global",
         "if",
-        "try",
         "as",
-        "continue",
+        "True",
+        "yield",
+        "raise",
+        "and",
+        "finally",
         "is",
+        "while",
+        "pass",
+        "from",
+        "False",
+        "elif",
+        "lambda",
+        "nonlocal",
+        "def",
+        "continue",
     )
     SOFT_KEYWORDS = (
-        "event",
+        "case",
         "template",
+        "match",
+        "event",
+        "alias",
         "attr",
+        "enamldef",
         "const",
-        "func",
-        "pragma",
-        "case",
-        "alias",
         "_",
-        "enamldef",
-        "match",
+        "pragma",
+        "func",
     )
 
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -33,7 +33,7 @@ dependencies = [
     "atom>=0.9.0",
     "kiwisolver>=1.2.0",
     "bytecode>=0.14.2",
-    "pegen>=0.1.0",
+    "pegen>=0.1.0,<0.2.0",
 ]
 dynamic=["version"]
 

diff --git a/releasenotes.rst b/releasenotes.rst
@@ -3,6 +3,11 @@ Enaml Release Notes
 
 Dates are written as DD/MM/YYYY
 
+XXXXX
+-----
+- fix bug in Enaml parser that was not showing proper location of syntax and
+  indentation errors in tracebacks when the error was in an Enaml file.
+
 0.16.1 - 05/05/2023
 -------------------
 - fix typo causing a crash in dock area PR #523

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -85,7 +85,7 @@ def validate_parser_is_up_to_date():
         last_source_modif
     ), (
         "Generated parser appears outdated compared to its sources, "
-        "re-generate it using enaml/core/parser/generate_enaml_parser.enaml"
+        "re-generate it using `python -m enaml.core.parser.generate_enaml_parser`"
     )