fixed non-letter 'commands' fixes #107

TexSoup/reader.py

@@ -505,12 +505,12 @@ def read_command(buf, n_required_args=-1, n_optional_args=-1, skip=0,
     ('item', [])
     >>> buf.peek()
     ' aaa '
-    """
-    # Broken because abcd is incorrectly tokenized with leading space
+    
     # >>> buf = Buffer(tokenize(categorize('\\sect abcd')))
     # >>> _ = next(buf)
-    # >>> peek_command(buf)
-    # ('sect', ('a',), 2)
+    # >>> read_command(buf)
+    # ('sect', ('a',))
+    """
     for _ in range(skip):
         next(buf)
 

TexSoup/tokens.py

@@ -102,7 +102,7 @@ def wrap(f):
 
 @token('escaped_symbols')
 def tokenize_escaped_symbols(text, prev=None):
-    r"""Process an escaped symbol.
+    r"""Process an escaped symbol or a known punctuation command.
 
     :param Buffer text: iterator over line, with current position
 
@@ -120,7 +120,8 @@ def tokenize_escaped_symbols(text, prev=None):
             and text.peek(1) \
             and text.peek(1).category in (
                 CC.Escape, CC.GroupBegin, CC.GroupEnd, CC.MathSwitch,
-                CC.Comment):
+                CC.Alignment, CC.Macro, CC.Superscript, CC.Subscript,
+                CC.Active, CC.Comment, CC.Other):
         result = text.forward(2)
         result.category = TC.EscapedComment
         return result

tests/test_parser.py

@@ -404,6 +404,21 @@ def test_math_environment_whitespace():
     assert str(soup.notescaped) == r'\begin{notescaped}\gamma = \beta\end{notescaped}'
 
 
+def test_non_letter_commands():
+    """
+    Tests that non-letters are still captured as an escaped sequence
+    (whether valid or not).
+    """
+    for punctuation in '!@#$%^&*_+-=~`<>,./?;:|':
+        tex = rf"""
+        \begin{{document}}
+        \lstinline{{\{punctuation} Word [a-z]+}}
+        \end{{document}}
+        """
+        soup = TexSoup(tex)
+        assert str(soup) == tex
+
+
 def test_math_environment_escape():
     """Tests $ escapes in math environment."""
     soup = TexSoup(r"$ \$ $")