Add Repeated option and port re to regex module.

clitable.py

@@ -27,7 +27,7 @@
 
 import copy
 import os
-import re
+import regex
 import threading
 import copyable_regex_object
 import textfsm
@@ -314,7 +314,7 @@ def _ParseCmdItem(self, cmd_input, template_file=None):
   def _PreParse(self, key, value):
     """Executed against each field of each row read from index table."""
     if key == 'Command':
-      return re.sub(r'(\[\[.+?\]\])', self._Completion, value)
+      return regex.sub(r'(\[\[.+?\]\])', self._Completion, value)
     else:
       return value
 

clitable_test.py

@@ -22,7 +22,7 @@
 
 import copy
 import os
-import re
+import regex
 import unittest
 
 import clitable
@@ -104,11 +104,11 @@ def setUp(self):
   def testCompletion(self):
     """Tests '[[]]' syntax replacement."""
     indx = clitable.CliTable()
-    self.assertEqual('abc', re.sub(r'(\[\[.+?\]\])', indx._Completion, 'abc'))
+    self.assertEqual('abc', regex.sub(r'(\[\[.+?\]\])', indx._Completion, 'abc'))
     self.assertEqual('a(b(c)?)?',
-                     re.sub(r'(\[\[.+?\]\])', indx._Completion, 'a[[bc]]'))
+                     regex.sub(r'(\[\[.+?\]\])', indx._Completion, 'a[[bc]]'))
     self.assertEqual('a(b(c)?)? de(f)?',
-                     re.sub(r'(\[\[.+?\]\])', indx._Completion,
+                     regex.sub(r'(\[\[.+?\]\])', indx._Completion,
                             'a[[bc]] de[[f]]'))
 
   def testRepeatRead(self):

copyable_regex_object.py

@@ -16,16 +16,16 @@
 
 """Work around a regression in Python 2.6 that makes RegexObjects uncopyable."""
 
-import re
+import regex
 
 
 class CopyableRegexObject(object):
-  """Like a re.RegexObject, but can be copied."""
+  """Like a regex.RegexObject, but can be copied."""
   # pylint: disable=C6409
 
   def __init__(self, pattern):
     self.pattern = pattern
-    self.regex = re.compile(pattern)
+    self.regex = regex.compile(pattern)
 
   def match(self, *args, **kwargs):
     return self.regex.match(*args, **kwargs)

examples/repeated_basic_example

@@ -0,0 +1,7 @@
+This is an example to demonstrate the usage of the 'repeated' keyword, which enables one variable to have multiple captures on one line.
+
+
+normaldata1.1 normaldata1.2 key1.1:data1.1, key1.2:data1.2, key1.3:data1.3, normaldata1.3 normaldata1.3
+normaldata2.1 normaldata2.2 key2.1:data2.1, key2.2:data2.2, normaldata2.3 normaldata2.4
+normaldata3.1 normaldata3.2 normaldata3.3 normaldata3.4
+normaldata4.1 normaldata4.2 key4.1:data4.1, key4.2:data4.2, key4.3:data4.3, normaldata4.3 normaldata4.3

examples/repeated_basic_template

@@ -0,0 +1,12 @@
+Value normaldata1 (\S+)
+Value normaldata2 (\S+)
+Value normaldata3 (\S+)
+Value normaldata4 (\S+)
+Value Repeated keything (\S+)
+Value Repeated valuedata (\S+)
+Value Repeated unusedRepeated (\S+)
+Value List unused (\S+)
+
+
+Start
+  ^${normaldata1}\s+${normaldata2} (${keything}:${valuedata},? )*${normaldata3}\s+${normaldata4} -> Record

setup.py

@@ -45,6 +45,6 @@
           'Programming Language :: Python :: 2',
           'Programming Language :: Python :: 3',
           'Topic :: Software Development :: Libraries'],
-      requires=['six'],
+      requires=['six', 'regex'],
       py_modules=['clitable', 'textfsm', 'copyable_regex_object',
                   'texttable', 'terminal'])

terminal.py

@@ -28,7 +28,7 @@
 import fcntl
 import getopt
 import os
-import re
+import regex
 import struct
 import sys
 import termios
@@ -100,7 +100,7 @@
 ANSI_END = '\002'
 
 
-sgr_re = re.compile(r'(%s?\033\[\d+(?:;\d+)*m%s?)' % (
+sgr_re = regex.compile(r'(%s?\033\[\d+(?:;\d+)*m%s?)' % (
     ANSI_START, ANSI_END))
 
 
@@ -159,12 +159,12 @@ def AnsiText(text, command_list=None, reset=True):
 
 def StripAnsiText(text):
   """Strip ANSI/SGR escape sequences from text."""
-  return sgr_re.sub('', text)
+  return sgr_regex.sub('', text)
 
 
 def EncloseAnsiText(text):
   """Enclose ANSI/SGR escape sequences with ANSI_START and ANSI_END."""
-  return sgr_re.sub(lambda x: ANSI_START + x.group(1) + ANSI_END, text)
+  return sgr_regex.sub(lambda x: ANSI_START + x.group(1) + ANSI_END, text)
 
 
 def TerminalSize():
@@ -195,15 +195,15 @@ def LineWrap(text, omit_sgr=False):
 
   def _SplitWithSgr(text_line):
     """Tokenise the line so that the sgr sequences can be omitted."""
-    token_list = sgr_re.split(text_line)
+    token_list = sgr_regex.split(text_line)
     text_line_list = []
     line_length = 0
     for (index, token) in enumerate(token_list):
       # Skip null tokens.
       if token is '':
         continue
 
-      if sgr_re.match(token):
+      if sgr_regex.match(token):
         # Add sgr escape sequences without splitting or counting length.
         text_line_list.append(token)
         text_line = ''.join(token_list[index +1:])

textfsm.py

@@ -32,7 +32,7 @@
 
 import getopt
 import inspect
-import re
+import regex
 import string
 import sys
 
@@ -127,6 +127,22 @@ def GetOption(cls, name):
     """Returns the class of the requested option name."""
     return getattr(cls, name)
 
+  class Repeated(OptionBase):
+    """Will use regex module's 'captures' behavior to get all repeated
+     values instead of just the last value as re would."""
+
+    def OnAssignVar(self):
+      self.value.value = self.value.values_list
+
+    def OnCreateOptions(self):
+      self.value.value = []
+
+    def OnClearVar(self):
+      self.value.value = []
+
+    def OnClearAllVar(self):
+      self.value.value = []
+
   class Required(OptionBase):
     """The Value must be non-empty for the row to be recorded."""
 
@@ -194,8 +210,8 @@ def OnAssignVar(self):
           match = None
       # If the List-value regex has match-groups defined, add the resulting dict to the list
       # Otherwise, add the string that was matched
-      if match and match.groupdict():
-          self._value.append(match.groupdict())
+      if match and match.capturesdict():
+          self._value.append({x: match.capturesdict()[x][-1] for x in match.capturesdict()})
       else:
           self._value.append(self.value.value)
 
@@ -240,12 +256,17 @@ def __init__(self, fsm=None, max_name_len=48, options_class=None):
     self.options = []
     self.regex = None
     self.value = None
+    self.values_list = None
     self.fsm = fsm
     self._options_cls = options_class
 
   def AssignVar(self, value):
     """Assign a value to this Value."""
-    self.value = value
+    try:
+      self.value = value[-1]
+    except IndexError:
+      self.value = ''
+    self.values_list = value
     # Call OnAssignVar on options.
     _ = [option.OnAssignVar() for option in self.options]
 
@@ -306,18 +327,18 @@ def Parse(self, value):
       raise TextFSMTemplateError(
           "Invalid Value name '%s' or name too long." % self.name)
 
-    if (not re.match(r'^\(.*\)$', self.regex) or
+    if (not regex.match(r'^\(.*\)$', self.regex) or
         self.regex.count('(') != self.regex.count(')')):
       raise TextFSMTemplateError(
           "Value '%s' must be contained within a '()' pair." % self.regex)
 
-    self.template = re.sub(r'^\(', '(?P<%s>' % self.name, self.regex)
+    self.template = regex.sub(r'^\(', '(?P<%s>' % self.name, self.regex)
 
     # Compile and store the regex object only on List-type values for use in nested matching
     if any(map(lambda x: isinstance(x, TextFSMOptions.List), self.options)):
         try:
-            self.compiled_regex = re.compile(self.regex)
-        except re.error as e:
+            self.compiled_regex = regex.compile(self.regex)
+        except regex.error as e:
             raise TextFSMTemplateError(str(e))
 
   def _AddOption(self, name):
@@ -360,12 +381,12 @@ def __str__(self):
 
 
 class CopyableRegexObject(object):
-  """Like a re.RegexObject, but can be copied."""
+  """Like a regex.RegexObject, but can be copied."""
   # pylint: disable=C6409
 
   def __init__(self, pattern):
     self.pattern = pattern
-    self.regex = re.compile(pattern)
+    self.regex = regex.compile(pattern)
 
   def match(self, *args, **kwargs):
     return self.regex.match(*args, **kwargs)
@@ -402,7 +423,7 @@ class TextFSMRule(object):
     line_num: Integer row number of Value.
   """
   # Implicit default is '(regexp) -> Next.NoRecord'
-  MATCH_ACTION = re.compile(r'(?P<match>.*)(\s->(?P<action>.*))')
+  MATCH_ACTION = regex.compile(r'(?P<match>.*)(\s->(?P<action>.*))')
 
   # The structure to the right of the '->'.
   LINE_OP = ('Continue', 'Next', 'Error')
@@ -418,11 +439,11 @@ class TextFSMRule(object):
   NEWSTATE_RE = r'(?P<new_state>\w+|\".*\")'
 
   # Compound operator (line and record) with optional new state.
-  ACTION_RE = re.compile(r'\s+%s(\s+%s)?$' % (OPERATOR_RE, NEWSTATE_RE))
+  ACTION_RE = regex.compile(r'\s+%s(\s+%s)?$' % (OPERATOR_RE, NEWSTATE_RE))
   # Record operator with optional new state.
-  ACTION2_RE = re.compile(r'\s+%s(\s+%s)?$' % (RECORD_OP_RE, NEWSTATE_RE))
+  ACTION2_RE = regex.compile(r'\s+%s(\s+%s)?$' % (RECORD_OP_RE, NEWSTATE_RE))
   # Default operators with optional new state.
-  ACTION3_RE = re.compile(r'(\s+%s)?$' % (NEWSTATE_RE))
+  ACTION3_RE = regex.compile(r'(\s+%s)?$' % (NEWSTATE_RE))
 
   def __init__(self, line, line_num=-1, var_map=None):
     """Initialise a new rule object.
@@ -468,7 +489,7 @@ def __init__(self, line, line_num=-1, var_map=None):
     try:
       # Work around a regression in Python 2.6 that makes RE Objects uncopyable.
       self.regex_obj = CopyableRegexObject(self.regex)
-    except re.error:
+    except regex.error:
       raise TextFSMTemplateError(
           "Invalid regular expression: '%s'. Line: %s." %
           (self.regex, self.line_num))
@@ -512,7 +533,7 @@ def __init__(self, line, line_num=-1, var_map=None):
 
     # Check that an error message is present only with the 'Error' operator.
     if self.line_op != 'Error' and self.new_state:
-      if not re.match(r'\w+', self.new_state):
+      if not regex.match(r'\w+', self.new_state):
         raise TextFSMTemplateError(
             'Alphanumeric characters only in state names. Line: %s.'
             % (self.line_num))
@@ -551,8 +572,8 @@ class TextFSM(object):
   """
   # Variable and State name length.
   MAX_NAME_LEN = 48
-  comment_regex = re.compile(r'^\s*#')
-  state_name_re = re.compile(r'^(\w+)$')
+  comment_regex = regex.compile(r'^\s*#')
+  state_name_re = regex.compile(r'^(\w+)$')
   _DEFAULT_OPTIONS = TextFSMOptions
 
   def __init__(self, template, options_class=_DEFAULT_OPTIONS):
@@ -659,7 +680,7 @@ def _AppendRecord(self):
     self._ClearRecord()
 
   def _Parse(self, template):
-    """Parses template file for FSM structure.
+    """Parses template file for FSM structuregex.
 
     Args:
       template: Valid template file.
@@ -928,7 +949,7 @@ def _CheckLine(self, line):
     for rule in self._cur_state:
       matched = self._CheckRule(rule, line)
       if matched:
-        for value in matched.groupdict():
+        for value in matched.capturesdict():
           self._AssignVar(matched, value)
 
         if self._Operations(rule, line):
@@ -965,7 +986,7 @@ def _AssignVar(self, matched, value):
     """
     _value = self._GetValue(value)
     if _value is not None:
-        _value.AssignVar(matched.group(value))
+        _value.AssignVar(matched.captures(value))
 
   def _Operations(self, rule, line):
     """Operators on the data record.

textfsm_test.py

@@ -513,15 +513,22 @@ def testParseTextToDicts(self):
     # Tests 'Filldown' and 'Required' options.
     data = 'two\none'
     result = t.ParseTextToDicts(data)
-    self.assertEqual(str(result), "[{'hoo': 'two', 'boo': 'one'}]")
+    try:
+      self.assertEqual(str(result), "[{'hoo': 'two', 'boo': 'one'}]")
+    except AssertionError:
+      self.assertEqual(str(result), "[{'boo': 'one', 'hoo': 'two'}]")
 
     t = textfsm.TextFSM(StringIO(tplt))
     # Matching two lines. Two records returned due to 'Filldown' flag.
     data = 'two\none\none'
     t.Reset()
     result = t.ParseTextToDicts(data)
-    self.assertEqual(
-        str(result), "[{'hoo': 'two', 'boo': 'one'}, {'hoo': 'two', 'boo': 'one'}]")
+    try:
+      self.assertEqual(
+          str(result), "[{'hoo': 'two', 'boo': 'one'}, {'hoo': 'two', 'boo': 'one'}]")
+    except AssertionError:
+      self.assertEqual(
+        str(result), "[{'boo': 'one', 'hoo': 'two'}, {'boo': 'one', 'hoo': 'two'}]")
 
     # Multiple Variables and options.
     tplt = ('Value Required,Filldown boo (one)\n'
@@ -531,8 +538,12 @@ def testParseTextToDicts(self):
     t = textfsm.TextFSM(StringIO(tplt))
     data = 'two\none\none'
     result = t.ParseTextToDicts(data)
-    self.assertEqual(
-        str(result), "[{'hoo': 'two', 'boo': 'one'}, {'hoo': 'two', 'boo': 'one'}]")
+    try:
+      self.assertEqual(
+          str(result), "[{'hoo': 'two', 'boo': 'one'}, {'hoo': 'two', 'boo': 'one'}]")
+    except AssertionError:
+      self.assertEqual(
+        str(result), "[{'boo': 'one', 'hoo': 'two'}, {'boo': 'one', 'hoo': 'two'}]")
 
   def testParseNullText(self):
 
@@ -797,7 +808,7 @@ def testEnd(self):
 
   def testInvalidRegexp(self):
 
-    tplt = 'Value boo (.$*)\n\nStart\n  ^$boo -> Next\n'
+    tplt = 'Value boo ([(\S+]))\n\nStart\n  ^$boo -> Next\n'
     self.assertRaises(textfsm.TextFSMTemplateError,
                       textfsm.TextFSM, StringIO(tplt))
 
@@ -846,5 +857,27 @@ def testFillup(self):
         str(result))
 
 
+  def testRepeated(self):
+    """Repeated option should work ok."""
+    tplt = """Value Repeated repeatedKey (\S+)
+Value Repeated repeatedValue (\S+)
+Value normalData (\S+)
+Value normalData2 (\S+)
+Value Repeated repeatedUnused (\S+)
+
+Start
+  ^${normalData} (${repeatedKey}:${repeatedValue} )*${normalData2} -> Record"""
+
+    data = """
+normal1 key1:value1 key2:value2 key3:value3 normal2 \n
+normal1 normal2 """
+
+    t = textfsm.TextFSM(StringIO(tplt))
+    result = t.ParseText(data)
+    self.assertEqual(
+      "[[['key1', 'key2', 'key3'], ['value1', 'value2', 'value3'], 'normal1', 'normal2', []],"
+      + " [[], [], 'normal1', 'normal2', []]]",
+      str(result))
+
 if __name__ == '__main__':
   unittest.main()