Manually factor left recursion into right recursion in the grammar fi…

…le for a more useful parse tree, 15x smaller grammar file, and faster parsing time

custom-parser/grammar-factoring/1-ebnf-to-json.js

@@ -3,6 +3,6 @@ import fs from 'fs';
 
 const filePath = process.argv[2] || 'MySQLFull.ebnf';
 let grammar = fs.readFileSync(filePath, 'utf8');
+grammar = grammar.replaceAll('%', 'fragment__F')
 let RULES = Grammars.W3C.getRules(grammar);
-
-console.log(JSON.stringify(RULES, null, 2));
+console.log(JSON.stringify(RULES, null, 2).replaceAll('fragment__F', '%'));

custom-parser/grammar-factoring/2-cli.py

@@ -1,7 +1,7 @@
 import json
 import sys
 import argparse
-from ebnfutils import eliminate_left_recursion, encode_as_ebnf, factor_common_prefixes
+from ebnfutils import eliminate_left_recursion, encode_as_ebnf, factor_common_prefixes, expand_grammar
 
 class CustomArgumentParser(argparse.ArgumentParser):
     def error(self, message):
@@ -14,7 +14,7 @@ def error(self, message):
 parser.add_argument(
     'mode',
     type=str,
-    choices=['lr', 'cp', 'all'],
+    choices=['lr', 'expand', 'cp', 'all'],
     help=(
         'Specify the mode. Options are:\n'
         "* 'lr' for left recursion elimination\n"
@@ -48,9 +48,10 @@ def error(self, message):
 # print(f"Selected mode: {args.mode}")
 # print(f"Filename: {args.filename}")
 
-if args.filename is None or args.mode not in ["lr", "cp", "all"]:
+if args.filename is None or args.mode not in ["expand", "lr", "cp", "all"]:
     print("Usage: python ebnf-to-right-recursive.py <mode> <filename> [--format json|ebnf]")
     print("Mode can be one of:")
+    print("* 'expand' for expansion of * ? + symbols")
     print("* 'lr' for left recursion elimination")
     print("* 'cp' for factoring common prefixes")
     print("* 'all' for both")
@@ -68,6 +69,10 @@ def error(self, message):
     sys.exit(1)
 
 updated_grammar = input_grammar
+if args.mode == "expand" or args.mode == "all":
+    grammar, new_rules = expand_grammar(updated_grammar)
+    updated_grammar = grammar
+
 if args.mode == "lr" or args.mode == "all":
     updated_grammar = eliminate_left_recursion(updated_grammar)
 

custom-parser/grammar-factoring/3-phpize-grammar.php

@@ -61,6 +61,21 @@ function export_as_php_var($var) {
     $compressed_grammar[$rule_index_by_name[$rule["name"]]] = $new_branches;
 }
 
+// Compress the fragment rules names – they take a lot of disk space and are
+// inlined in the final parse tree anyway.
+$last_fragment = 1;
+foreach($rules_ids as $id => $name) {
+    if(
+        $name[0] === '%' || 
+        str_ends_with($name, '_zero_or_one') || 
+        str_ends_with($name, '_zero_or_more') || 
+        str_ends_with($name, '_one_or_more')
+    ) {
+        $rules_ids[$id] = '%f' . $last_fragment;
+        ++$last_fragment;
+    }
+}
+
 $full_grammar = [
     "rules_offset" => $rules_offset,
     "rules_names" => $rules_ids,