diff --git a/monticore-generator/src/main/java/de/monticore/codegen/cd2java/mill/MillDecorator.java b/monticore-generator/src/main/java/de/monticore/codegen/cd2java/mill/MillDecorator.java index d1f024ce92..3598923aee 100644 --- a/monticore-generator/src/main/java/de/monticore/codegen/cd2java/mill/MillDecorator.java +++ b/monticore-generator/src/main/java/de/monticore/codegen/cd2java/mill/MillDecorator.java @@ -100,7 +100,7 @@ public ASTCDClass decorate(List packageList) { List classList = cd.getCDElementList() .stream() .filter(x -> x instanceof ASTCDClass) - .map(x -> (ASTCDClass) x) + .map(ASTCDClass.class::cast) .filter(x -> !x.getModifier().isAbstract()) .filter(this::checkIncludeInMill) .map(x -> x.deepClone()) diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGenerator.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGenerator.java index 6d14ca0d33..e13de5bb05 100644 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGenerator.java +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGenerator.java @@ -6,6 +6,8 @@ import de.monticore.cd.codegen.CDGenerator; import de.monticore.cd.codegen.CdUtilsPrinter; import de.monticore.cd.codegen.TopDecorator; +import de.monticore.cd.methodtemplates.CD4C; +import de.monticore.cdbasis._ast.ASTCDClass; import de.monticore.cdbasis._ast.ASTCDCompilationUnit; import de.monticore.codegen.cd2java.AbstractService; import de.monticore.codegen.cd2java.DecorationHelper; @@ -15,6 +17,7 @@ import de.monticore.codegen.cd2java._parser.ParserService; import de.monticore.codegen.parser.antlr.AntlrTool; import de.monticore.codegen.parser.antlr.Grammar2Antlr; +import de.monticore.codegen.parser.antlr.Grammar2ParseVisitor; import de.monticore.generating.GeneratorEngine; import de.monticore.generating.GeneratorSetup; import de.monticore.generating.templateengine.GlobalExtensionManagement; @@ -63,8 +66,8 @@ public static void generateFullParser( MCPath templatePath, File targetDir) { - generateParser(glex, astGrammar, symbolTable, handcodedPath, templatePath, targetDir); - generateParserWrapper(glex, astClassDiagram, handcodedPath, templatePath, targetDir); + ASTCDClass parseVisitorClass = generateParser(glex, astGrammar, symbolTable, handcodedPath, templatePath, targetDir); + generateParserWrapper(glex, astClassDiagram, parseVisitorClass, handcodedPath, templatePath, targetDir); } @@ -76,7 +79,7 @@ public static void generateFullParser( * @param astGrammar - grammar AST * @param targetDir - target file */ - public static void generateParser( + public static ASTCDClass generateParser( GlobalExtensionManagement glex, ASTMCGrammar astGrammar, IGrammar_WithConceptsGlobalScope symbolTable, @@ -84,7 +87,7 @@ public static void generateParser( MCPath templatePath, File targetDir) { - generateParser(glex, astGrammar, symbolTable, handcodedPath, templatePath, targetDir, true, Languages.JAVA); + return generateParser(glex, astGrammar, symbolTable, handcodedPath, templatePath, targetDir, true, Languages.JAVA); } @@ -97,7 +100,7 @@ public static void generateParser( * @param targetDir - target file * @param embeddedJavaCode - embed Java Code */ - public static void generateParser( + public static ASTCDClass generateParser( GlobalExtensionManagement glex, ASTMCGrammar astGrammar, IGrammar_WithConceptsGlobalScope symbolTable, @@ -131,7 +134,7 @@ public static void generateParser( if (astGrammar.isComponent()) { ParserInfoGenerator.generateParserInfoForComponent(astGrammar, setup, genHelper.getParserPackage(), lang); Log.info("No parser generation for the grammar " + astGrammar.getName(), LOG); - return; + return null; } Log.debug("Start parser generation for the grammar " + astGrammar.getName(), LOG); @@ -148,6 +151,15 @@ public static void generateParser( astGrammar.getName() + "AntlrParser" + suffix + ".g4"); new GeneratorEngine(setup).generate("parser.Parser", parserPath, astGrammar, grammar2Antlr, suffix); + // 1.5 ParseVisitor + CD4C.init(setup); + traverser = Grammar_WithConceptsMill.traverser(); + Grammar2ParseVisitor parserVisitor = new Grammar2ParseVisitor(glex, genHelper, grammarInfo, grammar2Antlr.getTmpNameDict()); + traverser.add4Grammar(parserVisitor); + traverser.setGrammarHandler(parserVisitor); + astGrammar.accept(traverser); + + // 2. Lexer suffix = GeneratorEngine.existsHandwrittenClass(handcodedPath, genHelper.getParserPackage()+"."+astGrammar.getName()+"AntlrLexer")?"TOP":""; @@ -162,7 +174,14 @@ public static void generateParser( Path gLexer = Paths.get(targetDir.getPath(), lexerPath.toString()); AntlrTool antlrTool = new AntlrTool( - new String[] { outputLang, "-o", antlrPath.toString(), "-Xexact-output-dir", "-no-listener", gLexer.toString(), gParser.toString() }, + new String[] { outputLang, + "-o", antlrPath.toString(), + "-package", genHelper.getParserPackage(), + "-Xexact-output-dir", + "-no-listener", + "-visitor", + gLexer.toString().replace('\\', '/'), // fix windows paths unicode \\u + gParser.toString().replace('\\', '/') }, grammarSymbol, grammar2Antlr.getTmpNameDict() ); @@ -174,6 +193,8 @@ public static void generateParser( Log.debug("End parser generation for the grammar " + astGrammar.getName(), LOG); ParserInfoGenerator.generateParserInfo(astGrammar, setup, antlrTool.getNonTerminalToParserStates(), genHelper.getParserPackage(), lang); + + return parserVisitor.getVisitorClass(); } /** @@ -185,6 +206,7 @@ public static void generateParser( public static void generateParserWrapper( GlobalExtensionManagement glex, ASTCDCompilationUnit astClassDiagram, + ASTCDClass parseVisitorClass, MCPath handcodedPath, MCPath templatePath, File targetDir) { @@ -195,6 +217,11 @@ public static void generateParserWrapper( ASTCDCompilationUnit decoratedCD = parserCDDecorator.decorate(astClassDiagram); + if (parseVisitorClass != null) { + // add the ASTBuilder class if exists + decoratedCD.getCDDefinition().getCDPackagesList().iterator().next().addCDElement(parseVisitorClass); + } + TopDecorator topDecorator = new TopDecorator(handcodedPath); topDecorator.decorate(decoratedCD); diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/ASTConstructionActions.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/ASTConstructionActions.java deleted file mode 100644 index 20c61b900d..0000000000 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/ASTConstructionActions.java +++ /dev/null @@ -1,247 +0,0 @@ -/* (c) https://github.com/MontiCore/monticore */ - -package de.monticore.codegen.parser.antlr; - -import de.monticore.codegen.cd2java.DecorationHelper; -import de.monticore.codegen.cd2java._ast.ast_class.ASTConstants; -import de.monticore.codegen.mc2cd.TransformationHelper; -import de.monticore.codegen.parser.ParserGeneratorHelper; -import de.monticore.codegen.parser.MCGrammarInfo; -import de.monticore.grammar.MCGrammarSymbolTableHelper; -import de.monticore.grammar.grammar._ast.*; -import de.monticore.grammar.grammar._symboltable.MCGrammarSymbol; -import de.se_rwth.commons.Joiners; -import de.se_rwth.commons.StringTransformations; - -import java.util.Optional; - -public class ASTConstructionActions { - - protected ParserGeneratorHelper parserGenHelper; - - protected MCGrammarSymbol symbolTable; - - public ASTConstructionActions(ParserGeneratorHelper parserGenHelper) { - this.parserGenHelper = parserGenHelper; - this.symbolTable = parserGenHelper.getGrammarSymbol(); - } - - protected String getConstantClassName(MCGrammarSymbol symbol) { - return Joiners.DOT.join(symbol.getFullName().toLowerCase(), - ASTConstants.AST_PACKAGE, - ASTConstants.AST_CONSTANTS + symbol.getName()); - } - - public String getConstantInConstantGroupMultipleEntries(ASTConstant constant, - ASTConstantGroup constgroup) { - String tmp = ""; - if (constgroup.isPresentUsageName()) { - String constfile; - String constantname; - Optional ruleGrammar = MCGrammarSymbolTableHelper - .getMCGrammarSymbol(constgroup.getEnclosingScope()); - if (ruleGrammar.isPresent()) { - constfile = getConstantClassName(ruleGrammar.get()); - constantname = parserGenHelper.getConstantNameForConstant(constant); - } - else { - constfile = getConstantClassName(symbolTable); - constantname = parserGenHelper.getConstantNameForConstant(constant); - } - - // Add as attribute to AST - tmp = "_builder.set%uname%(%constfile%.%constantname%);"; - - tmp = tmp.replaceAll("%uname%", - StringTransformations.capitalize(constgroup.getUsageName())); - - tmp = tmp.replaceAll("%constfile%", constfile); - - tmp = tmp.replaceAll("%constantname%", constantname); - } - - return tmp; - } - - public String getActionAfterConstantInEnumProdSingle(ASTConstant c) { - return "ret = true ;"; - } - - public String getConstantInConstantGroupSingleEntry(ASTConstant constant, - ASTConstantGroup constgroup) { - String tmp = ""; - - if (constgroup.isPresentUsageName()) { - // Add as attribute to AST - tmp = "_builder.set%uname%(true);"; - - tmp = tmp.replaceAll("%uname%", - StringTransformations.capitalize(constgroup.getUsageName())); - } else { - if (constgroup.getConstantList().size() == 1) { - // both == null and #constants == 1 -> use constant string as name - tmp = "_builder.set%cname%(true);"; - tmp = tmp.replaceAll("%cname%", StringTransformations.capitalize(constgroup.getConstantList().get(0).getHumanName())); - } else { - // both == null and #constants > 1 -> user wants to ignore token in AST - } - } - - return tmp; - } - - public String getActionForRuleBeforeRuleBody(ASTClassProd a) { - StringBuilder b = new StringBuilder(); - String type = TransformationHelper - .getQualifiedName(a.getSymbol()); - Optional grammar = MCGrammarSymbolTableHelper - .getMCGrammarSymbol(a.getEnclosingScope()); - - // Setup builder - b.append("// getActionForAltBeforeRuleBody\n"); - b.append(type + "Builder _builder = " + parserGenHelper.getQualifiedGrammarName().toLowerCase() - + "." + parserGenHelper.getGrammarSymbol().getName() + "Mill." - + StringTransformations.uncapitalize(a.getName()) + "Builder();\n"); - - return b.toString(); - } - - public String getActionForAltBeforeRuleBody(String className, ASTAlt a) { - StringBuilder b = new StringBuilder(); - String type = TransformationHelper - .getQualifiedName(symbolTable.getProdWithInherited(className).get()); - Optional grammar = MCGrammarSymbolTableHelper - .getMCGrammarSymbol(a.getEnclosingScope()); - String name = grammar.isPresent() - ? grammar.get().getName() - : symbolTable.getProdWithInherited(className).get().getName(); - - // Setup builder - b.append("// getActionForAltBeforeRuleBody\n"); - b.append(type + "Builder _builder = " + parserGenHelper.getQualifiedGrammarName().toLowerCase() - + "." + parserGenHelper.getGrammarSymbol().getName() + "Mill." - + StringTransformations.uncapitalize(className) + "Builder();\n"); - - return b.toString(); - } - - - public String getActionForLexerRuleNotIteratedAttribute(ASTNonTerminal a) { - - String tmp = "_builder.set%u_usage%(convert" + a.getName() + "($%tmp%));"; - - // Replace templates - tmp = tmp.replaceAll("%u_usage%", - StringTransformations.capitalize(parserGenHelper.getUsageName(a))); - tmp = tmp.replaceAll("%tmp%", parserGenHelper.getTmpVarNameForAntlrCode(a)); - - return tmp; - - } - - public String getActionForLexerRuleIteratedAttribute(ASTNonTerminal a) { - - String tmpname = parserGenHelper.getTmpVarNameForAntlrCode(a); - String tmp = " addToIteratedAttributeIfNotNull(_builder.get%u_usage%(), convert" + a.getName() - + "($%tmp%));"; - - // Replace templates - tmp = tmp.replaceAll("%u_usage%", - StringTransformations.capitalize(MCGrammarInfo.getListName(a))); - tmp = tmp.replaceAll("%tmp%", tmpname); - - return tmp; - } - - public String getActionForInternalRuleIteratedAttribute(ASTNonTerminal a) { - - String tmp = "addToIteratedAttributeIfNotNull(_builder.get%u_usage%(), _localctx.%tmp%.ret);"; - if (symbolTable.getProdWithInherited(a.getName()).get().isIsEnum()) { - tmp = "addToIteratedAttributeIfNotNull(_builder.get%u_usage%(), _localctx.%tmp%.ret);"; - } - - // Replace templates - tmp = tmp.replaceAll("%u_usage%", - StringTransformations.capitalize(MCGrammarInfo.getListName(a))); - tmp = tmp.replaceAll("%tmp%", parserGenHelper.getTmpVarNameForAntlrCode(a)); - - return tmp; - } - - public String getActionForInternalRuleNotIteratedAttribute(ASTNonTerminal a) { - - String tmp = "_builder.set%u_usage%(_localctx.%tmp%.ret);"; - - // Replace templates - tmp = tmp.replaceAll("%u_usage%", - StringTransformations.capitalize(parserGenHelper.getUsageName(a))); - tmp = tmp.replaceAll("%tmp%", parserGenHelper.getTmpVarNameForAntlrCode(a)); - - return tmp; - } - - public String getActionForInternalRuleNotIteratedLeftRecursiveAttribute(ASTNonTerminal a) { - - SourcePositionActions sourcePositionBuilder = new SourcePositionActions(parserGenHelper); - StringBuilder b = new StringBuilder(); - b.append("// Action code for left recursive rule \n"); - // Setup builder - b.append("// getActionForInternalRuleNotIteratedLeftRecursiveAttribute \n"); - b.append(sourcePositionBuilder.endPosition()); - b.append("\n_localctx." + parserGenHelper.getTmpVarName(a) + ".ret = _builder.uncheckedBuild();"); - b.append("\n_builder=" + parserGenHelper.getQualifiedGrammarName().toLowerCase() - + "." + parserGenHelper.getGrammarSymbol().getName() + "Mill." - + - StringTransformations.uncapitalize(a.getName()) + "Builder();\n"); - b.append(sourcePositionBuilder.startPositionForLeftRecursiveRule(a)); - - - return b.toString(); - } - - /** - * Nothing to do for ignore - */ - public String getActionForTerminalIgnore(ASTTerminal a) { - return ""; - } - - public String getActionForTerminalNotIteratedAttribute(ASTITerminal a) { - if (!a.isPresentUsageName()) { - return ""; - } - String usageName = StringTransformations.capitalize(a.getUsageName()); - String value = a.getName(); - return "_builder.set" + usageName + "(\"" + value + "\");"; - } - - public String getActionForTerminalIteratedAttribute(ASTITerminal a) { - if (!a.isPresentUsageName()) { - return ""; - } - String usageName = StringTransformations.capitalize(a.getUsageName()) + DecorationHelper.GET_SUFFIX_LIST; - String value = a.getName(); - return "_builder.get" + usageName + "().add(\"" + value + "\");"; - } - - public String getActionForKeyTerminalNotIteratedAttribute(ASTKeyTerminal a) { - if (!a.isPresentUsageName()) { - return ""; - } - String usageName = StringTransformations.capitalize(a.getUsageName()); - return "_builder.set" + usageName + "(_input.LT(-1).getText());"; - } - - public String getActionForKeyTerminalIteratedAttribute(ASTKeyTerminal a) { - if (!a.isPresentUsageName()) { - return ""; - } - String usageName = StringTransformations.capitalize(a.getUsageName()) + DecorationHelper.GET_SUFFIX_LIST; - return "_builder.get" + usageName + "().add(_input.LT(-1).getText());"; - } - - public String getBuildAction() { - return "\n_localctx.ret = _builder.uncheckedBuild();"; - } - -} diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/AntlrTool.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/AntlrTool.java index bb4b71df56..911f60ec9e 100644 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/AntlrTool.java +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/AntlrTool.java @@ -66,7 +66,7 @@ protected void createMessage(ANTLRMessage message, boolean isError) { for (int i = 0; i < args.length; i++) { if (args[i] instanceof String) { String name = StringTransformations.capitalize((String) args[i]); - Optional rule = grammarSymbol==null?grammarSymbol.getProd(name):Optional.empty(); + Optional rule = grammarSymbol!=null?grammarSymbol.getProd(name):Optional.empty(); if (rule.isPresent()) { args[i] = name; if (i == 0) { diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/AttributeCardinalityConstraint.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/AttributeCardinalityConstraint.java index daa64593d7..458c0c2217 100644 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/AttributeCardinalityConstraint.java +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/AttributeCardinalityConstraint.java @@ -18,11 +18,11 @@ * MinMax-constraint checks */ public class AttributeCardinalityConstraint { - + protected ParserGeneratorHelper parserGenHelper; - + protected MCGrammarSymbol symbolTable; - + public AttributeCardinalityConstraint(ParserGeneratorHelper parserGenHelper) { this.parserGenHelper = parserGenHelper; this.symbolTable = parserGenHelper.getGrammarSymbol(); @@ -35,18 +35,18 @@ public String addActionForRuleBeforeRuleBody(ASTClassProd ast) { for (AdditionalAttributeSymbol att : prodSymbol.getSpannedScope().getLocalAdditionalAttributeSymbols()) { String usageName = att.getName(); if (TransformationHelper.getMax(att).isPresent() - || MCGrammarSymbolTableHelper.getMin(att).isPresent()) { + || MCGrammarSymbolTableHelper.getMin(att).isPresent()) { ret.append("\n" + "int " + getCounterName(usageName) + "=0;"); } } return ret.toString(); } - + public String addActionForRuleAfterRuleBody(ASTClassProd ast) { StringBuilder ret = new StringBuilder(); ProdSymbol prodSymbol = ast.getSymbol(); for (AdditionalAttributeSymbol att : prodSymbol.getSpannedScope().getLocalAdditionalAttributeSymbols()) { - + String usageName = att.getName(); Optional min = MCGrammarSymbolTableHelper.getMin(att); Optional max = TransformationHelper.getMax(att); @@ -54,13 +54,13 @@ public String addActionForRuleAfterRuleBody(ASTClassProd ast) { if (min.isPresent()) { String runtimemessage = "\"0xA7017" + getGeneratedErrorCode(ast) + " Invalid minimal occurence for %attributename% in rule %rulename% : Should be %reference% but is \"+%value%+\"!\""; - + runtimemessage = runtimemessage.replaceAll("%attributename%", usageName); runtimemessage = runtimemessage.replaceAll("%rulename%", ast.getName()); runtimemessage = runtimemessage.replaceAll("%value%", getCounterName(usageName)); runtimemessage = runtimemessage.replaceAll("%reference%", ParserGeneratorHelper.formatAttributeValue(min)); - + String message = "if (!checkMin(" + getCounterName(usageName) + "," @@ -95,20 +95,20 @@ public String addActionForRuleAfterRuleBody(ASTClassProd ast) { } } } - + return ret.toString(); } - + public String addActionForNonTerminal(ASTNonTerminal ast) { StringBuilder ret = new StringBuilder(); - + String usageName = parserGenHelper.getUsageName(ast); - + Optional rule = MCGrammarSymbolTableHelper.getEnclosingRule(ast); if (!rule.isPresent()) { return ret.toString(); } - + Optional att = rule.get().getSpannedScope().resolveAdditionalAttributeLocally(usageName); if (att.isPresent() && (TransformationHelper.getMax(att.get()).isPresent() || MCGrammarSymbolTableHelper.getMin(att.get()).isPresent())) { diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2Antlr.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2Antlr.java index 1dcac29a28..e6bd160d26 100644 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2Antlr.java +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2Antlr.java @@ -7,9 +7,9 @@ import com.google.common.collect.Lists; import de.monticore.ast.ASTNode; import de.monticore.codegen.mc2cd.TransformationHelper; +import de.monticore.codegen.parser.MCGrammarInfo; import de.monticore.codegen.parser.ParserGeneratorHelper; import de.monticore.grammar.DirectLeftRecursionDetector; -import de.monticore.codegen.parser.MCGrammarInfo; import de.monticore.grammar.MCGrammarSymbolTableHelper; import de.monticore.grammar.PredicatePair; import de.monticore.grammar.grammar._ast.*; @@ -20,7 +20,6 @@ import de.monticore.grammar.grammar._visitor.GrammarTraverser; import de.monticore.grammar.grammar._visitor.GrammarVisitor2; import de.monticore.grammar.grammar_withconcepts.Grammar_WithConceptsMill; -import de.monticore.grammar.grammar_withconcepts._ast.ASTAction; import de.se_rwth.commons.JavaNamesHelper; import de.se_rwth.commons.StringTransformations; import de.se_rwth.commons.logging.Log; @@ -35,7 +34,7 @@ public class Grammar2Antlr implements GrammarVisitor2, GrammarHandler { protected MCGrammarSymbol grammarEntry; - GrammarTraverser traverser; + protected GrammarTraverser traverser; /** * This list is used for the detection of the left recursion @@ -44,12 +43,8 @@ public class Grammar2Antlr implements GrammarVisitor2, GrammarHandler { protected DirectLeftRecursionDetector leftRecursionDetector = new DirectLeftRecursionDetector(); - protected SourcePositionActions positionActions; - protected AttributeCardinalityConstraint attributeConstraints; - protected ASTConstructionActions astActions; - protected List productionAntlrCode = Lists.newArrayList(); protected StringBuilder codeSection; @@ -65,33 +60,27 @@ public class Grammar2Antlr implements GrammarVisitor2, GrammarHandler { protected Map> tmpNameDict = new LinkedHashMap<>(); public Grammar2Antlr( - ParserGeneratorHelper parserGeneratorHelper, - MCGrammarInfo grammarInfo) { + ParserGeneratorHelper parserGeneratorHelper, + MCGrammarInfo grammarInfo) { Preconditions.checkArgument(parserGeneratorHelper.getGrammarSymbol() != null); + this.attributeConstraints = new AttributeCardinalityConstraint(parserGeneratorHelper); this.grammarEntry = parserGeneratorHelper.getGrammarSymbol(); this.grammarInfo = grammarInfo; this.parserHelper = parserGeneratorHelper; - astActions = new ASTConstructionActions(parserGeneratorHelper); - attributeConstraints = new AttributeCardinalityConstraint(parserGeneratorHelper); - positionActions = new SourcePositionActions(parserGeneratorHelper); - this.embeddedJavaCode = true; } public Grammar2Antlr( - ParserGeneratorHelper parserGeneratorHelper, - MCGrammarInfo grammarInfo, - boolean embeddedJavaCode) { + ParserGeneratorHelper parserGeneratorHelper, + MCGrammarInfo grammarInfo, + boolean embeddedJavaCode) { Preconditions.checkArgument(parserGeneratorHelper.getGrammarSymbol() != null); + this.attributeConstraints = new AttributeCardinalityConstraint(parserGeneratorHelper); this.grammarEntry = parserGeneratorHelper.getGrammarSymbol(); this.grammarInfo = grammarInfo; this.parserHelper = parserGeneratorHelper; - astActions = new ASTConstructionActions(parserGeneratorHelper); - attributeConstraints = new AttributeCardinalityConstraint(parserGeneratorHelper); - positionActions = new SourcePositionActions(parserGeneratorHelper); - this.embeddedJavaCode = embeddedJavaCode; } @@ -190,39 +179,23 @@ public void handle(ASTClassProd ast) { // Start code codeSection for rules addToCodeSection(ruleName); List subRules = grammarInfo - .getSubRulesForParsing(ast.getName()); + .getSubRulesForParsing(ast.getName()); if (embeddedJavaCode) { - addToCodeSection(" returns [", classnameFromRulenameorInterfacename, " ret = ", - getDefaultValue(ruleByName), "]\n", options); - // Add actions - if (ast.isPresentAction() && ast.getAction() instanceof ASTAction) { + if (ast.isPresentAction()) { addToAction(ParserGeneratorHelper.getText(ast.getAction())); } // Action at beginning of rule @init - addToAction(astActions.getActionForRuleBeforeRuleBody(ast)); - // Action for determining positions - addToAction(positionActions.startPosition()); - // Action for determining positions of comments (First set position) - addToAction("setActiveBuilder(_builder);\n"); - addToAction(attributeConstraints.addActionForRuleBeforeRuleBody(ast)); - if (!isActionEmpty()) { addToCodeSection("@init"); addActionToCodeSection(); } // Action at end of rule - addToAction(positionActions.endPosition()); addToAction(attributeConstraints.addActionForRuleAfterRuleBody(ast)); - if (subRules != null && !subRules.isEmpty()) { - addToAction("\nif (_localctx.ret == null)"); - } - addToAction(astActions.getBuildAction()); - if (!isActionEmpty()) { addToCodeSection("\n@after"); addActionToCodeSection(); @@ -253,9 +226,6 @@ public void handle(ASTClassProd ast) { startCodeSection(); String subRuleVar = "subRuleVar" + i; addToCodeSection("(" + subRuleVar + " = " + getRuleNameForAntlr(x.getClassname())); - if (embeddedJavaCode) { - addToCodeSection(" {$ret = $" + subRuleVar + ".ret;}"); - } addToCodeSection(") | \n"); endCodeSection(); i++; @@ -270,7 +240,7 @@ public void handle(ASTClassProd ast) { createAntlrCodeForAlts(alts); addDummyRules(ast.getName(), ruleName, - classnameFromRulenameorInterfacename); + classnameFromRulenameorInterfacename); addToAntlrCode(";"); @@ -289,21 +259,22 @@ public void handle(ASTEnumProd ast) { // Head of Rule addToCodeSection(ruleName + " returns [" - + getQualifiedName(ruleByName) + " ret = " - + getDefaultValue(ruleByName) + "] "); + + getQualifiedName(ruleByName) + " ret = " + + getDefaultValue(ruleByName) + "] "); addToCodeSection("\n: "); String sep = ""; + int index = 0; for (ASTConstant c : ast.getConstantList()) { addToCodeSection(sep); - addToCodeSection("\n", parserHelper.getLexSymbolName(c.getName())); + addToCodeSection("\n", "e_" + index++ + "=" + parserHelper.getLexSymbolName(c.getName())); if (embeddedJavaCode) { String temp1 = ""; temp1 += "$ret = " + getQualifiedName(ruleByName) - + "." - + parserHelper.getConstantNameForConstant(c) + ";"; + + "." + + parserHelper.getConstantNameForConstant(c) + ";"; if (!temp1.isEmpty()) { addToCodeSection("\n{" + temp1 + "}"); @@ -328,29 +299,30 @@ public void handle(ASTConstantGroup ast) { boolean iterated = false; if (ast.isPresentSymbol()) { iterated = TransformationHelper - .isConstGroupIterated(ast.getSymbol()); + .isConstGroupIterated(ast.getSymbol()); } + + // No += refs for constant groups + String label = "="; + // One entry leads to boolean isMethods if (!iterated) { ASTConstant x = ast.getConstantList().get(0); addToCodeSection("("); + String tmpName = parserHelper.getTmpVarName(x); if (x.isPresentKeyConstant()) { - addToCodeSection(createKeyPredicate(x.getKeyConstant().getStringList())); + addToCodeSection(createKeyPredicate(x.getKeyConstant().getStringList(), tmpName + label)); } else if (x.isPresentTokenConstant()) { - addToCodeSection(parserHelper.getLexSymbolName(x.getTokenConstant().getString())); + addToCodeSection(tmpName + label + parserHelper.getLexSymbolName(x.getTokenConstant().getString())); } else if (!grammarInfo.isKeyword(x.getName(), grammarEntry)) { - addToCodeSection(parserHelper.getLexSymbolName(x.getName())); + addToCodeSection(tmpName + label +parserHelper.getLexSymbolName(x.getName())); } else if (grammarInfo.getKeywordRules().contains(x.getName())) { - addToCodeSection(parserHelper.getKeyRuleName(x.getName())); + addToCodeSection(tmpName + label +parserHelper.getKeyRuleName(x.getName())); } else { - addToCodeSection(parserHelper.getLexSymbolName(x.getName())); + addToCodeSection(tmpName + label +parserHelper.getLexSymbolName(x.getName())); } - if (embeddedJavaCode) { - addToAction(astActions.getConstantInConstantGroupSingleEntry(x, ast)); - addActionToCodeSectionWithNewLine(); - } addToCodeSection(")", printIteration(ast.getIteration())); } @@ -360,23 +332,19 @@ public void handle(ASTConstantGroup ast) { addToCodeSection("("); String del = ""; for (Iterator iter = ast.getConstantList().iterator(); iter - .hasNext(); ) { + .hasNext(); ) { addToCodeSection(del); ASTConstant x = iter.next(); + String tmpName = parserHelper.getTmpVarName(x); if (x.isPresentKeyConstant()) { - addToCodeSection(createKeyPredicate(x.getKeyConstant().getStringList())); + addToCodeSection(createKeyPredicate(x.getKeyConstant().getStringList(), tmpName + label)); } else if (!grammarInfo.isKeyword(x.getName(), grammarEntry)) { - addToCodeSection(parserHelper.getLexSymbolName(x.getName())); + addToCodeSection(tmpName + label + parserHelper.getLexSymbolName(x.getName())); } else if (grammarInfo.getKeywordRules().contains(x.getName())) { - addToCodeSection(parserHelper.getKeyRuleName(x.getName())); + addToCodeSection(tmpName + label + parserHelper.getKeyRuleName(x.getName())); } else { - addToCodeSection(parserHelper.getLexSymbolName(x.getName())); - } - - if (embeddedJavaCode) { - addToAction(astActions.getConstantInConstantGroupMultipleEntries(x, ast)); - addActionToCodeSectionWithNewLine(); + addToCodeSection(tmpName + label + parserHelper.getLexSymbolName(x.getName())); } del = "|\n"; @@ -427,10 +395,7 @@ public void handle(ASTLexBlock ast) { addToCodeSection("("); if (ast.isPresentOption()) { addToCodeSection("\noptions {", ast.getOption().getID(), "=", ast.getOption() - .getValue(), ";}"); - } - if (embeddedJavaCode && ast.isPresentInitAction()) { - addToCodeSection("{", ParserGeneratorHelper.getText(ast.getInitAction()), "}"); + .getValue(), ";}"); } endCodeSection(); @@ -497,16 +462,15 @@ public void handle(ASTBlock a) { addToCodeSection("\n }"); } - // Print init actions - if (embeddedJavaCode && a.isPresentInitAction()) { - addToCodeSection("{" + ParserGeneratorHelper.getText(a.getInitAction()) + "}"); - } - endCodeSection(); + ruleIteratedStack.push((!ruleIteratedStack.isEmpty() && ruleIteratedStack.peek()) || isIterated(a.getIteration())); + // Visit all alternatives createAntlrCodeForAlts(a.getAltList()); + ruleIteratedStack.pop(); + // Start of Block with iteration startCodeSection(); addToCodeSection("\n)" + printIteration(a.getIteration())); @@ -538,8 +502,12 @@ public void visit(ASTTerminal ast) { if (keywords.containsKey(ast.getName())) { addToCodeSection("("); String seperator = ""; + int nokeywordindex = 0; // we use an index for (String replaceString: keywords.get(ast.getName())) { addToCodeSection(seperator); + addToCodeSection(parserHelper.getTmpVarName(ast) + "_nk" + nokeywordindex++); + addToCodeSection(ast.isPresentSymbol() && ast.getSymbol().isIsList() ? "+=" : "="); + if (grammarInfo.getKeywordRules().contains(replaceString)) { addToCodeSection(parserHelper.getKeyRuleName(replaceString)); } else { @@ -549,22 +517,10 @@ public void visit(ASTTerminal ast) { } addToCodeSection(")"); } else { + addToCodeSection(parserHelper.getTmpVarName(ast)); + addToCodeSection((ast.isPresentSymbol() && ast.getSymbol().isIsList() ? "+=" : "=")); addToCodeSection(rulename); } - - if (embeddedJavaCode) { - // Add Actions - if (isAttribute) { - if (ast.getSymbol().isIsList()) { - addToAction(astActions.getActionForTerminalIteratedAttribute(ast)); - } else { - addToAction(astActions.getActionForTerminalNotIteratedAttribute(ast)); - } - } else { - addToAction(astActions.getActionForTerminalIgnore(ast)); - } - } - addActionToCodeSection(); if (isAttribute && isListOrOpt) { @@ -576,13 +532,13 @@ public void visit(ASTTerminal ast) { } - protected String createKeyPredicate(List stringList) { + protected String createKeyPredicate(List stringList, String tmpNamePlusLbl) { String rulename = "("; String sep = ""; for (String key : stringList) { rulename += sep; sep = " | "; - rulename += parserHelper.getKeyRuleName(key); + rulename += tmpNamePlusLbl + parserHelper.getKeyRuleName(key); } rulename += ")"; @@ -594,36 +550,27 @@ public void visit(ASTKeyTerminal ast) { startCodeSection("ASTKeyTerminal " + ast.getName()); addToCodeSection("("); - String rulename = createKeyPredicate(ast.getKeyConstant().getStringList()); - if (grammarEntry.getReplacedKeywordsWithInherited().containsKey(ast.getName())) { + String labelAssignment = ast.isPresentSymbol() && ast.getSymbol().isIsList() ? "+=" : "="; + + String tmpVarName = parserHelper.getTmpVarName(ast); + var replacedAdditionalKeywords = grammarEntry.getReplacedKeywordsWithInherited(); + if (replacedAdditionalKeywords.containsKey(ast.getName())) { + handleTerminal(labelAssignment, tmpVarName, replacedAdditionalKeywords, ast.getName()); + } else if (ast.getKeyConstant().sizeStrings() == 1) { + // Extra case for one-long strings + addToCodeSection(tmpVarName + labelAssignment + parserHelper.getKeyRuleName(ast.getKeyConstant().getString(0))); + } else { + // replace keyword is currently not implemented for multiple key addToCodeSection("("); String seperator = ""; - for (String replaceString: grammarEntry.getAdditionalKeywords().get(ast.getName())) { + int nkIndex = 0; + for (String keyString : ast.getKeyConstant().getStringList()) { addToCodeSection(seperator); - if (grammarInfo.getKeywordRules().contains(replaceString)) { - addToCodeSection(parserHelper.getKeyRuleName(replaceString)); - } else { - addToCodeSection(parserHelper.getLexSymbolName(replaceString)); - } - seperator = " | "; + addToCodeSection(tmpVarName + "_key" + nkIndex++ + labelAssignment + parserHelper.getKeyRuleName(keyString)); + seperator = "|"; } addToCodeSection(")"); - } else { - addToCodeSection(rulename); - } - - if (embeddedJavaCode) { - boolean isAttribute = ast.isPresentUsageName(); - boolean isList = ast.isPresentSymbol() && ast.getSymbol().isIsList(); - // Add Actions - if (isAttribute) { - if (isList) { - addToAction(astActions.getActionForKeyTerminalIteratedAttribute(ast)); - } else { - addToAction(astActions.getActionForKeyTerminalNotIteratedAttribute(ast)); - } - } } addActionToCodeSection(); @@ -634,40 +581,36 @@ public void visit(ASTKeyTerminal ast) { } + void handleTerminal(String labelAssignment, String tmpVarName, Map> replacedAdditionalKeywords, String name) { + addToCodeSection("("); + String seperator = ""; + int nokeywordindex = 0; + for (String replaceString: replacedAdditionalKeywords.get(name)) { + addToCodeSection(seperator); + addToCodeSection(tmpVarName + "_nk" + nokeywordindex++); + addToCodeSection(labelAssignment); + if (grammarInfo.getKeywordRules().contains(replaceString)) { + addToCodeSection(parserHelper.getKeyRuleName(replaceString)); + } else { + addToCodeSection(parserHelper.getLexSymbolName(replaceString)); + } + seperator = " | "; + } + addToCodeSection(")"); + } + @Override public void visit(ASTTokenTerminal ast) { - startCodeSection("ASTTokenTerminal " + ast.getName()); addToCodeSection("("); - if (grammarEntry.getReplacedKeywordsWithInherited().containsKey(ast.getName())) { - addToCodeSection("("); - String seperator = ""; - for (String replaceString: grammarEntry.getAdditionalKeywords().get(ast.getName())) { - addToCodeSection(seperator); - if (grammarInfo.getKeywordRules().contains(replaceString)) { - addToCodeSection(parserHelper.getKeyRuleName(replaceString)); - } else { - addToCodeSection(parserHelper.getLexSymbolName(replaceString)); - } - seperator = " | "; - } - addToCodeSection(")"); + String labelAssignment = ast.isPresentSymbol() && ast.getSymbol().isIsList() ? "+=" : "="; + String tmpVarName = parserHelper.getTmpVarName(ast); + var replacedAdditionalKeywords = grammarEntry.getReplacedKeywordsWithInherited(); + if (replacedAdditionalKeywords.containsKey(ast.getName())) { + handleTerminal(labelAssignment, tmpVarName, replacedAdditionalKeywords, ast.getName()); } else { - addToCodeSection(parserHelper.getLexSymbolName(ast.getTokenConstant().getString())); - } - - if (embeddedJavaCode) { - boolean isAttribute = ast.isPresentUsageName(); - boolean isList = ast.isPresentSymbol() && ast.getSymbol().isIsList(); - // Add Actions - if (isAttribute) { - if (isList) { - addToAction(astActions.getActionForTerminalIteratedAttribute(ast)); - } else { - addToAction(astActions.getActionForTerminalNotIteratedAttribute(ast)); - } - } + addToCodeSection(tmpVarName + labelAssignment + parserHelper.getLexSymbolName(ast.getTokenConstant().getString())); } addActionToCodeSection(); @@ -786,7 +729,7 @@ public void visit(ASTNonTerminal ast) { Optional prod = grammarEntry.getProdWithInherited(ast.getName()); if (!prod.isPresent()) { Log.error("0xA2201 Production symbol for " + ast.getName() + " couldn't be resolved.", - ast.get_SourcePositionStart()); + ast.get_SourcePositionStart()); } // Lexer Rule if (prod.get().isIsLexerProd()) { @@ -794,12 +737,12 @@ public void visit(ASTNonTerminal ast) { } // Other Rule called else if (prod.get().isParserProd() - || - prod.get().isIsInterface() - || - prod.get().isIsAbstract() - || - prod.get().isIsEnum()) { + || + prod.get().isIsInterface() + || + prod.get().isIsAbstract() + || + prod.get().isIsEnum()) { addCodeForRuleReference(ast); } @@ -884,6 +827,7 @@ public PredicatePair getPredicatePair() { public List createAntlrCodeForInterface(ProdSymbol interfaceRule) { clearAntlrCode(); + parserHelper.resetTmpVarNames(); String interfacename = interfaceRule.getName(); // Dummy rules @@ -893,9 +837,6 @@ public List createAntlrCodeForInterface(ProdSymbol interfaceRule) { startCodeSection(interfaceRule.getName()); addToAntlrCode(getRuleNameForAntlr(interfacename)); - if (embeddedJavaCode) { - addToAntlrCode(" returns [" + usageName + " ret]"); - } addToAntlrCode(": "); List alts = new ArrayList<>(); @@ -905,8 +846,8 @@ public List createAntlrCodeForInterface(ProdSymbol interfaceRule) { // Append sorted alternatives Collections.sort(alts, (p2, p1) -> - new Integer(p1.getPredicatePair().getRuleReference().isPresentPrio() ? p1.getPredicatePair().getRuleReference().getPrio() : "0").compareTo( - new Integer(p2.getPredicatePair().getRuleReference().isPresentPrio() ? p2.getPredicatePair().getRuleReference().getPrio() : "0"))); + new Integer(p1.getPredicatePair().getRuleReference().isPresentPrio() ? p1.getPredicatePair().getRuleReference().getPrio() : "0").compareTo( + new Integer(p2.getPredicatePair().getRuleReference().isPresentPrio() ? p2.getPredicatePair().getRuleReference().getPrio() : "0"))); for (NodePair entry : alts) { addToAntlrCode(del); @@ -923,61 +864,20 @@ public List createAntlrCodeForInterface(ProdSymbol interfaceRule) { // Left recursive rule ASTAlt alt = (ASTAlt) entry.getAlternative(); String className = entry.getPredicatePair().getClassname(); - if (embeddedJavaCode) { - // Generate code for left recursive alternatives - addToAction(astActions.getActionForAltBeforeRuleBody(className, alt)); - // Action for determining positions - addToAction(positionActions.startPosition()); - // Action for determining positions of comments (First set position) - addToAction("setActiveBuilder(_builder);\n"); - if (!leftRecursionDetector.isAlternativeLeftRecursive(alt, interfacename)) { - // If the alternative is left-recursive, the code may only - // be saved later. With left-recursive rules, Antlr expects the first - // element to be the recursive element. - // Code is also not allowed at this point. - // But for the other rules it is necessary that the code - // is saved first - addActionToCodeSection(); - endCodeSection(); - } - } + alt.accept(getTraverser()); - if (embeddedJavaCode) { - addToAction(positionActions.endPosition()); - addToAction(astActions.getBuildAction()); - addActionToCodeSection(); - endCodeSection(); - } } else { if (left && entry.getAlternative() instanceof ASTClassProd && ((ASTClassProd) entry.getAlternative()).getAltList().size() == 1) { ASTAlt alt = ((ASTClassProd) entry.getAlternative()).getAltList().get(0); String className = entry.getPredicatePair().getClassname(); - if (embeddedJavaCode) { - // Generate code for left recursive alternatives - addToAction(astActions.getActionForAltBeforeRuleBody(className, alt)); - // Action for determining positions - addToAction(positionActions.startPosition()); - // Action for determining positions of comments (First set position) - addToAction("setActiveBuilder(_builder);\n"); - startCodeSection(); - addActionToCodeSection(); - endCodeSection(); - } alt.accept(getTraverser()); - addToAction(positionActions.endPosition()); - addToAction(astActions.getBuildAction()); - addActionToCodeSectionWithNewLine(); endCodeSection(); } else { // normal rule startCodeSection(); String tmpVar = parserHelper.getTmpVarName(entry.getAlternative()); addToCodeSection(tmpVar + "=" - + getRuleNameForAntlr(entry.getPredicatePair().getClassname())); - if (embeddedJavaCode) { - // Action for AntLR4 - addToCodeSection("\n{$ret=$" + tmpVar + ".ret;}"); - } + + getRuleNameForAntlr(entry.getPredicatePair().getClassname())); endCodeSection(); } } @@ -989,6 +889,8 @@ public List createAntlrCodeForInterface(ProdSymbol interfaceRule) { addToAntlrCode(";"); + tmpNameDict.put(interfaceRule.getAstNode(), new LinkedHashMap<>(parserHelper.getTmpVariables())); + return getAntlrCode(); } @@ -1048,7 +950,11 @@ protected void addCodeForLexerRule(ASTNonTerminal ast) { // AntLR2 -> AntLR4: Replace : by = // tmp = "( %tmp%=%rulename% %initaction% %actions%"; - addToCodeSection(parserHelper.getTmpVarName(ast), "=", ast.getName()); + // In case the AST has a list of this rule (*/+), we use += instead of = in the g4 rule + boolean isRuleIterated = getASTMax(ast); + + String tmpName = parserHelper.getTmpVarName(ast); + addToCodeSection(tmpName, isRuleIterated ? "+=" : "=", ast.getName()); if (embeddedJavaCode) { // Add Actions @@ -1058,12 +964,6 @@ protected void addCodeForLexerRule(ASTNonTerminal ast) { addToAction(attributeConstraints.addActionForNonTerminal(ast)); String attributename = ast.isPresentUsageName() ? ast.getUsageName() : StringTransformations.uncapitalize(ast.getName()); List rcs = scope.get().getSpannedScope().resolveRuleComponentDownMany(attributename); - if (!rcs.isEmpty() - && rcs.get(0).isIsList()) { - addToAction(astActions.getActionForLexerRuleIteratedAttribute(ast)); - } else { - addToAction(astActions.getActionForLexerRuleNotIteratedAttribute(ast)); - } } addActionToCodeSection(); @@ -1075,23 +975,23 @@ protected void addCodeForLexerRule(ASTNonTerminal ast) { if (ast.isPlusKeywords()) { addToAntlrCode("/* Automatically added keywords " + grammarInfo.getKeywords() - + " */"); + + " */"); ArrayList keys = Lists.newArrayList(grammarInfo.getKeywords()); keys.removeAll(grammarInfo.getKeywordRules()); for (String y : keys) { addToAntlrCode(" | "); ASTTerminal term = Grammar_WithConceptsMill.terminalBuilder() - .setName(y).build(); + .setName(y).build(); if (ast.isPresentSymbol()) { RuleComponentSymbol componentSymbol = ast.getSymbol(); Optional rule = MCGrammarSymbolTableHelper - .getEnclosingRule(componentSymbol); + .getEnclosingRule(componentSymbol); term.setUsageName(ParserGeneratorHelper.getUsageName(ast)); if (rule.isPresent()) { - addActionForKeyword(term, rule.get(), componentSymbol.isIsList()); + addActionForKeyword(term, rule.get(), componentSymbol.isIsList(), tmpName + (isRuleIterated?"+=":"=")); } } } @@ -1107,6 +1007,9 @@ protected String embedded(ASTNonTerminal ast) { return ""; } + // Stack for Block iterations (Prod)* => should result in tmp += + protected Stack ruleIteratedStack = new Stack<>(); + /** * Print code for references to other rules in the same grammar * @@ -1120,10 +1023,10 @@ protected void addCodeForRuleReference(ASTNonTerminal ast) { } boolean isLeftRecursive = false; if (scope.get().getName().equals(ast.getName()) - && !altList.isEmpty()) { + && !altList.isEmpty()) { // Check if rule is left recursive isLeftRecursive = leftRecursionDetector - .isAlternativeLeftRecursive(altList.get(0), ast); + .isAlternativeLeftRecursive(altList.get(0), ast); } startCodeSection(); @@ -1137,22 +1040,14 @@ protected void addCodeForRuleReference(ASTNonTerminal ast) { String tmpVarName = parserHelper.getTmpVarName(ast); - addToCodeSection(braceopen, " ", tmpVarName, "=", getRuleNameForAntlr(ast.getName())); + // In case the AST has a list of this rule (*/+), we use += instead of = in the g4 rule + boolean isRuleIterated = getASTMax(ast); + + String label = isRuleIterated ? "+=":"="; + addToCodeSection(braceopen, " ", tmpVarName, label, getRuleNameForAntlr(ast.getName())); if (embeddedJavaCode) { - if (isLeftRecursive) { - addToAction(astActions - .getActionForInternalRuleNotIteratedLeftRecursiveAttribute(ast)); - } addToAction(attributeConstraints.addActionForNonTerminal(ast)); - String attributename = ast.isPresentUsageName() ? ast.getUsageName() : StringTransformations.uncapitalize(ast.getName()); - List rcs = scope.get().getSpannedScope().resolveRuleComponentDownMany(attributename); - if (!rcs.isEmpty() && rcs.get(0).isIsList()) { - addToAction(astActions.getActionForInternalRuleIteratedAttribute(ast)); - } else { - addToAction(astActions.getActionForInternalRuleNotIteratedAttribute(ast)); - } - addActionToCodeSection(); } @@ -1162,7 +1057,23 @@ protected void addCodeForRuleReference(ASTNonTerminal ast) { } - protected void addActionForKeyword(ASTTerminal keyword, ProdSymbol rule, boolean isList) { + boolean getASTMax(ASTNonTerminal ast) { + String usageName = ParserGeneratorHelper.getUsageName(ast); + Optional max = ast.getEnclosingScope().getLocalAdditionalAttributeSymbols() + .stream() + .filter(e -> e.getName().equals(usageName)) + .map(TransformationHelper::getMax) + .map(e->e.orElse(1)) + .findFirst(); + + boolean isRuleIterated = (!ruleIteratedStack.isEmpty() && ruleIteratedStack.peek()) || isIterated(ast.getIteration()); + + if (max.isPresent() && max.get() == 1) + return false; // an astrule forcefully set the cardinality + return isRuleIterated; + } + + protected void addActionForKeyword(ASTTerminal keyword, ProdSymbol rule, boolean isList, String tmpNamePlusLbl) { startCodeSection(); @@ -1174,25 +1085,7 @@ protected void addActionForKeyword(ASTTerminal keyword, ProdSymbol rule, boolean // No actions in predicates // Template engine cannot be used for substition in rare cases - addToCodeSection(rulename); // + " %initaction% %actions% ) %iteration% "; - - if (embeddedJavaCode) { - boolean isAttribute = (keyword.isPresentUsageName()); - - // Add Actions - if (isAttribute) { - if (isList) { - addToAction(astActions.getActionForTerminalIteratedAttribute(keyword)); - } else { - addToAction(astActions.getActionForTerminalNotIteratedAttribute(keyword)); - } - } else { - addToAction(astActions.getActionForTerminalIgnore(keyword)); - } - - addActionToCodeSection(); - } - + addToCodeSection(tmpNamePlusLbl + rulename); // + " %initaction% %actions% ) %iteration% "; addToCodeSection(")", printIteration(keyword.getIteration())); endCodeSection(); @@ -1374,4 +1267,8 @@ public GrammarTraverser getTraverser() { public void setTraverser(GrammarTraverser traverser) { this.traverser = traverser; } -} \ No newline at end of file + + public boolean isIterated(int iteration) { + return iteration == ASTConstantsGrammar.STAR || iteration == ASTConstantsGrammar.PLUS; + } +} diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2ParseVisitor.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2ParseVisitor.java new file mode 100644 index 0000000000..e2e73c0391 --- /dev/null +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2ParseVisitor.java @@ -0,0 +1,801 @@ +/* (c) https://github.com/MontiCore/monticore */ +package de.monticore.codegen.parser.antlr; + +import de.monticore.ast.ASTNode; +import de.monticore.cd.facade.*; +import de.monticore.cd.methodtemplates.CD4C; +import de.monticore.cd4code.CD4CodeMill; +import de.monticore.cd4codebasis._ast.ASTCDConstructor; +import de.monticore.cd4codebasis._ast.ASTCDMethod; +import de.monticore.cdbasis.CDBasisMill; +import de.monticore.cdbasis._ast.ASTCDClass; +import de.monticore.codegen.cd2java._ast.ast_class.ASTConstants; +import de.monticore.codegen.mc2cd.TransformationHelper; +import de.monticore.codegen.parser.MCGrammarInfo; +import de.monticore.codegen.parser.ParserGeneratorHelper; +import de.monticore.generating.templateengine.GlobalExtensionManagement; +import de.monticore.generating.templateengine.HookPoint; +import de.monticore.generating.templateengine.StringHookPoint; +import de.monticore.generating.templateengine.TemplateHookPoint; +import de.monticore.grammar.MCGrammarSymbolTableHelper; +import de.monticore.grammar.PredicatePair; +import de.monticore.grammar.grammar.GrammarMill; +import de.monticore.grammar.grammar._ast.*; +import de.monticore.grammar.grammar._symboltable.MCGrammarSymbol; +import de.monticore.grammar.grammar._symboltable.ProdSymbol; +import de.monticore.grammar.grammar._symboltable.RuleComponentSymbol; +import de.monticore.grammar.grammar._visitor.GrammarHandler; +import de.monticore.grammar.grammar._visitor.GrammarTraverser; +import de.monticore.grammar.grammar._visitor.GrammarVisitor2; +import de.monticore.grammar.grammar_withconcepts.Grammar_WithConceptsMill; +import de.monticore.types.MCTypeFacade; +import de.monticore.types.mcbasictypes._ast.ASTMCType; +import de.se_rwth.commons.Joiners; +import de.se_rwth.commons.Names; +import de.se_rwth.commons.StringTransformations; +import de.se_rwth.commons.logging.Log; +import org.antlr.v4.runtime.BufferedTokenStream; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.RuleNode; +import org.antlr.v4.runtime.tree.TerminalNode; + +import javax.annotation.Nullable; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static de.monticore.cd.codegen.CD2JavaTemplates.EMPTY_BODY; + +/** + * Generates a class, which transforms the ANTLR ParseTree into an AST + * For moe information, see de.monticore.antlr4.MCBuildVisitor + *

+ * Parser-Preconditions, actions and concepts will still be present in the parser. + * (Although without access to the AST-builders) + */ +public class Grammar2ParseVisitor implements GrammarVisitor2, GrammarHandler { + + protected GrammarTraverser grammarTraverser; + + protected final GlobalExtensionManagement glex; + protected final ParserGeneratorHelper parserGeneratorHelper; + protected final MCGrammarInfo grammarInfo; + + protected final Stack stack = new Stack<>(); + protected final Map> tmpNameDict; + + + protected CDMethodFacade cdMethodFacade = CDMethodFacade.getInstance(); + protected CDConstructorFacade cdConstructorFacade = CDConstructorFacade.getInstance(); + protected CDAttributeFacade cdAttributeFacade = CDAttributeFacade.getInstance(); + protected CDParameterFacade cdParameterFacade = CDParameterFacade.getInstance(); + protected MCTypeFacade mcTypeFacade = MCTypeFacade.getInstance(); + + protected ASTCDClass visitorClass; + protected String antlrParserName; + protected String millName; + + protected Map tmpNames; + + protected ASTMCType astNodeType = mcTypeFacade.createQualifiedType(ASTNode.class.getName()); + protected ASTMCType objectType = mcTypeFacade.createQualifiedType(Object.class.getName()); + + protected Stack ruleIteratedStack = new Stack<>(); + + public Grammar2ParseVisitor(GlobalExtensionManagement glex, ParserGeneratorHelper parserGeneratorHelper, MCGrammarInfo grammarInfo, Map> tmpNameDict) { + this.glex = glex; + this.parserGeneratorHelper = parserGeneratorHelper; + this.grammarInfo = grammarInfo; + + this.tmpNameDict = tmpNameDict; + } + + public ASTCDClass getVisitorClass() { + return visitorClass; + } + + @Override + public GrammarTraverser getTraverser() { + return grammarTraverser; + } + + @Override + public void setTraverser(GrammarTraverser traverser) { + this.grammarTraverser = traverser; + } + + @Override + public void visit(ASTMCGrammar node) { + antlrParserName = StringTransformations.capitalize(node.getName()) + "AntlrParser"; + millName = parserGeneratorHelper.getQualifiedGrammarName().toLowerCase() + "." + StringTransformations.capitalize(node.getName()) + "Mill"; + visitorClass = CDBasisMill.cDClassBuilder() + .setName(node.getName() + "ASTBuildVisitor") + .setModifier(CDModifier.PUBLIC.build()) + .build(); + visitorClass.setCDInterfaceUsage( + CD4CodeMill.cDInterfaceUsageBuilder() + .addInterface( + mcTypeFacade.createBasicGenericTypeOf(node.getName() + "AntlrParserVisitor", "Object")) + .build()); + visitorClass.setCDExtendUsage(CDBasisMill.cDExtendUsageBuilder().addSuperclass(mcTypeFacade.createQualifiedType("de.monticore.antlr4.MCBuildVisitor")).build()); + + + ASTCDConstructor constructor = cdConstructorFacade.createConstructor(CDModifier.PUBLIC.build(), visitorClass.getName(), + cdParameterFacade.createParameter(String.class, "filename"), + cdParameterFacade.createParameter(BufferedTokenStream.class.getName(), "tokenStream") + ); + visitorClass.addCDMember(constructor); + glex.replaceTemplate(EMPTY_BODY, constructor, new TemplateHookPoint("_parser.visitor.Constructor")); + + CD4C.getInstance().addImport(visitorClass, "de.monticore.ast.ASTNode"); + CD4C.getInstance().addImport(visitorClass, String.join(".", parserGeneratorHelper.getQualifiedGrammarName().toLowerCase(), "_ast.*")); + CD4C.getInstance().addImport(visitorClass, millName); + CD4C.getInstance().addImport(visitorClass, "org.antlr.v4.runtime.tree.*"); + CD4C.getInstance().addImport(visitorClass, "org.antlr.v4.runtime.Token"); + grammarInfo.getGrammarSymbol().getAllSuperGrammars().forEach(superg -> + CD4C.getInstance().addImport(visitorClass, String.join(".", superg.getFullName().toLowerCase(), "_ast.*"))); + + visitorClass.addCDMember(cdAttributeFacade.createAttribute(CDModifier.PUBLIC.build(), mcTypeFacade.createIntType(), "depth")); + + // add visit(ParseTree) -> tree.accept + ASTCDMethod m; + visitorClass.addCDMember((m = cdMethodFacade.createMethod(CDModifier.PUBLIC.build(), objectType, "visit", cdParameterFacade.createParameter(ParseTree.class, "tree")))); + glex.replaceTemplate(EMPTY_BODY, m, new TemplateHookPoint("_parser.visitor.VisitTree")); + // add visitChildren(RuleNode) -> EXC + visitorClass.addCDMember((m = cdMethodFacade.createMethod(CDModifier.PUBLIC.build(), astNodeType, "visitChildren", cdParameterFacade.createParameter(RuleNode.class, "node")))); + glex.replaceTemplate(EMPTY_BODY, m, new TemplateHookPoint("_parser.visitor.NotImplemented")); + // add visitErrorNode + visitorClass.addCDMember((m = cdMethodFacade.createMethod(CDModifier.PUBLIC.build(), astNodeType, "visitErrorNode", cdParameterFacade.createParameter(ErrorNode.class, "node")))); + glex.replaceTemplate(EMPTY_BODY, m, new TemplateHookPoint("_parser.visitor.NotImplemented")); + // add visitTerminal + visitorClass.addCDMember((m = cdMethodFacade.createMethod(CDModifier.PUBLIC.build(), astNodeType, "visitTerminal", cdParameterFacade.createParameter(TerminalNode.class, "node")))); + glex.replaceTemplate(EMPTY_BODY, m, new TemplateHookPoint("_parser.visitor.NotImplemented")); + + // noKeyword and splittoken introduces terminals, which require a visit method + List pseudoProductions = new ArrayList<>(grammarInfo.getSplitRules().values()); + for (String s : grammarInfo.getGrammarSymbol().getKeywordRulesWithInherited()) { + pseudoProductions.add(parserGeneratorHelper.getKeyRuleName(s)); + } + for (String s : pseudoProductions) { + + String ruleNameCap = StringTransformations.capitalize(s); + + ASTCDMethod method = cdMethodFacade.createMethod(CDModifier.PUBLIC.build(), + astNodeType, + "visit" + ruleNameCap, + cdParameterFacade.createParameter(antlrParserName + "." + ruleNameCap + "Context", "ctx")); + glex.replaceTemplate(EMPTY_BODY, method, new TemplateHookPoint("_parser.visitor.NotImplemented")); + + visitorClass.addCDMember(method); + } + + } + + + @Override + public void traverse(ASTMCGrammar node) { + // Modified traversal: We traverse over all class and interface productions of this grammar and its super grammars + node.getSymbol().getProdsWithInherited().values().forEach(symbol -> symbol.getAstNode().accept(getTraverser())); + } + + @Override + public void handle(ASTExternalProd node) { + // nothing + } + + @Override + public void handle(ASTClassProd ast) { + if (ast.getSymbol().isIsIndirectLeftRecursive()) { + // No CodeGen + return; + } + String rulename = Grammar2Antlr.getRuleNameForAntlr(ast.getName()); + String rulenameCap = StringTransformations.capitalize(rulename); + + tmpNames = tmpNameDict.get(ast); + if (tmpNames == null) { + // No parser generation for this node (it is most likely overriden) + return; + } + // Rules extending, etc. + List subRules = grammarInfo.getSubRulesForParsing(ast.getName()); + + ASTCDMethod method = cdMethodFacade.createMethod(CDModifier.PUBLIC.build(), + astNodeType, + "visit" + rulenameCap, + cdParameterFacade.createParameter(antlrParserName + "." + rulenameCap + "Context", "ctx")); + + ParseVisitorEntry root = new ParseVisitorEntry(); + stack.push(root); + + List alts = parserGeneratorHelper.getAlternatives(ast); + + if (subRules != null && !subRules.isEmpty()) { + int i = 0; + for (PredicatePair ignored : subRules) { + ParseVisitorEntry subruleAlt = new ParseVisitorEntry(); + + root.alternatives.add(subruleAlt); + + ParseVisitorEntry e = new ParseVisitorEntry(); + e.subrule = "subRuleVar" + i++; + + subruleAlt.blockComponents.add(e); + subruleAlt.condition = "ctx." + e.subrule + " != null"; + } + } + + alts.forEach(alt -> alt.accept(getTraverser())); + + root.isRoot = true; + + this.glex.replaceTemplate(EMPTY_BODY, method, new TemplateHookPoint("_parser.visitor.VisitClassProd", + ast.getName(), millName, root, + ast.isPresentAction() ? Optional.of(GrammarMill.prettyPrint(ast.getAction(), false)) : Optional.empty())); + visitorClass.addCDMember(method); + + + stack.pop(); + + } + + @Override + public void handle(ASTEnumProd ast) { + String rulename = Grammar2Antlr.getRuleNameForAntlr(ast.getName()); + String rulenameCap = StringTransformations.capitalize(rulename); + String enumClass = MCGrammarSymbolTableHelper.getQualifiedName(ast, ast.getSymbol(), "AST", ""); + ASTCDMethod method = cdMethodFacade.createMethod(CDModifier.PUBLIC.build(), + mcTypeFacade.createQualifiedType(enumClass), + "visit" + rulenameCap, + cdParameterFacade.createParameter(antlrParserName + "." + rulenameCap + "Context", "ctx")); + + visitorClass.addCDMember(method); + + List enumConstants = ast.getConstantList().stream().map(parserGeneratorHelper::getConstantNameForConstant) + .map(cn -> enumClass + "." + cn).collect(Collectors.toList()); + + glex.replaceTemplate(EMPTY_BODY, method, new TemplateHookPoint("_parser.visitor.VisitEnum", enumConstants)); + } + + @Override + public void handle(ASTInterfaceProd ast) { + handleInterfaceOrAbstract(ast, ast.getSymbol(), ast.getName()); + } + + @Override + public void handle(ASTAbstractProd ast) { + handleInterfaceOrAbstract(ast, ast.getSymbol(), ast.getName()); + } + + public void handleInterfaceOrAbstract(ASTProd ast, ProdSymbol symbol, String name) { + String rulename = Grammar2Antlr.getRuleNameForAntlr(name); + String rulenameCap = StringTransformations.capitalize(rulename); + + tmpNames = tmpNameDict.get(ast); + + List alts = new ArrayList<>(); + boolean isLeft = collectAlts(symbol, alts); + alts.sort(AltEntry::compareTo); + + for (AltEntry alt : alts) { + + ParseVisitorEntry e = new ParseVisitorEntry(); + if (alt.getRuleNode() instanceof ASTAlt) { + // Left recursive rule + stack.push(e); + alt.getRuleNode().accept(getTraverser()); + stack.pop(); + alt.setParseVisitorEntry(e.getAlternatives().get(0)); + } else if (isLeft && alt.getRuleNode() instanceof ASTClassProd && ((ASTClassProd) alt.getRuleNode()).getAltList().size() == 1) { + stack.push(e); + ((ASTClassProd) alt.getRuleNode()).getAlt(0).accept(getTraverser()); + stack.pop(); + if (e.getAlternatives().size() != 1) { + Log.error("0xA0393 Unspected count of alternatives. Expected 1, butfound " + e.getAlternatives().size(), + alt.getRuleNode().get_SourcePositionStart()); + return; + } + alt.setParseVisitorEntry(e.getAlternatives().get(0)); + } else { + // normal rule - like a NonTerminal + String tmpVar = tmpNames.get(alt.getRuleNode()); + + alt.setParseVisitorEntry(e); + e.tmpName = tmpVar; + e.condition = "ctx." + tmpVar + " != null"; + alt.setSimpleReference(true); + } + } + + + ASTCDMethod method = cdMethodFacade.createMethod(CDModifier.PUBLIC.build(), + astNodeType, + "visit" + rulenameCap, + cdParameterFacade.createParameter(antlrParserName + "." + rulenameCap + "Context", "ctx")); + glex.replaceTemplate(EMPTY_BODY, method, new TemplateHookPoint("_parser.visitor.VisitInterface", name, millName, alts)); + + visitorClass.addCDMember(method); + } + + protected boolean collectAlts(ProdSymbol prodSymbol, List alts) { + boolean isLeft = false; + List interfaces = grammarInfo.getSubRulesForParsing(prodSymbol.getName()); + for (PredicatePair interf : interfaces) { + Optional symbol = parserGeneratorHelper.getGrammarSymbol().getSpannedScope().resolveProd(interf.getClassname()); + if (symbol.isEmpty()) { + continue; + } + ProdSymbol superSymbol = symbol.get(); + if (!prodSymbol.isPresentAstNode()) { + continue; + } + if (superSymbol.isIsIndirectLeftRecursive()) { + isLeft = true; + if (superSymbol.isClass()) { + List localAlts = ((ASTClassProd) superSymbol.getAstNode()).getAltList(); + for (ASTAlt alt : localAlts) { + alts.add(new AltEntry(alt, (ASTParserProd) superSymbol.getAstNode(), interf)); + } + } else if (prodSymbol.isIsInterface()) { + collectAlts(superSymbol, alts); + } + } else { + alts.add(new AltEntry(superSymbol.getAstNode(), superSymbol.getAstNode(), interf)); + } + } + return isLeft; + } + + public static class AltEntry implements Comparable { + protected final ASTNode ruleNode; + protected final ASTProd builderNode; + protected final PredicatePair pair; + + protected ParseVisitorEntry parseVisitorEntry; + + protected final int prio; + + protected boolean simpleReference; + + public AltEntry(ASTNode ruleNode, ASTProd builderNode, PredicatePair pair) { + this.ruleNode = ruleNode; + this.builderNode = builderNode; + this.pair = pair; + this.prio = pair.getRuleReference().isPresentPrio() ? Integer.parseInt(pair.getRuleReference().getPrio()) : 0; + } + + @Override + public int compareTo(AltEntry o) { + return Integer.compare(o.prio, this.prio); // Highest priority first + } + + public ASTNode getRuleNode() { + return ruleNode; + } + + public ASTProd getBuilderNode() { + return builderNode; + } + + public PredicatePair getPair() { + return pair; + } + + public ParseVisitorEntry getParseVisitorEntry() { + return parseVisitorEntry; + } + + public void setParseVisitorEntry(ParseVisitorEntry parseVisitorEntry) { + this.parseVisitorEntry = parseVisitorEntry; + } + + public void setSimpleReference(boolean simpleReference) { + this.simpleReference = simpleReference; + } + + public boolean isSimpleReference() { + return simpleReference; + } + + public String getBuilderNodeName() { + return this.getBuilderNode().getName(); + } + } + + protected Set convertMethods = new HashSet<>(); + + @Override + public void handle(ASTLexProd node) { + if (node.getSymbol().isIsExternal()) return; + // add convert + if (convertMethods.contains(node.getName())) return; + convertMethods.add(node.getName()); + + ASTMCType retType = mcTypeFacade.createStringType(); + HookPoint hookPoint = new StringHookPoint("return t.getText();" + "/*" + node.getSymbol().getFullName() + "*/"); + String tokenParamName = "t"; + if (node.isPresentVariable()) { + if (node.getTypeList() == null || node.getTypeList().isEmpty()) { + switch (node.getVariable()) { + case "int": + retType = mcTypeFacade.createIntType(); + hookPoint = new StringHookPoint("return Integer.parseInt(t.getText());"); + break; + case "boolean": + retType = mcTypeFacade.createBooleanType(); + hookPoint = new StringHookPoint("return (t.getText().equals(\"1\")||t.getText().equals(\"start\")||t.getText().equals(\"on\")||t.getText().equals(\"true\"));"); + break; + case "byte": + retType = mcTypeFacade.createQualifiedType("byte"); + hookPoint = new StringHookPoint("return Byte.parseByte(t.getText());"); + break; + case "char": + retType = mcTypeFacade.createQualifiedType("char"); + hookPoint = new StringHookPoint("return t.getText().charAt(0);"); + break; + case "float": + retType = mcTypeFacade.createQualifiedType("float"); + hookPoint = new StringHookPoint("return Float.parseFloat(t.getText());"); + break; + case "double": + retType = mcTypeFacade.createQualifiedType("double"); + hookPoint = new StringHookPoint("return Double.parseDouble(t.getText());"); + break; + case "long": + retType = mcTypeFacade.createQualifiedType("long"); + hookPoint = new StringHookPoint("return Long.parseLong(t.getText());"); + break; + case "short": + retType = mcTypeFacade.createQualifiedType("short"); + hookPoint = new StringHookPoint("return Short.parseShort(t.getText());"); + break; + case "card": + retType = mcTypeFacade.createQualifiedType("int"); + hookPoint = new StringHookPoint("return t.getText().equals(\"*\")?-1 : Integer.parseInt(t.getText());"); + break; + default: + Log.warn( + "0xA1061 No function for " + node.getVariable() + " registered, will treat it as string!"); + } + } else if (node.isPresentBlock()) { // specific function + retType = mcTypeFacade.createQualifiedType(Names.constructQualifiedName(node.getTypeList())); + hookPoint = new StringHookPoint(Grammar_WithConceptsMill.prettyPrint(node.getBlock(), true)); + tokenParamName = node.getVariable(); + } + } + ASTCDMethod method = cdMethodFacade.createMethod(CDModifier.PROTECTED.build(), + retType, + "convert" + StringTransformations.capitalize(node.getName()), + cdParameterFacade.createParameter("Token", tokenParamName) + ); + glex.replaceTemplate(EMPTY_BODY, method, hookPoint); + visitorClass.addCDMember(method); + + } + + + @Override + public void handle(ASTAlt node) { + ParseVisitorEntry prev = stack.peek(); + ParseVisitorEntry alt = new ParseVisitorEntry(); + prev.alternatives.add(alt); + stack.push(alt); + GrammarHandler.super.handle(node); + stack.pop(); + + alt.blockComponents.forEach(c -> alt.allOptConditions.addAll(c.allOptConditions)); + + List nonOptEntries = alt.blockComponents.stream().filter(e -> e.condition != null).filter(f -> !f.ruleOptional).collect(Collectors.toList()); + if (nonOptEntries.isEmpty()) { + if (alt.allOptConditions.stream().noneMatch(Objects::nonNull)) + alt.condition = null; + else + alt.condition = ("/* alt all opt */ (" + String.join("||", alt.allOptConditions) + ")"); + } else { + alt.condition = "(" + nonOptEntries.stream().map(e -> e.condition).filter(Objects::nonNull).collect(Collectors.joining(" && ")) + ")"; + } + + // remove conditions, as it has been moved upwards + nonOptEntries.forEach(e -> e.condition = null); + + if (nonOptEntries.isEmpty() && alt.blockComponents.size() == 1) + alt.blockComponents.forEach(e -> e.condition = null); // remove sub ones only iff one non-default is there + + + if (alt.condition != null) + alt.condition += " /*from alt*/"; + + if (alt.blockComponents.isEmpty()) { + // Add an empty alt for the generation + prev.alternatives.remove(alt); + } + + } + + + @Override + public void handle(ASTBlock node) { + ParseVisitorEntry blockEntry = new ParseVisitorEntry(); + + blockEntry.ruleOptional = node.getIteration() == ASTConstantsGrammar.QUESTION || node.getIteration() == ASTConstantsGrammar.STAR; + ruleIteratedStack.push((!ruleIteratedStack.isEmpty() && ruleIteratedStack.peek()) || node.getIteration() == ASTConstantsGrammar.STAR || node.getIteration() == ASTConstantsGrammar.PLUS); + + ParseVisitorEntry prev = stack.peek(); + prev.blockComponents.add(blockEntry); + stack.push(blockEntry); + + GrammarHandler.super.handle(node); + + stack.pop(); + + ruleIteratedStack.pop(); + + blockEntry.alternatives.forEach(c -> blockEntry.allOptConditions.addAll(c.allOptConditions)); + + // no else for (A | B)* + if (node.getIteration() == ASTConstantsGrammar.PLUS || node.getIteration() == ASTConstantsGrammar.STAR) { + blockEntry.alternatives.forEach(c -> c.setNoAltElseRec(true)); + } + + // A | B => A || B + List nonOptEntries = blockEntry.alternatives.stream().filter(e -> e.condition != null).filter(f -> !f.ruleOptional).collect(Collectors.toList()); + if (nonOptEntries.isEmpty()) { + if (blockEntry.allOptConditions.stream().anyMatch(Objects::nonNull)) + blockEntry.condition = "/* block all opt */ (" + String.join("||", blockEntry.allOptConditions) + ")"; + else + blockEntry.condition = null; + } else { + blockEntry.condition = nonOptEntries.stream().map(e -> e.condition).filter(Objects::nonNull).collect(Collectors.joining("||")); + blockEntry.condition += " /*from block*/"; + } + + if (blockEntry.alternatives.isEmpty()) { + prev.blockComponents.remove(blockEntry); + } + + } + + @Override + public void handle(ASTNonTerminal node) { + ParseVisitorEntry e = new ParseVisitorEntry(); + + ProdSymbol ps = grammarInfo.getGrammarSymbol().getProdWithInherited(node.getName()).get(); + + // Only handle Lexer, Parser, interface, abstract and enum rules, but no external, ... rules + if (!ps.isIsLexerProd() && !ps.isParserProd() + && + !ps.isIsInterface() + && + !ps.isIsAbstract() + && + !ps.isIsEnum()) { + // External -> skip + // external rule called (first+second version) + return; + } + // In case the AST has a list of this rule (*/+), we use += instead of = in the g4 rule + String usageName = ParserGeneratorHelper.getUsageName(node); + Optional max = node.getEnclosingScope().getLocalAdditionalAttributeSymbols() + .stream() + .filter(astrule -> astrule.getName().equals(usageName)) + .map(TransformationHelper::getMax) + .map(o -> o.orElse(1)) + .findFirst(); + + boolean isRuleIterated = (!ruleIteratedStack.isEmpty() && ruleIteratedStack.peek()) || node.getIteration() == ASTConstantsGrammar.STAR || node.getIteration() == ASTConstantsGrammar.PLUS; + + if (max.isPresent() && max.get() == 1) + isRuleIterated = false; // an astrule forcefully set the cardinality + + + e.usageName = ParserGeneratorHelper.getUsageName(node); + // Use the FQN when casting + MCGrammarSymbol pdGrammar = ((ASTMCGrammar) ps.getEnclosingScope().getAstNode()).getSymbol(); + e.cast = Joiners.DOT.join(pdGrammar.getFullName().toLowerCase(), + ASTConstants.AST_PACKAGE) + ".AST" + StringTransformations.capitalize(node.getName()); + if (ps.isIsExternal()) + e.cast += "Ext"; + e.tmpName = tmpNames.get(node) != null ? tmpNames.get(node) : "TMPNAME_IS_NULL"; + e.astList = node.getSymbol().isIsList(); + e.astOptional = node.getSymbol().isIsOptional(); + e.ruleOptional = node.getIteration() == ASTConstantsGrammar.STAR || node.getIteration() == ASTConstantsGrammar.QUESTION; + e.ruleList = isRuleIterated; + e.isLexNT = ps.isIsLexerProd(); + + if (e.isLexNT) { + e.convert = "convert" + StringTransformations.capitalize(node.getName()); + } + + if (e.ruleList) + e.condition = "!ctx." + e.tmpName + ".isEmpty()"; + else + e.condition = "ctx." + e.tmpName + " != null"; + + e.allOptConditions.add(e.condition); + + stack.peek().blockComponents.add(e); + } + + + @Override + public void handle(ASTKeyTerminal node) { + this.handleTerminal(node, node::getSymbol, node.getIteration()); + } + + @Override + public void handle(ASTTokenTerminal node) { + this.handleTerminal(node, node::getSymbol, node.getIteration()); + } + + @Override + public void handle(ASTTerminal node) { + this.handleTerminal(node, node::getSymbol, node.getIteration()); + } + + protected String getRuleName(String name) { + if (name.isEmpty()) { + return ""; + } else if (grammarInfo.isKeyword(name, parserGeneratorHelper.getGrammarSymbol()) && grammarInfo.getKeywordRules().contains(name)) { + return parserGeneratorHelper.getKeyRuleName(name); + } else { + return parserGeneratorHelper.getLexSymbolName(name.intern()); + } + } + + protected void handleTerminal(ASTITerminal node, Supplier symbolSup, int iteration) { + String rulename = getRuleName(node.getName()); + + ParseVisitorEntry e = new ParseVisitorEntry(); + e.constantLexName = rulename; + stack.peek().blockComponents.add(e); + @Nullable String tmpName = tmpNames.get(node); + Map> replacedKeywords = grammarInfo.getGrammarSymbol().getReplacedKeywordsWithInherited(); + if (tmpName == null) { + Log.error("0xA0392 Missing tmpname " + visitorClass.getName(), node.get_SourcePositionStart()); + throw new IllegalStateException("Missing tmpname " + node.getEnclosingScope().getName()); + } else if (replacedKeywords.containsKey(node.getName())) { + int nokeywordindex = 0; + Map entries = new HashMap<>(); + + for (String ignored : replacedKeywords.get(node.getName())) { + // Make a new alternative out of each possible keyword + String prodname = tmpName + "_nk" + nokeywordindex++; + ParseVisitorEntry alte = new ParseVisitorEntry(); + alte.constantLexName = prodname; + alte.tmpName = prodname; + alte.condition = "ctx." + prodname + " != null /*renk*/"; + entries.put(prodname, alte); + // we might be able to combine multiple conditions together here + } + + e.alternatives.addAll(entries.values()); + + // Turn this an alternative + e.condition = "(" + entries.values().stream().map(ParseVisitorEntry::getCondition) + .collect(Collectors.joining("||")) + ")"; + } else if (node instanceof ASTKeyTerminal && ((ASTKeyTerminal) node).getKeyConstant().sizeStrings() == 1) { + e.condition = "ctx." + tmpName + " != null /*cc*/"; + } else if (node instanceof ASTKeyTerminal) { + int keyIndex = 0; + for (String ignored : ((ASTKeyTerminal) node).getKeyConstant().getStringList()) { + ParseVisitorEntry alte = new ParseVisitorEntry(); + String prodname = tmpName + "_key" + keyIndex++; + alte.constantLexName = prodname; + alte.tmpName = prodname; + alte.condition = "ctx." + prodname + " != null /*key1+*/"; + e.alternatives.add(alte); + } + e.condition = "(" + e.alternatives.stream().map(ParseVisitorEntry::getCondition).collect(Collectors.joining("||")) + ") /* key-constant */"; + } else { + e.condition = "ctx." + tmpName + " != null /*kk*/"; + } + e.allOptConditions.add(e.condition); + e.ruleOptional = iteration == ASTConstantsGrammar.STAR || iteration == ASTConstantsGrammar.QUESTION; + + // we can ignore Terminals without a usage name for the 2nd part here + if (node.isPresentUsageName()) { + RuleComponentSymbol symbol = symbolSup.get(); + e.astList = symbol.isIsList(); + e.ruleList = symbol.isIsList(); + + e.astOptional = symbol.isIsOptional(); + e.usageName = node.getUsageName(); + e.tmpName = tmpName; + if (symbol.isIsList()) { + e.astList = true; + + String usageName = node.getUsageName(); + Optional max = node.getEnclosingScope().getLocalAdditionalAttributeSymbols() + .stream() + .filter(astrule -> astrule.getName().equals(usageName)) + .map(TransformationHelper::getMax) + .map(o -> o.orElse(1)) + .findFirst(); + + if (max.isPresent() && max.get() == 1) + e.astList = false; // an astrule forcefully set the cardinality + } + + e.alternatives.forEach(ea -> { + ea.astList = e.astList; + ea.ruleList = e.ruleList; + ea.usageName = e.usageName; + }); + } + + // Note: If Actions within epeated blocks should be considered, + // a separate transformation would be necessary + // (and at least one (non)terminal within the repetition would require an implicit usage-name) + } + + @Override + public void handle(ASTConstantGroup node) { + boolean iterated = TransformationHelper + .isConstGroupIterated(node.getSymbol()); + if (!iterated) { + ASTConstant x = node.getConstantList().get(0); + + ParseVisitorEntry e = new ParseVisitorEntry(); + stack.peek().blockComponents.add(e); + String tmpName = tmpNames.get(x); + e.condition = "ctx." + tmpName + " != null"; + e.usageName = (node.isPresentUsageName()) ? node.getUsageName() : x.getHumanName(); + e.constantValue = "true"; + e.ruleOptional = node.getIteration() == ASTConstantsGrammar.STAR || node.getIteration() == ASTConstantsGrammar.QUESTION; + + e.allOptConditions.add(e.condition); + } else { + Optional ruleGrammar = MCGrammarSymbolTableHelper + .getMCGrammarSymbol(node.getEnclosingScope()); + String constfile = getConstantClassName(ruleGrammar.get()); + + ParseVisitorEntry e = new ParseVisitorEntry(); + stack.peek().blockComponents.add(e); + e.usageName = node.getUsageName(); + e.ruleOptional = node.getIteration() == ASTConstantsGrammar.STAR || node.getIteration() == ASTConstantsGrammar.QUESTION; + e.ruleList = node.getIteration() == ASTConstantsGrammar.STAR || node.getIteration() == ASTConstantsGrammar.PLUS; + + List values = new ArrayList<>(); + for (ASTConstant x : node.getConstantList()) { + String constantName = parserGeneratorHelper.getConstantNameForConstant(x); + String tmpName = tmpNames.get(x); + String ruleName = tmpName == null ? (getRuleName(x) + "()") : tmpName; + values.add(new String[]{"ctx." + ruleName + " != null", constfile + "." + constantName}); + e.allOptConditions.add("ctx." + ruleName + " != null"); + + } + e.condition = "(" + String.join("||", e.allOptConditions) + ")"; + e.constantValues = values; + } + + } + + protected String getRuleName(ASTConstant constant) { + if (constant.isPresentKeyConstant()) { + if (constant.getKeyConstant().getStringList().size() > 1) { + Log.error("0xA0394 Unexpected key constant string list: " + constant.getKeyConstant().getStringList(), constant.get_SourcePositionStart()); + return "invalid getruleName"; + } + return parserGeneratorHelper.getKeyRuleName(constant.getKeyConstant().getString(0)); + } else if (constant.isPresentTokenConstant()) { + return parserGeneratorHelper.getLexSymbolName(constant.getTokenConstant().getString()); + } else if (!grammarInfo.isKeyword(constant.getName(), parserGeneratorHelper.getGrammarSymbol())) { + return parserGeneratorHelper.getLexSymbolName(constant.getName()); + } else if (grammarInfo.getKeywordRules().contains(constant.getName())) { + return parserGeneratorHelper.getKeyRuleName(constant.getName()); + } else { + return parserGeneratorHelper.getLexSymbolName(constant.getName()); + } + } + + + protected String getASTPackage(MCGrammarSymbol symbol) { + return Joiners.DOT.join(symbol.getFullName().toLowerCase(), + ASTConstants.AST_PACKAGE); + } + + protected String getConstantClassName(MCGrammarSymbol symbol) { + return Joiners.DOT.join(getASTPackage(symbol), ASTConstants.AST_CONSTANTS + symbol.getName()); + } +} diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/ParseVisitorEntry.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/ParseVisitorEntry.java new file mode 100644 index 0000000000..d54b92028f --- /dev/null +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/ParseVisitorEntry.java @@ -0,0 +1,162 @@ +/* (c) https://github.com/MontiCore/monticore */ +package de.monticore.codegen.parser.antlr; + +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings("unused") +public class ParseVisitorEntry { + // Common + public boolean astList, astOptional; + + public boolean ruleList, ruleOptional; + + public String usageName; + + public String condition; + + // root + public boolean isRoot; + + // NonTermRef + public String cast; + public String tmpName; + public boolean isLexNT; + public String action; + + // SubRules (extends, etc.) + public String subrule; + + // Token + String constantLexName; + + String convert; // convert function, used by lex + + // ConstantGroup + String constantValue; + List constantValues; + + // ( A | B)* => no else + boolean noAltElse = false; + + public List alternatives = new ArrayList<>(); + + List blockComponents = new ArrayList<>(); + + + List allOptConditions = new ArrayList<>(); + + + public boolean isAlternative() { + return !this.alternatives.isEmpty(); + } + + public boolean isRoot() { + return isRoot; + } + + public List getAlternatives() { + return alternatives; + } + + public boolean isList() { + return astList; + } + + public String getUsageName() { + return usageName; + } + + public String getCondition() { + return condition; + } + + public String getCast() { + return cast; + } + + public String getTmpName() { + return tmpName; + } + + public boolean isLexNT() { + return isLexNT; + } + + public String getConstantLexName() { + return constantLexName; + } + + public String getConvert() { + return convert; + } + + public List getBlockComponents() { + return blockComponents; + } + + public List getAllOptConditions() { + return allOptConditions; + } + + public boolean isAstList() { + return astList; + } + + public boolean isAstOptional() { + return astOptional; + } + + public boolean isRuleList() { + return ruleList; + } + + public boolean isRuleOptional() { + return ruleOptional; + } + + public String getConstantValue() { + return constantValue; + } + + public List getConstantValues() { + return constantValues; + } + + public boolean isNoAltElse() { + return noAltElse; + } + + public String getAction() { + return action; + } + + public String getSubrule() { + return subrule; + } + + @Override + public String toString() { + return "\nParseVisitorEntry{" + + "astList=" + astList + + ", astOptional=" + astOptional + + ", ruleList=" + ruleList + + ", ruleOptional=" + ruleOptional + + ", usageName='" + usageName + '\'' + + ", condition='" + condition + '\'' + + ", cast='" + cast + '\'' + + ", tmpName='" + tmpName + '\'' + + ", isLexNT=" + isLexNT + + ", constantLexName='" + constantLexName + '\'' + + ", alternatives=" + alternatives + + ", blockComponents=" + blockComponents + + ", allOptConditions=" + allOptConditions + + '}' + '\n'; + } + + // Set the noAltElse recursively on all alts + public void setNoAltElseRec(boolean noAltElse) { + this.noAltElse = noAltElse; + this.blockComponents.forEach(r->r.alternatives.forEach(rr->rr.setNoAltElseRec(noAltElse))); + } +} diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/SourcePositionActions.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/SourcePositionActions.java deleted file mode 100644 index 8d8b1627a1..0000000000 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/SourcePositionActions.java +++ /dev/null @@ -1,46 +0,0 @@ -/* (c) https://github.com/MontiCore/monticore */ - -package de.monticore.codegen.parser.antlr; - -import de.monticore.ast.ASTNode; -import de.monticore.codegen.parser.ParserGeneratorHelper; -import de.monticore.grammar.grammar._ast.ASTNonTerminal; - -/** - * Adds source code positions building up code to the parsers - */ -public class SourcePositionActions { - - protected ParserGeneratorHelper parserGenHelper; - - /** - * Constructor for de.monticore.codegen.parser.antlr.SourcePositionActions - * - * @param parserGenHelper - */ - public SourcePositionActions(ParserGeneratorHelper parserGenHelper) { - this.parserGenHelper = parserGenHelper; - } - - /** - * Create a mc.ast.SourcePosition at the beginning of a rule - */ - public String startPosition() { - return "_builder.set_SourcePositionStart( computeStartPosition(_input.LT(1)));\n"; - } - - public String endPosition() { - // Fetch last token to determine position - return "_builder.set_SourcePositionEnd(computeEndPosition(_input.LT(-1)));"; - } - - public String startPositionForLeftRecursiveRule(ASTNonTerminal a) { - return "_builder.set_SourcePositionStart(_localctx." - + parserGenHelper.getTmpVarName(a) - + ".ret.get_SourcePositionStart());\n"; - } - - public String endPositionForLeftRecursiveRule(ASTNonTerminal a) { - return "_builder.set_SourcePositionEnd( computeStartPosition(_input.LT(1)));\n"; - } -} diff --git a/monticore-generator/src/main/resources/_parser/ParseRule.ftl b/monticore-generator/src/main/resources/_parser/ParseRule.ftl index 49a4073213..aba08781ed 100644 --- a/monticore-generator/src/main/resources/_parser/ParseRule.ftl +++ b/monticore-generator/src/main/resources/_parser/ParseRule.ftl @@ -1,8 +1,8 @@ <#-- (c) https://github.com/MontiCore/monticore --> ${tc.signature("grammarName", "astClassName", "parseRuleNameJavaCompatible")} ${grammarName}AntlrParser parser = create(fileName); - ${astClassName} ast; - ast = parser.${parseRuleNameJavaCompatible}().ret; + ${astClassName} astPV; + var prc = parser.${parseRuleNameJavaCompatible}(); if (parser.hasErrors()) { setError(true); return Optional.empty(); @@ -14,4 +14,8 @@ ${tc.signature("grammarName", "astClassName", "parseRuleNameJavaCompatible")} Log.error("Expected EOF but found token " + currentToken, parser.computeStartPosition(currentToken)); return Optional.empty(); } - return Optional.of(ast); \ No newline at end of file + // Build ast + ${grammarName}ASTBuildVisitor buildVisitor = new ${grammarName}ASTBuildVisitor(parser.getFilename(), (org.antlr.v4.runtime.CommonTokenStream)parser.getTokenStream()); + astPV = (${astClassName})prc.accept(buildVisitor); + buildVisitor.addFinalComments(astPV, prc); + return Optional.of(astPV); \ No newline at end of file diff --git a/monticore-generator/src/main/resources/_parser/ParseRuleReader.ftl b/monticore-generator/src/main/resources/_parser/ParseRuleReader.ftl index f72e119c3b..214ce587cb 100644 --- a/monticore-generator/src/main/resources/_parser/ParseRuleReader.ftl +++ b/monticore-generator/src/main/resources/_parser/ParseRuleReader.ftl @@ -1,8 +1,8 @@ <#-- (c) https://github.com/MontiCore/monticore --> ${tc.signature("grammarName","astClassName", "parseRuleNameJavaCompatible")} ${grammarName}AntlrParser parser = create(reader); - ${astClassName} ast; - ast = parser.${parseRuleNameJavaCompatible}().ret; + ${astClassName} astPV; + var prc = parser.${parseRuleNameJavaCompatible}(); if (parser.hasErrors()) { setError(true); return Optional.empty(); @@ -14,4 +14,8 @@ ${tc.signature("grammarName","astClassName", "parseRuleNameJavaCompatible")} Log.error("Expected EOF but found token " + currentToken, parser.computeStartPosition(currentToken)); return Optional.empty(); } - return Optional.of(ast); \ No newline at end of file + // Build ast + ${grammarName}ASTBuildVisitor buildVisitor = new ${grammarName}ASTBuildVisitor(parser.getFilename(), (org.antlr.v4.runtime.CommonTokenStream)parser.getTokenStream()); + astPV = (${astClassName})prc.accept(buildVisitor); + buildVisitor.addFinalComments(astPV, prc); + return Optional.of(astPV); \ No newline at end of file diff --git a/monticore-generator/src/main/resources/_parser/visitor/Constructor.ftl b/monticore-generator/src/main/resources/_parser/visitor/Constructor.ftl new file mode 100644 index 0000000000..000f70eb06 --- /dev/null +++ b/monticore-generator/src/main/resources/_parser/visitor/Constructor.ftl @@ -0,0 +1,2 @@ +<#-- (c) https://github.com/MontiCore/monticore --> +super(filename, tokenStream); \ No newline at end of file diff --git a/monticore-generator/src/main/resources/_parser/visitor/NotImplemented.ftl b/monticore-generator/src/main/resources/_parser/visitor/NotImplemented.ftl new file mode 100644 index 0000000000..b1e5088fdd --- /dev/null +++ b/monticore-generator/src/main/resources/_parser/visitor/NotImplemented.ftl @@ -0,0 +1,3 @@ +<#-- (c) https://github.com/MontiCore/monticore --> +${tc.signature()} +throw new IllegalStateException(); diff --git a/monticore-generator/src/main/resources/_parser/visitor/TreeEntry.ftl b/monticore-generator/src/main/resources/_parser/visitor/TreeEntry.ftl new file mode 100644 index 0000000000..733b067d8f --- /dev/null +++ b/monticore-generator/src/main/resources/_parser/visitor/TreeEntry.ftl @@ -0,0 +1,90 @@ +<#-- (c) https://github.com/MontiCore/monticore --> +${tc.signature("prodname", "t")} +<#assign withCondition=(t.getCondition())??> +<#if t.isAlternative()> + // alt + <#list t.getAlternatives() as altt> + <@recursiveTreeEntry pProdName=prodname pT=altt/> + <#sep> <#if !altt.isNoAltElse()>else /* alt else ${altt.isNoAltElse()?c} */ + +<#elseif t.isLexNT()> + <#if withCondition> if (${t.getCondition()}) { + <#if t.isRuleList()> + ctx.${t.getTmpName()}.forEach(e -> _builder.add${t.getUsageName()?cap_first}( ${t.getConvert()}(e) )); + <#elseif t.isAstList()> + _builder.add${t.getUsageName()?cap_first}( ${t.getConvert()}(ctx.${t.getTmpName()}) ); + <#else> + _builder.set${t.getUsageName()?cap_first}( ${t.getConvert()}(ctx.${t.getTmpName()}) ); + + <#if withCondition> } +<#elseif (t.getSubrule())??> + var subruleast = (ASTNode) ctx.${t.getSubrule()}.accept(this); + // copy comments onto the subrule + subruleast.addAll_PreComments(_builder.get_PreCommentList()); + subruleast.addAll_PostComments(_builder.get_PostCommentList()); + return subruleast; +<#elseif (t.getCast())??> + <#if withCondition> if (${t.getCondition()}) { + <#if t.isRuleList()> + ctx.${t.getTmpName()}.forEach(e -> _builder.add${t.getUsageName()?cap_first}( (${t.getCast()}) e.accept(this) )); + <#elseif t.isAstList()> + _builder.add${t.getUsageName()?cap_first}( (${t.getCast()}) ctx.${t.getTmpName()}.accept(this) ); + <#else> + _builder.set${t.getUsageName()?cap_first}( (${t.getCast()}) ctx.${t.getTmpName()}.accept(this) ); + + <#if withCondition> } +<#elseif (t.getConstantLexName())??> + // getConstantLexName + <#if withCondition> if (${t.getCondition()}) { + <#if (t.getUsageName())??> + <#assign access = (t.getTmpName())!"${t.getConstantLexName()}() /*notmpname*/"> + <#if t.isAstList() && t.isRuleList()> // ast & rulelist + ctx.${access}.forEach(e -> _builder.add${t.getUsageName()?cap_first}( e.getText() )); + <#elseif t.isAstList()> // astlist + _builder.add${t.getUsageName()?cap_first}(ctx.${access}.getText()); + <#else> + _builder.set${t.getUsageName()?cap_first}(ctx.${access}.getText()); + + <#else> + // lex prod ref without attr + + <#if withCondition> } +<#elseif (t.getBlockComponents()?size > 0 || t.isRoot())> + // Start block + <#if withCondition> if (${t.getCondition()}) { <#else> // Block without condition + <#list t.getBlockComponents() as b> + <@recursiveTreeEntry pProdName=prodname pT=b/> + + <#if withCondition> } + // end block +<#elseif (t.getConstantValue())??> + // Start ConstantGroup + <#if withCondition> if (${t.getCondition()}) { + _builder.set${t.getUsageName()?cap_first}(${t.getConstantValue()}); + <#if withCondition> } + // End ConstantGroup +<#elseif (t.getConstantValues())??> + // Start ConstantGroup-m + <#if withCondition> if (${t.getCondition()}) { + <#list t.getConstantValues() as cv> + if (${cv[0]}) { + _builder.set${t.getUsageName()?cap_first}(${cv[1]}); + } <#sep> <#if !t.isRuleList()> else /* cg else */ <#else> /* iterated cg => no else */ + + <#if withCondition> } + // End ConstantGroup-m +<#elseif (t.getAction())??> + // Action + ${t.getAction()} + +<#else> + Log.error("Unhandled parse builder option"); + throw new IllegalStateException("Unhandled parse builder option"); + /* Unhandled TreeEntry + ${t} + */ + + +<#macro recursiveTreeEntry pProdName, pT> + ${tc.includeArgs("_parser.visitor.TreeEntry", [pProdName, pT])} + \ No newline at end of file diff --git a/monticore-generator/src/main/resources/_parser/visitor/VisitClassProd.ftl b/monticore-generator/src/main/resources/_parser/visitor/VisitClassProd.ftl new file mode 100644 index 0000000000..31a6cbd936 --- /dev/null +++ b/monticore-generator/src/main/resources/_parser/visitor/VisitClassProd.ftl @@ -0,0 +1,18 @@ +<#-- (c) https://github.com/MontiCore/monticore --> +${tc.signature("prodname", "mill", "treeroot", "action")} +String prefix = com.google.common.base.Strings.repeat("| ", depth++); + +<#if action.isPresent()> + // action + ${action.get()} + // end action + + +var _builder = ${mill}.${prodname?uncap_first}Builder(); +setSourcePos(_builder, ctx); +handlePreComments(_builder, ctx); + +${tc.includeArgs("_parser.visitor.TreeEntry", [prodname, treeroot])} +depth--; +handleInnerComments(_builder, ctx); +return _builder.uncheckedBuild(); diff --git a/monticore-generator/src/main/resources/_parser/visitor/VisitEnum.ftl b/monticore-generator/src/main/resources/_parser/visitor/VisitEnum.ftl new file mode 100644 index 0000000000..b8d4e93e9d --- /dev/null +++ b/monticore-generator/src/main/resources/_parser/visitor/VisitEnum.ftl @@ -0,0 +1,9 @@ +<#-- (c) https://github.com/MontiCore/monticore --> +${tc.signature("enumconsts")} + +<#list enumconsts as c> + if (ctx.e_${c?index} != null) + return ${c}; + <#sep>else + +throw new IllegalStateException("Unhandled enum constant during AST construction. Please report this error"); diff --git a/monticore-generator/src/main/resources/_parser/visitor/VisitInterface.ftl b/monticore-generator/src/main/resources/_parser/visitor/VisitInterface.ftl new file mode 100644 index 0000000000..7055095034 --- /dev/null +++ b/monticore-generator/src/main/resources/_parser/visitor/VisitInterface.ftl @@ -0,0 +1,38 @@ +<#-- (c) https://github.com/MontiCore/monticore --> +${tc.signature("prodname", "mill", "alts")} +String prefix = com.google.common.base.Strings.repeat("| ", depth++); +if (debug) +System.err.println(prefix+"Visit expr ${prodname}"); +if (debug) +System.err.println(prefix+ctx.start.getInputStream().getText(new org.antlr.v4.runtime.misc.Interval(ctx.start.getStartIndex(), ctx.stop.getStopIndex()))); + +<#list alts> + <#items as alt> + if (${(alt.getParseVisitorEntry().getCondition())}) { + <#if alt.isSimpleReference()> + ASTNode n= (ASTNode) ctx.${alt.getParseVisitorEntry().getTmpName()}.accept(this); + depth--; + + return n; + <#else> + var _builder = ${mill}.${alt.getBuilderNodeName()?uncap_first}Builder(); + setSourcePos(_builder, ctx); + handlePreComments(_builder, ctx); + + ${tc.includeArgs("_parser.visitor.TreeEntry", [prodname, alt.getParseVisitorEntry()])} + + depth--; + handleInnerComments(_builder, ctx); + return _builder.uncheckedBuild(); + + } + <#sep>else + + else { + throw new IllegalStateException("Unable to parse interface. Please report this error."); // This should never happen + } + <#else > + depth--; + return null; // empty production + + diff --git a/monticore-generator/src/main/resources/_parser/visitor/VisitTree.ftl b/monticore-generator/src/main/resources/_parser/visitor/VisitTree.ftl new file mode 100644 index 0000000000..d4e1e231e8 --- /dev/null +++ b/monticore-generator/src/main/resources/_parser/visitor/VisitTree.ftl @@ -0,0 +1,3 @@ +<#-- (c) https://github.com/MontiCore/monticore --> +${tc.signature()} +return tree.accept(this); diff --git a/monticore-generator/src/main/resources/parser/Lexer.ftl b/monticore-generator/src/main/resources/parser/Lexer.ftl index 778f9824d3..bc4cbe96fd 100644 --- a/monticore-generator/src/main/resources/parser/Lexer.ftl +++ b/monticore-generator/src/main/resources/parser/Lexer.ftl @@ -9,12 +9,6 @@ options { } -@lexer::header { -<#if genHelper.isJava()> - package ${genHelper.getParserPackage()}; - -} - ${tc.includeArgs("parser.LexerMember", [antlrGenerator, parserHelper.getGrammarSymbol().getName()])} <#list genHelper.getLexSymbolsWithInherited() as lexSymbol> diff --git a/monticore-generator/src/main/resources/parser/LexerMember.ftl b/monticore-generator/src/main/resources/parser/LexerMember.ftl index 09e831c656..e929ded5d9 100644 --- a/monticore-generator/src/main/resources/parser/LexerMember.ftl +++ b/monticore-generator/src/main/resources/parser/LexerMember.ftl @@ -19,14 +19,5 @@ public void setMCParser(${parserName}AntlrParser in) { this._monticore_parser = in; } -protected void storeComment(){ - if (getCompiler() != null) { - de.monticore.ast.Comment _comment = new de.monticore.ast.Comment(getText()); - de.se_rwth.commons.SourcePosition startPos = new de.se_rwth.commons.SourcePosition(_tokenStartLine, _tokenStartCharPositionInLine, getCompiler().getFilename()); - _comment.set_SourcePositionStart(startPos); - _comment.set_SourcePositionEnd(getCompiler().computeEndPosition(startPos, getText())); - getCompiler().addComment(_comment); - } -} } diff --git a/monticore-generator/src/main/resources/parser/ParserHeader.ftl b/monticore-generator/src/main/resources/parser/ParserHeader.ftl index 52b888c3ee..48c5189198 100644 --- a/monticore-generator/src/main/resources/parser/ParserHeader.ftl +++ b/monticore-generator/src/main/resources/parser/ParserHeader.ftl @@ -4,9 +4,6 @@ ${tc.signature("suffix")} <#assign genHelper = glex.getGlobalVar("parserHelper")> parser grammar ${ast.getName()}AntlrParser${suffix}; @parser::header { -<#if genHelper.isJava()> -package ${genHelper.getParserPackage()}; - <#if genHelper.isEmbeddedJavaCode()> import de.monticore.antlr4.*; import de.monticore.parser.*; diff --git a/monticore-grammar/build.gradle b/monticore-grammar/build.gradle index e4fe8a9f6a..c4ac6d7dd7 100644 --- a/monticore-grammar/build.gradle +++ b/monticore-grammar/build.gradle @@ -237,6 +237,7 @@ task checkArtifacts {} configurations { MLC } dependencies { MLC(group: 'de.monticore.lang', name: 'mlc-gradle', version: "$version") + MLC(project(':monticore-runtime')) // Ensure the proper runtime is used for MLC } StringJoiner joiner = new StringJoiner(" ") configurations.compileClasspath.resolve().each { joiner.add(it.toString()) } diff --git a/monticore-grammar/src/main/grammars/de/monticore/Cardinality.mc4 b/monticore-grammar/src/main/grammars/de/monticore/Cardinality.mc4 index fd6767f06b..ad66771412 100644 --- a/monticore-grammar/src/main/grammars/de/monticore/Cardinality.mc4 +++ b/monticore-grammar/src/main/grammars/de/monticore/Cardinality.mc4 @@ -29,25 +29,12 @@ component grammar Cardinality */ Cardinality = "[" - ( many:["*"] {_builder.setLowerBound(0);_builder.setUpperBound(0);} + ( many:["*"] | lowerBoundLit:NatLiteral - { _builder.setLowerBound( - _builder.getLowerBoundLit().getValue()); - _builder.setUpperBound(_builder.getLowerBound()); } ( ".." ( upperBoundLit:NatLiteral - ( {_builder.setUpperBound( - _builder.getUpperBoundLit().getValue());}) | - noUpperLimit:["*"] {_builder.setUpperBound(0);} ) )? + noUpperLimit:["*"] ) )? ) "]"; - /** - A Cardinality Object stores the bounds as integers. - */ - - astrule Cardinality = - lowerBound:int - upperBound:int; - } diff --git a/monticore-grammar/src/main/grammars/de/monticore/MCBasics.mc4 b/monticore-grammar/src/main/grammars/de/monticore/MCBasics.mc4 index 79833eb684..1d07234757 100644 --- a/monticore-grammar/src/main/grammars/de/monticore/MCBasics.mc4 +++ b/monticore-grammar/src/main/grammars/de/monticore/MCBasics.mc4 @@ -38,15 +38,13 @@ component grammar MCBasics { Comments are ignored by the parser. */ token SL_COMMENT = - "//" (~('\n' | '\r' ))* : ->skip - {storeComment();}; + "//" (~('\n' | '\r' ))* : -> channel(HIDDEN); /** A multi line comment in Java style. The comments are not nested. Comments are ignored by the parser. */ token ML_COMMENT = - "/*" .*? "*/" : -> skip - {storeComment();}; + "/*" .*? "*/" : -> channel(HIDDEN); } diff --git a/monticore-grammar/src/main/grammars/de/monticore/types/MCArrayTypes.mc4 b/monticore-grammar/src/main/grammars/de/monticore/types/MCArrayTypes.mc4 index 8f5a12947a..9b8a73e1b6 100644 --- a/monticore-grammar/src/main/grammars/de/monticore/types/MCArrayTypes.mc4 +++ b/monticore-grammar/src/main/grammars/de/monticore/types/MCArrayTypes.mc4 @@ -25,10 +25,12 @@ component grammar MCArrayTypes /** ASTArrayType introduces array for arbitrary types */ MCArrayType implements MCType = - MCType - ("[" "]" {_builder.setDimensions(_builder.getDimensions()+1);} )+; + MCType (dimT:"[" "]" )+; // counter dimensions counts the array depth astrule MCArrayType = - dimensions:int; + method public int getDimensions() { + return this.sizeDimT(); + } + ; } diff --git a/monticore-grammar/src/main/java/de/monticore/cardinality/_ast/ASTCardinality.java b/monticore-grammar/src/main/java/de/monticore/cardinality/_ast/ASTCardinality.java new file mode 100644 index 0000000000..7061d8a751 --- /dev/null +++ b/monticore-grammar/src/main/java/de/monticore/cardinality/_ast/ASTCardinality.java @@ -0,0 +1,36 @@ +/* (c) https://github.com/MontiCore/monticore */ + +package de.monticore.cardinality._ast; + + +import de.monticore.cardinality.CardinalityMill; + +public class ASTCardinality extends ASTCardinalityTOP { + + public int getLowerBound() { + if (this.isMany()) { + return 0; + } else { + return this.getLowerBoundLit().getValue(); + } + } + + public void setLowerBound(int i) { + setLowerBoundLit(CardinalityMill.natLiteralBuilder().setDigits(Integer.toString(i)).build()); + } + + public int getUpperBound() { + if (this.isMany() || this.isNoUpperLimit()) { + return 0; + } else if (this.isPresentUpperBoundLit()) { + return this.getUpperBoundLit().getValue(); + } + return this.getLowerBoundLit().getValue(); + } + + public void setUpperBound(int i) { + setUpperBoundLit(CardinalityMill.natLiteralBuilder().setDigits(Integer.toString(i)).build()); + } +} + + diff --git a/monticore-grammar/src/main/java/de/monticore/types/MCTypeFacade.java b/monticore-grammar/src/main/java/de/monticore/types/MCTypeFacade.java index f8496516a5..09c343dfcb 100644 --- a/monticore-grammar/src/main/java/de/monticore/types/MCTypeFacade.java +++ b/monticore-grammar/src/main/java/de/monticore/types/MCTypeFacade.java @@ -230,9 +230,13 @@ public ASTMCBasicGenericType createBasicGenericTypeOf(final String name, String. */ public ASTMCArrayType createArrayType(final ASTMCType type, int dimension) { + List dimList = Lists.newArrayList(); + for (int i=0; i ast = CombineExpressionsWithLiteralsMill.parser().parse_StringExpression(expr); + assertTrue(ast.isPresent()); + } + + @Test + public void parseBigExpr2() throws IOException { + + String expr = + "((f1 + f2) * (f1 + f3) * (f1 + f4) / (f1 + f5) + (f1 + f6) / (f1 + f7) + (f1 + f8) * (f1 + f9) + (f1 + f10) + (f1 + f11) / (f1 + f12) + (f1 + f13) + (f + f14) +\n" + + " (f1 + f3) * (f1 + f4) / (f1 + f5) + (f1 + f6) / (f1 + f7) + (f1 + f8) * (f1 + f9) + (f1 + f10) + (f1 + f11) / (f1 + f12) + (f1 + f13) + (f + f14) +\n" + + " (f2 + f3) * (f2 + f4) * (f2 + f5) * (f2 + f6) / (f2 + f7) / (f2 + f8) * (f2 + f9) + (f2 + f10) * (f2 + f11) * (f2 + f12) * (f2 + f13) * (f2 + f14))"; + + + Optional ast = CombineExpressionsWithLiteralsMill.parser().parse_StringExpression(expr); + assertTrue(ast.isPresent()); + } + + + +} \ No newline at end of file diff --git a/monticore-grammar/src/test/java/de/monticore/types/check/SynthesizeSymTypeFromMCArrayTypesTest.java b/monticore-grammar/src/test/java/de/monticore/types/check/SynthesizeSymTypeFromMCArrayTypesTest.java index 1710eaac2d..2ddd9ffa24 100644 --- a/monticore-grammar/src/test/java/de/monticore/types/check/SynthesizeSymTypeFromMCArrayTypesTest.java +++ b/monticore-grammar/src/test/java/de/monticore/types/check/SynthesizeSymTypeFromMCArrayTypesTest.java @@ -144,7 +144,7 @@ public void symTypeFromAST_ReturnTest3() throws IOException { @Test public void symTypeFrom_AST_ArrayTest() throws IOException { ASTMCType prim = parser.parse_StringMCType("int").get(); - ASTMCArrayType asttype = MCArrayTypesMill.mCArrayTypeBuilder().setMCType(prim).setDimensions(2).build(); + ASTMCArrayType asttype = MCArrayTypesMill.mCArrayTypeBuilder().setMCType(prim).addDimT("[]").addDimT("[]").build(); asttype.accept(traverser); Assertions.assertEquals("int[][]", tc.symTypeFromAST(asttype).printFullName()); } @@ -152,7 +152,7 @@ public void symTypeFrom_AST_ArrayTest() throws IOException { @Test public void symTypeFrom_AST_ArrayTest2() throws IOException { ASTMCType person = parser.parse_StringMCType("Person").get(); - ASTMCArrayType asttype = MCArrayTypesMill.mCArrayTypeBuilder().setMCType(person).setDimensions(1).build(); + ASTMCArrayType asttype = MCArrayTypesMill.mCArrayTypeBuilder().setMCType(person).addDimT("[]").build(); asttype.accept(traverser); Assertions.assertEquals("Person[]", tc.symTypeFromAST(asttype).printFullName()); } diff --git a/monticore-grammar/src/test/resources/de/monticore/comments/CommentsTest.jlight b/monticore-grammar/src/test/resources/de/monticore/comments/CommentsTest.jlight index b2c9e4327f..1b38f8316e 100644 --- a/monticore-grammar/src/test/resources/de/monticore/comments/CommentsTest.jlight +++ b/monticore-grammar/src/test/resources/de/monticore/comments/CommentsTest.jlight @@ -1,8 +1,8 @@ // (c) https://github.com/MontiCore/monticore -public /* after doStuff:mod */ void /* after doStuff:type */ doStuff() /* after doStuff:name */ { - // First doStuff (missing) - int /* after i:type */ i /* after i:name */ = /* after i:op */ 12 /* after i:val */; +public /* t2 */ void /* t4 */ doStuff() /* t6 */ { + // First doStuff + int /* pre name */ i /* pre op */ = /* pre value */ 12 /* after value */; // after line - // Final doStuff (missing) + // Final doStuff } -// After doStuff (missing) +// After doStuff diff --git a/monticore-runtime/src/main/java/de/monticore/antlr4/MCBuildVisitor.java b/monticore-runtime/src/main/java/de/monticore/antlr4/MCBuildVisitor.java new file mode 100644 index 0000000000..023d4dfcd1 --- /dev/null +++ b/monticore-runtime/src/main/java/de/monticore/antlr4/MCBuildVisitor.java @@ -0,0 +1,227 @@ +/* (c) https://github.com/MontiCore/monticore */ +package de.monticore.antlr4; + +import de.monticore.ast.ASTNode; +import de.monticore.ast.ASTNodeBuilder; +import de.monticore.ast.Comment; +import de.se_rwth.commons.SourcePosition; +import org.antlr.v4.runtime.BufferedTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.misc.IntervalSet; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.function.Consumer; + +/** + * BuildVisitors are used in the second stage of the two-phase-parsers. + * Phase one results in an ANTLR parse tree (without any AST-specific actions), + * and phase is done by ANTLR {@link org.antlr.v4.runtime.tree.ParseTreeVisitor}-instances of this class building the + * MontiCore-AST representation of the input. + *

+ * This class mainly provides {@link SourcePosition} and {@link Comment} related functionality. + * The concrete ParseTree-to-AST transformation is handled in the generated instances of this class. + */ +public abstract class MCBuildVisitor { + + protected String fileName; + protected BufferedTokenStream bufferedTokenStream; + + /** + * A flag for further debugging of comment-handling. + * Not printed by default due to performance + */ + public static boolean debug = false; + + public MCBuildVisitor(String fileName, BufferedTokenStream bufferedTokenStream) { + this.fileName = fileName; + this.bufferedTokenStream = bufferedTokenStream; + } + + public String getFileName() { + return this.fileName; + } + + protected > T setSourcePos(T builder, ParserRuleContext ctx) { + builder.set_SourcePositionStart(computePosition(ctx.start)); + // ctx.stop may be null for e.g. NoWSLast2 = {noSpace()}? ; + builder.set_SourcePositionEnd(computeEndPosition(ctx.stop == null ? ctx.start : ctx.stop)); + return builder; + } + + + public de.se_rwth.commons.SourcePosition computePosition(Token token) { + if (token == null) { + return null; + } + int line = token.getLine(); + int column = token.getCharPositionInLine(); + return new de.se_rwth.commons.SourcePosition(line, column, getFileName()); + } + + + public SourcePosition computeEndPosition(@Nonnull Token token) { + int line = token.getLine(); + int column = token.getCharPositionInLine(); + String text = token.getText(); + if (text == null) { + throw new IllegalArgumentException("0xA0708 text was null!"); + } else if ("\n".equals(text)) { + column += text.length(); + } else if (text.indexOf("\n") == -1) { + column += text.length(); + } else { + String[] splitted = text.split("\n", -1); + line += splitted.length - 1; + // +1: if there is 1 character on the last line, sourcepos must + // be 2... + column = splitted[splitted.length - 1].length() + 1; + } + return new de.se_rwth.commons.SourcePosition(line, column, getFileName()); + } + + protected int pevPreCommentTokenIndex = 0; + + // tokens + protected int tokenIndex = -1; + + protected int commentChannel = Token.HIDDEN_CHANNEL; + + protected IntervalSet commentSet = new IntervalSet(); + + /** + * Collect fresh tokens on the {@link MCBuildVisitor#commentChannel} channel and store them as pre-comments + * @param builder the builder to store the pre-comments on + * @param ctx the context object to walk up to + */ + protected void handlePreComments(ASTNodeBuilder builder, ParserRuleContext ctx) { + // Handle the pre-comments before the rule (such as JavaDoc comments on methods) + int start = ctx.start.getTokenIndex(); + + // We might have to skip over token definitions + if (tokenIndex != -1) + start = tokenIndex; + + // Expand left side to include hidden pres + @Nullable + List hiddenLeft = bufferedTokenStream.getHiddenTokensToLeft(start, this.commentChannel); + if (hiddenLeft != null) { + start = Math.min(start, hiddenLeft.stream().map(Token::getTokenIndex).min(Integer::compare).orElse(Integer.MAX_VALUE)); + } + + + doComments(builder::add_PreComment, start, ctx.start.getTokenIndex()); + pevPreCommentTokenIndex = ctx.start.getTokenIndex(); + } + + /** + * Collect all fresh tokens of the same line on the {@link MCBuildVisitor#commentChannel} channel and store them as + * post-comments + * @param builder the builder to store the post-comments on + * @param ctx the context object in which line we check + */ + protected void handleInnerComments(ASTNodeBuilder builder, ParserRuleContext ctx) { + // Productions with only predicates might not have a stop + if (ctx.getStop() == null) return; + // Handle all comments within the bounds of a rule + int start = ctx.getStart().getTokenIndex(); + int stop = ctx.getStop().getTokenIndex(); + + // Consume all hidden tokens/comments on the same line (but only if the next real token is not in this line) + @Nullable List hiddenRight = bufferedTokenStream.getHiddenTokensToRight(stop, this.commentChannel); + if (hiddenRight != null) { + int rightStop = stop; + // Find the rightmost hidden index on the same line + for (Token hr : hiddenRight) { + if (hr.getLine() != ctx.stop.getLine()) break; + rightStop = Math.max(rightStop, hr.getTokenIndex()); + } + if (rightStop != stop && bufferedTokenStream.size() >= rightStop) { + // Check if the next real is not on the same line + Token t = bufferedTokenStream.get(rightStop + 1); + if (t.getLine() != ctx.stop.getLine()) { + // Move the stop index to the index of the right-most hidden token in our stop token line + stop = rightStop; + } + } + } + + // we skip already found tokens + doComments(builder::add_PostComment, start, stop); + } + + /** + * Finally, add all comment-tokens after the final AST-element as post-comments to the element. + * @param node the final ASTNode + * @param ctx the parse tree + */ + public void addFinalComments(ASTNode node, ParserRuleContext ctx) { + // Productions with only predicates might not have a stop + if (ctx.getStop() == null) return; + int stop = ctx.getStop().getTokenIndex(); + // Expand right side to include hidden pres + @Nullable + List hiddenRight = bufferedTokenStream.getHiddenTokensToRight(stop, this.commentChannel); + if (hiddenRight != null) { + stop = Math.max(stop, hiddenRight.stream().map(Token::getTokenIndex).min(Integer::compare).orElse(Integer.MAX_VALUE)); + } + this.doComments(node::add_PostComment, ctx.start.getTokenIndex(), stop); + } + + /** + * Find all (not yet found) hidden/comment tokens + * and calls the consumer for each + * + * @param addConsumer + * @param start {@link IntervalSet#complement(int, int)} + * @param stop {@link IntervalSet#complement(int, int)} + */ + protected void doComments(Consumer addConsumer, int start, int stop) { + tokenIndex = stop; + @Nullable + IntervalSet actual = commentSet.complement(start, stop); + + // Add the interval to the set of checked token indizes + commentSet.add(start, stop); + + if (debug) + System.err.println("Everything from " + start + "(" + bufferedTokenStream.get(start).getText() + ") " + + " to " + stop + "(" + bufferedTokenStream.get(stop).getText() + ") " + + " in " + Thread.currentThread().getStackTrace()[3].getMethodName() + " > " + Thread.currentThread().getStackTrace()[2].getMethodName()); + + if (actual == null) return; + + // For every (not yet seen) index within the range + for (int tokenIndex : actual.toList()) { + // fetch the token + Token t = bufferedTokenStream.get(tokenIndex); + if (isComment(t)) { + // And add the comment iff a comment token is used + if (debug) + System.err.println("found " + t.getText() + " at " + t.getTokenIndex()); + addConsumer.accept(createComment(t)); + } + } + } + + protected boolean isComment(Token token) { + return token.getChannel() == this.commentChannel; + } + + + /** + * Create a new MontiCore comment from an ANTLR token. + * Sets the start and end position, as well as the text. + * + * @param token the token + * @return the comment + */ + protected Comment createComment(Token token) { + de.monticore.ast.Comment _comment = new de.monticore.ast.Comment(token.getText()); + _comment.set_SourcePositionStart(computePosition(token)); + _comment.set_SourcePositionEnd(computeEndPosition(token)); + return _comment; + } +} diff --git a/monticore-test/01.experiments/generator/src/main/grammars/SuperParser.mc4 b/monticore-test/01.experiments/generator/src/main/grammars/SuperParser.mc4 index b191f20c7e..ac1c682265 100644 --- a/monticore-test/01.experiments/generator/src/main/grammars/SuperParser.mc4 +++ b/monticore-test/01.experiments/generator/src/main/grammars/SuperParser.mc4 @@ -5,5 +5,5 @@ grammar SuperParser { ParserSuperClass = "superparser.ModifiedMCParserBase" } - P = "..." { doCustomMethod(); }; + P = {doCustomMethod()}? "..." ; } diff --git a/monticore-test/01.experiments/generator/src/main/java/superparser/ModifiedMCParserBase.java b/monticore-test/01.experiments/generator/src/main/java/superparser/ModifiedMCParserBase.java index e6593030da..38cf92d04f 100644 --- a/monticore-test/01.experiments/generator/src/main/java/superparser/ModifiedMCParserBase.java +++ b/monticore-test/01.experiments/generator/src/main/java/superparser/ModifiedMCParserBase.java @@ -14,8 +14,9 @@ public ModifiedMCParserBase() { public static int customCalled; - public void doCustomMethod() { + public boolean doCustomMethod() { customCalled++; + return true; } } diff --git a/monticore-test/01.experiments/prettyPrinters/src/test/java/de/monticore/prettyprint/TestPrettyPrinterTest.java b/monticore-test/01.experiments/prettyPrinters/src/test/java/de/monticore/prettyprint/TestPrettyPrinterTest.java index 0e30b1b921..5aab595d7e 100644 --- a/monticore-test/01.experiments/prettyPrinters/src/test/java/de/monticore/prettyprint/TestPrettyPrinterTest.java +++ b/monticore-test/01.experiments/prettyPrinters/src/test/java/de/monticore/prettyprint/TestPrettyPrinterTest.java @@ -8,8 +8,8 @@ import de.monticore.testprettyprinters._ast.ASTToBeReplacedKeyword; import de.monticore.testprettyprinters._parser.TestPrettyPrintersParser; import de.monticore.testprettyprinters._prettyprint.TestPrettyPrintersFullPrettyPrinter; -import de.se_rwth.commons.Joiners; import de.se_rwth.commons.logging.Log; +import de.se_rwth.commons.logging.LogStub; import org.junit.*; import java.io.IOException; @@ -25,7 +25,7 @@ public class TestPrettyPrinterTest extends PPTestClass { @BeforeClass public static void setup() { TestPrettyPrintersMill.init(); - Log.init(); + LogStub.init(); Log.enableFailQuick(false); } diff --git a/monticore-test/it/src/main/grammars/mc/common/Basics.mc4 b/monticore-test/it/src/main/grammars/mc/common/Basics.mc4 index 519db5d335..36ad7b18c6 100644 --- a/monticore-test/it/src/main/grammars/mc/common/Basics.mc4 +++ b/monticore-test/it/src/main/grammars/mc/common/Basics.mc4 @@ -24,8 +24,7 @@ grammar Basics { "//" (~('\n' | '\r' ) )* - : -> skip - {storeComment();} + : -> channel(HIDDEN) ; token ML_COMMENT = @@ -35,8 +34,7 @@ grammar Basics { '\n' | '\r' ) )* - "*/" : -> skip - {storeComment();} + "*/" : -> channel(HIDDEN) ; fragment token ESC = diff --git a/monticore-test/it/src/main/grammars/mc/feature/ScopesExample.mc4 b/monticore-test/it/src/main/grammars/mc/feature/ScopesExample.mc4 deleted file mode 100644 index 3487fc3017..0000000000 --- a/monticore-test/it/src/main/grammars/mc/feature/ScopesExample.mc4 +++ /dev/null @@ -1,59 +0,0 @@ -/* (c) https://github.com/MontiCore/monticore */ - -package mc.feature; - -grammar ScopesExample extends mc.grammar.lexicals.ItTestLexicals { - - token CHAR = '\'' ( 'a'..'z' | 'A'..'Z' ) '\''; - - token INT = ('0'..'9')+ ; - - token FLOAT = INT '.' INT; - - token STRING = '"' ('a'..'z' | 'A'..'Z')* '"'; - - //a very simplified grammar. - ClassDefinition = (public:["public"] | private:["private"] | protected:["protected"])? - (final:["final"])? - "class" Name ("extends" superclass:Name)? "{" - body:ClassBody - "}"; - - ClassBody = - ( fields:Field | methods:Method | nestedClasses:ClassDefinition)*; - - - - Method = (public:["public"] | private:["private"] | protected:["protected"])? - "method" Name ("returns" returnType:Name)? "{" - body:MethodBody - "}"; - - - MethodBody = (fields:Field | assigns:Assignement | increases:Increase | calls:MethodCall)*; - - - Field = (public:["public"] | private:["private"] | protected:["protected"])? - "field" type:Name Name ";"; - - //actions are not used at the moment cause classgenwithingrammar cannot generate attributes of any type :( - Assignement = varname:Name "=" (value:STRING - | var:Name - | bool:"false" {_builder.setType("Boolean");} - | bool:"true" {_builder.setType("Boolean");} - | int:INT {_builder.setType("Int");} - | float:FLOAT {_builder.setType("Float");} - | call:MethodCall) ";"; - - - Increase = varname:Name "+" "+" ";"; - - MethodCall= methodName:Name "(" ")" ";"; - - - //create places for type infos filled while parsing... - //not used at the moment cause classgenwithingrammar cannot generate attributes of any type :( - astrule Assignement = - type:Name; - -} diff --git a/monticore-test/it/src/main/grammars/mc/feature/classgenwithingrammar/Type.mc4 b/monticore-test/it/src/main/grammars/mc/feature/classgenwithingrammar/Type.mc4 index 55d728d897..2fe419f2d1 100644 --- a/monticore-test/it/src/main/grammars/mc/feature/classgenwithingrammar/Type.mc4 +++ b/monticore-test/it/src/main/grammars/mc/feature/classgenwithingrammar/Type.mc4 @@ -7,7 +7,7 @@ grammar Type extends mc.common.Basics { // ---- astrule Type = - sub:ASTSubRule1 min=1 max = 3 + sub:ASTSubRule1 min=2 max = 3 sub2:ASTSubRule2* additional:Name additionalMore:Name* diff --git a/monticore-test/it/src/main/grammars/mc/feature/semanticpredicate/SemPredWithInterface.mc4 b/monticore-test/it/src/main/grammars/mc/feature/semanticpredicate/SemPredWithInterface.mc4 index 224d6b2bb6..947a0d95ad 100644 --- a/monticore-test/it/src/main/grammars/mc/feature/semanticpredicate/SemPredWithInterface.mc4 +++ b/monticore-test/it/src/main/grammars/mc/feature/semanticpredicate/SemPredWithInterface.mc4 @@ -25,8 +25,8 @@ grammar SemPredWithInterface extends mc.common.Basics { first: boolean; FirstRun implements { first }? I = - { _builder.setFirst(true); first = false;} "foo"; + {first = false;} "foo"; OtherRun implements { !first }? I = - {_builder.setFirst(false);} "foo"; + "foo"; } diff --git a/monticore-test/it/src/main/grammars/mc/grammar/ItTestGrammar.mc4 b/monticore-test/it/src/main/grammars/mc/grammar/ItTestGrammar.mc4 index 8f88faea42..b80bf0804c 100644 --- a/monticore-test/it/src/main/grammars/mc/grammar/ItTestGrammar.mc4 +++ b/monticore-test/it/src/main/grammars/mc/grammar/ItTestGrammar.mc4 @@ -485,8 +485,7 @@ component grammar ItTestGrammar extends mc.grammar.literals.ItTestLiterals { GenericType = (Name& || ".")+ ("<" (GenericType || ",")+ ">")? - {_builder.setDimension(0);} ("[" "]" - {_builder.setDimension(_builder.getDimension()+1);} )*; + (dimT:"[" "]" )*; astrule GenericType = Dimension:int diff --git a/monticore-test/it/src/main/grammars/mc/grammar/common/ItTestCommon.mc4 b/monticore-test/it/src/main/grammars/mc/grammar/common/ItTestCommon.mc4 index 953d1dc077..c50b74db84 100644 --- a/monticore-test/it/src/main/grammars/mc/grammar/common/ItTestCommon.mc4 +++ b/monticore-test/it/src/main/grammars/mc/grammar/common/ItTestCommon.mc4 @@ -86,20 +86,15 @@ grammar ItTestCommon extends mc.grammar.types.ItTestTypes { Cardinality = "[" ( - many:["*"] {_builder.setLowerBound(0);_builder.setUpperBound(0);} + many:["*"] | lowerBoundLit:IntLiteral - { - _builder.setLowerBound(_builder.getLowerBoundLit().getValue()); - _builder.setUpperBound(_builder.getLowerBound()); - } ( ".." ( upperBoundLit:IntLiteral - ({_builder.setUpperBound(_builder.getUpperBoundLit().getValue());}) | - noUpperLimit:["*"] {_builder.setUpperBound(0);} + noUpperLimit:["*"] ) )? ) diff --git a/monticore-test/it/src/main/grammars/mc/grammar/types/ItTestTypes.mc4 b/monticore-test/it/src/main/grammars/mc/grammar/types/ItTestTypes.mc4 index f10e7fa65a..e12c21fc05 100644 --- a/monticore-test/it/src/main/grammars/mc/grammar/types/ItTestTypes.mc4 +++ b/monticore-test/it/src/main/grammars/mc/grammar/types/ItTestTypes.mc4 @@ -156,8 +156,7 @@ grammar ItTestTypes extends mc.grammar.literals.ItTestLiterals { ( ( ( - "[" "]" - {_builder.setDimensions(_builder.getDimensions()+1);} + dimT:"[" "]" )+ ) ); @@ -172,8 +171,7 @@ grammar ItTestTypes extends mc.grammar.literals.ItTestLiterals { PrimitiveArrayType extends ArrayType implements Type = componentType:PrimitiveType ( - "[" "]" - {_builder.setDimensions(_builder.getDimensions()+1);} + dimT:"[" "]" )+; /** ASTVoidType represents the return type "void". diff --git a/monticore-test/it/src/main/grammars/org/nest/commons/Commons.mc4 b/monticore-test/it/src/main/grammars/org/nest/commons/Commons.mc4 index d045104e6d..321c5e63c0 100644 --- a/monticore-test/it/src/main/grammars/org/nest/commons/Commons.mc4 +++ b/monticore-test/it/src/main/grammars/org/nest/commons/Commons.mc4 @@ -10,13 +10,7 @@ grammar Commons extends mc.grammar.types.ItTestTypes { ('\n' | '\r' ('\n' )? )? - : {_channel = HIDDEN; - if (getCompiler() != null) { - de.monticore.ast.Comment _comment = new de.monticore.ast.Comment(getText()); - _comment.set_SourcePositionStart(new de.se_rwth.commons.SourcePosition(getLine(), getCharPositionInLine())); - _comment.set_SourcePositionEnd(getCompiler().computeEndPosition(getToken())); - getCompiler().addComment(_comment); - }}; + : {_channel = HIDDEN;}; token NEWLINE = ('\r' '\n' | '\r' | '\n' ); token WS = (' ' | '\t'):{_channel = HIDDEN;}; diff --git a/monticore-test/it/src/main/java/mc/feature/semanticpredicate/sempredwithinterface/_ast/ASTFirstRunBuilder.java b/monticore-test/it/src/main/java/mc/feature/semanticpredicate/sempredwithinterface/_ast/ASTFirstRunBuilder.java new file mode 100644 index 0000000000..66fd748c1b --- /dev/null +++ b/monticore-test/it/src/main/java/mc/feature/semanticpredicate/sempredwithinterface/_ast/ASTFirstRunBuilder.java @@ -0,0 +1,13 @@ +/* (c) https://github.com/MontiCore/monticore */ +package mc.feature.semanticpredicate.sempredwithinterface._ast; + +public class ASTFirstRunBuilder extends ASTFirstRunBuilderTOP { + + public ASTFirstRunBuilder() { + super(); + setFirst(true); + } + +} + + diff --git a/monticore-test/it/src/main/java/mc/feature/semanticpredicate/sempredwithinterface/_ast/ASTOtherRunBuilder.java b/monticore-test/it/src/main/java/mc/feature/semanticpredicate/sempredwithinterface/_ast/ASTOtherRunBuilder.java new file mode 100644 index 0000000000..cadd834738 --- /dev/null +++ b/monticore-test/it/src/main/java/mc/feature/semanticpredicate/sempredwithinterface/_ast/ASTOtherRunBuilder.java @@ -0,0 +1,13 @@ +/* (c) https://github.com/MontiCore/monticore */ +package mc.feature.semanticpredicate.sempredwithinterface._ast; + +public class ASTOtherRunBuilder extends ASTOtherRunBuilderTOP { + + public ASTOtherRunBuilder() { + super(); + setFirst(false); + } + +} + + diff --git a/monticore-test/it/src/test/java/mc/feature/classgenwithingrammar/ParserTest.java b/monticore-test/it/src/test/java/mc/feature/classgenwithingrammar/ParserTest.java index a5e4a49d9b..a7461e5293 100644 --- a/monticore-test/it/src/test/java/mc/feature/classgenwithingrammar/ParserTest.java +++ b/monticore-test/it/src/test/java/mc/feature/classgenwithingrammar/ParserTest.java @@ -33,11 +33,11 @@ public void test() throws IOException { Assertions.assertTrue(hasError); } - // Test that too many Hallo and Welt are detected in one go + // Test that the last Hallo is too much @Test public void test2() throws IOException { - boolean hasError = parse("Hallo Hallo Hallo Hallo Welt "); + boolean hasError = parse("Hallo Hallo Hallo Hallo "); Assertions.assertTrue(hasError); } @@ -51,7 +51,16 @@ public void test3() throws IOException { Assertions.assertTrue(Log.getFindings().isEmpty()); } - + + // Tests that one hallo is issing + @Test + public void test4() throws IOException { + + boolean hasError = parse("Hallo "); + + Assertions.assertTrue(hasError); + } + // Test that one Welt is too much @Test public void testl() throws IOException {