From a1922042226e82b2211dcc55205d44ba7516133b Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 4 Oct 2024 11:28:58 +0200 Subject: [PATCH 1/4] Move conversion of legacy semantic token types into general converter --- .../src/main/rascal/VSCodeRascal.rsc | 26 +---------------- .../main/rascal/lang/textmate/Conversion.rsc | 29 ++++++++++++++++++- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc b/rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc index 3e1325a..6eeb329 100644 --- a/rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc +++ b/rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc @@ -21,31 +21,7 @@ private Grammar getRscGrammar() { : p[attributes = attributes + \tag("category"(category))]; return visit (rsc) { - - // The following mapping is based on: - // - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59 - // - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142 - // With updates based on: - // - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695 - // - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152 - case \tag("category"("Normal")) => \tag("category"("source")) - case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type) - case \tag("category"("Identifier")) => \tag("category"("variable")) - case \tag("category"("Variable")) => \tag("category"("variable")) - case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant) - case \tag("category"("Comment")) => \tag("category"("comment")) - case \tag("category"("Todo")) => \tag("category"("comment")) - case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string) - case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid")) - case \tag("category"("MetaVariable")) => \tag("category"("variable")) - case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other) - case \tag("category"("MetaSkipped")) => \tag("category"("string")) - case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter) - case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text) - case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text) - case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text) - - // With additional hot-patching as discussed: + // Temporarily hot-patch Rascal's own grammar as discussed here: // - https://github.com/SWAT-engineering/rascal-textmate/pull/6 case p: prod(label("integer", sort("Literal")), _, _) => setCategory(p, "constant.numeric") case p: prod(label("real", sort("Literal")), _, _) => setCategory(p, "constant.numeric") diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc index 5e959b9..622d1e5 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc @@ -56,10 +56,37 @@ TmGrammar toTmGrammar(RscGrammar rsc, str name, NameGeneration nameGeneration = RscGrammar preprocess(RscGrammar rsc) { // Replace occurrences of singleton ranges with just the corresponding // literal. This makes it easier to identify delimiters. - return visit (rsc) { + rsc = visit (rsc) { case \char-class([range(char, char)]) => d when d := \lit(""), isDelimiter(d) } + + // Replace legacy semantic token types with TextMate scopes based on: + // - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59 + // - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142 + // With updates based on: + // - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695 + // - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152 + rsc = visit (rsc) { + case \tag("category"("Normal")) => \tag("category"("source")) + case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type) + case \tag("category"("Identifier")) => \tag("category"("variable")) + case \tag("category"("Variable")) => \tag("category"("variable")) + case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant) + case \tag("category"("Comment")) => \tag("category"("comment")) + case \tag("category"("Todo")) => \tag("category"("comment")) + case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string) + case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid")) + case \tag("category"("MetaVariable")) => \tag("category"("variable")) + case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other) + case \tag("category"("MetaSkipped")) => \tag("category"("string")) + case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter) + case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text) + case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text) + case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text) + } + + return rsc; } @synoposis{ From 3648bfc5c84157d63bd9702803a01ecf52887deb Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 4 Oct 2024 11:29:38 +0200 Subject: [PATCH 2/4] Add conversion of current semantic token types to TextMate scopes --- .../main/rascal/lang/textmate/Conversion.rsc | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc index 622d1e5..2ed11d0 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc @@ -61,6 +61,35 @@ RscGrammar preprocess(RscGrammar rsc) { when d := \lit(""), isDelimiter(d) } + // Replace current semantic token types with TextMate scopes based on: + // - https://github.com/microsoft/vscode/blob/9f3a7b5bc8a2758584b33d0385b227f25ae8d3fb/src/vs/platform/theme/common/tokenClassificationRegistry.ts#L543-L571 + rsc = visit (rsc) { + case \tag("category"("comment")) => \tag("category"("comment")) + case \tag("category"("string")) => \tag("category"("string")) + case \tag("category"("keyword")) => \tag("category"("keyword.control")) + case \tag("category"("number")) => \tag("category"("constant.numeric")) + case \tag("category"("regexp")) => \tag("category"("constant.regexp")) + case \tag("category"("operator")) => \tag("category"("keyword.operator")) + case \tag("category"("namespace")) => \tag("category"("entity.name.namespace")) + case \tag("category"("type")) => \tag("category"("support.type")) // Alternative: support.type + case \tag("category"("struct")) => \tag("category"("entity.name.type.struct")) + case \tag("category"("class")) => \tag("category"("entity.name.type.class")) // Alternative: support.class + case \tag("category"("interface")) => \tag("category"("entity.name.type.interface")) + case \tag("category"("enum")) => \tag("category"("entity.name.type.enum")) + case \tag("category"("typeParameter")) => \tag("category"("entity.name.type.parameter")) + case \tag("category"("function")) => \tag("category"("entity.name.function")) // Alternative: support.function + case \tag("category"("method")) => \tag("category"("entity.name.function.member")) // Alternative: support.function + case \tag("category"("macro")) => \tag("category"("entity.name.function.preprocessor")) + case \tag("category"("variable")) => \tag("category"("variable.other.readwrite")) // Alternative: entity.name.variable + case \tag("category"("parameter")) => \tag("category"("variable.parameter")) + case \tag("category"("property")) => \tag("category"("variable.other.property")) + case \tag("category"("enumMember")) => \tag("category"("variable.other.enummember")) + case \tag("category"("event")) => \tag("category"("variable.other.event")) + case \tag("category"("decorator")) => \tag("category"("entity.name.decorator")) // Alternative: entity.name.function + // Note: Categories types `member` and `label` are deprecated/undefined + // and therefore excluded from this mapping + } + // Replace legacy semantic token types with TextMate scopes based on: // - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59 // - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142 From 80349e5e6e7d9f26b27f595d428b732f65ed1551 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 4 Oct 2024 11:30:12 +0200 Subject: [PATCH 3/4] Update tests --- .../lang/textmate/conversiontests/Pico.rsc | 4 +- .../lang/textmate/conversiontests/Pico.test | 6 +- .../lang/textmate/conversiontests/Rascal.rsc | 42 ++++++------ .../lang/textmate/conversiontests/Rascal.test | 10 +-- .../textmate/conversiontests/RascalClass.rsc | 6 +- .../conversiontests/RascalComment.rsc | 4 +- .../conversiontests/RascalComment.test | 66 +++++++++---------- .../conversiontests/RascalConcrete.rsc | 12 ++-- .../conversiontests/RascalStringConstant.rsc | 2 +- .../conversiontests/RascalStringConstant.test | 50 +++++++------- .../conversiontests/RascalStringLiteral.rsc | 10 +-- .../conversiontests/RascalStringLiteral.test | 26 ++++---- .../textmate/conversiontests/RascalTag.rsc | 4 +- .../textmate/conversiontests/RascalTag.test | 30 ++++----- 14 files changed, 136 insertions(+), 136 deletions(-) diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc index 3c8d076..d57b3d8 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc @@ -14,8 +14,8 @@ Grammar rsc = preprocess(grammar(#Program)); list[ConversionUnit] units = [ unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("-"),lit(","),lit(")"),lit("("),lit("+"),lit("||"),lit(":="),lit("\""),lit(";")})],{}), false, false, , ), - unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("Comment"))}), false, false, , ), - unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("Comment"))}), false, true, , ), + unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("comment"))}), false, false, , ), + unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("comment"))}), false, true, , ), unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("do"),lit("declare"),lit("fi"),lit("else"),lit("end"),lit("od"),lit("nil-type"),lit("begin"),lit("natural"),lit("then"),lit("if"),lit("while"),lit("string")})],{\tag("category"("keyword.control"))}), false, false, , ) ]; diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test index 64242dd..569a2c8 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test @@ -1,13 +1,13 @@ # SYNTAX TEST "Pico" %% foo bar -# ^^^^^^^^^^ Comment +# ^^^^^^^^^^ comment %% foo % bar -# ^^^^^^^^^^^^ Comment +# ^^^^^^^^^^^^ comment %% do -# ^^^^^ Comment +# ^^^^^ comment do # ^^ keyword.control \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Rascal.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Rascal.rsc index 8f06b8a..2c3446f 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Rascal.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Rascal.rsc @@ -14,27 +14,27 @@ Grammar rsc = preprocess(grammar(#Module)); list[ConversionUnit] units = [ unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit(","),lit(")"),lit("("),lit("%"),lit("\<==\>"),lit("\<\<="),lit("!="),lit("\>="),lit("://"),lit("&="),lit("\<-"),lit("-="),lit("*="),lit("+="),lit("..."),lit("/="),lit("!:="),lit("$"),lit("!\<\<"),lit("=\>"),lit("!\>\>"),lit("||"),lit("\>\>"),lit("::"),lit("&&"),lit(":="),lit("#"),lit("?="),lit("\<:"),lit("==\>"),lit("^"),lit(";"),lit("{")})],{}), false, false, , ), - unit(rsc, prod(label("stderrOutput",lex("Output")),[conditional(lit("⚠"),{\begin-of-line()}),\iter-star(\char-class([range(1,9),range(11,12),range(14,1114111)])),lit("\n")],{\tag("category"("StdErr"))}), false, false, , ), - unit(rsc, prod(label("stdoutOutput",lex("Output")),[conditional(lit("≫"),{\begin-of-line()}),\iter-star(\char-class([range(1,9),range(11,12),range(14,1114111)])),lit("\n")],{\tag("category"("StdOut"))}), false, false, , ), - unit(rsc, prod(label("resultOutput",lex("Output")),[lit("⇨"),\iter-star(\char-class([range(1,9),range(11,12),range(14,1114111)])),lit("\n")],{\tag("category"("Result"))}), false, false, , ), - unit(rsc, prod(label("bq",lex("ConcretePart")),[lit("\\`")],{\tag("category"("MetaSkipped"))}), false, false, , ), - unit(rsc, prod(label("bs",lex("ConcretePart")),[lit("\\\\")],{\tag("category"("MetaSkipped"))}), false, false, , ), - unit(rsc, prod(label("gt",lex("ConcretePart")),[lit("\\\>")],{\tag("category"("MetaSkipped"))}), false, false, , ")),just(lit("\\\>"))>), - unit(rsc, prod(label("hole",lex("ConcretePart")),[label("hole",sort("ConcreteHole"))],{\tag("category"("MetaVariable"))}), true, true, , "))>), - unit(rsc, prod(label("lt",lex("ConcretePart")),[lit("\\\<")],{\tag("category"("MetaSkipped"))}), false, false, , ), - unit(rsc, prod(label("text",lex("ConcretePart")),[conditional(iter(\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)])),{\not-follow(\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)]))})],{\tag("category"("MetaSkipped"))}), false, false, , ), - unit(rsc, prod(lex("Char"),[\char-class([range(1,31),range(33,33),range(35,38),range(40,44),range(46,59),range(61,61),range(63,90),range(94,1114111)])],{\tag("category"("Constant"))}), false, true, , ), - unit(rsc, prod(lex("Char"),[lex("UnicodeEscape")],{\tag("category"("Constant"))}), false, false, , ), - unit(rsc, prod(lex("Char"),[lit("\\"),\char-class([range(32,32),range(34,34),range(39,39),range(45,45),range(60,60),range(62,62),range(91,93),range(98,98),range(102,102),range(110,110),range(114,114),range(116,116)])],{\tag("category"("Constant"))}), false, false, , ), - unit(rsc, prod(label("default",sort("Tag")),[lit("@"),layouts("LAYOUTLIST"),label("name",lex("Name")),layouts("LAYOUTLIST"),label("contents",lex("TagString"))],{\tag("Folded"()),\tag("category"("Comment"))}), true, true, , ), - unit(rsc, prod(label("expression",sort("Tag")),[lit("@"),layouts("LAYOUTLIST"),label("name",lex("Name")),layouts("LAYOUTLIST"),lit("="),layouts("LAYOUTLIST"),conditional(label("expression",sort("Expression")),{\not-follow(lit("@"))})],{\tag("Folded"()),\tag("category"("Comment"))}), true, true, , ), - unit(rsc, prod(lex("MidStringChars"),[lit("\>"),\iter-star(lex("StringCharacter")),lit("\<")],{\tag("category"("Constant"))}), false, true, , ")),just(lit("\<"))>), - unit(rsc, prod(lex("PostStringChars"),[lit("\>"),\iter-star(lex("StringCharacter")),lit("\"")],{\tag("category"("Constant"))}), false, true, , ")),just(lit("\""))>), - unit(rsc, prod(lex("Comment"),[lit("//"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\not-follow(\char-class([range(9,9),range(13,13),range(32,32),range(160,160),range(5760,5760),range(8192,8202),range(8239,8239),range(8287,8287),range(12288,12288)])),\end-of-line()})],{\tag("category"("Comment"))}), false, false, , ), - unit(rsc, prod(lex("Comment"),[lit("/*"),\iter-star(alt({\char-class([range(1,41),range(43,1114111)]),conditional(lit("*"),{\not-follow(lit("/"))})})),lit("*/")],{\tag("category"("Comment"))}), false, true, , ), - unit(rsc, prod(lex("CaseInsensitiveStringConstant"),[lit("\'"),label("chars",\iter-star(lex("StringCharacter"))),lit("\'")],{\tag("category"("Constant"))}), false, true, , ), - unit(rsc, prod(lex("PreStringChars"),[lit("\""),\iter-star(lex("StringCharacter")),lit("\<")],{\tag("category"("Constant"))}), false, true, , ), - unit(rsc, prod(lex("StringConstant"),[lit("\""),label("chars",\iter-star(lex("StringCharacter"))),lit("\"")],{\tag("category"("Constant"))}), false, true, , ), + unit(rsc, prod(label("stderrOutput",lex("Output")),[conditional(lit("⚠"),{\begin-of-line()}),\iter-star(\char-class([range(1,9),range(11,12),range(14,1114111)])),lit("\n")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("stdoutOutput",lex("Output")),[conditional(lit("≫"),{\begin-of-line()}),\iter-star(\char-class([range(1,9),range(11,12),range(14,1114111)])),lit("\n")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("resultOutput",lex("Output")),[lit("⇨"),\iter-star(\char-class([range(1,9),range(11,12),range(14,1114111)])),lit("\n")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("bq",lex("ConcretePart")),[lit("\\`")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("bs",lex("ConcretePart")),[lit("\\\\")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("gt",lex("ConcretePart")),[lit("\\\>")],{\tag("category"("string"))}), false, false, , ")),just(lit("\\\>"))>), + unit(rsc, prod(label("hole",lex("ConcretePart")),[label("hole",sort("ConcreteHole"))],{\tag("category"("variable"))}), true, true, , "))>), + unit(rsc, prod(label("lt",lex("ConcretePart")),[lit("\\\<")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("text",lex("ConcretePart")),[conditional(iter(\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)])),{\not-follow(\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)]))})],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(lex("Char"),[\char-class([range(1,31),range(33,33),range(35,38),range(40,44),range(46,59),range(61,61),range(63,90),range(94,1114111)])],{\tag("category"("string"))}), false, true, , ), + unit(rsc, prod(lex("Char"),[lex("UnicodeEscape")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(lex("Char"),[lit("\\"),\char-class([range(32,32),range(34,34),range(39,39),range(45,45),range(60,60),range(62,62),range(91,93),range(98,98),range(102,102),range(110,110),range(114,114),range(116,116)])],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("default",sort("Tag")),[lit("@"),layouts("LAYOUTLIST"),label("name",lex("Name")),layouts("LAYOUTLIST"),label("contents",lex("TagString"))],{\tag("Folded"()),\tag("category"("comment"))}), true, true, , ), + unit(rsc, prod(label("expression",sort("Tag")),[lit("@"),layouts("LAYOUTLIST"),label("name",lex("Name")),layouts("LAYOUTLIST"),lit("="),layouts("LAYOUTLIST"),conditional(label("expression",sort("Expression")),{\not-follow(lit("@"))})],{\tag("Folded"()),\tag("category"("comment"))}), true, true, , ), + unit(rsc, prod(lex("MidStringChars"),[lit("\>"),\iter-star(lex("StringCharacter")),lit("\<")],{\tag("category"("string"))}), false, true, , ")),just(lit("\<"))>), + unit(rsc, prod(lex("PostStringChars"),[lit("\>"),\iter-star(lex("StringCharacter")),lit("\"")],{\tag("category"("string"))}), false, true, , ")),just(lit("\""))>), + unit(rsc, prod(lex("Comment"),[lit("//"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\not-follow(\char-class([range(9,9),range(13,13),range(32,32),range(160,160),range(5760,5760),range(8192,8202),range(8239,8239),range(8287,8287),range(12288,12288)])),\end-of-line()})],{\tag("category"("comment"))}), false, false, , ), + unit(rsc, prod(lex("Comment"),[lit("/*"),\iter-star(alt({\char-class([range(1,41),range(43,1114111)]),conditional(lit("*"),{\not-follow(lit("/"))})})),lit("*/")],{\tag("category"("comment"))}), false, true, , ), + unit(rsc, prod(lex("CaseInsensitiveStringConstant"),[lit("\'"),label("chars",\iter-star(lex("StringCharacter"))),lit("\'")],{\tag("category"("string"))}), false, true, , ), + unit(rsc, prod(lex("PreStringChars"),[lit("\""),\iter-star(lex("StringCharacter")),lit("\<")],{\tag("category"("string"))}), false, true, , ), + unit(rsc, prod(lex("StringConstant"),[lit("\""),label("chars",\iter-star(lex("StringCharacter"))),lit("\"")],{\tag("category"("string"))}), false, true, , ), unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("lexical"),lit("loc"),lit("test"),lit("lrel"),lit("throws"),lit("clear"),lit("top-down-break"),lit("module"),lit("any"),lit("int"),lit("quit"),lit("bottom-up-break"),lit("o"),lit("anno"),lit("true"),lit("public"),lit("keyword"),lit("for"),lit("tuple"),lit("bracket"),lit("bag"),lit("it"),lit("visit"),lit("do"),lit("data"),lit("layout"),lit("bool"),lit("edit"),lit("join"),lit("is"),lit("import"),lit("view"),lit("in"),lit("rat"),lit("modules"),lit("continue"),lit("left"),lit("num"),lit("assert"),lit("throw"),lit("one"),lit("help"),lit("default"),lit("all"),lit("global"),lit("syntax"),lit("false"),lit("finally"),lit("private"),lit("mod"),lit("java"),lit("node"),lit("start"),lit("set"),lit("if"),lit("bottom-up"),lit("right"),lit("variable"),lit("map"),lit("10"),lit("on"),lit("break"),lit("dynamic"),lit("solve"),lit("fail"),lit("unimport"),lit("outermost"),lit("real"),lit("list"),lit("insert"),lit("innermost"),lit("declarations"),lit("else"),lit("rel"),lit("function"),lit("notin"),lit("filter"),lit("datetime"),lit("catch"),lit("try"),lit("renaming"),lit("tag"),lit("has"),lit("top-down"),lit("Z"),lit("when"),lit("type"),lit("append"),lit("extend"),lit("non-assoc"),lit("assoc"),lit("switch"),lit("void"),lit("history"),lit("T"),lit("while"),lit("str"),lit("value"),lit("undeclare"),lit("case"),lit("alias"),lit("return"),lit("0")})],{\tag("category"("keyword.control"))}), false, false, , ) ]; diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Rascal.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Rascal.test index 12ac071..69c1a32 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Rascal.test +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Rascal.test @@ -1,11 +1,11 @@ # SYNTAX TEST "Rascal" "foo bar" -# ^^^^^^^^^ Constant +# ^^^^^^^^^ string "foobar" -# ^^^^^ ^^^^^ Constant -# ^^^^^ -Constant +# ^^^^^ ^^^^^ string +# ^^^^^ -string ### TODO: The following test shows that, currently, multi-line strings are ### disabled. This is because the converter determines that: @@ -18,6 +18,6 @@ ### improved in the future. "foo -# ^^^^ -Constant +# ^^^^ -string bar" -# ^^^^ -Constant \ No newline at end of file +# ^^^^ -string \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalClass.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalClass.rsc index 0ac24c4..438e3d5 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalClass.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalClass.rsc @@ -41,9 +41,9 @@ Grammar rsc = preprocess(grammar(#Class)); list[ConversionUnit] units = [ unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("-"),lit(")"),lit("("),lit("!"),lit("||"),lit("&&")})],{}), false, false, , ), - unit(rsc, prod(lex("Char"),[\char-class([range(1,31),range(33,33),range(35,38),range(40,44),range(46,59),range(61,61),range(63,90),range(94,1114111)])],{\tag("category"("Constant"))}), false, true, , ), - unit(rsc, prod(lex("Char"),[lex("UnicodeEscape")],{\tag("category"("Constant"))}), false, false, , ), - unit(rsc, prod(lex("Char"),[lit("\\"),\char-class([range(32,32),range(34,34),range(39,39),range(45,45),range(60,60),range(62,62),range(91,93),range(98,98),range(102,102),range(110,110),range(114,114),range(116,116)])],{\tag("category"("Constant"))}), false, false, , ), + unit(rsc, prod(lex("Char"),[\char-class([range(1,31),range(33,33),range(35,38),range(40,44),range(46,59),range(61,61),range(63,90),range(94,1114111)])],{\tag("category"("string"))}), false, true, , ), + unit(rsc, prod(lex("Char"),[lex("UnicodeEscape")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(lex("Char"),[lit("\\"),\char-class([range(32,32),range(34,34),range(39,39),range(45,45),range(60,60),range(62,62),range(91,93),range(98,98),range(102,102),range(110,110),range(114,114),range(116,116)])],{\tag("category"("string"))}), false, false, , ), unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("10"),lit("0")})],{\tag("category"("keyword.control"))}), false, false, , ) ]; diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalComment.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalComment.rsc index a6a3dc3..0b564ce 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalComment.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalComment.rsc @@ -18,8 +18,8 @@ lexical Comment Grammar rsc = preprocess(grammar(#Comment)); list[ConversionUnit] units = [ - unit(rsc, prod(lex("Comment"),[lit("//"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\not-follow(\char-class([range(9,9),range(13,13),range(32,32),range(160,160),range(5760,5760),range(8192,8202),range(8239,8239),range(8287,8287),range(12288,12288)])),\end-of-line()})],{\tag("category"("Comment"))}), false, false, , ), - unit(rsc, prod(lex("Comment"),[lit("/*"),\iter-star(alt({\char-class([range(1,41),range(43,1114111)]),conditional(lit("*"),{\not-follow(lit("/"))})})),lit("*/")],{\tag("category"("Comment"))}), false, true, , ) + unit(rsc, prod(lex("Comment"),[lit("//"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\not-follow(\char-class([range(9,9),range(13,13),range(32,32),range(160,160),range(5760,5760),range(8192,8202),range(8239,8239),range(8287,8287),range(12288,12288)])),\end-of-line()})],{\tag("category"("comment"))}), false, false, , ), + unit(rsc, prod(lex("Comment"),[lit("/*"),\iter-star(alt({\char-class([range(1,41),range(43,1114111)]),conditional(lit("*"),{\not-follow(lit("/"))})})),lit("*/")],{\tag("category"("comment"))}), false, true, , ) ]; test bool analyzeTest() = doAnalyzeTest(rsc, units, name = "RascalComment"); diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalComment.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalComment.test index bc74ee2..ebcc4c3 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalComment.test +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalComment.test @@ -1,75 +1,75 @@ # SYNTAX TEST "RascalComment" // foo bar -# ^^^^^^^^^^ Comment +# ^^^^^^^^^^ comment foo // bar -# ^^^^ -Comment -# ^^^^^^ Comment +# ^^^^ -comment +# ^^^^^^ comment /* foo bar */ -# ^^^^^^^^^^^^^ Comment +# ^^^^^^^^^^^^^ comment foo /* bar */ -# ^^^^ -Comment -# ^^^^^^^^^ Comment +# ^^^^ -comment +# ^^^^^^^^^ comment /* foo */ bar -# ^^^^^^^^^ Comment -# ^^^^ -Comment +# ^^^^^^^^^ comment +# ^^^^ -comment /* *** */ -# ^^^^^^^^^ Comment +# ^^^^^^^^^ comment /* * / */ -# ^^^^^^^^^ Comment +# ^^^^^^^^^ comment /* /* */ foo -# ^^^^^^^^ Comment -# ^^^^ -Comment +# ^^^^^^^^ comment +# ^^^^ -comment /* /* */ */ -# ^^^^^^^^ Comment -# ^^^ -Comment +# ^^^^^^^^ comment +# ^^^ -comment /* // */ foo -# ^^^^^^^^ Comment -# ^^^^ -Comment +# ^^^^^^^^ comment +# ^^^^ -comment // /* */ foo -## ^^^^^^^^^^^^ Comment +## ^^^^^^^^^^^^ comment /* foo -# ^^^^^^ Comment +# ^^^^^^ comment bar */ -# ^^^^^^ Comment +# ^^^^^^ comment foo /* -# ^^^^ -Comment -# ^^ Comment +# ^^^^ -comment +# ^^ comment bar */ -# ^^^^^^ Comment +# ^^^^^^ comment /* foo -# ^^^^^^ Comment +# ^^^^^^ comment */ bar -# ^^ Comment -# ^^^^ -Comment +# ^^ comment +# ^^^^ -comment /* -# ^^ Comment +# ^^ comment * -# ^ Comment +# ^ comment ** -# ^^ Comment +# ^^ comment */ -# ^^ Comment +# ^^ comment /* -# ^^ Comment +# ^^ comment * -# ^ Comment +# ^ comment / -# ^ Comment +# ^ comment */ -# ^^ Comment \ No newline at end of file +# ^^ comment \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalConcrete.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalConcrete.rsc index 133a1fd..ee6df07 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalConcrete.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalConcrete.rsc @@ -31,12 +31,12 @@ Grammar rsc = preprocess(grammar(#Concrete)); list[ConversionUnit] units = [ unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("\n"),lit("\'")})],{}), false, false, , ), - unit(rsc, prod(label("bq",lex("ConcretePart")),[lit("\\`")],{\tag("category"("MetaSkipped"))}), false, false, , ), - unit(rsc, prod(label("bs",lex("ConcretePart")),[lit("\\\\")],{\tag("category"("MetaSkipped"))}), false, false, , ), - unit(rsc, prod(label("gt",lex("ConcretePart")),[lit("\\\>")],{\tag("category"("MetaSkipped"))}), false, false, , ")),just(lit("\\\>"))>), - unit(rsc, prod(label("hole",lex("ConcretePart")),[label("hole",sort("ConcreteHole"))],{\tag("category"("MetaVariable"))}), false, false, , "))>), - unit(rsc, prod(label("lt",lex("ConcretePart")),[lit("\\\<")],{\tag("category"("MetaSkipped"))}), false, false, , ), - unit(rsc, prod(label("text",lex("ConcretePart")),[conditional(iter(\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)])),{\not-follow(\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)]))})],{\tag("category"("MetaSkipped"))}), false, false, , ) + unit(rsc, prod(label("bq",lex("ConcretePart")),[lit("\\`")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("bs",lex("ConcretePart")),[lit("\\\\")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("gt",lex("ConcretePart")),[lit("\\\>")],{\tag("category"("string"))}), false, false, , ")),just(lit("\\\>"))>), + unit(rsc, prod(label("hole",lex("ConcretePart")),[label("hole",sort("ConcreteHole"))],{\tag("category"("variable"))}), false, false, , "))>), + unit(rsc, prod(label("lt",lex("ConcretePart")),[lit("\\\<")],{\tag("category"("string"))}), false, false, , ), + unit(rsc, prod(label("text",lex("ConcretePart")),[conditional(iter(\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)])),{\not-follow(\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)]))})],{\tag("category"("string"))}), false, false, , ) ]; test bool analyzeTest() = doAnalyzeTest(rsc, units, name = "RascalConcrete"); diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringConstant.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringConstant.rsc index c8cf144..3f94eca 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringConstant.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringConstant.rsc @@ -31,7 +31,7 @@ Grammar rsc = preprocess(grammar(#StringConstant)); list[ConversionUnit] units = [ unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("\n"),lit("\'"),lit("\\")})],{}), false, false, , ), - unit(rsc, prod(lex("StringConstant"),[lit("\""),label("chars",\iter-star(lex("StringCharacter"))),lit("\"")],{\tag("category"("Constant"))}), false, true, , ), + unit(rsc, prod(lex("StringConstant"),[lit("\""),label("chars",\iter-star(lex("StringCharacter"))),lit("\"")],{\tag("category"("string"))}), false, true, , ), unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("10"),lit("0")})],{\tag("category"("keyword.control"))}), false, false, , ) ]; diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringConstant.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringConstant.test index 88c1a48..eb1a728 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringConstant.test +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringConstant.test @@ -1,59 +1,59 @@ # SYNTAX TEST "RascalStringConstant" "foo bar" -# ^^^^^^^^^ Constant +# ^^^^^^^^^ string foo "bar" -# ^^^^ -Constant -# ^^^^^ Constant +# ^^^^ -string +# ^^^^^ string "foo" bar -# ^^^^^ Constant -# ^^^^ -Constant +# ^^^^^ string +# ^^^^ -string "foo\"bar" -# ^^^^^^^^^^ Constant +# ^^^^^^^^^^ string "foo\\"bar -# ^^^^^^^ Constant -# ^^^ -Constant +# ^^^^^^^ string +# ^^^ -string "foo\\\"bar" -# ^^^^^^^^^^^^ Constant +# ^^^^^^^^^^^^ string "foo\\\\"bar -# ^^^^^^^^^ Constant -# ^^^ -Constant +# ^^^^^^^^^ string +# ^^^ -string "foo -# ^^^^ Constant +# ^^^^ string bar" -# ^^^^ Constant +# ^^^^ string "foo -# ^^^^ Constant +# ^^^^ string \" -# ^^ Constant +# ^^ string bar" -# ^^^^ Constant +# ^^^^ string "foo -# ^^^^ Constant +# ^^^^ string \\" -# ^^^ Constant +# ^^^ string bar -# ^^^ -Constant +# ^^^ -string "foo -# ^^^ Constant +# ^^^ string \\\" -# ^^^^ Constant +# ^^^^ string bar" -# ^^^^ Constant +# ^^^^ string "foo -# ^^^^ Constant +# ^^^^ string \\\\" -# ^^^ Constant +# ^^^ string bar -# ^^^ -Constant \ No newline at end of file +# ^^^ -string \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringLiteral.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringLiteral.rsc index 89999bd..ef27235 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringLiteral.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringLiteral.rsc @@ -69,11 +69,11 @@ syntax Expression Grammar rsc = preprocess(grammar(#StringLiteral)); list[ConversionUnit] units = [ - unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("-"),lit(","),lit(")"),lit("("),lit("\n"),lit("\'"),lit("\<="),lit("\\"),lit("\>="),lit(";"),lit("{")})],{}), false, false, , ), - unit(rsc, prod(lex("PostStringChars"),[lit("\>"),\iter-star(lex("StringCharacter")),lit("\"")],{\tag("category"("Constant"))}), false, true, , ")),just(lit("\""))>), - unit(rsc, prod(lex("MidStringChars"),[lit("\>"),\iter-star(lex("StringCharacter")),lit("\<")],{\tag("category"("Constant"))}), false, true, , ")),just(lit("\<"))>), - unit(rsc, prod(lex("PreStringChars"),[lit("\""),\iter-star(lex("StringCharacter")),lit("\<")],{\tag("category"("Constant"))}), false, true, , ), - unit(rsc, prod(lex("StringConstant"),[lit("\""),label("chars",\iter-star(lex("StringCharacter"))),lit("\"")],{\tag("category"("Constant"))}), false, true, , ), + unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("-"),lit(","),lit(")"),lit("("),lit("\n"),lit("\'"),lit("\<="),lit("}"),lit("\\"),lit("\>="),lit(";"),lit("{")})],{}), false, false, , ), + unit(rsc, prod(lex("MidStringChars"),[lit("\>"),\iter-star(lex("StringCharacter")),lit("\<")],{\tag("category"("string"))}), false, true, , ")),just(lit("\<"))>), + unit(rsc, prod(lex("PostStringChars"),[lit("\>"),\iter-star(lex("StringCharacter")),lit("\"")],{\tag("category"("string"))}), false, true, , ")),just(lit("\""))>), + unit(rsc, prod(lex("PreStringChars"),[lit("\""),\iter-star(lex("StringCharacter")),lit("\<")],{\tag("category"("string"))}), false, true, , ), + unit(rsc, prod(lex("StringConstant"),[lit("\""),label("chars",\iter-star(lex("StringCharacter"))),lit("\"")],{\tag("category"("string"))}), false, true, , ), unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("for"),lit("do"),lit("if"),lit("10"),lit("else"),lit("while"),lit("0")})],{\tag("category"("keyword.control"))}), false, false, , ) ]; diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringLiteral.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringLiteral.test index df30656..3269b8a 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringLiteral.test +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalStringLiteral.test @@ -1,27 +1,27 @@ # SYNTAX TEST "RascalStringLiteral" "foofoofoo"bar -# ^^^^^ Constant -# ^^^ -Constant +# ^^^^^ string +# ^^^ -string "foo"bar -# ^^^^^ Constant -# ^^^ -Constant +# ^^^^^ string +# ^^^ -string <5, 6>" -# ^^^^^ -Constant -# ^^ Constant +# ^^^^^ -string +# ^^ string ## TODO: Improve this? (Probably very hard to do with TextMate...) "foo <5 > 6> bar" -# ^^^^^^ Constant -# ^^^^^ -Constant -# ^^^^^^ Constant \ No newline at end of file +# ^^^^^^ string +# ^^^^^ -string +# ^^^^^^ string \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalTag.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalTag.rsc index c80dcda..7cf92fc 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalTag.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalTag.rsc @@ -41,8 +41,8 @@ Grammar rsc = preprocess(grammar(#Tag)); list[ConversionUnit] units = [ unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("="),lit("\\"),lit(";"),lit("{")})],{}), false, false, , ), - unit(rsc, prod(label("default",sort("Tag")),[lit("@"),layouts("LAYOUTLIST"),label("name",lex("Name")),layouts("LAYOUTLIST"),label("contents",lex("TagString"))],{\tag("Folded"()),\tag("category"("Comment"))}), true, true, , ), - unit(rsc, prod(label("expression",sort("Tag")),[lit("@"),layouts("LAYOUTLIST"),label("name",lex("Name")),layouts("LAYOUTLIST"),lit("="),layouts("LAYOUTLIST"),conditional(label("expression",sort("Expression")),{\not-follow(lit("@"))})],{\tag("Folded"()),\tag("category"("Comment"))}), false, true, , ) + unit(rsc, prod(label("default",sort("Tag")),[lit("@"),layouts("LAYOUTLIST"),label("name",lex("Name")),layouts("LAYOUTLIST"),label("contents",lex("TagString"))],{\tag("Folded"()),\tag("category"("comment"))}), true, true, , ), + unit(rsc, prod(label("expression",sort("Tag")),[lit("@"),layouts("LAYOUTLIST"),label("name",lex("Name")),layouts("LAYOUTLIST"),lit("="),layouts("LAYOUTLIST"),conditional(label("expression",sort("Expression")),{\not-follow(lit("@"))})],{\tag("Folded"()),\tag("category"("comment"))}), false, true, , ) ]; test bool analyzeTest() = doAnalyzeTest(rsc, units, name = "RascalTag"); diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalTag.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalTag.test index c75087b..e9a8f4f 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalTag.test +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/RascalTag.test @@ -1,23 +1,23 @@ # SYNTAX TEST "RascalTag" @memo foo -# ^^^^^ Comment -# ^^^ -Comment +# ^^^^^ comment +# ^^^ -comment @synopsis{foo} bar -# ^^^^^^^^^^^^^^ Comment -# ^^^^ -Comment +# ^^^^^^^^^^^^^^ comment +# ^^^^ -comment @synopsis{ -# ^^^^^^^^^^ Comment +# ^^^^^^^^^^ comment foo bar -# ^^^^^^^^^ Comment +# ^^^^^^^^^ comment } baz -# ^ Comment -# ^^^^ -Comment +# ^ comment +# ^^^^ -comment @category={;} -# ^^^^^^^^^^^^^ Comment +# ^^^^^^^^^^^^^ comment ### TODO: The following tests show that, currently, no newlines are allowed ### between `@` and the tag name. Starting from commit 6ba991d, there is a TODO @@ -26,16 +26,16 @@ ### generated rules" @ -# ^ -Comment +# ^ -comment memo -# ^^^^ -Comment +# ^^^^ -comment @ -# ^ -Comment +# ^ -comment synopsis{foo} -# ^^^^^^^^^^^^^ -Comment +# ^^^^^^^^^^^^^ -comment @ -# ^ -Comment +# ^ -comment category={;} -# ^^^^^^^^^^^^^ -Comment \ No newline at end of file +# ^^^^^^^^^^^^^ -comment \ No newline at end of file From 5c21d9d9f28a2ebfead8ff9becb3b77844a1635b Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 4 Oct 2024 14:11:59 +0200 Subject: [PATCH 4/4] Split preprocessor in separate functions --- .../main/rascal/lang/textmate/Conversion.rsc | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc index 2ed11d0..c8e3d9c 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc @@ -54,16 +54,24 @@ TmGrammar toTmGrammar(RscGrammar rsc, str name, NameGeneration nameGeneration = } RscGrammar preprocess(RscGrammar rsc) { - // Replace occurrences of singleton ranges with just the corresponding - // literal. This makes it easier to identify delimiters. - rsc = visit (rsc) { + rsc = replaceSingletonRanges(rsc); + rsc = replaceCurrentSemanticTokenTypes(rsc); + rsc = replaceLegacySemanticTokenTypes(rsc); + return rsc; +} + +// Replace occurrences of singleton ranges with just the corresponding literal. +// This makes it easier to identify delimiters. +private RscGrammar replaceSingletonRanges(RscGrammar rsc) + = visit (rsc) { case \char-class([range(char, char)]) => d when d := \lit(""), isDelimiter(d) - } + }; - // Replace current semantic token types with TextMate scopes based on: - // - https://github.com/microsoft/vscode/blob/9f3a7b5bc8a2758584b33d0385b227f25ae8d3fb/src/vs/platform/theme/common/tokenClassificationRegistry.ts#L543-L571 - rsc = visit (rsc) { +// Replace current semantic token types with TextMate scopes based on: +// - https://github.com/microsoft/vscode/blob/9f3a7b5bc8a2758584b33d0385b227f25ae8d3fb/src/vs/platform/theme/common/tokenClassificationRegistry.ts#L543-L571 +private RscGrammar replaceCurrentSemanticTokenTypes(RscGrammar rsc) + = visit (rsc) { case \tag("category"("comment")) => \tag("category"("comment")) case \tag("category"("string")) => \tag("category"("string")) case \tag("category"("keyword")) => \tag("category"("keyword.control")) @@ -88,15 +96,16 @@ RscGrammar preprocess(RscGrammar rsc) { case \tag("category"("decorator")) => \tag("category"("entity.name.decorator")) // Alternative: entity.name.function // Note: Categories types `member` and `label` are deprecated/undefined // and therefore excluded from this mapping - } - - // Replace legacy semantic token types with TextMate scopes based on: - // - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59 - // - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142 - // With updates based on: - // - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695 - // - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152 - rsc = visit (rsc) { + }; + +// Replace legacy semantic token types with TextMate scopes based on: +// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59 +// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142 +// With updates based on: +// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695 +// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152 +private RscGrammar replaceLegacySemanticTokenTypes(RscGrammar rsc) + = visit (rsc) { case \tag("category"("Normal")) => \tag("category"("source")) case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type) case \tag("category"("Identifier")) => \tag("category"("variable")) @@ -113,10 +122,7 @@ RscGrammar preprocess(RscGrammar rsc) { case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text) case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text) case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text) - } - - return rsc; -} + }; @synoposis{ Analyzes Rascal grammar `rsc`. Returns a list of productions, in the form of