Skip to content

Commit

Permalink
Merge pull request #21 from SWAT-engineering/convert-semantic-token-t…
Browse files Browse the repository at this point in the history 
…ypes-to-textmate-scopes

Convert semantic token types to TextMate scopes
  • Loading branch information
@sungshik
sungshik authored Oct 4, 2024
2 parents f05f035 + 5c21d9d commit 09aab8c
Show file tree
Hide file tree
Showing 16 changed files with 204 additions and 166 deletions.
26 changes: 1 addition & 25 deletions rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,31 +21,7 @@ private Grammar getRscGrammar() {
: p[attributes = attributes + \tag("category"(category))];

return visit (rsc) {

// The following mapping is based on:
// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
// With updates based on:
// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
case \tag("category"("Normal")) => \tag("category"("source"))
case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type)
case \tag("category"("Identifier")) => \tag("category"("variable"))
case \tag("category"("Variable")) => \tag("category"("variable"))
case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant)
case \tag("category"("Comment")) => \tag("category"("comment"))
case \tag("category"("Todo")) => \tag("category"("comment"))
case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string)
case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid"))
case \tag("category"("MetaVariable")) => \tag("category"("variable"))
case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other)
case \tag("category"("MetaSkipped")) => \tag("category"("string"))
case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text)
case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text)
case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text)

// With additional hot-patching as discussed:
// Temporarily hot-patch Rascal's own grammar as discussed here:
// - https://github.com/SWAT-engineering/rascal-textmate/pull/6
case p: prod(label("integer", sort("Literal")), _, _) => setCategory(p, "constant.numeric")
case p: prod(label("real", sort("Literal")), _, _) => setCategory(p, "constant.numeric")
Expand Down
72 changes: 67 additions & 5 deletions rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,75 @@ TmGrammar toTmGrammar(RscGrammar rsc, str name, NameGeneration nameGeneration =
}

RscGrammar preprocess(RscGrammar rsc) {
// Replace occurrences of singleton ranges with just the corresponding
// literal. This makes it easier to identify delimiters.
return visit (rsc) {
rsc = replaceSingletonRanges(rsc);
rsc = replaceCurrentSemanticTokenTypes(rsc);
rsc = replaceLegacySemanticTokenTypes(rsc);
return rsc;
}

// Replace occurrences of singleton ranges with just the corresponding literal.
// This makes it easier to identify delimiters.
private RscGrammar replaceSingletonRanges(RscGrammar rsc)
= visit (rsc) {
case \char-class([range(char, char)]) => d
when d := \lit("<stringChar(char)>"), isDelimiter(d)
}
}
};

// Replace current semantic token types with TextMate scopes based on:
// - https://github.com/microsoft/vscode/blob/9f3a7b5bc8a2758584b33d0385b227f25ae8d3fb/src/vs/platform/theme/common/tokenClassificationRegistry.ts#L543-L571
private RscGrammar replaceCurrentSemanticTokenTypes(RscGrammar rsc)
= visit (rsc) {
case \tag("category"("comment")) => \tag("category"("comment"))
case \tag("category"("string")) => \tag("category"("string"))
case \tag("category"("keyword")) => \tag("category"("keyword.control"))
case \tag("category"("number")) => \tag("category"("constant.numeric"))
case \tag("category"("regexp")) => \tag("category"("constant.regexp"))
case \tag("category"("operator")) => \tag("category"("keyword.operator"))
case \tag("category"("namespace")) => \tag("category"("entity.name.namespace"))
case \tag("category"("type")) => \tag("category"("support.type")) // Alternative: support.type
case \tag("category"("struct")) => \tag("category"("entity.name.type.struct"))
case \tag("category"("class")) => \tag("category"("entity.name.type.class")) // Alternative: support.class
case \tag("category"("interface")) => \tag("category"("entity.name.type.interface"))
case \tag("category"("enum")) => \tag("category"("entity.name.type.enum"))
case \tag("category"("typeParameter")) => \tag("category"("entity.name.type.parameter"))
case \tag("category"("function")) => \tag("category"("entity.name.function")) // Alternative: support.function
case \tag("category"("method")) => \tag("category"("entity.name.function.member")) // Alternative: support.function
case \tag("category"("macro")) => \tag("category"("entity.name.function.preprocessor"))
case \tag("category"("variable")) => \tag("category"("variable.other.readwrite")) // Alternative: entity.name.variable
case \tag("category"("parameter")) => \tag("category"("variable.parameter"))
case \tag("category"("property")) => \tag("category"("variable.other.property"))
case \tag("category"("enumMember")) => \tag("category"("variable.other.enummember"))
case \tag("category"("event")) => \tag("category"("variable.other.event"))
case \tag("category"("decorator")) => \tag("category"("entity.name.decorator")) // Alternative: entity.name.function
// Note: Categories types `member` and `label` are deprecated/undefined
// and therefore excluded from this mapping
};

// Replace legacy semantic token types with TextMate scopes based on:
// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
// With updates based on:
// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
private RscGrammar replaceLegacySemanticTokenTypes(RscGrammar rsc)
= visit (rsc) {
case \tag("category"("Normal")) => \tag("category"("source"))
case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type)
case \tag("category"("Identifier")) => \tag("category"("variable"))
case \tag("category"("Variable")) => \tag("category"("variable"))
case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant)
case \tag("category"("Comment")) => \tag("category"("comment"))
case \tag("category"("Todo")) => \tag("category"("comment"))
case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string)
case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid"))
case \tag("category"("MetaVariable")) => \tag("category"("variable"))
case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other)
case \tag("category"("MetaSkipped")) => \tag("category"("string"))
case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text)
case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text)
case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text)
};

@synoposis{
Analyzes Rascal grammar `rsc`. Returns a list of productions, in the form of
Expand Down
4 changes: 2 additions & 2 deletions rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ Grammar rsc = preprocess(grammar(#Program));

list[ConversionUnit] units = [
unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("-"),lit(","),lit(")"),lit("("),lit("+"),lit("||"),lit(":="),lit("\""),lit(";")})],{}), false, false, <nothing(),nothing()>, <nothing(),nothing()>),
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("Comment"))}), false, false, <nothing(),nothing()>, <just(lit("%%")),nothing()>),
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("Comment"))}), false, true, <nothing(),nothing()>, <just(lit("%")),just(lit("%"))>),
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("comment"))}), false, false, <nothing(),nothing()>, <just(lit("%%")),nothing()>),
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("comment"))}), false, true, <nothing(),nothing()>, <just(lit("%")),just(lit("%"))>),
unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("do"),lit("declare"),lit("fi"),lit("else"),lit("end"),lit("od"),lit("nil-type"),lit("begin"),lit("natural"),lit("then"),lit("if"),lit("while"),lit("string")})],{\tag("category"("keyword.control"))}), false, false, <nothing(),nothing()>, <nothing(),nothing()>)
];

Expand Down
6 changes: 3 additions & 3 deletions rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
# SYNTAX TEST "Pico"

%% foo bar
# ^^^^^^^^^^ Comment
# ^^^^^^^^^^ comment

%% foo % bar
# ^^^^^^^^^^^^ Comment
# ^^^^^^^^^^^^ comment

%% do
# ^^^^^ Comment
# ^^^^^ comment

do
# ^^ keyword.control
Loading

0 comments on commit 09aab8c

Please sign in to comment.