diff --git a/src/vs/editor/common/model/bracketPairColorizer/ast.ts b/src/vs/editor/common/model/bracketPairColorizer/ast.ts index 81382d797b889..4fa3bf3ccb74e 100644 --- a/src/vs/editor/common/model/bracketPairColorizer/ast.ts +++ b/src/vs/editor/common/model/bracketPairColorizer/ast.ts @@ -20,7 +20,7 @@ export type AstNode = PairAstNode | ListAstNode | BracketAstNode | InvalidBracke abstract class BaseAstNode { abstract readonly kind: AstNodeKind; abstract readonly children: readonly AstNode[]; - abstract readonly unopenedBrackets: SmallImmutableSet; + abstract readonly missingBracketIds: SmallImmutableSet; /** * In case of a list, determines the height of the (2,3) tree. @@ -55,7 +55,6 @@ abstract class BaseAstNode { export class PairAstNode extends BaseAstNode { public static create( - category: SmallImmutableSet, openingBracket: BracketAstNode, child: AstNode | null, closingBracket: BracketAstNode | null @@ -71,7 +70,7 @@ export class PairAstNode extends BaseAstNode { children.push(closingBracket); } - return new PairAstNode(length, category, children, child ? child.unopenedBrackets : SmallImmutableSet.getEmpty()); + return new PairAstNode(length, children, child ? child.missingBracketIds : SmallImmutableSet.getEmpty()); } get kind(): AstNodeKind.Pair { @@ -82,7 +81,7 @@ export class PairAstNode extends BaseAstNode { } canBeReused( - expectedClosingCategories: SmallImmutableSet, + openedBracketIds: SmallImmutableSet, endLineDidChange: boolean ) { if (this.closingBracket === null) { @@ -96,7 +95,7 @@ export class PairAstNode extends BaseAstNode { return false; } - if (expectedClosingCategories.intersects(this.unopenedBrackets)) { + if (openedBracketIds.intersects(this.missingBracketIds)) { return false; } @@ -105,7 +104,6 @@ export class PairAstNode extends BaseAstNode { flattenLists(): PairAstNode { return PairAstNode.create( - this.category, this.openingBracket.flattenLists(), this.child && this.child.flattenLists(), this.closingBracket && this.closingBracket.flattenLists() @@ -138,9 +136,8 @@ export class PairAstNode extends BaseAstNode { private constructor( length: Length, - public readonly category: SmallImmutableSet, public readonly children: readonly AstNode[], - public readonly unopenedBrackets: SmallImmutableSet + public readonly missingBracketIds: SmallImmutableSet ) { super(length); } @@ -148,9 +145,8 @@ export class PairAstNode extends BaseAstNode { clone(): PairAstNode { return new PairAstNode( this.length, - this.category, clone(this.children), - this.unopenedBrackets + this.missingBracketIds ); } } @@ -172,10 +168,10 @@ export class ListAstNode extends BaseAstNode { return new ListAstNode(lengthZero, 0, items, SmallImmutableSet.getEmpty()); } else { let length = items[0].length; - let unopenedBrackets = items[0].unopenedBrackets; + let unopenedBrackets = items[0].missingBracketIds; for (let i = 1; i < items.length; i++) { length = lengthAdd(length, items[i].length); - unopenedBrackets = unopenedBrackets.merge(items[i].unopenedBrackets); + unopenedBrackets = unopenedBrackets.merge(items[i].missingBracketIds); } return new ListAstNode(length, items[0].listHeight + 1, items, unopenedBrackets); } @@ -187,7 +183,7 @@ export class ListAstNode extends BaseAstNode { get children(): readonly AstNode[] { return this._items; } - get unopenedBrackets(): SmallImmutableSet { + get missingBracketIds(): SmallImmutableSet { return this._unopenedBrackets; } @@ -201,7 +197,7 @@ export class ListAstNode extends BaseAstNode { } canBeReused( - expectedClosingCategories: SmallImmutableSet, + openedBracketIds: SmallImmutableSet, endLineDidChange: boolean ): boolean { if (this._items.length === 0) { @@ -209,7 +205,7 @@ export class ListAstNode extends BaseAstNode { return true; } - if (expectedClosingCategories.intersects(this.unopenedBrackets)) { + if (openedBracketIds.intersects(this.missingBracketIds)) { return false; } @@ -219,7 +215,7 @@ export class ListAstNode extends BaseAstNode { } return lastChild.canBeReused( - expectedClosingCategories, + openedBracketIds, endLineDidChange ); } @@ -238,7 +234,7 @@ export class ListAstNode extends BaseAstNode { } clone(): ListAstNode { - return new ListAstNode(this.length, this.listHeight, clone(this._items), this.unopenedBrackets); + return new ListAstNode(this.length, this.listHeight, clone(this._items), this.missingBracketIds); } private handleChildrenChanged(): void { @@ -248,10 +244,10 @@ export class ListAstNode extends BaseAstNode { } let length = items[0].length; - let unopenedBrackets = items[0].unopenedBrackets; + let unopenedBrackets = items[0].missingBracketIds; for (let i = 1; i < items.length; i++) { length = lengthAdd(length, items[i].length); - unopenedBrackets = unopenedBrackets.merge(items[i].unopenedBrackets); + unopenedBrackets = unopenedBrackets.merge(items[i].missingBracketIds); } this._length = length; this._unopenedBrackets = unopenedBrackets; @@ -372,12 +368,12 @@ export class TextAstNode extends BaseAstNode { get children(): readonly AstNode[] { return emptyArray; } - get unopenedBrackets(): SmallImmutableSet { + get missingBracketIds(): SmallImmutableSet { return SmallImmutableSet.getEmpty(); } canBeReused( - expectedClosingCategories: SmallImmutableSet, + openedBracketIds: SmallImmutableSet, endLineDidChange: boolean ) { // Don't reuse text from a line that got changed. @@ -422,7 +418,7 @@ export class BracketAstNode extends BaseAstNode { return emptyArray; } - get unopenedBrackets(): SmallImmutableSet { + get missingBracketIds(): SmallImmutableSet { return SmallImmutableSet.getEmpty(); } @@ -456,18 +452,18 @@ export class InvalidBracketAstNode extends BaseAstNode { return emptyArray; } - public readonly unopenedBrackets: SmallImmutableSet; + public readonly missingBracketIds: SmallImmutableSet; - constructor(pairsWith: SmallImmutableSet, length: Length) { + constructor(closingBrackets: SmallImmutableSet, length: Length) { super(length); - this.unopenedBrackets = pairsWith; + this.missingBracketIds = closingBrackets; } canBeReused( - expectedClosingCategories: SmallImmutableSet, + openedBracketIds: SmallImmutableSet, endLineDidChange: boolean ) { - return !expectedClosingCategories.intersects(this.unopenedBrackets); + return !openedBracketIds.intersects(this.missingBracketIds); } flattenLists(): InvalidBracketAstNode { diff --git a/src/vs/editor/common/model/bracketPairColorizer/bracketPairColorizer.ts b/src/vs/editor/common/model/bracketPairColorizer/bracketPairColorizer.ts index 9a8a7c0d0d701..a87776198792a 100644 --- a/src/vs/editor/common/model/bracketPairColorizer/bracketPairColorizer.ts +++ b/src/vs/editor/common/model/bracketPairColorizer/bracketPairColorizer.ts @@ -115,8 +115,8 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider private initialAstWithoutTokens: AstNode | undefined; private astWithTokens: AstNode | undefined; - private readonly denseKeyProvider = new DenseKeyProvider(); - private readonly brackets = new LanguageAgnosticBracketTokens([], this.denseKeyProvider); + private readonly denseKeyProvider = new DenseKeyProvider(); + private readonly brackets = new LanguageAgnosticBracketTokens(this.denseKeyProvider); public didLanguageChange(languageId: LanguageId): boolean { return this.brackets.didLanguageChange(languageId); diff --git a/src/vs/editor/common/model/bracketPairColorizer/brackets.ts b/src/vs/editor/common/model/bracketPairColorizer/brackets.ts index a292c92fad5eb..ddd6e61ffa695 100644 --- a/src/vs/editor/common/model/bracketPairColorizer/brackets.ts +++ b/src/vs/editor/common/model/bracketPairColorizer/brackets.ts @@ -4,94 +4,70 @@ *--------------------------------------------------------------------------------------------*/ import { escapeRegExpCharacters } from 'vs/base/common/strings'; -import { SmallImmutableSet, DenseKeyProvider } from 'vs/editor/common/model/bracketPairColorizer/smallImmutableSet'; +import { toLength } from 'vs/editor/common/model/bracketPairColorizer/length'; +import { SmallImmutableSet, DenseKeyProvider, identityKeyProvider } from 'vs/editor/common/model/bracketPairColorizer/smallImmutableSet'; import { LanguageId } from 'vs/editor/common/modes'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { BracketAstNode } from './ast'; -import { toLength } from './length'; -import { Token, TokenKind } from './tokenizer'; +import { OpeningBracketId, Token, TokenKind } from './tokenizer'; export class BracketTokens { - static createFromLanguage(languageId: LanguageId, customBracketPairs: readonly [string, string][], denseKeyProvider: DenseKeyProvider): BracketTokens { - const brackets = [...(LanguageConfigurationRegistry.getColorizedBracketPairs(languageId))]; + static createFromLanguage(languageId: LanguageId, denseKeyProvider: DenseKeyProvider): BracketTokens { + function getId(languageId: LanguageId, openingText: string): OpeningBracketId { + return denseKeyProvider.getKey(`${languageId}:::${openingText}`); + } - const tokens = new BracketTokens(); + const brackets = [...(LanguageConfigurationRegistry.getColorizedBracketPairs(languageId))]; - // Used for detecting shared brackets. - const categoryCache = new Map(); + const closingBrackets = new Map, first: OpeningBracketId }>(); + const openingBrackets = new Set(); - let newOpeningBracketCategory = 0; - let parseBracket = (openingBracket: string, closingBracket: string) => { - let currCategory = newOpeningBracketCategory; + for (const [openingText, closingText] of brackets) { + openingBrackets.add(openingText); - let existingOpeningBracketCategory = categoryCache.get(openingBracket); - if (existingOpeningBracketCategory === undefined) { - // Opening bracket is new, so create a new category for it and add it. - tokens.addBracket(languageId, openingBracket, TokenKind.OpeningBracket, newOpeningBracketCategory, denseKeyProvider); - categoryCache.set(openingBracket, newOpeningBracketCategory); - newOpeningBracketCategory++; - } else { - // Opening bracket exists already, remember its category. - currCategory = existingOpeningBracketCategory; + let info = closingBrackets.get(closingText); + const openingTextId = getId(languageId, openingText); + if (!info) { + info = { openingBrackets: SmallImmutableSet.getEmpty(), first: openingTextId }; + closingBrackets.set(closingText, info); } - // Add closing bracket with opening bracket's category as a new value to pair with. - tokens.addBracket(languageId, closingBracket, TokenKind.ClosingBracket, currCategory, denseKeyProvider); - }; + info.openingBrackets = info.openingBrackets.add(openingTextId, identityKeyProvider); + } - for (const pair of brackets) { - parseBracket(pair[0], pair[1]); + const map = new Map(); + + for (const [closingText, info] of closingBrackets) { + const length = toLength(0, closingText.length); + map.set(closingText, new Token( + length, + TokenKind.ClosingBracket, + info.first, + info.openingBrackets, + BracketAstNode.create(length) + )); } - for (const pair of customBracketPairs) { - parseBracket(pair[0], pair[1]); + for (const openingText of openingBrackets) { + const length = toLength(0, openingText.length); + const openingTextId = getId(languageId, openingText); + map.set(openingText, new Token( + length, + TokenKind.OpeningBracket, + openingTextId, + SmallImmutableSet.getEmpty().add(openingTextId, identityKeyProvider), + BracketAstNode.create(length) + )); } - return tokens; + return new BracketTokens(map); } private hasRegExp = false; private _regExpGlobal: RegExp | null = null; - private readonly map = new Map(); - - private addBracket(languageId: LanguageId, value: string, kind: TokenKind, category: number, denseKeyProvider: DenseKeyProvider): void { - const length = toLength(0, value.length); - - // A language can have at most 1000 bracket pairs. - const offsetCategory = languageId * 1000 + category; - - let newCategory = SmallImmutableSet.getEmpty(); - let mergedPairsWith = SmallImmutableSet.getEmpty(); - - const existingBracket = this.map.get(value); - switch (kind) { - case TokenKind.OpeningBracket: - if (existingBracket !== undefined) { - // Do not update existing opening brackets. - return; - } - // Add category to new set. - newCategory = newCategory.add(offsetCategory, denseKeyProvider); - break; - case TokenKind.ClosingBracket: - if (existingBracket !== undefined) { - // Closing bracket exists already, add to its pairsWith set. - mergedPairsWith = existingBracket.pairsWith || mergedPairsWith; - } - mergedPairsWith = mergedPairsWith.add(offsetCategory, denseKeyProvider); - break; - } - this.map.set(value, - new Token( - length, - kind, - newCategory, - mergedPairsWith, - languageId, - BracketAstNode.create(length) - ) - ); - } + constructor( + private readonly map: Map + ) { } getRegExpStr(): string | null { if (this.isEmpty) { @@ -128,7 +104,7 @@ export class BracketTokens { export class LanguageAgnosticBracketTokens { private readonly languageIdToBracketTokens: Map = new Map(); - constructor(private readonly customBracketPairs: readonly [string, string][], private readonly denseKeyProvider: DenseKeyProvider) { + constructor(private readonly denseKeyProvider: DenseKeyProvider) { } public didLanguageChange(languageId: LanguageId): boolean { @@ -136,14 +112,14 @@ export class LanguageAgnosticBracketTokens { if (!existing) { return false; } - const newRegExpStr = BracketTokens.createFromLanguage(languageId, this.customBracketPairs, this.denseKeyProvider).getRegExpStr(); + const newRegExpStr = BracketTokens.createFromLanguage(languageId, this.denseKeyProvider).getRegExpStr(); return existing.getRegExpStr() !== newRegExpStr; } getSingleLanguageBracketTokens(languageId: LanguageId): BracketTokens { let singleLanguageBracketTokens = this.languageIdToBracketTokens.get(languageId); if (!singleLanguageBracketTokens) { - singleLanguageBracketTokens = BracketTokens.createFromLanguage(languageId, this.customBracketPairs, this.denseKeyProvider); + singleLanguageBracketTokens = BracketTokens.createFromLanguage(languageId, this.denseKeyProvider); this.languageIdToBracketTokens.set(languageId, singleLanguageBracketTokens); } return singleLanguageBracketTokens; diff --git a/src/vs/editor/common/model/bracketPairColorizer/parser.ts b/src/vs/editor/common/model/bracketPairColorizer/parser.ts index 6580e16f69671..40082e9d4ab23 100644 --- a/src/vs/editor/common/model/bracketPairColorizer/parser.ts +++ b/src/vs/editor/common/model/bracketPairColorizer/parser.ts @@ -9,7 +9,7 @@ import { SmallImmutableSet } from './smallImmutableSet'; import { lengthGetLineCount, lengthIsZero, lengthLessThanEqual } from './length'; import { concat23Trees } from './concat23Trees'; import { NodeReader } from './nodeReader'; -import { Tokenizer, TokenKind } from './tokenizer'; +import { OpeningBracketId, Tokenizer, TokenKind } from './tokenizer'; export function parseDocument(tokenizer: Tokenizer, edits: TextEditInfo[], oldNode: AstNode | undefined): AstNode { const parser = new Parser(tokenizer, edits, oldNode); @@ -58,7 +58,7 @@ class Parser { } private parseList( - expectedClosingCategories: SmallImmutableSet, + openedBracketIds: SmallImmutableSet, ): AstNode | null { const items = new Array(); @@ -67,12 +67,12 @@ class Parser { if ( !token || (token.kind === TokenKind.ClosingBracket && - token.pairsWith?.intersects(expectedClosingCategories)) + token.bracketIds.intersects(openedBracketIds)) ) { break; } - const child = this.parseChild(expectedClosingCategories); + const child = this.parseChild(openedBracketIds); if (child.kind === AstNodeKind.List && child.children.length === 0) { continue; } @@ -85,7 +85,7 @@ class Parser { } private parseChild( - expectingClosingCategories: SmallImmutableSet, + openedBracketIds: SmallImmutableSet, ): AstNode { if (this.oldNodeReader) { const maxCacheableLength = this.positionMapper.getDistanceToNextChange(this.tokenizer.offset); @@ -96,7 +96,7 @@ class Parser { } const endLineDidChange = lengthGetLineCount(curNode.length) === lengthGetLineCount(maxCacheableLength); - const canBeReused = curNode.canBeReused(expectingClosingCategories, endLineDidChange); + const canBeReused = curNode.canBeReused(openedBracketIds, endLineDidChange); return canBeReused; }); @@ -114,31 +114,29 @@ class Parser { switch (token.kind) { case TokenKind.ClosingBracket: - return new InvalidBracketAstNode(token.pairsWith ?? SmallImmutableSet.getEmpty(), token.length); + return new InvalidBracketAstNode(token.bracketIds, token.length); case TokenKind.Text: return token.astNode as TextAstNode; case TokenKind.OpeningBracket: - const set = expectingClosingCategories.merge(token.category ?? SmallImmutableSet.getEmpty()); + const set = openedBracketIds.merge(token.bracketIds); const child = this.parseList(set); const nextToken = this.tokenizer.peek(); if ( nextToken && nextToken.kind === TokenKind.ClosingBracket && - nextToken.pairsWith?.intersects(token.category ?? SmallImmutableSet.getEmpty()) + (nextToken.bracketId === token.bracketId || nextToken.bracketIds.intersects(token.bracketIds)) ) { this.tokenizer.read(); return PairAstNode.create( - token.category ?? SmallImmutableSet.getEmpty(), token.astNode as BracketAstNode, child, nextToken.astNode as BracketAstNode ); } else { return PairAstNode.create( - token.category ?? SmallImmutableSet.getEmpty(), token.astNode as BracketAstNode, child, null diff --git a/src/vs/editor/common/model/bracketPairColorizer/smallImmutableSet.ts b/src/vs/editor/common/model/bracketPairColorizer/smallImmutableSet.ts index 5b1d4845be00d..793c600a69e89 100644 --- a/src/vs/editor/common/model/bracketPairColorizer/smallImmutableSet.ts +++ b/src/vs/editor/common/model/bracketPairColorizer/smallImmutableSet.ts @@ -37,7 +37,7 @@ export class SmallImmutableSet { ) { } - public add(value: T, keyProvider: DenseKeyProvider): SmallImmutableSet { + public add(value: T, keyProvider: IDenseKeyProvider): SmallImmutableSet { const key = keyProvider.getKey(value); let idx = key >> 5; // divided by 32 if (idx === 0) { @@ -59,7 +59,7 @@ export class SmallImmutableSet { return SmallImmutableSet.create(this.items, newItems); } - public has(value: T, keyProvider: DenseKeyProvider): boolean { + public has(value: T, keyProvider: IDenseKeyProvider): boolean { const key = keyProvider.getKey(value); let idx = key >> 5; // divided by 32 if (idx === 0) { @@ -129,6 +129,16 @@ export class SmallImmutableSet { } } +export interface IDenseKeyProvider { + getKey(value: T): number; +} + +export const identityKeyProvider: IDenseKeyProvider = { + getKey(value: number) { + return value; + } +}; + /** * Assigns values a unique incrementing key. */ diff --git a/src/vs/editor/common/model/bracketPairColorizer/tokenizer.ts b/src/vs/editor/common/model/bracketPairColorizer/tokenizer.ts index 9d2dd73e396ca..4383207dc827d 100644 --- a/src/vs/editor/common/model/bracketPairColorizer/tokenizer.ts +++ b/src/vs/editor/common/model/bracketPairColorizer/tokenizer.ts @@ -7,7 +7,7 @@ import { NotSupportedError } from 'vs/base/common/errors'; import { LineTokens } from 'vs/editor/common/core/lineTokens'; import { ITextModel } from 'vs/editor/common/model'; import { SmallImmutableSet } from 'vs/editor/common/model/bracketPairColorizer/smallImmutableSet'; -import { LanguageId, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes'; +import { StandardTokenType, TokenMetadata } from 'vs/editor/common/modes'; import { BracketAstNode, TextAstNode } from './ast'; import { BracketTokens, LanguageAgnosticBracketTokens } from './brackets'; import { lengthGetColumnCountIfZeroLineCount, Length, lengthAdd, lengthDiff, lengthToObj, lengthZero, toLength } from './length'; @@ -29,13 +29,24 @@ export const enum TokenKind { ClosingBracket = 2, } +export type OpeningBracketId = number; + export class Token { constructor( readonly length: Length, readonly kind: TokenKind, - readonly category: SmallImmutableSet | null, - readonly pairsWith: SmallImmutableSet | null, - readonly languageId: LanguageId, + /** + * If this token is an opening bracket, this is the id of the opening bracket. + * If this token is a closing bracket, this is the id of the first opening bracket that is closed by this bracket. + * Otherwise, it is -1. + */ + readonly bracketId: OpeningBracketId, + /** + * If this token is an opening bracket, this just contains `bracketId`. + * If this token is a closing bracket, this lists all opening bracket ids, that it closes. + * Otherwise, it is empty. + */ + readonly bracketIds: SmallImmutableSet, readonly astNode: BracketAstNode | TextAstNode | undefined, ) { } } @@ -231,7 +242,7 @@ class NonPeekableTextBufferTokenizer { } const length = lengthDiff(startLineIdx, startLineCharOffset, this.lineIdx, this.lineCharOffset); - return new Token(length, TokenKind.Text, null, null, -1, new TextAstNode(length)); + return new Token(length, TokenKind.Text, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length)); } } @@ -257,7 +268,7 @@ export class FastTokenizer implements Tokenizer { for (let i = 0; i < 60; i++) { smallTextTokens0Line.push( new Token( - toLength(0, i), TokenKind.Text, null, null, -1, + toLength(0, i), TokenKind.Text, -1, SmallImmutableSet.getEmpty(), new TextAstNode(toLength(0, i)) ) ); @@ -267,7 +278,7 @@ export class FastTokenizer implements Tokenizer { for (let i = 0; i < 60; i++) { smallTextTokens1Line.push( new Token( - toLength(1, i), TokenKind.Text, null, null, -1, + toLength(1, i), TokenKind.Text, -1, SmallImmutableSet.getEmpty(), new TextAstNode(toLength(1, i)) ) ); @@ -290,7 +301,7 @@ export class FastTokenizer implements Tokenizer { token = smallTextTokens0Line[colCount]; } else { const length = toLength(0, colCount); - token = new Token(length, TokenKind.Text, null, null, -1, new TextAstNode(length)); + token = new Token(length, TokenKind.Text, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length)); } } else { const lineCount = curLineCount - lastTokenEndLine; @@ -299,7 +310,7 @@ export class FastTokenizer implements Tokenizer { token = smallTextTokens1Line[colCount]; } else { const length = toLength(lineCount, colCount); - token = new Token(length, TokenKind.Text, null, null, -1, new TextAstNode(length)); + token = new Token(length, TokenKind.Text, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length)); } } tokens.push(token); @@ -320,7 +331,7 @@ export class FastTokenizer implements Tokenizer { const length = (lastTokenEndLine === curLineCount) ? toLength(0, offset - lastTokenEndOffset) : toLength(curLineCount - lastTokenEndLine, offset - lastLineBreakOffset); - tokens.push(new Token(length, TokenKind.Text, null, null, -1, new TextAstNode(length))); + tokens.push(new Token(length, TokenKind.Text, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length))); } this.length = toLength(curLineCount, offset - lastLineBreakOffset); diff --git a/src/vs/editor/test/common/model/bracketPairColorizer/brackets.test.ts b/src/vs/editor/test/common/model/bracketPairColorizer/brackets.test.ts index 7b5e5614a3b02..f1f774094e954 100644 --- a/src/vs/editor/test/common/model/bracketPairColorizer/brackets.test.ts +++ b/src/vs/editor/test/common/model/bracketPairColorizer/brackets.test.ts @@ -11,11 +11,10 @@ import { Token, TokenKind } from 'vs/editor/common/model/bracketPairColorizer/to import { LanguageIdentifier } from 'vs/editor/common/modes'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; - suite('Bracket Pair Colorizer - Brackets', () => { test('Basic', () => { const mode1 = new LanguageIdentifier('testMode1', 3); - const denseKeyProvider = new DenseKeyProvider(); + const denseKeyProvider = new DenseKeyProvider(); const getImmutableSet = (elements: number[]) => { let newSet = SmallImmutableSet.getEmpty(); elements.forEach(x => newSet = newSet.add(x, denseKeyProvider)); @@ -32,7 +31,7 @@ suite('Bracket Pair Colorizer - Brackets', () => { ] })); - const brackets = new LanguageAgnosticBracketTokens([], denseKeyProvider); + const brackets = new LanguageAgnosticBracketTokens(denseKeyProvider); const bracketsExpected = [ { text: '{', length: 1, kind: 'OpeningBracket', category: getImmutableSet([3000]), pairsWith: SmallImmutableSet.getEmpty(), languageId: 3 }, { text: '[', length: 1, kind: 'OpeningBracket', category: getImmutableSet([3001]), pairsWith: SmallImmutableSet.getEmpty(), languageId: 3 }, @@ -58,7 +57,6 @@ suite('Bracket Pair Colorizer - Brackets', () => { assert.deepStrictEqual(bracketsActual, bracketsExpected); disposableStore.dispose(); - }); }); @@ -69,13 +67,11 @@ function tokenToObject(token: Token | undefined, text: string): any { return { text: text, length: token.length, - category: token.category, - pairsWith: token.pairsWith, + category: token.bracketId, kind: { [TokenKind.ClosingBracket]: 'ClosingBracket', [TokenKind.OpeningBracket]: 'OpeningBracket', [TokenKind.Text]: 'Text', }[token.kind], - languageId: token.languageId, }; } diff --git a/src/vs/editor/test/common/model/bracketPairColorizer/concat23Trees.test.ts b/src/vs/editor/test/common/model/bracketPairColorizer/concat23Trees.test.ts index a1fc021818dc4..8321024fd8eba 100644 --- a/src/vs/editor/test/common/model/bracketPairColorizer/concat23Trees.test.ts +++ b/src/vs/editor/test/common/model/bracketPairColorizer/concat23Trees.test.ts @@ -33,12 +33,12 @@ suite('Bracket Pair Colorizer - mergeItems', () => { } } - if (!node1.unopenedBrackets.equals(node2.unopenedBrackets)) { + if (!node1.missingBracketIds.equals(node2.missingBracketIds)) { return false; } if (node1.kind === AstNodeKind.Pair && node2.kind === AstNodeKind.Pair) { - return node1.category === node2.category; + return true; } else if (node1.kind === node2.kind) { return true; } diff --git a/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts b/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts index 988b9a5633d9d..24a5dd3930764 100644 --- a/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts +++ b/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts @@ -18,7 +18,7 @@ import { createTextModel } from 'vs/editor/test/common/editorTestUtils'; suite('Bracket Pair Colorizer - Tokenizer', () => { test('Basic', () => { const mode1 = new LanguageIdentifier('testMode1', 2); - const denseKeyProvider = new DenseKeyProvider(); + const denseKeyProvider = new DenseKeyProvider(); const getImmutableSet = (elements: number[]) => { let newSet = SmallImmutableSet.getEmpty(); elements.forEach(x => newSet = newSet.add(x, denseKeyProvider)); @@ -35,10 +35,10 @@ suite('Bracket Pair Colorizer - Tokenizer', () => { const disposableStore = new DisposableStore(); disposableStore.add(TokenizationRegistry.register(mode1.language, document.getTokenizationSupport())); disposableStore.add(LanguageConfigurationRegistry.register(mode1, { - brackets: [['{', '}'], ['[', ']'], ['(', ')']], + brackets: [['{', '}'], ['[', ']'], ['(', ')'], ['begin', 'end']], })); - const brackets = new LanguageAgnosticBracketTokens([['begin', 'end']], denseKeyProvider); + const brackets = new LanguageAgnosticBracketTokens(denseKeyProvider); const model = createTextModel(document.getText(), {}, mode1); model.forceTokenization(model.getLineCount()); @@ -47,15 +47,15 @@ suite('Bracket Pair Colorizer - Tokenizer', () => { assert.deepStrictEqual(toArr(tokens, model), [ { category: null, pairsWith: null, kind: 'Text', languageId: -1, text: ' ', }, - { category: getImmutableSet([2000]), pairsWith: SmallImmutableSet.getEmpty(), kind: 'OpeningBracket', languageId: 2, text: '{', }, + { category: getImmutableSet([2000]), pairsWith: SmallImmutableSet.getEmpty(), kind: 'OpeningBracket', text: '{', }, { category: null, pairsWith: null, kind: 'Text', languageId: -1, text: ' ', }, - { category: SmallImmutableSet.getEmpty(), pairsWith: getImmutableSet([2000]), kind: 'ClosingBracket', languageId: 2, text: '}', }, + { category: SmallImmutableSet.getEmpty(), pairsWith: getImmutableSet([2000]), kind: 'ClosingBracket', text: '}', }, { category: null, pairsWith: null, kind: 'Text', languageId: -1, text: ' ', }, - { category: getImmutableSet([2003]), pairsWith: SmallImmutableSet.getEmpty(), kind: 'OpeningBracket', languageId: 2, text: 'begin', }, + { category: getImmutableSet([2003]), pairsWith: SmallImmutableSet.getEmpty(), kind: 'OpeningBracket', text: 'begin', }, { category: null, pairsWith: null, kind: 'Text', languageId: -1, text: ' ', }, - { category: SmallImmutableSet.getEmpty(), pairsWith: getImmutableSet([2003]), kind: 'ClosingBracket', languageId: 2, text: 'end', }, + { category: SmallImmutableSet.getEmpty(), pairsWith: getImmutableSet([2003]), kind: 'ClosingBracket', text: 'end', }, { category: null, pairsWith: null, kind: 'Text', languageId: -1, text: '\nhello{', }, - { category: SmallImmutableSet.getEmpty(), pairsWith: getImmutableSet([2000]), kind: 'ClosingBracket', languageId: 2, text: '}', } + { category: SmallImmutableSet.getEmpty(), pairsWith: getImmutableSet([2000]), kind: 'ClosingBracket', text: '}', } ]); disposableStore.dispose(); @@ -87,14 +87,13 @@ function toArr(tokens: Token[], model: TextModel): any[] { function tokenToObj(token: Token, offset: Length, model: TextModel): any { return { text: model.getValueInRange(lengthsToRange(offset, lengthAdd(offset, token.length))), - category: token.category, - pairsWith: token.pairsWith, + category: token.bracketId, + pairsWith: token.bracketIds, kind: { [TokenKind.ClosingBracket]: 'ClosingBracket', [TokenKind.OpeningBracket]: 'OpeningBracket', [TokenKind.Text]: 'Text', - }[token.kind], - languageId: token.languageId, + }[token.kind] }; }