Skip to content

Commit

Permalink
Allow colorized bracket pairs to share opening or closing brackets (#…
Browse files Browse the repository at this point in the history 
…132209)
  • Loading branch information
@DragWx
DragWx committed Sep 18, 2021
1 parent 54cdd9c commit 65cb8e5
Show file tree
Hide file tree
Showing 9 changed files with 235 additions and 107 deletions.
52 changes: 24 additions & 28 deletions src/vs/editor/common/model/bracketPairColorizer/ast.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/

import { tail } from 'vs/base/common/arrays';
import { DenseKeyProvider, SmallImmutableSet } from './smallImmutableSet';
import { SmallImmutableSet } from './smallImmutableSet';
import { lengthAdd, lengthZero, Length, lengthHash } from './length';

export const enum AstNodeKind {
Expand All @@ -20,7 +20,7 @@ export type AstNode = PairAstNode | ListAstNode | BracketAstNode | InvalidBracke
abstract class BaseAstNode {
abstract readonly kind: AstNodeKind;
abstract readonly children: readonly AstNode[];
abstract readonly unopenedBrackets: SmallImmutableSet<number>;
abstract readonly missingBracketIds: SmallImmutableSet<number>;

/**
* In case of a list, determines the height of the (2,3) tree.
Expand Down Expand Up @@ -55,7 +55,6 @@ abstract class BaseAstNode {

export class PairAstNode extends BaseAstNode {
public static create(
category: number,
openingBracket: BracketAstNode,
child: AstNode | null,
closingBracket: BracketAstNode | null
Expand All @@ -71,7 +70,7 @@ export class PairAstNode extends BaseAstNode {
children.push(closingBracket);
}

return new PairAstNode(length, category, children, child ? child.unopenedBrackets : SmallImmutableSet.getEmpty());
return new PairAstNode(length, children, child ? child.missingBracketIds : SmallImmutableSet.getEmpty());
}

get kind(): AstNodeKind.Pair {
Expand All @@ -82,7 +81,7 @@ export class PairAstNode extends BaseAstNode {
}

canBeReused(
expectedClosingCategories: SmallImmutableSet<number>,
openedBracketIds: SmallImmutableSet<number>,
endLineDidChange: boolean
) {
if (this.closingBracket === null) {
Expand All @@ -96,7 +95,7 @@ export class PairAstNode extends BaseAstNode {
return false;
}

if (expectedClosingCategories.intersects(this.unopenedBrackets)) {
if (openedBracketIds.intersects(this.missingBracketIds)) {
return false;
}

Expand All @@ -105,7 +104,6 @@ export class PairAstNode extends BaseAstNode {

flattenLists(): PairAstNode {
return PairAstNode.create(
this.category,
this.openingBracket.flattenLists(),
this.child && this.child.flattenLists(),
this.closingBracket && this.closingBracket.flattenLists()
Expand Down Expand Up @@ -138,19 +136,17 @@ export class PairAstNode extends BaseAstNode {

private constructor(
length: Length,
public readonly category: number,
public readonly children: readonly AstNode[],
public readonly unopenedBrackets: SmallImmutableSet<number>
public readonly missingBracketIds: SmallImmutableSet<number>
) {
super(length);
}

clone(): PairAstNode {
return new PairAstNode(
this.length,
this.category,
clone(this.children),
this.unopenedBrackets
this.missingBracketIds
);
}
}
Expand All @@ -172,10 +168,10 @@ export class ListAstNode extends BaseAstNode {
return new ListAstNode(lengthZero, 0, items, SmallImmutableSet.getEmpty());
} else {
let length = items[0].length;
let unopenedBrackets = items[0].unopenedBrackets;
let unopenedBrackets = items[0].missingBracketIds;
for (let i = 1; i < items.length; i++) {
length = lengthAdd(length, items[i].length);
unopenedBrackets = unopenedBrackets.merge(items[i].unopenedBrackets);
unopenedBrackets = unopenedBrackets.merge(items[i].missingBracketIds);
}
return new ListAstNode(length, items[0].listHeight + 1, items, unopenedBrackets);
}
Expand All @@ -187,7 +183,7 @@ export class ListAstNode extends BaseAstNode {
get children(): readonly AstNode[] {
return this._items;
}
get unopenedBrackets(): SmallImmutableSet<number> {
get missingBracketIds(): SmallImmutableSet<number> {
return this._unopenedBrackets;
}

Expand All @@ -201,15 +197,15 @@ export class ListAstNode extends BaseAstNode {
}

canBeReused(
expectedClosingCategories: SmallImmutableSet<number>,
openedBracketIds: SmallImmutableSet<number>,
endLineDidChange: boolean
): boolean {
if (this._items.length === 0) {
// might not be very helpful
return true;
}

if (expectedClosingCategories.intersects(this.unopenedBrackets)) {
if (openedBracketIds.intersects(this.missingBracketIds)) {
return false;
}

Expand All @@ -219,7 +215,7 @@ export class ListAstNode extends BaseAstNode {
}

return lastChild.canBeReused(
expectedClosingCategories,
openedBracketIds,
endLineDidChange
);
}
Expand All @@ -238,7 +234,7 @@ export class ListAstNode extends BaseAstNode {
}

clone(): ListAstNode {
return new ListAstNode(this.length, this.listHeight, clone(this._items), this.unopenedBrackets);
return new ListAstNode(this.length, this.listHeight, clone(this._items), this.missingBracketIds);
}

private handleChildrenChanged(): void {
Expand All @@ -248,10 +244,10 @@ export class ListAstNode extends BaseAstNode {
}

let length = items[0].length;
let unopenedBrackets = items[0].unopenedBrackets;
let unopenedBrackets = items[0].missingBracketIds;
for (let i = 1; i < items.length; i++) {
length = lengthAdd(length, items[i].length);
unopenedBrackets = unopenedBrackets.merge(items[i].unopenedBrackets);
unopenedBrackets = unopenedBrackets.merge(items[i].missingBracketIds);
}
this._length = length;
this._unopenedBrackets = unopenedBrackets;
Expand Down Expand Up @@ -372,12 +368,12 @@ export class TextAstNode extends BaseAstNode {
get children(): readonly AstNode[] {
return emptyArray;
}
get unopenedBrackets(): SmallImmutableSet<number> {
get missingBracketIds(): SmallImmutableSet<number> {
return SmallImmutableSet.getEmpty();
}

canBeReused(
expectedClosingCategories: SmallImmutableSet<number>,
openedBracketIds: SmallImmutableSet<number>,
endLineDidChange: boolean
) {
// Don't reuse text from a line that got changed.
Expand Down Expand Up @@ -422,7 +418,7 @@ export class BracketAstNode extends BaseAstNode {
return emptyArray;
}

get unopenedBrackets(): SmallImmutableSet<number> {
get missingBracketIds(): SmallImmutableSet<number> {
return SmallImmutableSet.getEmpty();
}

Expand Down Expand Up @@ -456,18 +452,18 @@ export class InvalidBracketAstNode extends BaseAstNode {
return emptyArray;
}

public readonly unopenedBrackets: SmallImmutableSet<number>;
public readonly missingBracketIds: SmallImmutableSet<number>;

constructor(category: number, length: Length, denseKeyProvider: DenseKeyProvider<number>) {
constructor(closingBrackets: SmallImmutableSet<number>, length: Length) {
super(length);
this.unopenedBrackets = SmallImmutableSet.getEmpty().add(category, denseKeyProvider);
this.missingBracketIds = closingBrackets;
}

canBeReused(
expectedClosingCategories: SmallImmutableSet<number>,
openedBracketIds: SmallImmutableSet<number>,
endLineDidChange: boolean
) {
return !expectedClosingCategories.intersects(this.unopenedBrackets);
return !openedBracketIds.intersects(this.missingBracketIds);
}

flattenLists(): InvalidBracketAstNode {
Expand Down
8 changes: 4 additions & 4 deletions src/vs/editor/common/model/bracketPairColorizer/bracketPairColorizer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
private initialAstWithoutTokens: AstNode | undefined;
private astWithTokens: AstNode | undefined;

private readonly brackets = new LanguageAgnosticBracketTokens([]);
private readonly denseKeyProvider = new DenseKeyProvider<number>();
private readonly denseKeyProvider = new DenseKeyProvider<string>();
private readonly brackets = new LanguageAgnosticBracketTokens(this.denseKeyProvider);

public didLanguageChange(languageId: LanguageId): boolean {
return this.brackets.didLanguageChange(languageId);
Expand Down Expand Up @@ -159,7 +159,7 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
// There are no token information yet
const brackets = this.brackets.getSingleLanguageBracketTokens(this.textModel.getLanguageIdentifier().id);
const tokenizer = new FastTokenizer(this.textModel.getValue(), brackets);
this.initialAstWithoutTokens = parseDocument(tokenizer, [], undefined, this.denseKeyProvider);
this.initialAstWithoutTokens = parseDocument(tokenizer, [], undefined);
this.astWithTokens = this.initialAstWithoutTokens.clone();
} else if (textModel.backgroundTokenizationState === BackgroundTokenizationState.Completed) {
// Skip the initial ast, as there is no flickering.
Expand Down Expand Up @@ -196,7 +196,7 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
const isPure = false;
const previousAstClone = isPure ? previousAst?.clone() : previousAst;
const tokenizer = new TextBufferTokenizer(this.textModel, this.brackets);
const result = parseDocument(tokenizer, edits, previousAstClone, this.denseKeyProvider);
const result = parseDocument(tokenizer, edits, previousAstClone);
return result;
}

Expand Down
81 changes: 50 additions & 31 deletions src/vs/editor/common/model/bracketPairColorizer/brackets.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,51 +4,70 @@
*--------------------------------------------------------------------------------------------*/

import { escapeRegExpCharacters } from 'vs/base/common/strings';
import { toLength } from 'vs/editor/common/model/bracketPairColorizer/length';
import { SmallImmutableSet, DenseKeyProvider, identityKeyProvider } from 'vs/editor/common/model/bracketPairColorizer/smallImmutableSet';
import { LanguageId } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { BracketAstNode } from './ast';
import { toLength } from './length';
import { Token, TokenKind } from './tokenizer';
import { OpeningBracketId, Token, TokenKind } from './tokenizer';

export class BracketTokens {
static createFromLanguage(languageId: LanguageId, customBracketPairs: readonly [string, string][]): BracketTokens {
static createFromLanguage(languageId: LanguageId, denseKeyProvider: DenseKeyProvider<string>): BracketTokens {
function getId(languageId: LanguageId, openingText: string): OpeningBracketId {
return denseKeyProvider.getKey(`${languageId}:::${openingText}`);
}

const brackets = [...(LanguageConfigurationRegistry.getColorizedBracketPairs(languageId))];

const tokens = new BracketTokens();
const closingBrackets = new Map</* closingText */ string, { openingBrackets: SmallImmutableSet<OpeningBracketId>, first: OpeningBracketId }>();
const openingBrackets = new Set</* openingText */ string>();

let idxOffset = 0;
for (const pair of brackets) {
tokens.addBracket(languageId, pair[0], TokenKind.OpeningBracket, idxOffset);
tokens.addBracket(languageId, pair[1], TokenKind.ClosingBracket, idxOffset);
idxOffset++;
for (const [openingText, closingText] of brackets) {
openingBrackets.add(openingText);

let info = closingBrackets.get(closingText);
const openingTextId = getId(languageId, openingText);
if (!info) {
info = { openingBrackets: SmallImmutableSet.getEmpty(), first: openingTextId };
closingBrackets.set(closingText, info);
}
info.openingBrackets = info.openingBrackets.add(openingTextId, identityKeyProvider);
}

for (const pair of customBracketPairs) {
idxOffset++;
tokens.addBracket(languageId, pair[0], TokenKind.OpeningBracket, idxOffset);
tokens.addBracket(languageId, pair[1], TokenKind.ClosingBracket, idxOffset);
const map = new Map<string, Token>();

for (const [closingText, info] of closingBrackets) {
const length = toLength(0, closingText.length);
map.set(closingText, new Token(
length,
TokenKind.ClosingBracket,
info.first,
info.openingBrackets,
BracketAstNode.create(length)
));
}

return tokens;
for (const openingText of openingBrackets) {
const length = toLength(0, openingText.length);
const openingTextId = getId(languageId, openingText);
map.set(openingText, new Token(
length,
TokenKind.OpeningBracket,
openingTextId,
SmallImmutableSet.getEmpty().add(openingTextId, identityKeyProvider),
BracketAstNode.create(length)
));
}

return new BracketTokens(map);
}

private hasRegExp = false;
private _regExpGlobal: RegExp | null = null;
private readonly map = new Map<string, Token>();

private addBracket(languageId: LanguageId, value: string, kind: TokenKind, idx: number): void {
const length = toLength(0, value.length);
this.map.set(value,
new Token(
length,
kind,
// A language can have at most 1000 bracket pairs.
languageId * 1000 + idx,
languageId,
BracketAstNode.create(length)
)
);
}
constructor(
private readonly map: Map<string, Token>
) { }

getRegExpStr(): string | null {
if (this.isEmpty) {
Expand Down Expand Up @@ -85,22 +104,22 @@ export class BracketTokens {
export class LanguageAgnosticBracketTokens {
private readonly languageIdToBracketTokens: Map<LanguageId, BracketTokens> = new Map();

constructor(private readonly customBracketPairs: readonly [string, string][]) {
constructor(private readonly denseKeyProvider: DenseKeyProvider<string>) {
}

public didLanguageChange(languageId: LanguageId): boolean {
const existing = this.languageIdToBracketTokens.get(languageId);
if (!existing) {
return false;
}
const newRegExpStr = BracketTokens.createFromLanguage(languageId, this.customBracketPairs).getRegExpStr();
const newRegExpStr = BracketTokens.createFromLanguage(languageId, this.denseKeyProvider).getRegExpStr();
return existing.getRegExpStr() !== newRegExpStr;
}

getSingleLanguageBracketTokens(languageId: LanguageId): BracketTokens {
let singleLanguageBracketTokens = this.languageIdToBracketTokens.get(languageId);
if (!singleLanguageBracketTokens) {
singleLanguageBracketTokens = BracketTokens.createFromLanguage(languageId, this.customBracketPairs);
singleLanguageBracketTokens = BracketTokens.createFromLanguage(languageId, this.denseKeyProvider);
this.languageIdToBracketTokens.set(languageId, singleLanguageBracketTokens);
}
return singleLanguageBracketTokens;
Expand Down
Loading

0 comments on commit 65cb8e5

Please sign in to comment.