Skip to content

Commit

Permalink
[23] Parsing details for markdown comments #2824 (#2831)
Browse files Browse the repository at this point in the history 
+ codeblock cannot interrupt a paragraph
+ separate handling of code blocks indented vs fenced
  + fence does not terminate indented code block
  + handle various forms of fences:
     - ` vs. ~
     - different fence lengths
     - no mixed fences
     - fences inside fenced region
+ handle (and require) escaping of [ ] inside references (method arguments)

fixes #2824
  • Loading branch information
@stephan-herrmann
stephan-herrmann authored Aug 18, 2024
1 parent e742bea commit 9b33f16
Show file tree
Hide file tree
Showing 4 changed files with 371 additions and 21 deletions.
37 changes: 32 additions & 5 deletions ...re.compiler.batch/src/org/eclipse/jdt/internal/compiler/parser/AbstractCommentParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,9 @@ protected boolean commentParse() {
consumeToken();
}

if (this.markdown && !Character.isWhitespace(nextCharacter) && nextCharacter != '/' && nextCharacter != '`') {
this.markdownHelper.recordText();
}
// Consume rules depending on the read character
switch (nextCharacter) {
case '@' :
Expand Down Expand Up @@ -469,8 +472,8 @@ protected boolean commentParse() {
this.textStart = this.index;
break;
}
} else if (nextCharacter == '`') {
this.markdownHelper.recordBackTick(this.lineStarted);
} else if (nextCharacter == '`' || nextCharacter == '~') {
this.markdownHelper.recordFenceChar(previousChar, nextCharacter, this.lineStarted);
}
}
if (isFormatterParser && nextCharacter == '<') {
Expand Down Expand Up @@ -650,12 +653,12 @@ protected Object parseArguments(Object receiver, boolean checkVerifySpaceOrEndCo
// Read possible additional type info
dim = 0;
isVarargs = false;
if (readToken() == TerminalTokens.TokenNameLBRACKET) {
if (readMarkdownEscapedToken(TerminalTokens.TokenNameLBRACKET)) {
// array declaration
while (readToken() == TerminalTokens.TokenNameLBRACKET) {
while (readMarkdownEscapedToken(TerminalTokens.TokenNameLBRACKET)) {
int dimStart = this.scanner.getCurrentTokenStartPosition();
consumeToken();
if (readToken() != TerminalTokens.TokenNameRBRACKET) {
if (!readMarkdownEscapedToken(TerminalTokens.TokenNameRBRACKET)) {
break nextArg;
}
consumeToken();
Expand Down Expand Up @@ -731,6 +734,10 @@ protected Object parseArguments(Object receiver, boolean checkVerifySpaceOrEndCo
}

// Something wrong happened => Invalid input
if (this.markdown) {
// skip over bogus token
this.currentTokenType = -1;
}
throw Scanner.invalidInput();
} finally {
// we have to make sure that this is reset to the previous value even if an exception occurs
Expand Down Expand Up @@ -3333,6 +3340,26 @@ protected int readToken() throws InvalidInputException {
return this.currentTokenType;
}

protected boolean readMarkdownEscapedToken(int expectedToken) throws InvalidInputException {
if (!this.markdown)
return readToken() == expectedToken;
if (this.currentTokenType < 0) {
this.tokenPreviousPosition = this.scanner.currentPosition;
if (peekChar() != '\\')
return false;
this.scanner.currentPosition++;
this.currentTokenType = this.scanner.getNextToken();
if (this.currentTokenType != expectedToken) {
this.scanner.currentPosition = this.tokenPreviousPosition;
this.currentTokenType = -1;
return false;
}
this.index = this.scanner.currentPosition;
this.lineStarted = true; // after having read a token, line is obviously started...
}
return this.currentTokenType == expectedToken;
}

protected int readTokenAndConsume() throws InvalidInputException {
int token = readToken();
consumeToken();
Expand Down
61 changes: 46 additions & 15 deletions ...e.compiler.batch/src/org/eclipse/jdt/internal/compiler/parser/IMarkdownCommentHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public interface IMarkdownCommentHelper {

void recordSlash(int nextIndex);

void recordBackTick(boolean lineStarted);
void recordFenceChar(char previous, char next, boolean lineStarted);

/**
* When at the beginning of a comment line, record that a whitespace was seen.
Expand All @@ -35,6 +35,8 @@ public interface IMarkdownCommentHelper {
*/
boolean recordSignificantLeadingSpace();

void recordText();

boolean isInCodeBlock();

/** Retrieve the start of the current text, possibly including significant leading whitespace. */
Expand All @@ -56,7 +58,6 @@ static IMarkdownCommentHelper create(AbstractCommentParser parser) {
return new NullMarkdownHelper();
}
}

}

class NullMarkdownHelper implements IMarkdownCommentHelper {
Expand All @@ -67,14 +68,18 @@ public void recordSlash(int nextIndex) {
// nop
}
@Override
public void recordBackTick(boolean lineStarted) {
public void recordFenceChar(char previous, char next, boolean lineStarted) {
// nop
}
@Override
public boolean recordSignificantLeadingSpace() {
return false;
}
@Override
public void recordText() {
// nop
}
@Override
public boolean isInCodeBlock() {
return false;
}
Expand All @@ -97,8 +102,13 @@ class MarkdownCommentHelper implements IMarkdownCommentHelper {
int slashCount = 0;
int leadingSpaces = 0;
int markdownLineStart = -1;
int backTickCount = 0;
boolean insideFence = false;
boolean insideIndentedCodeBlock = false;
boolean insideFencedCodeBlock = false;
char fenceChar;
int fenceCharCount;
int fenceLength;
boolean isBlankLine = true;
boolean previousIsBlankLine = true;

public MarkdownCommentHelper(int lineStart, int commonIndent) {
this.markdownLineStart = lineStart;
Expand All @@ -116,29 +126,48 @@ public void recordSlash(int nextIndex) {
}

@Override
public void recordBackTick(boolean lineStarted) {
if (this.backTickCount < 3) {
if (this.backTickCount == 0 && lineStarted) {
public void recordFenceChar(char previous, char next, boolean lineStarted) {
if (this.insideIndentedCodeBlock) {
return;
}
if (this.fenceCharCount == 0) {
if (lineStarted)
return;
}
if (++this.backTickCount == 3) {
this.insideFence^=true;
}
this.fenceChar = next;
this.fenceCharCount = 1;
return;
}
if (next != this.fenceChar || previous != next)
return;
int required = this.insideFencedCodeBlock ? this.fenceLength : 3;
if (++this.fenceCharCount == required) {
this.insideFencedCodeBlock^=true;
}
this.fenceLength = this.fenceCharCount;
}

@Override
public boolean recordSignificantLeadingSpace() {
if (this.markdownLineStart != -1) {
if (++this.leadingSpaces > this.commonIndent)
if (++this.leadingSpaces > this.commonIndent) {
if (!this.insideFencedCodeBlock && this.previousIsBlankLine && this.leadingSpaces - this.commonIndent >= 4)
this.insideIndentedCodeBlock = true;
return true;
}
}
return false;
}

@Override
public void recordText() {
this.isBlankLine = false;
if (this.leadingSpaces - this.commonIndent < 4)
this.insideIndentedCodeBlock = false;
}

@Override
public boolean isInCodeBlock() {
return this.insideFence || (this.leadingSpaces - this.commonIndent >= 4);
return this.insideIndentedCodeBlock || this.insideFencedCodeBlock;
}

@Override
Expand All @@ -156,10 +185,12 @@ public void resetLineStart() {

@Override
public void resetAtLineEnd() {
this.previousIsBlankLine = this.isBlankLine;
this.isBlankLine = true;
this.slashCount = 0;
this.leadingSpaces = 0;
this.markdownLineStart = -1;
this.backTickCount = 0;
this.fenceCharCount = 0;
// do not reset `insideFence`
}
}
31 changes: 30 additions & 1 deletion ...sts.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/MarkdownCommentsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ public void test010() {
public class X {
/// Some text here without the necessary tags for main method
/// @param arguments array of strings
/// @return java.lang.Str -- should not raise an error
/// no tags here
public static void main(String[] arguments) {
Expand All @@ -309,7 +310,7 @@ public static void main(String[] arguments) {
}
""", },
"----------\n" +
"1. ERROR in X.java (at line 6)\n" +
"1. ERROR in X.java (at line 7)\n" +
" public static void main(String[] arguments) {\n" +
" ^^^^^^^^^\n" +
"Javadoc: Missing tag for parameter arguments\n" +
Expand Down Expand Up @@ -596,4 +597,32 @@ public static void main(String[] args) {
this.reportMissingJavadocTags = bkup;
}
}
public void test021() {
// arrays in method reference lack escaping.
// TODO specific error message?
String bkup = this.reportMissingJavadocTags;
try {
this.reportMissingJavadocTags = CompilerOptions.IGNORE;
this.runNegativeTest(new String[] { "X.java",
"""
///
/// Reference to method with array parameter: [#main(String[])]
///
public class X {
public static void main(String[] args) {
System.out.println("Hello");
}
}
""", },
"----------\n" +
"1. ERROR in X.java (at line 2)\n" +
" /// Reference to method with array parameter: [#main(String[])]\n" +
" ^^^^^^^^\n" +
"Javadoc: Invalid parameters declaration\n" +
"----------\n",
JavacTestOptions.Excuse.EclipseWarningConfiguredAsError);
} finally {
this.reportMissingJavadocTags = bkup;
}
}
}
Loading

0 comments on commit 9b33f16

Please sign in to comment.