Skip to content

Commit

Permalink
A flag to build/reconcile Java model structure with ASTParser
Browse files Browse the repository at this point in the history 
Introduce a CompilationUnit.DOM_BASED_OPERATIONS flag which instead of
using the ECJ-derived parser uses ASTParser to create a DOM and then
turns the DOM into a JavaModel structure.

This has the benefit of:
* Simplifying the code as an crawling DOM is usually simpler than
dealing with lower-level parser
* allowing other parsers as backend if ASTParser is configured for it
  • Loading branch information
@mickaelistria
mickaelistria committed Jul 10, 2024
1 parent eb2e0c6 commit 975645c
Show file tree
Hide file tree
Showing 4 changed files with 1,607 additions and 108 deletions.
2 changes: 1 addition & 1 deletion org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/ASTHolderCUInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import org.eclipse.jdt.core.dom.CompilationUnit;

public class ASTHolderCUInfo extends CompilationUnitElementInfo {
int astLevel;
public int astLevel;
boolean resolveBindings;
int reconcileFlags;
Map<String, CategorizedProblem[]> problems = null;
Expand Down
283 changes: 206 additions & 77 deletions org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/CompilationUnit.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,73 @@
package org.eclipse.jdt.internal.core;

import java.io.IOException;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.eclipse.core.resources.*;
import org.eclipse.core.runtime.*;
import org.eclipse.jdt.core.*;
import org.eclipse.jdt.core.compiler.*;
import java.util.stream.Stream;

import org.eclipse.core.resources.IContainer;
import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IMarker;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Path;
import org.eclipse.core.runtime.PerformanceStats;
import org.eclipse.jdt.core.CompletionRequestor;
import org.eclipse.jdt.core.Flags;
import org.eclipse.jdt.core.IBuffer;
import org.eclipse.jdt.core.IBufferFactory;
import org.eclipse.jdt.core.ICodeAssist;
import org.eclipse.jdt.core.ICodeCompletionRequestor;
import org.eclipse.jdt.core.ICompilationUnit;
import org.eclipse.jdt.core.ICompletionRequestor;
import org.eclipse.jdt.core.IImportContainer;
import org.eclipse.jdt.core.IImportDeclaration;
import org.eclipse.jdt.core.IJavaElement;
import org.eclipse.jdt.core.IJavaModelMarker;
import org.eclipse.jdt.core.IJavaModelStatusConstants;
import org.eclipse.jdt.core.IJavaProject;
import org.eclipse.jdt.core.IMember;
import org.eclipse.jdt.core.IMethod;
import org.eclipse.jdt.core.IModuleDescription;
import org.eclipse.jdt.core.IOpenable;
import org.eclipse.jdt.core.IPackageDeclaration;
import org.eclipse.jdt.core.IPackageFragment;
import org.eclipse.jdt.core.IPackageFragmentRoot;
import org.eclipse.jdt.core.IProblemRequestor;
import org.eclipse.jdt.core.ISourceManipulation;
import org.eclipse.jdt.core.ISourceRange;
import org.eclipse.jdt.core.ISourceReference;
import org.eclipse.jdt.core.IType;
import org.eclipse.jdt.core.ITypeRoot;
import org.eclipse.jdt.core.IWorkingCopy;
import org.eclipse.jdt.core.JavaConventions;
import org.eclipse.jdt.core.JavaCore;
import org.eclipse.jdt.core.JavaModelException;
import org.eclipse.jdt.core.WorkingCopyOwner;
import org.eclipse.jdt.core.compiler.CategorizedProblem;
import org.eclipse.jdt.core.compiler.CharOperation;
import org.eclipse.jdt.core.compiler.IProblem;
import org.eclipse.jdt.core.dom.AST;
import org.eclipse.jdt.core.dom.ASTNode;
import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.internal.compiler.IProblemFactory;
import org.eclipse.jdt.internal.compiler.SourceElementParser;
import org.eclipse.jdt.internal.compiler.ast.CompilationUnitDeclaration;
import org.eclipse.jdt.internal.compiler.env.IElementInfo;
import org.eclipse.jdt.internal.compiler.impl.CompilerOptions;
import org.eclipse.jdt.internal.compiler.lookup.TypeConstants;
import org.eclipse.jdt.internal.compiler.problem.AbortCompilationUnit;
import org.eclipse.jdt.internal.compiler.problem.DefaultProblem;
import org.eclipse.jdt.internal.compiler.problem.DefaultProblemFactory;
import org.eclipse.jdt.internal.compiler.problem.ProblemSeverities;
import org.eclipse.jdt.internal.compiler.util.SuffixConstants;
import org.eclipse.jdt.internal.core.util.DeduplicationUtil;
import org.eclipse.jdt.internal.core.util.MementoTokenizer;
Expand All @@ -47,6 +99,8 @@
* @see ICompilationUnit
*/
public class CompilationUnit extends Openable implements ICompilationUnit, org.eclipse.jdt.internal.compiler.env.ICompilationUnit, SuffixConstants {
public static boolean DOM_BASED_OPERATIONS = Boolean.getBoolean(CompilationUnit.class.getSimpleName() + ".DOM_BASED_OPERATIONS"); //$NON-NLS-1$

private static final IImportDeclaration[] NO_IMPORTS = new IImportDeclaration[0];

protected final String name;
Expand Down Expand Up @@ -102,56 +156,18 @@ public void becomeWorkingCopy(IProgressMonitor monitor) throws JavaModelExceptio
protected boolean buildStructure(OpenableElementInfo info, final IProgressMonitor pm, Map<IJavaElement, IElementInfo> newElements, IResource underlyingResource) throws JavaModelException {
CompilationUnitElementInfo unitInfo = (CompilationUnitElementInfo) info;

// ensure buffer is opened
IBuffer buffer = getBufferManager().getBuffer(CompilationUnit.this);
if (buffer == null) {
openBuffer(pm, unitInfo); // open buffer independently from the info, since we are building the info
}

// generate structure and compute syntax problems if needed
CompilationUnitStructureRequestor requestor = new CompilationUnitStructureRequestor(this, unitInfo, newElements);
JavaModelManager.PerWorkingCopyInfo perWorkingCopyInfo = getPerWorkingCopyInfo();
IJavaProject project = getJavaProject();

boolean createAST;
boolean resolveBindings;
int reconcileFlags;
Map<String, CategorizedProblem[]> problems;
if (info instanceof ASTHolderCUInfo) {
ASTHolderCUInfo astHolder = (ASTHolderCUInfo) info;
createAST = astHolder.astLevel != NO_AST;
resolveBindings = astHolder.resolveBindings;
reconcileFlags = astHolder.reconcileFlags;
problems = astHolder.problems;
} else {
createAST = false;
resolveBindings = false;
reconcileFlags = 0;
problems = null;
}

boolean createAST = info instanceof ASTHolderCUInfo astHolder ? astHolder.astLevel != NO_AST : false;
boolean resolveBindings = info instanceof ASTHolderCUInfo astHolder ? astHolder.resolveBindings : false;
int reconcileFlags = info instanceof ASTHolderCUInfo astHolder ? astHolder.reconcileFlags : 0;
boolean computeProblems = perWorkingCopyInfo != null && perWorkingCopyInfo.isActive() && project != null && JavaProject.hasJavaNature(project.getProject());
IProblemFactory problemFactory = new DefaultProblemFactory();
Map<String, String> options = this.getOptions(true);
if (!computeProblems) {
// disable task tags checking to speed up parsing
options.put(JavaCore.COMPILER_TASK_TAGS, ""); //$NON-NLS-1$
}
CompilerOptions compilerOptions = new CompilerOptions(options);
compilerOptions.ignoreMethodBodies = (reconcileFlags & ICompilationUnit.IGNORE_METHOD_BODIES) != 0;
SourceElementParser parser = new SourceElementParser(
requestor,
problemFactory,
compilerOptions,
true/*report local declarations*/,
!createAST /*optimize string literals only if not creating a DOM AST*/);
parser.reportOnlyOneSyntaxError = !computeProblems;
parser.setMethodsFullRecovery(true);
parser.setStatementsRecovery((reconcileFlags & ICompilationUnit.ENABLE_STATEMENTS_RECOVERY) != 0);

if (!computeProblems && !resolveBindings && !createAST) // disable javadoc parsing if not computing problems, not resolving and not creating ast
parser.javadocParser.checkDocComment = false;
requestor.parser = parser;

// update timestamp (might be IResource.NULL_STAMP if original does not exist)
if (underlyingResource == null) {
Expand All @@ -161,44 +177,146 @@ protected boolean buildStructure(OpenableElementInfo info, final IProgressMonito
if (underlyingResource != null)
unitInfo.timestamp = underlyingResource.getModificationStamp();

// compute other problems if needed
CompilationUnitDeclaration compilationUnitDeclaration = null;
// ensure buffer is opened
IBuffer buffer = getBufferManager().getBuffer(CompilationUnit.this);
if (buffer == null) {
openBuffer(pm, unitInfo); // open buffer independently from the info, since we are building the info
}

CompilationUnit source = cloneCachingContents();
try {
if (computeProblems) {
if (problems == null) {
// report problems to the problem requestor
problems = new HashMap<>();
compilationUnitDeclaration = CompilationUnitProblemFinder.process(source, parser, this.owner, problems, createAST, reconcileFlags, pm);
try {
perWorkingCopyInfo.beginReporting();
for (CategorizedProblem[] categorizedProblems : problems.values()) {
if (categorizedProblems == null) continue;
for (CategorizedProblem categorizedProblem : categorizedProblems) {
perWorkingCopyInfo.acceptProblem(categorizedProblem);
Map<String, CategorizedProblem[]> problems = info instanceof ASTHolderCUInfo astHolder ? astHolder.problems : null;
if (DOM_BASED_OPERATIONS) {
ASTParser astParser = ASTParser.newParser(info instanceof ASTHolderCUInfo astHolder && astHolder.astLevel > 0 ? astHolder.astLevel : AST.getJLSLatest());
astParser.setWorkingCopyOwner(getOwner());
astParser.setSource(this instanceof ClassFileWorkingCopy ? source : this);
if (resolveBindings || computeProblems) {
astParser.setProject(getJavaProject());
} else {
// workaround https://github.com/eclipse-jdt/eclipse.jdt.core/issues/2204
// skips many operations, and prevents from conflicting classpath computation
astParser.setProject(null);
}
astParser.setStatementsRecovery((reconcileFlags & ICompilationUnit.ENABLE_STATEMENTS_RECOVERY) != 0);
astParser.setResolveBindings(computeProblems || resolveBindings);
astParser.setBindingsRecovery((reconcileFlags & ICompilationUnit.ENABLE_BINDINGS_RECOVERY) != 0);
astParser.setIgnoreMethodBodies((reconcileFlags & ICompilationUnit.IGNORE_METHOD_BODIES) != 0);
astParser.setCompilerOptions(options);
ASTNode dom = null;
try {
dom = astParser.createAST(pm);
} catch (AbortCompilationUnit e) {
var problem = e.problem;
if (problem == null && e.exception instanceof IOException ioEx) {
String path = source.getPath().toString();
String exceptionTrace = ioEx.getClass().getName() + ':' + ioEx.getMessage();
problem = new DefaultProblemFactory().createProblem(
path.toCharArray(),
IProblem.CannotReadSource,
new String[] { path, exceptionTrace },
new String[] { path, exceptionTrace },
ProblemSeverities.AbortCompilation | ProblemSeverities.Error | ProblemSeverities.Fatal,
0, 0, 1, 0);
}
if (problems != null) {
problems.put(Integer.toString(CategorizedProblem.CAT_BUILDPATH),
new CategorizedProblem[] { problem });
} else if (perWorkingCopyInfo != null) {
perWorkingCopyInfo.beginReporting();
perWorkingCopyInfo.acceptProblem(problem);
perWorkingCopyInfo.endReporting();
}
}
if (dom instanceof org.eclipse.jdt.core.dom.CompilationUnit newAST) {
if (computeProblems) {
IProblem[] interestingProblems = Arrays.stream(newAST.getProblems())
.filter(problem ->
!ignoreOptionalProblems()
|| !(problem instanceof DefaultProblem)
|| (problem instanceof DefaultProblem defaultProblem && (defaultProblem.severity & ProblemSeverities.Optional) == 0)
).toArray(IProblem[]::new);
if (perWorkingCopyInfo != null && problems == null) {
try {
perWorkingCopyInfo.beginReporting();
for (IProblem problem : interestingProblems) {
perWorkingCopyInfo.acceptProblem(problem);
}
} finally {
perWorkingCopyInfo.endReporting();
}
} finally {
perWorkingCopyInfo.endReporting();
} else if (interestingProblems.length > 0) {
problems.put(IJavaModelMarker.JAVA_MODEL_PROBLEM_MARKER, Stream.of(interestingProblems)
.filter(CategorizedProblem.class::isInstance)
.map(CategorizedProblem.class::cast)
.toArray(CategorizedProblem[]::new));
}
} else {
// collect problems
compilationUnitDeclaration = CompilationUnitProblemFinder.process(source, parser, this.owner, problems, createAST, reconcileFlags, pm);
}
} else {
compilationUnitDeclaration = parser.parseCompilationUnit(source, true /*full parse to find local elements*/, pm);
if (info instanceof ASTHolderCUInfo astHolder) {
astHolder.ast = newAST;
}
newAST.accept(new DOMToModelPopulator(newElements, this, unitInfo));
boolean structureKnown = true;
for (IProblem problem : newAST.getProblems()) {
structureKnown &= (IProblem.Syntax & problem.getID()) == 0;
}
unitInfo.setIsStructureKnown(structureKnown);
}
} else {
CompilerOptions compilerOptions = new CompilerOptions(options);
compilerOptions.ignoreMethodBodies = (reconcileFlags & ICompilationUnit.IGNORE_METHOD_BODIES) != 0;
CompilationUnitStructureRequestor requestor = new CompilationUnitStructureRequestor(this, unitInfo, newElements);
IProblemFactory problemFactory = new DefaultProblemFactory();
SourceElementParser parser = new SourceElementParser(
requestor,
problemFactory,
compilerOptions,
true/*report local declarations*/,
!createAST /*optimize string literals only if not creating a DOM AST*/);
parser.reportOnlyOneSyntaxError = !computeProblems;
parser.setMethodsFullRecovery(true);
parser.setStatementsRecovery((reconcileFlags & ICompilationUnit.ENABLE_STATEMENTS_RECOVERY) != 0);

if (!computeProblems && !resolveBindings && !createAST) // disable javadoc parsing if not computing problems, not resolving and not creating ast
parser.javadocParser.checkDocComment = false;
requestor.parser = parser;

// compute other problems if needed
CompilationUnitDeclaration compilationUnitDeclaration = null;
try {
if (computeProblems) {
if (problems == null) {
// report problems to the problem requestor
problems = new HashMap<>();
compilationUnitDeclaration = CompilationUnitProblemFinder.process(source, parser, this.owner, problems, createAST, reconcileFlags, pm);
try {
perWorkingCopyInfo.beginReporting();
for (CategorizedProblem[] categorizedProblems : problems.values()) {
if (categorizedProblems == null) continue;
for (CategorizedProblem categorizedProblem : categorizedProblems) {
perWorkingCopyInfo.acceptProblem(categorizedProblem);
}
}
} finally {
perWorkingCopyInfo.endReporting();
}
} else {
// collect problems
compilationUnitDeclaration = CompilationUnitProblemFinder.process(source, parser, this.owner, problems, createAST, reconcileFlags, pm);
}
} else {
compilationUnitDeclaration = parser.parseCompilationUnit(source, true /*full parse to find local elements*/, pm);
}

if (createAST) {
int astLevel = ((ASTHolderCUInfo) info).astLevel;
org.eclipse.jdt.core.dom.CompilationUnit cu = AST.convertCompilationUnit(astLevel, compilationUnitDeclaration, options, computeProblems, source, reconcileFlags, pm);
((ASTHolderCUInfo) info).ast = cu;
if (createAST) {
int astLevel = ((ASTHolderCUInfo) info).astLevel;
org.eclipse.jdt.core.dom.CompilationUnit cu = AST.convertCompilationUnit(astLevel, compilationUnitDeclaration, options, computeProblems, source, reconcileFlags, pm);
((ASTHolderCUInfo) info).ast = cu;
}
} finally {
if (compilationUnitDeclaration != null) {
unitInfo.hasFunctionalTypes = compilationUnitDeclaration.hasFunctionalTypes();
compilationUnitDeclaration.cleanUp();
}
}
} finally {
if (compilationUnitDeclaration != null) {
unitInfo.hasFunctionalTypes = compilationUnitDeclaration.hasFunctionalTypes();
compilationUnitDeclaration.cleanUp();
}
}

return unitInfo.isStructureKnown();
Expand Down Expand Up @@ -1129,7 +1247,7 @@ public org.eclipse.jdt.core.dom.CompilationUnit makeConsistent(int astLevel, boo
openWhenClosed(info, true, monitor);
org.eclipse.jdt.core.dom.CompilationUnit result = info.ast;
info.ast = null;
return result;
return astLevel != NO_AST ? result : null;
} else {
openWhenClosed(createElementInfo(), true, monitor);
return null;
Expand Down Expand Up @@ -1448,6 +1566,17 @@ public Map<String, String> getCustomOptions() {
if (this.owner != null) {
try {
Map<String, String> customOptions = this.getCompilationUnitElementInfo().getCustomOptions();
IJavaProject parentProject = getJavaProject();
Map<String, String> parentOptions = parentProject == null ? JavaCore.getOptions() : parentProject.getOptions(true);
if (JavaCore.ENABLED.equals(parentOptions.get(JavaCore.COMPILER_PB_ENABLE_PREVIEW_FEATURES)) &&
AST.newAST(parentOptions).apiLevel() < AST.getJLSLatest()) {
// Disable preview features for older Java releases as it causes the compiler to fail later
if (customOptions != null) {
customOptions.put(JavaCore.COMPILER_PB_ENABLE_PREVIEW_FEATURES, JavaCore.DISABLED);
} else {
customOptions = Map.of(JavaCore.COMPILER_PB_ENABLE_PREVIEW_FEATURES, JavaCore.DISABLED);
}
}
return customOptions == null ? Collections.emptyMap() : customOptions;
} catch (JavaModelException e) {
// do nothing
Expand Down
Loading

0 comments on commit 975645c

Please sign in to comment.