From a249c806af1b92da2514e75060d5e4ba67c867a1 Mon Sep 17 00:00:00 2001 From: Dmitrii Tikhomirov Date: Thu, 11 Apr 2024 16:39:44 -0700 Subject: [PATCH] step five --- .../main/java/com/example/MyProcessor.java | 8 + j2cl-tasks/pom.xml | 14 +- .../j2cl/build/provided/BytecodeTask.java | 21 +- .../j2cl/build/provided/JdtBytecodeTask.java | 185 ++++++++++++++++++ .../java/com/vertispan/j2cl/tools/Jdt.java | 87 ++++++++ 5 files changed, 305 insertions(+), 10 deletions(-) create mode 100644 j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/JdtBytecodeTask.java create mode 100644 j2cl-tasks/src/main/java/com/vertispan/j2cl/tools/Jdt.java diff --git a/j2cl-maven-plugin/src/it/annotation-processor-in-reactor/processor/src/main/java/com/example/MyProcessor.java b/j2cl-maven-plugin/src/it/annotation-processor-in-reactor/processor/src/main/java/com/example/MyProcessor.java index a83b1f6e..85eadbbd 100644 --- a/j2cl-maven-plugin/src/it/annotation-processor-in-reactor/processor/src/main/java/com/example/MyProcessor.java +++ b/j2cl-maven-plugin/src/it/annotation-processor-in-reactor/processor/src/main/java/com/example/MyProcessor.java @@ -46,6 +46,14 @@ public boolean process(Set annotations, RoundEnvironment Set elements = roundEnv.getElementsAnnotatedWith(annotationElt); Filer filer = processingEnv.getFiler(); + + System.out.println("Processing " + elements.size() + " elements"); + + elements.forEach(e -> { + System.out.println(" Element: " + e); + }); + + elements.stream().filter(e -> e.getKind().isInterface()).forEach(type -> { //emit a new class for that type, with no-arg string methods try { diff --git a/j2cl-tasks/pom.xml b/j2cl-tasks/pom.xml index bc063eb2..73f492ac 100644 --- a/j2cl-tasks/pom.xml +++ b/j2cl-tasks/pom.xml @@ -31,6 +31,12 @@ com.vertispan.j2cl gwt-incompatible-stripper ${j2cl.version} + + + com.vertispan.j2cl.external + org.eclipse.jdt.core + + com.vertispan.j2cl @@ -77,6 +83,12 @@ ${project.version} + + org.eclipse.jdt + org.eclipse.jdt.core + 3.37.0 + + com.google.turbine turbine @@ -95,4 +107,4 @@ test - \ No newline at end of file + diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java index b619b031..60551ceb 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java @@ -42,7 +42,7 @@ public class BytecodeTask extends TaskFactory { public static final PathMatcher NOT_BYTECODE = p -> !JAVA_BYTECODE.matches(p); public static final PathMatcher APT_PROCESSOR = p -> - p.equals(Paths.get("META-INF", "services", "javax.annotation.processing.Processor")); + p.equals(Paths.get("META-INF", "services", "javax.annotation.processing.Processor")); @Override public String getOutputType() { @@ -51,7 +51,8 @@ public String getOutputType() { @Override public String getTaskName() { - return "default"; + //return "default"; + return "javac"; } @Override @@ -87,7 +88,7 @@ public Task resolve(Project project, Config config) { List bytecodeClasspath = scope(project.getDependencies() .stream() - //.filter(p -> p.getProject().getJar() == null || p.getProject().hasSourcesMapped()) + .filter(p -> p.getProject().getJar() == null || p.getProject().hasSourcesMapped()) .filter(dependency -> dependency.getProject().getProcessors().isEmpty()).collect(Collectors.toSet()), com.vertispan.j2cl.build.task.Dependency.Scope.COMPILE) .stream() @@ -126,16 +127,15 @@ public Task resolve(Project project, Config config) { if (!inputSources.getFilesAndHashes().isEmpty()) { // At least one .java file in sources, compile it (otherwise skip this and just copy resource) - List classpathDirs = //Stream.concat( + List classpathDirs = Stream.concat( Stream.concat( bytecodeClasspath.stream().map(Input::getParentPaths).flatMap(Collection::stream).map(Path::toFile), - extraClasspath.stream() -/* , + extraClasspath.stream()), project.getDependencies() .stream() .filter(p -> !p.getProject().hasSourcesMapped()) .filter(p -> p.getProject().getJar() != null) - .map(p -> p.getProject().getJar())*/ + .map(p -> p.getProject().getJar()) ).collect(Collectors.toUnmodifiableList()); List sourcePaths = inputDirs.getParentPaths().stream().map(Path::toFile).collect(Collectors.toUnmodifiableList()); @@ -171,11 +171,14 @@ public Task resolve(Project project, Config config) { } protected Set maybeAddInReactorAptProcessor(List reactorProcessors, Set processors) { - if (processors.isEmpty()) { - return Collections.emptySet(); + + //??? BUG + if (reactorProcessors.isEmpty()) { + //return Collections.emptySet(); } Set existingProcessors = new HashSet<>(processors); reactorProcessors.forEach(input -> input.getFilesAndHashes().forEach(file -> { + System.out.println("CHECK FILE: " + file.getAbsolutePath()); try (Stream lines = Files.lines(file.getAbsolutePath())) { lines.forEach(line -> existingProcessors.add(line.trim())); } catch (IOException e) { diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/JdtBytecodeTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/JdtBytecodeTask.java new file mode 100644 index 00000000..1a99c31b --- /dev/null +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/JdtBytecodeTask.java @@ -0,0 +1,185 @@ +package com.vertispan.j2cl.build.provided; + +import com.google.auto.service.AutoService; +import com.google.j2cl.common.SourceUtils; +import com.vertispan.j2cl.build.task.CachedPath; +import com.vertispan.j2cl.build.task.Config; +import com.vertispan.j2cl.build.task.Input; +import com.vertispan.j2cl.build.task.OutputTypes; +import com.vertispan.j2cl.build.task.Project; +import com.vertispan.j2cl.build.task.TaskFactory; +import com.vertispan.j2cl.tools.Jdt; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@AutoService(TaskFactory.class) +public class JdtBytecodeTask extends BytecodeTask { + + @Override + public String getTaskName() { + return "default"; + //return "jdt"; + } + + @Override + public Task resolve(Project project, Config config) { + if (!project.hasSourcesMapped()) { + // instead, copy the bytecode+resources out of the jar so it can be used by downstream bytecode/apt tasks + Input existingUnpackedBytecode = input(project, OutputTypes.INPUT_SOURCES); + return context -> { + for (CachedPath entry : existingUnpackedBytecode.getFilesAndHashes()) { + Path outputFile = context.outputPath().resolve(entry.getSourcePath()); + Files.createDirectories(outputFile.getParent()); + Files.copy(entry.getAbsolutePath(), outputFile); + } + }; + } + + // TODO just use one input for both of these + // track the dirs (with all file changes) so that APT can see things it wants + Input inputDirs = input(project, OutputTypes.INPUT_SOURCES); + // track just java files (so we can just compile them) + Input inputSources = input(project, OutputTypes.INPUT_SOURCES).filter(JAVA_SOURCES); + // track resources so they are available to downstream processors on the classpath, as they would + // be if we had built a jar + Input resources = input(project, OutputTypes.INPUT_SOURCES).filter(NOT_BYTECODE); + + + scope(project.getDependencies() + .stream() + .filter(dependency -> dependency.getProject().getProcessors().isEmpty()).collect(Collectors.toSet()), + com.vertispan.j2cl.build.task.Dependency.Scope.COMPILE) + .stream() + //.filter(dependency -> !dependency.isJsZip()) + .forEach(dependency -> { + System.out.println("Dependency: " + dependency.getKey()); + }); + + List bytecodeClasspath = scope(project.getDependencies() + .stream() + .filter(dependency -> dependency.getProject().getProcessors().isEmpty()).collect(Collectors.toSet()), + com.vertispan.j2cl.build.task.Dependency.Scope.COMPILE) + .stream() + //.filter(dependency -> !dependency.isJsZip()) + .filter(dependency -> !dependency.getKey().toString().equals("com.vertispan.j2cl:jre:v20230718-1:jszip")) + .map(inputs(OutputTypes.BYTECODE)) + .collect(Collectors.toUnmodifiableList()); + + List inReactorProcessors = scope(project.getDependencies().stream().filter(dependency -> dependency.getProject().hasSourcesMapped() + && !dependency.getProject().isJsZip()).collect(Collectors.toSet()), + com.vertispan.j2cl.build.task.Dependency.Scope.COMPILE) + .stream() + .map(inputs(OutputTypes.BYTECODE)) + .map(input -> input.filter(APT_PROCESSOR)) + .collect(Collectors.toUnmodifiableList()); + + System.out.println("processors in " + project.getKey() + " " + inReactorProcessors.size()); + + inReactorProcessors.forEach(p -> { + System.out.println("Processor: " + p.getProject().getKey()); + }); + + //File bootstrapClasspath = config.getBootstrapClasspath(); + List extraClasspath = new ArrayList<>(config.getExtraClasspath()); + Set processors = new HashSet<>(); + project.getDependencies() + .stream() + .map(d -> d.getProject()) + .filter(p -> !p.getProcessors().isEmpty()) + .forEach(p -> { + processors.addAll(p.getProcessors()); + extraClasspath.add(p.getJar()); + }); + + return context -> { + /* we don't know if reactor dependency project is apt, before it's compiled, so we need to check it on the fly + 1) there are no processors in the project, so we pass empty set to javac, nothing happens + 2) there are only reactor processors, so we pass empty set to javac, they will be triggered by javac + 3) thee are both reactor and nonreactor processors, so we pass both to javac via set + 4) there are only nonreactor processors, we pass them to javac via set + */ + + System.out.println("CONTEXT: " + project.getKey() + " " + inReactorProcessors.size()); + + Set aptProcessors = maybeAddInReactorAptProcessor(inReactorProcessors, processors); + + aptProcessors.forEach(p -> { + System.out.println("APT Processor: " + p); + }); + + + + if (!inputSources.getFilesAndHashes().isEmpty()) { + // At least one .java file in sources, compile it (otherwise skip this and just copy resource) + + List classpathDirs = Stream.concat( + bytecodeClasspath.stream().map(Input::getParentPaths) + .flatMap(Collection::stream) + .map(path -> { + if (path.toString().contains("com.vertispan.j2cljre")) { + return path.resolve("bazelbin/jre/java/jre.js/javaemul"); + } + return path; + }) + .map(Path::toFile), + extraClasspath.stream().filter(f -> !f.getName().equals("jre-v20230718-1.jar")) + ) + + + .collect(Collectors.toUnmodifiableList()); + + + List sourcePaths = inputDirs.getParentPaths().stream().map(Path::toFile).collect(Collectors.toUnmodifiableList()); + File generatedClassesDir = getGeneratedClassesDir(context); + File classOutputDir = context.outputPath().toFile(); + + + sourcePaths.forEach(s -> { + System.out.println("Source Path: " + s.getAbsolutePath()); + }); + + classpathDirs.forEach(f -> { + System.out.println("Classpath Dir: " + f.getAbsolutePath()); + }); + + + Jdt javac = new Jdt(context, generatedClassesDir, sourcePaths, classpathDirs, classOutputDir, aptProcessors); + + // TODO convention for mapping to original file paths, provide FileInfo out of Inputs instead of Paths, + // automatically relativized? + List sources = inputSources.getFilesAndHashes() + .stream() + .filter(p -> !p.getAbsolutePath().toFile().getName().equals("moduleinfo.java")) + .map(p -> SourceUtils.FileInfo.create(p.getAbsolutePath().toString(), p.getSourcePath().toString())) + .collect(Collectors.toUnmodifiableList()); + + try { + if (!javac.compile(sources)) { + throw new RuntimeException("Failed to complete bytecode task, check log"); + } + } catch (Exception exception) { + exception.printStackTrace(); + throw exception; + } + } + + // Copy all resources, even .java files, so that this output is the source of truth as if this + // were freshly unpacked from a jar + for (CachedPath entry : resources.getFilesAndHashes()) { + Files.createDirectories(context.outputPath().resolve(entry.getSourcePath()).getParent()); + Files.copy(entry.getAbsolutePath(), context.outputPath().resolve(entry.getSourcePath())); + } + + }; + } + +} diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/tools/Jdt.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/tools/Jdt.java new file mode 100644 index 00000000..b5a41ce2 --- /dev/null +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/tools/Jdt.java @@ -0,0 +1,87 @@ +package com.vertispan.j2cl.tools; + +import com.google.j2cl.common.SourceUtils; +import com.vertispan.j2cl.build.task.BuildLog; +import org.eclipse.jdt.core.compiler.batch.BatchCompiler; + +import javax.lang.model.SourceVersion; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintWriter; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class Jdt { + + private final BuildLog log; + private final List javacOptions; + + public Jdt(BuildLog log, File generatedClassesPath, List sourcePaths, List classpath, File classesDirFile, Set processors) { + this.log = log; + this.javacOptions = new ArrayList<>(Arrays.asList("-encoding", "utf8")); + //this.javacOptions = new ArrayList<>(); + + + + if (generatedClassesPath == null) { + javacOptions.add("-proc:none"); + } + if (SourceVersion.latestSupported().compareTo(SourceVersion.RELEASE_11) >= 0) { + //none + } + javacOptions.add("-11"); + + if (!processors.isEmpty()) { + javacOptions.add("-processor"); + javacOptions.add(String.join(",", processors)); + System.out.println("processors: " + String.join(",", processors)); + + } else { + System.out.println("No processors"); + javacOptions.add("-proc:none"); + } + + javacOptions.add("-sourcepath"); + javacOptions.add(sourcePaths.stream().map(File::getAbsolutePath).collect(Collectors.joining(":"))); + + System.out.println("sourcepath: " + sourcePaths.stream().map(File::getAbsolutePath).collect(Collectors.joining(":"))); + + javacOptions.add("-classpath"); + + System.out.println("classpath: "); + classpath.stream().map(File::getAbsolutePath).forEach(e -> { + System.out.println(e); + }); + + javacOptions.add(classpath.stream().map(File::getAbsolutePath).collect(Collectors.joining(":"))); + + javacOptions.add("-d"); + javacOptions.add(classesDirFile.getAbsolutePath()); + javacOptions.add("-s"); + javacOptions.add(generatedClassesPath.getAbsolutePath()+"/test"); + javacOptions.add("-time"); + + System.out.println("-D " + classesDirFile.getAbsolutePath()); + System.out.println("-S " + generatedClassesPath.getAbsolutePath()); + + } + + public boolean compile(List modifiedJavaFiles) { + modifiedJavaFiles.forEach(f -> javacOptions.add(f.sourcePath())); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ByteArrayOutputStream errorOutputStream = new ByteArrayOutputStream(); + + BatchCompiler.compile("-help", new PrintWriter(outputStream), new PrintWriter(errorOutputStream), null); + + boolean result = BatchCompiler.compile(javacOptions.toArray(new String[javacOptions.size()]), new PrintWriter(outputStream), new PrintWriter(errorOutputStream), null); + + log.info(outputStream.toString(Charset.defaultCharset())); + log.error(errorOutputStream.toString(Charset.defaultCharset())); + return result; + } +}