From 6569f16298ced385f0e2af8f00e73fd055ecd10d Mon Sep 17 00:00:00 2001 From: Doug Simon Date: Thu, 14 Mar 2024 22:22:34 +0100 Subject: [PATCH 1/2] improve diagnostics for javac daemon failure --- common.json | 16 ++++++------ .../mxtool/compilerserver/CompilerDaemon.java | 26 +++++++------------ .../mxtool/compilerserver/JavacDaemon.java | 20 ++++++++++++-- src/mx/_impl/mx.py | 9 ++++--- 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/common.json b/common.json index 6d584f89..7f23ab6a 100644 --- a/common.json +++ b/common.json @@ -8,7 +8,7 @@ "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { - "galahad-jdk": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+10-725", "platformspecific": true, "extrabundles": ["static-libs"]}, + "galahad-jdk": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+14-1010", "platformspecific": true, "extrabundles": ["static-libs"]}, "oraclejdk11": {"name": "jpg-jdk", "version": "11.0.11", "build_id": "jdk-11.0.11+9", "platformspecific": true, "extrabundles": ["static-libs"] }, @@ -44,13 +44,13 @@ "labsjdk-ee-21Debug": {"name": "labsjdk", "version": "ee-21.0.2+13-jvmci-23.1-b33-debug", "platformspecific": true }, "labsjdk-ee-21-llvm": {"name": "labsjdk", "version": "ee-21.0.2+13-jvmci-23.1-b33-sulong", "platformspecific": true }, - "oraclejdk-latest": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+12", "platformspecific": true, "extrabundles": ["static-libs"]}, - "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-23+12-jvmci-b01", "platformspecific": true }, - "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-23+12-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-23+12-jvmci-b01-sulong", "platformspecific": true }, - "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-23+12-jvmci-b01", "platformspecific": true }, - "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-23+12-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-23+12-jvmci-b01-sulong", "platformspecific": true } + "oraclejdk-latest": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+13", "platformspecific": true, "extrabundles": ["static-libs"]}, + "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-23+13-jvmci-b01", "platformspecific": true }, + "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-23+13-jvmci-b01-debug", "platformspecific": true }, + "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-23+13-jvmci-b01-sulong", "platformspecific": true }, + "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-23+13-jvmci-b01", "platformspecific": true }, + "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-23+13-jvmci-b01-debug", "platformspecific": true }, + "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-23+13-jvmci-b01-sulong", "platformspecific": true } }, "eclipse": { diff --git a/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/CompilerDaemon.java b/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/CompilerDaemon.java index 1a2c5f74..968b9386 100644 --- a/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/CompilerDaemon.java +++ b/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/CompilerDaemon.java @@ -37,9 +37,9 @@ public abstract class CompilerDaemon { - protected void logf(String commandLine, Object... args) { + protected void logf(String format, Object... args) { if (verbose) { - System.err.printf(commandLine, args); + System.err.printf(format, args); } } @@ -106,18 +106,6 @@ interface Compiler { int compile(String[] args) throws Exception; } - String join(String delim, String[] strings) { - if (strings.length == 0) { - return ""; - } - StringBuilder sb = new StringBuilder(strings[0]); - for (int i = 1; i < strings.length; i++) { - sb.append(delim); - sb.append(strings[i]); - } - return sb.toString(); - } - public class Connection implements Runnable { private final Socket connectionSocket; @@ -136,7 +124,7 @@ public void run() { try { String commandLine = input.readLine(); if (commandLine == null || commandLine.length() == 0) { - logf("Shutting down\n"); + logf("Shutting down%n"); running = false; while (threadPool.getActiveCount() > 1) { threadPool.awaitTermination(50, TimeUnit.MILLISECONDS); @@ -146,10 +134,14 @@ public void run() { System.exit(0); } else { String[] args = commandLine.split("\u0000"); - logf("Compiling %s\n", join(" ", args)); + logf("Compiling %s%n", String.join(" ", args)); int result = compiler.compile(args); - logf("Result = %d\n", result); + if (result != 0 && args.length != 0 && args[0].startsWith("GET / HTTP")) { + // GR-52712 + System.err.printf("Failing compilation received on %s%n", connectionSocket); + } + logf("Result = %d%n", result); output.write(result + "\n"); } diff --git a/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/JavacDaemon.java b/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/JavacDaemon.java index 9e19a02e..e9ea6b0f 100644 --- a/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/JavacDaemon.java +++ b/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/JavacDaemon.java @@ -31,7 +31,23 @@ public class JavacDaemon extends CompilerDaemon { private final class JavacCompiler implements Compiler { public int compile(String[] args) throws Exception { final Object receiver = javacMainClass.getDeclaredConstructor().newInstance(); - return (Integer) compileMethod.invoke(receiver, new Object[]{args}); + int result = (Integer) compileMethod.invoke(receiver, (Object) args); + if (result != 0 && result != 1) { + // @formatter:off + /* + * com.sun.tools.javac.main.Main: + * + * public enum Result { + * OK(0), // Compilation completed with no errors. + * ERROR(1), // Completed but reported errors. + * CMDERR(2), // Bad command-line arguments + * SYSERR(3), // System error or resource exhaustion. + * ABNORMAL(4); // Compiler terminated abnormally + */ + // @formatter:on + System.err.printf("javac exited with exit code %d for args: '%s'%n", result, String.join("' '", args)); + } + return result; } } @@ -40,7 +56,7 @@ public int compile(String[] args) throws Exception { JavacDaemon() throws Exception { this.javacMainClass = Class.forName("com.sun.tools.javac.Main"); - this.compileMethod = javacMainClass.getMethod("compile", new Class[]{(new String[]{}).getClass()}); + this.compileMethod = javacMainClass.getMethod("compile", String[].class); } @Override diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index e53a2d63..3e237cee 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -8681,17 +8681,20 @@ def compile(self, compilerArgs): response = str(f.readline()) if response == '': # Compiler server process probably crashed - logv('[Compiler daemon process appears to have crashed]') + log('[Compiler daemon process appears to have crashed. ]') retcode = -1 else: retcode = int(response) s.close() if retcode: + detailed_retcode = str(subprocess.CalledProcessError(retcode, f'Compile with {self.name()}: ' + ' '.join(compilerArgs))) if _opts.verbose: if _opts.very_verbose: - retcode = str(subprocess.CalledProcessError(retcode, f'Compile with {self.name()}: ' + ' '.join(compilerArgs))) + retcode = detailed_retcode else: log('[exit code: ' + str(retcode) + ']') + elif retcode == 2: + retcode = detailed_retcode abort(retcode) return retcode @@ -19295,7 +19298,7 @@ def alarm_handler(signum, frame): _CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache')) # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue -version = VersionSpec("7.15.1") # list type subscripting is not available in Python <3.9 +version = VersionSpec("7.15.2") # GR-52712 - improve diagnostics for JavacDaemon _mx_start_datetime = datetime.utcnow() From 1bb875cb9c97a868bbe9782ada54e2e6a6dff641 Mon Sep 17 00:00:00 2001 From: Doug Simon Date: Fri, 15 Mar 2024 12:52:54 +0100 Subject: [PATCH 2/2] use protocol headers for requests to CompilerDaemon --- .../mxtool/compilerserver/CompilerDaemon.java | 26 ++++++++++++++++--- src/mx/_impl/mx.py | 15 ++++++++--- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/CompilerDaemon.java b/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/CompilerDaemon.java index 968b9386..501d9ff7 100644 --- a/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/CompilerDaemon.java +++ b/java/com.oracle.mxtool.compilerserver/src/com/oracle/mxtool/compilerserver/CompilerDaemon.java @@ -34,9 +34,19 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public abstract class CompilerDaemon { + // These values are used in mx.py so keep in sync. + public static final String REQUEST_HEADER_COMPILE = "MX DAEMON/COMPILE: "; + public static final String REQUEST_HEADER_SHUTDOWN = "MX DAEMON/SHUTDOWN"; + + /** + * The deamon will shut down after receiving this many requests with an unrecognized header. + */ + static final int MAX_UNRECOGNIZED_REQUESTS = 5; + protected void logf(String format, Object... args) { if (verbose) { System.err.printf(format, args); @@ -47,6 +57,7 @@ protected void logf(String format, Object... args) { private volatile boolean running; private ThreadPoolExecutor threadPool; private ServerSocket serverSocket; + private final AtomicInteger unrecognizedRequests = new AtomicInteger(); public void run(String[] args) throws Exception { int jobsArg = -1; @@ -122,8 +133,8 @@ public void run() { OutputStreamWriter output = new OutputStreamWriter(connectionSocket.getOutputStream(), "UTF-8"); try { - String commandLine = input.readLine(); - if (commandLine == null || commandLine.length() == 0) { + String request = input.readLine(); + if (request == null || request.equals(REQUEST_HEADER_SHUTDOWN)) { logf("Shutting down%n"); running = false; while (threadPool.getActiveCount() > 1) { @@ -132,7 +143,8 @@ public void run() { serverSocket.close(); // Just to be sure... System.exit(0); - } else { + } else if (request.startsWith(REQUEST_HEADER_COMPILE)) { + String commandLine = request.substring(REQUEST_HEADER_COMPILE.length()); String[] args = commandLine.split("\u0000"); logf("Compiling %s%n", String.join(" ", args)); @@ -144,6 +156,14 @@ public void run() { logf("Result = %d%n", result); output.write(result + "\n"); + } else { + System.err.printf("Unrecognized request (len=%d): \"%s\"%n", request.length(), request); + int unrecognizedRequestCount = unrecognizedRequests.incrementAndGet(); + if (unrecognizedRequestCount > MAX_UNRECOGNIZED_REQUESTS) { + System.err.printf("Shutting down after receiving %d unrecognized requests%n", unrecognizedRequestCount); + System.exit(0); + } + output.write("-1\n"); } } finally { // close IO streams, then socket diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index 3e237cee..b6a44563 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -8606,6 +8606,7 @@ class CompilerDaemon(Daemon): def __init__(self, jdk, jvmArgs, mainClass, toolJar, buildArgs=None): logv(f"Starting daemon for {jdk.java} [{', '.join(jvmArgs)}]") self.jdk = jdk + self.jvmArgs = jvmArgs if not buildArgs: buildArgs = [] build(buildArgs + ['--no-daemon', '--dependencies', 'com.oracle.mxtool.compilerserver']) @@ -8671,12 +8672,18 @@ def _noticePort(self, data): if m: self.port = int(m.group(1)) + # See: + # com.oracle.mxtool.compilerserver.CompilerDaemon.REQUEST_HEADER_COMPILE + # com.oracle.mxtool.compilerserver.CompilerDaemon.REQUEST_HEADER_SHUTDOWN + header_compile = "MX DAEMON/COMPILE: " + header_shutdown = "MX DAEMON/SHUTDOWN" + def compile(self, compilerArgs): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(('127.0.0.1', self.port)) - logv(f'Compile with {self.name()}: ' + ' '.join(compilerArgs)) + logv(f'Compile with {self.name()}: {" ".join(compilerArgs)}') commandLine = u'\x00'.join(compilerArgs) - s.send((commandLine + '\n').encode('utf-8')) + s.send((f'{CompilerDaemon.header_compile}{commandLine}\n').encode('utf-8')) f = s.makefile() response = str(f.readline()) if response == '': @@ -8702,7 +8709,7 @@ def compile(self, compilerArgs): def shutdown(self): if not self.closed: try: - self.connection.send('\n'.encode('utf8')) + self.connection.send(f'{CompilerDaemon.header_shutdown}\n'.encode('utf8')) self.connection.close() self.closed = True logv('[Stopped ' + str(self) + ']') @@ -8710,7 +8717,7 @@ def shutdown(self): logv('Error stopping ' + str(self) + ': ' + str(e)) def __str__(self): - return self.name() + ' on port ' + str(self.port) + ' for ' + str(self.jdk) + return f"{self.name()} on port {self.port} for {self.jdk} with VM args {self.jvmArgs}" class JavacDaemon(CompilerDaemon): def __init__(self, jdk, jvmArgs):