Logstash diagnostics added, human added to all pretty parameters, _te…

…mplate added, cat_fielddata changed to .txt extension

pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.elasticsearch</groupId>
     <artifactId>support-diagnostics</artifactId>
-    <version>2.0.7</version>
+    <version>2.0.8</version>
     <packaging>jar</packaging>
     <name>Support Diagnostics Utilities</name>
 

src/main/java/com/elastic/support/diagnostics/DiagnosticApp.java

@@ -43,7 +43,6 @@ public static void main(String[] args) throws Exception {
          long interval = inputs.getInterval() * 1000;
 
          for (int i = 1; i <= reps; i++) {
-            //diags.run(inputs);
             dc.runDiagnostic(ctx);
             System.out.println("Run " + i + " of " + reps + " completed.");
             if (reps > 1) {

src/main/java/com/elastic/support/diagnostics/InputParams.java

@@ -4,7 +4,7 @@
 
 public class InputParams {
 
-   @Parameter(names = {"-?", "--help"}, help = true)
+   @Parameter(names = {"-?", "--help"}, description = "Help contents.", help = true)
    private boolean help;
 
    @Parameter(names = {"-o", "--out", "--output", "--outputDir"}, description = "Fully qualified path to output directory or c for current working directory.")
@@ -13,10 +13,10 @@ public class InputParams {
    @Parameter(names = {"-h", "--host",}, description = "Required field.  Hostname, IP Address, or localhost.  HTTP access must be enabled.")
    private String host = "";
 
-   @Parameter(names = {"--port"}, description = "HTTP or HTTPS listening port. Defaults to 9200")
+   @Parameter(names = {"--port"}, description = "HTTP or HTTPS listening port. Defaults to 9200.")
    private int port = 9200;
 
-   @Parameter(names = {"-u", "--user"}, description = "Username")
+   @Parameter(names = {"-u", "--user"}, description = "Username.")
    private String username;
 
    @Parameter(names = {"-p", "--password", "--pwd"}, description = "Prompt for a password?  No password value required, only the option. Hidden from the command line on entry.", password = true)
@@ -50,6 +50,7 @@ public class InputParams {
    private boolean skipVerification = false;
 
    private boolean secured = false;
+   private boolean wasPortSet = false;
 
    public boolean isSkipVerification() {
       return skipVerification;
@@ -89,6 +90,7 @@ public int getPort() {
 
    public void setPort(int port) {
       this.port = port;
+      wasPortSet = true;
    }
 
    public String getUsername() {
@@ -189,6 +191,12 @@ public void setBzip(boolean bzip) {
    public String getUrl() {
       String protocol;
 
+      if(diagType.equalsIgnoreCase("logstash")){
+         if(! wasPortSet){
+            this.port = 9600;
+         }
+      }
+
       if (this.isSsl) {
          protocol = "https";
       } else {

src/main/java/com/elastic/support/diagnostics/RestModule.java

@@ -2,43 +2,27 @@
 
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.http.HttpResponse;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.utils.HttpClientUtils;
 import org.apache.http.conn.HttpHostConnectException;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-/*
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.client.RestClientException;
-import org.springframework.web.client.RestTemplate;
-*/
 
 import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
 
 public class RestModule {
 
-/*   RestTemplate restTemplate;
-   HttpEntity<String> request;*/
 
    HttpClient client;
 
    private final static Logger logger = LoggerFactory.getLogger(RestModule.class);
 
-/*   public RestModule(RestTemplate restTemplate, HttpEntity<String> request) {
-      this.restTemplate = restTemplate;
-      this.request = request;
-   }*/
-
    public RestModule(HttpClient client){
       this.client = client;
    }
@@ -60,7 +44,8 @@ public String submitRequest(String url){
          catch (Exception e){
             logger.error("Error handling response for " + url, e);
          } finally {
-            HttpClientUtils.closeQuietly(response);         }
+            HttpClientUtils.closeQuietly(response);
+         }
       }
       catch (HttpHostConnectException e){
          throw new RuntimeException("Error connecting to host " + url, e);
@@ -102,7 +87,9 @@ public void submitRequest(String url, String queryName, String destination){
          catch (Exception e){
             logger.error("Error writing response for " + queryName + " to disk.", e);
          } finally {
-            HttpClientUtils.closeQuietly(response);         }
+            HttpClientUtils.closeQuietly(response);
+            Thread.sleep(2000);
+         }
       }
       catch (Exception e){
          if (url.contains("_license")) {
@@ -133,32 +120,4 @@ private void checkResponseCode(String name, HttpResponse response){
       }
    }
 
-/*
-   public String submitRequest(String url) {
-
-      String result;
-      try {
-         ;
-         logger.debug("Submitting: " + url);
-         ResponseEntity<String> response = restTemplate.exchange(url, HttpMethod.GET, request, String.class);
-         result = response.getBody();
-         logger.debug(result);
-
-      } catch (RestClientException e) {
-         if (url.contains("_license")) {
-            logger.info("There were no licenses installed");
-            return "No licenses installed";
-         }
-         String msg = "Please check log file for additional details.";
-         logger.error("Error submitting request\n:", e);
-         if (e.getMessage().contains("401 Unauthorized")) {
-            msg = "Authentication failure: invalid login credentials.\n" + msg;
-         }
-         throw new RuntimeException(msg);
-      }
-
-      return result;
-
-   }
-   */
 }

src/main/java/com/elastic/support/diagnostics/commands/RestModuleSetupCmd.java

@@ -29,9 +29,6 @@ public boolean execute(DiagnosticContext context) {
          }
 
          DiagnosticRequestFactory diagnosticRequestFactory = new DiagnosticRequestFactory(connectTimeout, requestTimeout, isSecured, user, pass);
-         //RestTemplate restTemplate = new RestTemplate(diagnosticRequestFactory.getSslReqFactory());
-         //HttpEntity<String> request = configureAuth(context.getInputParams());
-         //RestModule restModule = new RestModule(restTemplate, request);
          HttpClient client = null;
          if (bypassVerify) {
             client = diagnosticRequestFactory.getUnverifiedSslClient();
@@ -51,21 +48,4 @@ public boolean execute(DiagnosticContext context) {
       return true;
    }
 
-/*   public HttpEntity<String> configureAuth(InputParams inputs) {
-
-      HttpHeaders headers = new HttpHeaders();
-
-      // If we need authentication
-      if (inputs.isSecured()) {
-         String plainCreds = inputs.getUsername()
-            + ":" + inputs.getPassword();
-         byte[] plainCredsBytes = plainCreds.getBytes();
-         byte[] base64CredsBytes = Base64.encodeBase64(plainCredsBytes);
-         String base64Creds = new String(base64CredsBytes);
-         headers.add("Authorization", "Basic " + base64Creds);
-      }
-
-      return new HttpEntity<>(headers);
-
-   }*/
 }

src/main/java/com/elastic/support/diagnostics/commands/RunClusterQueriesCmd.java

@@ -23,10 +23,9 @@ public boolean execute(DiagnosticContext context) {
       String queryName = null;
       String fileName = null;
 
-      String majorVersion = context.getVersion().substring(0, 1);
 
       List textFileExtensions = (List) config.get("textFileExtensions");
-      Map<String, String> statements = (Map<String, String>) config.get("restQueries-" + majorVersion);
+      Map<String, String> statements = (Map<String, String>) config.get("logstash");
       Set<Map.Entry<String, String>> entries = statements.entrySet();
 
       logger.debug("Generating full diagnostic.");
@@ -47,8 +46,6 @@ public boolean execute(DiagnosticContext context) {
 
             fileName = context.getTempDir() + SystemProperties.fileSeparator + queryName + ext;
             restModule.submitRequest(url, queryName, fileName);
-
-            //Files.write(Paths.get(fileName), result.getBytes());
       }
 
       return true;

src/main/java/com/elastic/support/diagnostics/commands/RunLogstashQueriesCmd.java

@@ -0,0 +1,43 @@
+package com.elastic.support.diagnostics.commands;
+
+import com.elastic.support.SystemProperties;
+import com.elastic.support.diagnostics.DiagnosticContext;
+import com.elastic.support.diagnostics.InputParams;
+import com.elastic.support.diagnostics.RestModule;
+
+import java.util.Map;
+import java.util.Set;
+
+public class RunLogstashQueriesCmd extends AbstractDiagnosticCmd {
+
+   public boolean execute(DiagnosticContext context) {
+
+      InputParams inputs = context.getInputParams();
+      Map config = context.getConfig();
+      RestModule restModule = context.getRestModule();
+      String queryName;
+      String fileName;
+
+      Map<String, String> statements = (Map<String, String>) config.get("logstash");
+      Set<Map.Entry<String, String>> entries = statements.entrySet();
+
+      logger.debug("Generating full diagnostic.");
+
+      for (Map.Entry<String, String> entry : entries) {
+         queryName = entry.getKey();
+         String query = entry.getValue();
+         logger.debug(": now processing " + queryName + ", " + query);
+         String url = inputs.getUrl() + "/" + query;
+         logger.info("Currently running the following query:" + queryName);
+
+         String ext = ".json";
+
+         fileName = context.getTempDir() + SystemProperties.fileSeparator + queryName + ext;
+         restModule.submitRequest(url, queryName, fileName);
+      }
+
+      return true;
+   }
+
+
+}

src/main/java/com/elastic/support/diagnostics/commands/SystemCallsCmd.java

@@ -8,60 +8,61 @@
 
 public class SystemCallsCmd extends AbstractDiagnosticCmd {
 
-    public boolean execute(DiagnosticContext context){
-
-            String os = checkOS();
-            Map<String, String> osCmds = (Map<String, String>) context.getConfig().get(os);
-
-            ProcessBuilder pb = new ProcessBuilder();
-            pb.redirectErrorStream(true);
-
-            Iterator<Map.Entry<String, String>> iter = osCmds.entrySet().iterator();
-            List cmds = new ArrayList();
-
-            try {
-                while (iter.hasNext()) {
-                    Map.Entry<String, String> entry = (Map.Entry<String, String>) iter.next();
-                    String cmdLabel = entry.getKey();
-                    String cmdText = entry.getValue();
-
-                    // One off hack for process limits
-                    if (cmdLabel.equals("proc-limit")){
-                        //cmdText.replaceAll("PID",  );
-                        cmdText = "cat /proc/" + context.getPid() + "/limits";
-                    }
-
-                    StringTokenizer st = new StringTokenizer(cmdText, " ");
-                    while (st.hasMoreTokens()) {
-                        cmds.add(st.nextToken());
-                    }
-
-                    pb.redirectOutput(new File(context.getTempDir() + SystemProperties.fileSeparator + cmdLabel + ".txt"));
-                    pb.command(cmds);
-                    Process pr = pb.start();
-                    pr.waitFor();
-                    cmds.clear();
-
-                }
-            } catch (Exception e) {
-                logger.error("Error processing system commands", e);
+   public boolean execute(DiagnosticContext context) {
+
+      String os = checkOS();
+      Map<String, String> osCmds = (Map<String, String>) context.getConfig().get(os);
+
+      ProcessBuilder pb = new ProcessBuilder();
+      pb.redirectErrorStream(true);
+
+      Iterator<Map.Entry<String, String>> iter = osCmds.entrySet().iterator();
+      List cmds = new ArrayList();
+
+      while (iter.hasNext()) {
+
+         try {
+            Map.Entry<String, String> entry =  iter.next();
+            String cmdLabel = entry.getKey();
+            String cmdText = entry.getValue();
+
+            // One off hack for process limits
+            if (cmdLabel.equals("proc-limit")) {
+               cmdText = "cat /proc/" + context.getPid() + "/limits";
             }
 
-        return true;
-    }
-
-    public String checkOS() {
-        String osName = SystemProperties.osName.toLowerCase();
-        if (osName.contains("windows")) {
-            return "winOS";
-        } else if (osName.contains("linux")) {
-            return "linuxOS";
-        } else if (osName.contains("darwin") || osName.contains("mac os x")) {
-            return "macOS";
-        } else {
-            logger.error("Failed to detect operating system!");
-            throw new RuntimeException("Unsupported OS");
-        }
-    }
+            StringTokenizer st = new StringTokenizer(cmdText, " ");
+            while (st.hasMoreTokens()) {
+               cmds.add(st.nextToken());
+            }
+
+            pb.redirectOutput(new File(context.getTempDir() + SystemProperties.fileSeparator + cmdLabel + ".txt"));
+            pb.command(cmds);
+            Process pr = pb.start();
+            pr.waitFor();
+         } catch (Exception e) {
+            logger.error("Error processing system command", e);
+         } finally {
+            cmds.clear();
+         }
+      }
+
+
+      return true;
+   }
+
+   public String checkOS() {
+      String osName = SystemProperties.osName.toLowerCase();
+      if (osName.contains("windows")) {
+         return "winOS";
+      } else if (osName.contains("linux")) {
+         return "linuxOS";
+      } else if (osName.contains("darwin") || osName.contains("mac os x")) {
+         return "macOS";
+      } else {
+         logger.error("Failed to detect operating system!");
+         throw new RuntimeException("Unsupported OS");
+      }
+   }
 
 }

src/main/resources/chains.yml

@@ -20,3 +20,11 @@ remote:
   - com.elastic.support.diagnostics.commands.GenerateManifestCmd
   - com.elastic.support.diagnostics.commands.ArchiveResultsCmd
   - com.elastic.support.diagnostics.commands.CleanupCmd
+
+logstash:
+  - com.elastic.support.diagnostics.commands.DirectorySetupCmd
+  - com.elastic.support.diagnostics.commands.RestModuleSetupCmd
+  - com.elastic.support.diagnostics.commands.RunLogstashQueriesCmd
+  - com.elastic.support.diagnostics.commands.ArchiveResultsCmd
+  - com.elastic.support.diagnostics.commands.CleanupCmd
+