Added support of "Globs" attribute

- Dealing with the case when unit contains "Globs" data and no (or null) "Files", in this case graph command should scan for matching files using globs. See sourcegraph/srclib#240

.gitmodules

@@ -17,3 +17,6 @@
 	path = testdata/case/java-android-gradle-sample
 	url = https://github.com/sgtest/java-android-gradle-sample
 	branch = master
+[submodule "testdata/case/java-with-srcfile"]
+	path = testdata/case/java-with-srcfile
+	url = https://github.com/sgtest/java-with-srcfile

src/main/java/com/sourcegraph/javagraph/AndroidCoreProject.java

@@ -8,10 +8,7 @@
 import java.io.IOException;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 
 /**
  * Set of rules to compile Android's libcore (https://android.googlesource.com/platform/libcore/)
@@ -93,8 +90,12 @@ public static SourceUnit createSourceUnit(String subdir) throws Exception {
         unit.Type = "JavaArtifact";
         unit.Name = "AndroidCore";
         unit.Dir = subdir;
-        List<String> directories = new ArrayList<>();
-        getSourceFilesAndDirectories(PathUtil.CWD.resolve(subdir), unit.Files, directories);
+
+        List<String> files = new LinkedList<>();
+        List<String> directories = new LinkedList<>();
+
+        getSourceFilesAndDirectories(PathUtil.CWD.resolve(subdir), files, directories);
+        unit.Files = files;
         unit.Data.put("AndroidCoreSubdir", subdir);
         return unit;
     }

src/main/java/com/sourcegraph/javagraph/GraphCommand.java

@@ -3,6 +3,7 @@
 import com.beust.jcommander.Parameter;
 import com.google.gson.Gson;
 import org.apache.commons.lang3.StringUtils;
+import org.codehaus.plexus.util.DirectoryScanner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -11,10 +12,7 @@
 import java.io.Reader;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
+import java.util.*;
 
 public class GraphCommand {
 
@@ -61,7 +59,11 @@ public void Execute() {
             Grapher grapher = new Grapher(proj,
                     rawGraph);
             LOGGER.debug("Starting graph collection");
-            Collection<String> files = new ArrayList<>(unit.Files);
+            Collection<String> files = new ArrayList<>();
+            if (unit.Files != null) {
+                files.addAll(unit.Files);
+            }
+            files.addAll(collectFilesUsingGlobs(unit.Globs));
             Collection<String> extraFiles = (Collection<String>) unit.Data.get("ExtraSourceFiles");
             if (extraFiles != null) {
                 files.addAll(extraFiles);
@@ -97,6 +99,30 @@ public void Execute() {
         JSONUtil.writeJSON(graph);
     }
 
+    /**
+     * Collects files using globs if any
+     * @param globs globs
+     * @return list of files matching given globs
+     */
+    private Collection<String> collectFilesUsingGlobs(List<String> globs) {
+        if (globs == null || globs.isEmpty()) {
+            return Collections.emptyList();
+        }
+        LOGGER.info("Collecting files using globs");
+        final DirectoryScanner directoryScanner = new DirectoryScanner();
+        String globsArray[] = new String[globs.size()];
+        globs.toArray(globsArray);
+        directoryScanner.setIncludes(globsArray);
+        directoryScanner.setExcludes(new String[] {".gradle-srclib/**", ".m2-srclib/**"});
+        directoryScanner.setBasedir(PathUtil.CWD.toString());
+        directoryScanner.scan();
+        Collection<String> files = new LinkedList<>();
+        for (String fileName : directoryScanner.getIncludedFiles()) {
+            files.add(PathUtil.concat(PathUtil.CWD, fileName).toString());
+        }
+        return files;
+    }
+
     /**
      * Javadoc object
      */

src/main/java/com/sourcegraph/javagraph/MavenProject.java

@@ -353,6 +353,7 @@ public static Collection<SourceUnit> findAllSourceUnits(String repoUri) throws I
         Collection<SourceUnit> ret = new ArrayList<>();
         for (BuildAnalysis.BuildInfo info : infos) {
             SourceUnit unit = new SourceUnit();
+            unit.Files = new LinkedList<>();
             unit.Name = info.getName();
             unit.Dir = info.projectDir;
             unit.Files.addAll(info.sources);

src/main/java/com/sourcegraph/javagraph/ScanCommand.java

@@ -158,6 +158,10 @@ private static void normalize(Collection<SourceUnit> units) {
     private static void splitInternalAndExternalFiles(Collection<String> files,
                                                       List<String> internal,
                                                       List<String> external) {
+
+        if (files == null) {
+            return;
+        }
         for (String file : files) {
             Path p = PathUtil.CWD.resolve(file).toAbsolutePath();
             if (p.startsWith(PathUtil.CWD)) {

src/main/java/com/sourcegraph/javagraph/SourceUnit.java

@@ -30,7 +30,15 @@ public class SourceUnit {
     /**
      * List of files that produce source units
      */
-    List<String> Files = new LinkedList<>();
+    List<String> Files;
+
+
+    /**
+     * Globs is a list of patterns that match files that make up this source
+     * unit. It is used to detect when the source unit definition is out of date
+     * (e.g., when a file matches the glob but is not in the Files list).
+     */
+    List<String> Globs;
 
     /**
      * Source unit directory
@@ -42,7 +50,6 @@ public class SourceUnit {
      */
     List<RawDependency> Dependencies = new LinkedList<>();
 
-    // TODO(rameshvarun): Globs entry
     /**
      * Source unit raw data
      */

testdata/expected/program/java-with-srcfile/com.sourcegraph/java-with-srcfile/JavaArtifact.depresolve.json

@@ -0,0 +1,39 @@
+[
+  {
+    "Raw": {
+      "groupID": "com.beust",
+      "artifactID": "jcommander",
+      "version": "1.30",
+      "scope": "compile",
+      "file": ".gradle-srclib/caches/modules-2/files-2.1/com.beust/jcommander/1.30/c440b30a944ba199751551aee393f8aa03b3c327/jcommander-1.30.jar"
+    },
+    "Target": {
+      "ToRepoCloneURL": "[email protected]:cbeust/jcommander.git",
+      "ToUnit": "com.beust/jcommander",
+      "ToUnitType": "JavaArtifact",
+      "ToVersionString": "1.30"
+    }
+  },
+  {
+    "Raw": {
+      "groupID": "com.beust",
+      "artifactID": "jcommander",
+      "version": "1.30",
+      "scope": "testCompile",
+      "file": ".gradle-srclib/caches/modules-2/files-2.1/com.beust/jcommander/1.30/c440b30a944ba199751551aee393f8aa03b3c327/jcommander-1.30.jar"
+    },
+    "Target": {
+      "ToRepoCloneURL": "[email protected]:cbeust/jcommander.git",
+      "ToUnit": "com.beust/jcommander",
+      "ToUnitType": "JavaArtifact",
+      "ToVersionString": "1.30"
+    }
+  },
+  {
+    "Target": {
+      "ToRepoCloneURL": "hg.openjdk.java.net/jdk8/jdk8/jdk",
+      "ToUnit": ".",
+      "ToUnitType": "Java"
+    }
+  }
+]

testdata/expected/program/java-with-srcfile/com.sourcegraph/java-with-srcfile/JavaArtifact.graph.json

@@ -0,0 +1,179 @@
+{
+  "Defs": [
+    {
+      "Path": "foo/bar/baz",
+      "Name": "baz",
+      "Kind": "package",
+      "File": "",
+      "DefStart": 0,
+      "DefEnd": 0,
+      "Data": {
+        "JavaKind": "PACKAGE",
+        "Package": "foo.bar.baz"
+      },
+      "TreePath": "foo/bar/baz"
+    },
+    {
+      "Path": "foo/bar/baz/Qux/:init",
+      "Name": "\u003cinit\u003e",
+      "Kind": "func",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "DefStart": 76,
+      "DefEnd": 79,
+      "Exported": true,
+      "Data": {
+        "JavaKind": "CONSTRUCTOR",
+        "TypeExpression": "()void",
+        "Package": "foo.bar.baz",
+        "Modifiers": [
+          "public"
+        ]
+      },
+      "TreePath": "foo/bar/baz/Qux/:init"
+    },
+    {
+      "Path": "foo/bar/baz/Qux:type",
+      "Name": "Qux",
+      "Kind": "type",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "DefStart": 63,
+      "DefEnd": 183,
+      "Exported": true,
+      "Data": {
+        "JavaKind": "CLASS",
+        "TypeExpression": "foo.bar.baz.Qux",
+        "Package": "foo.bar.baz",
+        "Modifiers": [
+          "public"
+        ]
+      },
+      "TreePath": "foo/bar/baz/Qux:type"
+    },
+    {
+      "Path": "foo/bar/baz/Qux:type/main:java.lang.String[]",
+      "Name": "main",
+      "Kind": "func",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "DefStart": 85,
+      "DefEnd": 181,
+      "Exported": true,
+      "Data": {
+        "JavaKind": "METHOD",
+        "TypeExpression": "(java.lang.String[])void",
+        "Package": "foo.bar.baz",
+        "Modifiers": [
+          "public",
+          "static"
+        ]
+      },
+      "TreePath": "foo/bar/baz/Qux:type/main:java.lang.String[]"
+    },
+    {
+      "Path": "foo/bar/baz/Qux:type/main:java.lang.String[]/args",
+      "Name": "args",
+      "Kind": "var",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "DefStart": 109,
+      "DefEnd": 122,
+      "Local": true,
+      "Data": {
+        "JavaKind": "PARAMETER",
+        "TypeExpression": "java.lang.String[]",
+        "Package": "foo.bar.baz",
+        "Modifiers": []
+      },
+      "TreePath": "foo/bar/baz/Qux:type/main:java.lang.String[]/args"
+    },
+    {
+      "Path": "foo/bar/baz/Qux:type/main:java.lang.String[]/jc",
+      "Name": "jc",
+      "Kind": "var",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "DefStart": 145,
+      "DefEnd": 178,
+      "Local": true,
+      "Data": {
+        "JavaKind": "LOCAL_VARIABLE",
+        "TypeExpression": "com.beust.jcommander.JCommander",
+        "Package": "foo.bar.baz",
+        "Modifiers": []
+      },
+      "TreePath": "foo/bar/baz/Qux:type/main:java.lang.String[]/jc"
+    }
+  ],
+  "Refs": [
+    {
+      "DefRepo": "github.com/cbeust/jcommander",
+      "DefUnitType": "JavaArtifact",
+      "DefUnit": "com.beust/jcommander",
+      "DefPath": "com/beust/jcommander/JCommander:type",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 145,
+      "End": 155
+    },
+    {
+      "DefRepo": "github.com/cbeust/jcommander",
+      "DefUnitType": "JavaArtifact",
+      "DefUnit": "com.beust/jcommander",
+      "DefPath": "com/beust/jcommander/JCommander:type",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 165,
+      "End": 175
+    },
+    {
+      "DefRepo": "github.com/cbeust/jcommander",
+      "DefUnitType": "JavaArtifact",
+      "DefUnit": "com.beust/jcommander",
+      "DefPath": "com/beust/jcommander/JCommander:type",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 50,
+      "End": 60
+    },
+    {
+      "DefPath": "foo/bar/baz/Qux:type/main:java.lang.String[]/args",
+      "Def": true,
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 116,
+      "End": 120
+    },
+    {
+      "DefPath": "foo/bar/baz/Qux:type/main:java.lang.String[]/jc",
+      "Def": true,
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 156,
+      "End": 158
+    },
+    {
+      "DefPath": "foo/bar/baz/Qux:type/main:java.lang.String[]",
+      "Def": true,
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 104,
+      "End": 108
+    },
+    {
+      "DefPath": "foo/bar/baz/Qux:type",
+      "Def": true,
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 76,
+      "End": 79
+    },
+    {
+      "DefRepo": "hg.openjdk.java.net/jdk8/jdk8/jdk",
+      "DefUnitType": "Java",
+      "DefUnit": ".",
+      "DefPath": "java/lang/Exception:type",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 131,
+      "End": 140
+    },
+    {
+      "DefRepo": "hg.openjdk.java.net/jdk8/jdk8/jdk",
+      "DefUnitType": "Java",
+      "DefUnit": ".",
+      "DefPath": "java/lang/String:type",
+      "File": "src/main/java/foo/bar/baz/Qux.java",
+      "Start": 109,
+      "End": 115
+    }
+  ]
+}

testdata/expected/program/java-with-srcfile/com.sourcegraph/java-with-srcfile/JavaArtifact.unit.json

@@ -0,0 +1 @@
+{"Name":"com.sourcegraph/java-with-srcfile","Type":"JavaArtifact","Repo":"github.com/sgtest/java-with-srcfile","Globs":["**/*.java"],"Files":null,"Dependencies":[{"artifactID":"jcommander","file":".gradle-srclib/caches/modules-2/files-2.1/com.beust/jcommander/1.30/c440b30a944ba199751551aee393f8aa03b3c327/jcommander-1.30.jar","groupID":"com.beust","scope":"compile","version":"1.30"},{"artifactID":"jcommander","file":".gradle-srclib/caches/modules-2/files-2.1/com.beust/jcommander/1.30/c440b30a944ba199751551aee393f8aa03b3c327/jcommander-1.30.jar","groupID":"com.beust","scope":"testCompile","version":"1.30"}],"Data":{"ClassPath":[".gradle-srclib/caches/modules-2/files-2.1/com.beust/jcommander/1.30/c440b30a944ba199751551aee393f8aa03b3c327/jcommander-1.30.jar"],"Description":"Test srclib-java for gradle support.","GradleFile":"build.gradle","SourcePath":[["com.sourcegraph/java-with-srcfile","0.0.1-SNAPSHOT","src/main/java"],["com.sourcegraph/java-with-srcfile","0.0.1-SNAPSHOT","src/test/java"]],"SourceVersion":"1.8"}}