Merge pull request #79 from ngageoint/elasticsearch-geoshape-upgrade

Issue 67: Update Elasticsearch and improve geoshape indexing

.gitignore

@@ -13,6 +13,7 @@ target/
 /log/
 /output/
 /scripts/
+/elasticsearch/
 # When executing tests in alphabetical order, Maven generates temporary
 # files with names like this:
 #

.travis.yml

@@ -13,11 +13,14 @@ env:
     # This is the encrypted COVERITY_SCAN_TOKEN, created via the
     # `travis encrypt` command using the project repo's public key.
     - secure: "v5ixqTeb74y0vRuPcDbe3C28GDDYvqyEXA2dt+9UVU6GG7WpnmpkBf05gI1dIhp51lBhwx9WSlFBtzho+KdCBmNY/CzBRhVHe/lCQYK9Hb6uGPvuwBvC0WjJgJXsVrLFjppeRhcf+OAweVQ3uw2RPMDRvKIVMUcO1BTFjjJl6REJXNUdzGS57MtH2mmRyOEz250EwgqUELZvcOytG7fNrjMJKVK2nSsoxi0BqZIpItTWPWWeQ1wi1FplJ18A2qtD+MPfAGNSB+/a+r0Av+VCT2eGl06ZyZAzP3q/vG5IYjQ3AJsSPqcZUt4ms+2us1+kwuzXIILjzZmcfImu29+y/thndU5E5b2v+nZ4H69CUCc5OmKW2RwozLNmBIUhO0n+35va/J7FiPIqm3pwxCz5vWA3YTHDADxnIYe7+9uY/+dOK/AvP5fyu7u07vuF3liKNBdrX7ylP3kYc7FXGmYl8wCZv31iy1yTtndQ9qKef7bo8lM9Cdh39KyowrygH+Um7pr9gqf2S9jn99nQ3bib32fBWgBkLpJRwhZYHPUupZjZfgu/9woby0DuriuHZKMqZd7QUawYz6wXGlhzu78x5Tohlj1pGBwHYdcJ/Tm3PiEpyH4aYQLffkjGHJAcCW5tO8QbB0qrLYWC8xVMWuFz1TpSBRXOqVYdBfIa2UZDtOU="
+    # Default Elasticsearch heap size can be too large for Travis
+    - ES_JAVA_OPTS="-Xms256m -Xmx512m"
 
   matrix:
     - MODULE='berkeleyje'
     - MODULE='cassandra'
     - MODULE='es' ARGS='-DthreadCount=1'
+    - MODULE='es' ARGS='-DthreadCount=1 -Pelasticsearch2'
     - MODULE='hadoop-parent/janusgraph-hadoop-2'
     - MODULE='hbase-parent/janusgraph-hbase-098'
     - MODULE='hbase-parent/janusgraph-hbase-10'
@@ -39,6 +42,7 @@ matrix:
     # Currently broken due to too many log statements (exceeds 4MB)
     # https://travis-ci.org/JanusGraph/janusgraph/jobs/197472453
     - env: MODULE='es' ARGS='-DthreadCount=1'
+    - env: MODULE='es' ARGS='-DthreadCount=1 -Pelasticsearch2'
 
     # Currently broken due to too many log statements (exceeds 4MB)
     # https://travis-ci.org/JanusGraph/janusgraph/jobs/197672947

docs/hadoop.txt

@@ -83,6 +83,7 @@ giraph.maxMessagesInMemory=100000
 spark.master=local[*]
 spark.executor.memory=1g
 spark.serializer=org.apache.spark.serializer.KryoSerializer
+spark.kryo.registrator=org.janusgraph.hadoop.serialize.JanusGraphKryoRegistrator
 ----
 
 [source, gremlin]

docs/lucene.txt

@@ -21,7 +21,7 @@ In the above configuration, the index backend is named `search`. Replace `search
 === Feature Support
 
 * *Full-Text*: Supports all `Text` predicates to search for text properties that matches a given word, prefix or regular expression.
-* *Geo*: Supports the `Geo.WITHIN` condition to search for points that fall within a given geographic shape. Only supports points for indexing and circles and boxes for querying.
+* *Geo*: Supports `Geo` predicates to search for geo properties that are intersecting, within, or contained in a given query geometry. Supports points, lines and polygons for indexing. Supports circles and boxes for querying point properties and all shapes for querying non-point properties. Note that JTS is required when using line and polygon shapes (see <<search-predicates#geoshape,Geoshape documentation>> for more information).
 * *Numeric Range*: Supports all numeric comparisons in `Compare`.
 * *Temporal*: Nanosecond granularity temporal indexing.
 

docs/searchpredicates.txt

@@ -30,8 +30,14 @@ See <<text-search>> for more information about full-text and string search.
 
 === Geo Predicate
 
-The `Geo` enum specifies the geo-location predicate `geoWithin` which holds true if one geometric object contains the other.
+The `Geo` enum specifies geo-location predicates.
 
+* `geoIntersect` which holds true if the two geometric objects have at least one point in common (opposite of `geoDisjoint`).
+* `geoWithin` which holds true if one geometric object contains the other.
+* `geoDisjoint` which holds true if the two geometric objects have no points in common (opposite of `geoIntersect`).
+* `geoContains` which holds true if one geometric object is contained by the other.
+
+See <<geo-search>> for more information about geo search.
 
 === Query Examples
 
@@ -80,10 +86,15 @@ While JanusGraph's composite indexes support any data type that can be stored in
 
 Additional data types will be supported in the future.
 
+[[geoshape]]
 === Geoshape Data Type
-The Geoshape data type supports representing a point, circle or box. However all index backends currently only support indexing points.
+The Geoshape data type supports representing a point, circle, box, line, polygon, multi-point, multi-line and multi-polygon. Index backends currently support indexing points, lines and polygons. Indexing multi-point, multi-line and multi-polygon properties has not been tested.
 Geospatial index lookups are only supported via mixed indexes.
 
+.Note:
+[NOTE]
+The Java Topology Suite (JTS) is currently required to use line, polygon, multi-point, multi-line and multi-polygon geometries. JTS is not included in JanusGraph distributions by default due to its LGPL license. Users must download the JTS JAR file separately and include it in the classpath when full geometry support is required.
+
 To construct a Geoshape use the following methods:
 
 [source, java]
@@ -93,8 +104,10 @@ Geoshape.point(37.97, 23.72)
 Geoshape.circle(37.97, 23.72, 50)
 //SW lat, SW lng, NE lat, NE lng
 Geoshape.box(37.97, 23.72, 38.97, 24.72)
+//WKT (requires JTS for polygon, linestring and multi-point/line/polygon geometries)
+Geoshape.fromWkt("POLYGON ((35.4 48.9, 35.6 48.9, 35.6 49.1, 35.4 49.1, 35.4 48.9))")
 
-In addition when importing a graph via GraphSON Point may be represented by:
+In addition, when importing a graph via GraphSON the geometry may be represented by GeoJSON:
 [source, java]
  //string
 "37.97, 23.72"
@@ -117,7 +130,7 @@ In addition when importing a graph via GraphSON Point may be represented by:
   "coordinates": [125.6, 10.1]
 }
 
-link:http://geojson.org/[GeoJSON] may be specified as Point, Circle or Polygon. However polygons must form a box.
+link:http://geojson.org/[GeoJSON] may be specified as Point, Circle, LineString or Polygon. Polygons must be closed.
 Note that unlike the JanusGraph API GeoJSON specifies coordinates as lng lat.
 
 === Collections

docs/solr.txt

@@ -8,7 +8,7 @@ Solr is the popular, blazing fast open source enterprise search platform from th
 JanusGraph supports http://lucene.apache.org/solr/[Solr] as an index backend.  Here are some of the Solr features supported by JanusGraph:
 
 * *Full-Text*: Supports all `Text` predicates to search for text properties that matches a given word, prefix or regular expression.
-* *Geo*: Supports the `Geo.WITHIN` condition to search for points that fall within a given circle. Only supports points for indexing and circles for querying.
+* *Geo*: Supports all `Geo` predicates to search for geo properties that are intersecting, within, disjoint to or contained in a given query geometry. Supports points, lines and polygons for indexing. Supports circles, boxes and polygons for querying point properties and all shapes for querying non-point properties. Note that JTS is required when using line and polygon shapes (see <<search-predicates#geoshape,Geoshape documentation>> for more information).
 * *Numeric Range*: Supports all numeric comparisons in `Compare`.
 * *TTL*: Supports automatically expiring indexed elements.
 * *Temporal*: Millisecond granularity temporal indexing.

docs/textsearch.txt

@@ -81,7 +81,28 @@ mgmt.commit()
 
 Note that the data will be stored in the index twice, once for exact matching and once for fuzzy matching.
 
-=== Field Mapping
+[[geo-search]]
+===Geo Mapping
+
+By default, JanusGraph supports indexing geo properties with point type and querying geo properties by circle or box. To index a non-point geo property with support for querying by any geoshape type, specify the mapping as `Mapping.PREFIX_TREE`:
+
+[source, gremlin]
+mgmt = graph.openManagement()
+name = mgmt.makePropertyKey('border').dataType(Geoshape.class).make()
+mgmt.buildIndex('borderIndex', Vertex.class).addKey(name, Mapping.PREFIX_TREE.asParameter()).buildMixedIndex("search")
+mgmt.commit()
+
+Additional parameters can be specified to tune the configuration of the underlying prefix tree mapping. These optional parameters include the number of levels used in the prefix tree as well as the associated precision.
+
+[source, gremlin]
+mgmt = graph.openManagement()
+name = mgmt.makePropertyKey('border').dataType(Geoshape.class).make()
+mgmt.buildIndex('borderIndex', Vertex.class).addKey(name, Mapping.PREFIX_TREE.asParameter(), Parameter.of("index-geo-max-levels", 18), Parameter.of("index-geo-dist-error-pct", 0.0125)).buildMixedIndex("search")
+mgmt.commit()
+
+Note that some indexing backends (e.g. Solr) may require additional external schema configuration to support and tune indexing non-point properties.
+
+== Field Mapping
 
 [[index-local-field-mapping]]
 ==== Individual Field Mapping

docs/versions.txt

@@ -13,5 +13,6 @@ JanusGraph.
 [options="header"]
 |==========================
 | JanusGraph | Cassandra | HBase | Bigtable | Elasticsearch | Solr | TinkerPop
-| 0.1.0 | 1.2.z, 2.0.z, 2.1.z | 0.98.z, 1.0.z, 1.1.z, 1.2.z | 0.9.z | 1.5.z |  5.2.z | 3.2.z
-|==========================
+| 0.1.0 | 1.2.z, 2.0.z, 2.1.z | 0.98.z, 1.0.z, 1.1.z, 1.2.z | 0.9.z | 2.z,5.z* |  5.2.z | 3.2.z
+
+*Only the REST client is compatible with both Elasticsearch 2.z and 5.z. The transport client is only compatibile with Elasticsearch 2.z.

janusgraph-core/pom.xml

@@ -70,6 +70,11 @@
             <groupId>com.spatial4j</groupId>
             <artifactId>spatial4j</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.vividsolutions</groupId>
+            <artifactId>jts</artifactId>
+            <optional>true</optional>
+        </dependency>
         <dependency>
             <groupId>commons-collections</groupId>
             <artifactId>commons-collections</artifactId>
@@ -102,6 +107,12 @@
            <groupId>com.google.code.findbugs</groupId>
            <artifactId>jsr305</artifactId>
        </dependency>
+       <!-- The Noggit JSON parsing library is needed for GeoJSON parsing -->
+       <dependency>
+           <groupId>org.noggit</groupId>
+           <artifactId>noggit</artifactId>
+           <version>0.6</version>
+       </dependency>
     </dependencies>
     <build>
         <directory>${basedir}/target</directory>

janusgraph-core/src/main/java/org/janusgraph/core/attribute/Geo.java

@@ -83,7 +83,7 @@ public JanusGraphPredicate negate() {
     },
 
     /**
-     * Whether one geographic region is completely contains within another
+     * Whether one geographic region is completely within another
      */
     WITHIN {
         @Override
@@ -104,6 +104,34 @@ public boolean hasNegation() {
             return false;
         }
 
+        @Override
+        public JanusGraphPredicate negate() {
+            throw new UnsupportedOperationException();
+        }
+    },
+
+    /**
+     * Whether one geographic region completely contains another
+     */
+    CONTAINS {
+        @Override
+        public boolean test(Object value, Object condition) {
+            Preconditions.checkArgument(condition instanceof Geoshape);
+            if (value == null) return false;
+            Preconditions.checkArgument(value instanceof Geoshape);
+            return ((Geoshape) value).contains((Geoshape) condition);
+        }
+
+        @Override
+        public String toString() {
+            return "contains";
+        }
+
+        @Override
+        public boolean hasNegation() {
+            return false;
+        }
+
         @Override
         public JanusGraphPredicate negate() {
             throw new UnsupportedOperationException();
@@ -137,4 +165,7 @@ public static <V> P<V> geoDisjoint(final V value) {
     public static <V> P<V> geoWithin(final V value) {
         return new P(Geo.WITHIN, value);
     }
+    public static <V> P<V> geoContains(final V value) {
+        return new P(Geo.CONTAINS, value);
+    }
 }