Add support for date_nanos and tests. (#337) (opensearch-project#1976)

* Add support for `date_nanos` and tests. (#337) * Add support for `date_nanos` and tests. Signed-off-by: Yury-Fridlyand <[email protected]> * Add more IT. Signed-off-by: Yury-Fridlyand <[email protected]> --------- Signed-off-by: Yury-Fridlyand <[email protected]> * Typo fix in IT. Signed-off-by: Yury-Fridlyand <[email protected]> * Address PR feedback. Signed-off-by: Yury-Fridlyand <[email protected]> * Spotless Signed-off-by: Yury-Fridlyand <[email protected]> --------- Signed-off-by: Yury-Fridlyand <[email protected]> (cherry picked from commit 752da21) Signed-off-by: Yury-Fridlyand <[email protected]>
Bit-Quill · Aug 22, 2023 · 9469b68 · 9469b68
1 parent 8313c34
commit 9469b68
Show file tree

Hide file tree

Showing 10 changed files with 152 additions and 25 deletions.
diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java
@@ -50,6 +50,7 @@ public void test_nonnumeric_data_types() throws IOException {
         schema("text_value", "string"),
         schema("binary_value", "binary"),
         schema("date_value", "timestamp"),
+        schema("date_nanos_value", "timestamp"),
         schema("ip_value", "ip"),
         schema("object_value", "struct"),
         schema("nested_value", "array"),

diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java
@@ -58,17 +58,31 @@ public void typeof_opensearch_types() throws IOException {
     verifyDataRows(response,
         rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE"));
 
-    response = executeQuery(String.format("source=%s | eval "
-        + "`text` = typeof(text_value), `date` = typeof(date_value),"
-        + "`boolean` = typeof(boolean_value), `object` = typeof(object_value),"
-        + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value),"
-        + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)"
-        // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported
-        //+ ", `nested` = typeof(nested_value)"
-        + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`",
-            TEST_INDEX_DATATYPE_NONNUMERIC));
-    verifyDataRows(response,
-        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
-                "IP", "BINARY", "GEO_POINT"));
+    response =
+        executeQuery(
+            String.format(
+                "source=%s | eval `text` = typeof(text_value), `date` = typeof(date_value),"
+                    + " `date_nanos` = typeof(date_nanos_value),`boolean` = typeof(boolean_value),"
+                    + " `object` = typeof(object_value),`keyword` = typeof(keyword_value), `ip` ="
+                    + " typeof(ip_value),`binary` = typeof(binary_value), `geo_point` ="
+                    + " typeof(geo_point_value)"
+                    // TODO activate this test once `ARRAY` type supported, see
+                    // ExpressionAnalyzer::isTypeNotSupported
+                    // + ", `nested` = typeof(nested_value)"
+                    + " | fields `text`, `date`, `date_nanos`, `boolean`, `object`, `keyword`,"
+                    + " `ip`, `binary`, `geo_point`",
+                TEST_INDEX_DATATYPE_NONNUMERIC));
+    verifyDataRows(
+        response,
+        rows(
+            "TEXT",
+            "TIMESTAMP",
+            "TIMESTAMP",
+            "BOOLEAN",
+            "OBJECT",
+            "KEYWORD",
+            "IP",
+            "BINARY",
+            "GEO_POINT"));
   }
 }
diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java
@@ -6,6 +6,7 @@
 
 package org.opensearch.sql.sql;
 
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS;
 import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT;
 import static org.opensearch.sql.util.MatcherUtils.rows;
@@ -30,6 +31,7 @@ public class DateTimeFormatsIT extends SQLIntegTestCase {
   public void init() throws Exception {
     super.init();
     loadIndex(Index.DATE_FORMATS);
+    loadIndex(Index.DATA_TYPE_NONNUMERIC);
   }
 
   @Test
@@ -120,6 +122,94 @@ public void testNumericFormats() {
         rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5"));
   }
 
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithFormats() {
+    String query =
+        String.format("SELECT hour_minute_second_OR_t_time" + " FROM %s", TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result, rows("09:07:42"), rows("07:07:42.123456789"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithFunctions() {
+    // in memory funcs
+    String query =
+        String.format(
+            "SELECT"
+                + " hour_minute_second_OR_t_time > TIME '08:07:00',"
+                + " hour_minute_second_OR_t_time < TIME '08:07:00',"
+                + " hour_minute_second_OR_t_time = t_time_no_millis,"
+                + " hour_minute_second_OR_t_time <> strict_t_time,"
+                + " hour_minute_second_OR_t_time >= t_time"
+                + " FROM %s",
+            TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(
+        result,
+        schema("hour_minute_second_OR_t_time > TIME '08:07:00'", null, "boolean"),
+        schema("hour_minute_second_OR_t_time < TIME '08:07:00'", null, "boolean"),
+        schema("hour_minute_second_OR_t_time = t_time_no_millis", null, "boolean"),
+        schema("hour_minute_second_OR_t_time <> strict_t_time", null, "boolean"),
+        schema("hour_minute_second_OR_t_time >= t_time", null, "boolean"));
+    verifyDataRows(
+        result, rows(true, false, true, false, true), rows(false, true, false, true, false));
+    // push down
+    query =
+        String.format(
+            "SELECT hour_minute_second_OR_t_time"
+                + " FROM %s WHERE hour_minute_second_OR_t_time > TIME '08:07:00'",
+            TEST_INDEX_DATE_FORMATS);
+    result = executeQuery(query);
+    verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result, rows("09:07:42"));
+    query =
+        String.format(
+            "SELECT hour_minute_second_OR_t_time"
+                + " FROM %s WHERE hour_minute_second_OR_t_time < TIME '08:07:00'",
+            TEST_INDEX_DATE_FORMATS);
+    result = executeQuery(query);
+    verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result, rows("07:07:42.123456789"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosOrderBy() {
+    String query =
+        String.format(
+            "SELECT hour_minute_second_OR_t_time"
+                + " FROM %s ORDER BY hour_minute_second_OR_t_time ASC",
+            TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result, rows("07:07:42.123456789"), rows("09:07:42"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosGroupBy() {
+    String query =
+        String.format(
+            "SELECT count(*)" + " FROM %s GROUP BY hour_minute_second_OR_t_time",
+            TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result, schema("count(*)", null, "integer"));
+    verifyDataRows(result, rows(1), rows(1));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithNanos() {
+    String query =
+        String.format("SELECT date_nanos_value" + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC);
+    JSONObject result = executeQuery(query);
+    verifySchema(result, schema("date_nanos_value", null, "timestamp"));
+    verifyDataRows(result, rows("2019-03-24 01:34:46.123456789"));
+  }
+
   protected JSONObject executeQuery(String query) throws IOException {
     Request request = new Request("POST", QUERY_API_ENDPOINT);
     request.setJsonEntity(String.format(Locale.ROOT, "{\n" + "  \"query\": \"%s\"\n" + "}", query));

diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java
@@ -47,14 +47,29 @@ public void typeof_opensearch_types() {
     verifyDataRows(response,
         rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE"));
 
-    response = executeJdbcRequest(String.format("SELECT typeof(text_value),"
-        + "typeof(date_value), typeof(boolean_value), typeof(object_value), typeof(keyword_value),"
-        + "typeof(ip_value), typeof(binary_value), typeof(geo_point_value)"
-        // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported
-        //+ ", typeof(nested_value)"
-        + " from %s;", TEST_INDEX_DATATYPE_NONNUMERIC));
-    verifyDataRows(response,
-        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
-                "IP", "BINARY", "GEO_POINT"));
+    response =
+        executeJdbcRequest(
+            String.format(
+                "SELECT typeof(text_value),typeof(date_value), typeof(date_nanos_value),"
+                    + " typeof(boolean_value), typeof(object_value),"
+                    + " typeof(keyword_value),typeof(ip_value), typeof(binary_value),"
+                    + " typeof(geo_point_value)"
+                    // TODO activate this test once `ARRAY` type supported, see
+                    // ExpressionAnalyzer::isTypeNotSupported
+                    // + ", typeof(nested_value)"
+                    + " from %s;",
+                TEST_INDEX_DATATYPE_NONNUMERIC));
+    verifyDataRows(
+        response,
+        rows(
+            "TEXT",
+            "TIMESTAMP",
+            "TIMESTAMP",
+            "BOOLEAN",
+            "OBJECT",
+            "KEYWORD",
+            "IP",
+            "BINARY",
+            "GEO_POINT"));
   }
 }
diff --git a/integ-test/src/test/resources/datatypes.json b/integ-test/src/test/resources/datatypes.json
@@ -1,2 +1,2 @@
 {"index":{"_id":"1"}}
-{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" :  "Smith"}, {"first" : "Alice", "last" :  "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }}
+{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" :  "Smith"}, {"first" : "Alice", "last" :  "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }}