From 9469b6810f0ff537426f2be413ab68bceff0662d Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Mon, 21 Aug 2023 09:40:29 -0700 Subject: [PATCH] Add support for `date_nanos` and tests. (#337) (#1976) * Add support for `date_nanos` and tests. (#337) * Add support for `date_nanos` and tests. Signed-off-by: Yury-Fridlyand * Add more IT. Signed-off-by: Yury-Fridlyand --------- Signed-off-by: Yury-Fridlyand * Typo fix in IT. Signed-off-by: Yury-Fridlyand * Address PR feedback. Signed-off-by: Yury-Fridlyand * Spotless Signed-off-by: Yury-Fridlyand --------- Signed-off-by: Yury-Fridlyand (cherry picked from commit 752da2154d337c74ce60752fe8e47b97ea182bb3) Signed-off-by: Yury-Fridlyand --- .../org/opensearch/sql/ppl/DataTypeIT.java | 1 + .../opensearch/sql/ppl/SystemFunctionIT.java | 38 +++++--- .../opensearch/sql/sql/DateTimeFormatsIT.java | 90 +++++++++++++++++++ .../opensearch/sql/sql/SystemFunctionIT.java | 33 +++++-- integ-test/src/test/resources/datatypes.json | 2 +- .../src/test/resources/date_formats.json | 2 +- .../datatypes_index_mapping.json | 5 +- .../date_formats_index_mapping.json | 2 +- .../data/type/OpenSearchDataType.java | 3 + .../data/type/OpenSearchDataTypeTest.java | 1 + 10 files changed, 152 insertions(+), 25 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 9911c35d8f..53a3b13d0e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -50,6 +50,7 @@ public void test_nonnumeric_data_types() throws IOException { schema("text_value", "string"), schema("binary_value", "binary"), schema("date_value", "timestamp"), + schema("date_nanos_value", "timestamp"), schema("ip_value", "ip"), schema("object_value", "struct"), schema("nested_value", "array"), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index de13aa5488..f569f5cd54 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -58,17 +58,31 @@ public void typeof_opensearch_types() throws IOException { verifyDataRows(response, rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE")); - response = executeQuery(String.format("source=%s | eval " - + "`text` = typeof(text_value), `date` = typeof(date_value)," - + "`boolean` = typeof(boolean_value), `object` = typeof(object_value)," - + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value)," - + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)" - // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported - //+ ", `nested` = typeof(nested_value)" - + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`", - TEST_INDEX_DATATYPE_NONNUMERIC)); - verifyDataRows(response, - rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", - "IP", "BINARY", "GEO_POINT")); + response = + executeQuery( + String.format( + "source=%s | eval `text` = typeof(text_value), `date` = typeof(date_value)," + + " `date_nanos` = typeof(date_nanos_value),`boolean` = typeof(boolean_value)," + + " `object` = typeof(object_value),`keyword` = typeof(keyword_value), `ip` =" + + " typeof(ip_value),`binary` = typeof(binary_value), `geo_point` =" + + " typeof(geo_point_value)" + // TODO activate this test once `ARRAY` type supported, see + // ExpressionAnalyzer::isTypeNotSupported + // + ", `nested` = typeof(nested_value)" + + " | fields `text`, `date`, `date_nanos`, `boolean`, `object`, `keyword`," + + " `ip`, `binary`, `geo_point`", + TEST_INDEX_DATATYPE_NONNUMERIC)); + verifyDataRows( + response, + rows( + "TEXT", + "TIMESTAMP", + "TIMESTAMP", + "BOOLEAN", + "OBJECT", + "KEYWORD", + "IP", + "BINARY", + "GEO_POINT")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index fc05e502c5..c0a66ec92e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -30,6 +31,7 @@ public class DateTimeFormatsIT extends SQLIntegTestCase { public void init() throws Exception { super.init(); loadIndex(Index.DATE_FORMATS); + loadIndex(Index.DATA_TYPE_NONNUMERIC); } @Test @@ -120,6 +122,94 @@ public void testNumericFormats() { rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5")); } + @Test + @SneakyThrows + public void testDateNanosWithFormats() { + String query = + String.format("SELECT hour_minute_second_OR_t_time" + " FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, rows("09:07:42"), rows("07:07:42.123456789")); + } + + @Test + @SneakyThrows + public void testDateNanosWithFunctions() { + // in memory funcs + String query = + String.format( + "SELECT" + + " hour_minute_second_OR_t_time > TIME '08:07:00'," + + " hour_minute_second_OR_t_time < TIME '08:07:00'," + + " hour_minute_second_OR_t_time = t_time_no_millis," + + " hour_minute_second_OR_t_time <> strict_t_time," + + " hour_minute_second_OR_t_time >= t_time" + + " FROM %s", + TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema( + result, + schema("hour_minute_second_OR_t_time > TIME '08:07:00'", null, "boolean"), + schema("hour_minute_second_OR_t_time < TIME '08:07:00'", null, "boolean"), + schema("hour_minute_second_OR_t_time = t_time_no_millis", null, "boolean"), + schema("hour_minute_second_OR_t_time <> strict_t_time", null, "boolean"), + schema("hour_minute_second_OR_t_time >= t_time", null, "boolean")); + verifyDataRows( + result, rows(true, false, true, false, true), rows(false, true, false, true, false)); + // push down + query = + String.format( + "SELECT hour_minute_second_OR_t_time" + + " FROM %s WHERE hour_minute_second_OR_t_time > TIME '08:07:00'", + TEST_INDEX_DATE_FORMATS); + result = executeQuery(query); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, rows("09:07:42")); + query = + String.format( + "SELECT hour_minute_second_OR_t_time" + + " FROM %s WHERE hour_minute_second_OR_t_time < TIME '08:07:00'", + TEST_INDEX_DATE_FORMATS); + result = executeQuery(query); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, rows("07:07:42.123456789")); + } + + @Test + @SneakyThrows + public void testDateNanosOrderBy() { + String query = + String.format( + "SELECT hour_minute_second_OR_t_time" + + " FROM %s ORDER BY hour_minute_second_OR_t_time ASC", + TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, rows("07:07:42.123456789"), rows("09:07:42")); + } + + @Test + @SneakyThrows + public void testDateNanosGroupBy() { + String query = + String.format( + "SELECT count(*)" + " FROM %s GROUP BY hour_minute_second_OR_t_time", + TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, schema("count(*)", null, "integer")); + verifyDataRows(result, rows(1), rows(1)); + } + + @Test + @SneakyThrows + public void testDateNanosWithNanos() { + String query = + String.format("SELECT date_nanos_value" + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC); + JSONObject result = executeQuery(query); + verifySchema(result, schema("date_nanos_value", null, "timestamp")); + verifyDataRows(result, rows("2019-03-24 01:34:46.123456789")); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 584cdd05dd..c6a3ec734d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -47,14 +47,29 @@ public void typeof_opensearch_types() { verifyDataRows(response, rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE")); - response = executeJdbcRequest(String.format("SELECT typeof(text_value)," - + "typeof(date_value), typeof(boolean_value), typeof(object_value), typeof(keyword_value)," - + "typeof(ip_value), typeof(binary_value), typeof(geo_point_value)" - // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported - //+ ", typeof(nested_value)" - + " from %s;", TEST_INDEX_DATATYPE_NONNUMERIC)); - verifyDataRows(response, - rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", - "IP", "BINARY", "GEO_POINT")); + response = + executeJdbcRequest( + String.format( + "SELECT typeof(text_value),typeof(date_value), typeof(date_nanos_value)," + + " typeof(boolean_value), typeof(object_value)," + + " typeof(keyword_value),typeof(ip_value), typeof(binary_value)," + + " typeof(geo_point_value)" + // TODO activate this test once `ARRAY` type supported, see + // ExpressionAnalyzer::isTypeNotSupported + // + ", typeof(nested_value)" + + " from %s;", + TEST_INDEX_DATATYPE_NONNUMERIC)); + verifyDataRows( + response, + rows( + "TEXT", + "TIMESTAMP", + "TIMESTAMP", + "BOOLEAN", + "OBJECT", + "KEYWORD", + "IP", + "BINARY", + "GEO_POINT")); } } diff --git a/integ-test/src/test/resources/datatypes.json b/integ-test/src/test/resources/datatypes.json index ea3290ee64..70ddd28763 100644 --- a/integ-test/src/test/resources/datatypes.json +++ b/integ-test/src/test/resources/datatypes.json @@ -1,2 +1,2 @@ {"index":{"_id":"1"}} -{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} +{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} diff --git a/integ-test/src/test/resources/date_formats.json b/integ-test/src/test/resources/date_formats.json index 13d46a0e8c..2ff0c867a3 100644 --- a/integ-test/src/test/resources/date_formats.json +++ b/integ-test/src/test/resources/date_formats.json @@ -1,4 +1,4 @@ {"index": {}} {"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "1961-04-12", "custom_time_parser_check": "85476321", "incomplete_1" : 1984, "incomplete_2": null, "incomplete_custom_date": 1999, "incomplete_custom_time" : 10, "incorrect" : null, "epoch_sec" : 42, "epoch_milli" : 42, "custom_no_delimiter_date" : "19841020", "custom_no_delimiter_time" : "102030", "custom_no_delimiter_ts" : "19841020153548"} {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "09:07:00", "custom_time_parser_check": "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T07:07:42.123456789Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "09:07:00", "custom_time_parser_check": "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"} diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json index 8c1759b369..5908114201 100644 --- a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json @@ -13,10 +13,13 @@ "binary_value": { "type": "binary" }, - "date_value": { + "date_value": { "type" : "date", "format": "yyyy-MM-dd HH:mm:ss" }, + "date_nanos_value": { + "type" : "date_nanos" + }, "ip_value": { "type": "ip" }, diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json index 65811f8d9e..0b6daaacb4 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json @@ -298,7 +298,7 @@ "format": "yyyy-MM-dd||epoch_millis" }, "hour_minute_second_OR_t_time" : { - "type" : "date", + "type" : "date_nanos", "format": "hour_minute_second||t_time" }, "custom_timestamp" : { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 273b980d2a..01585600d0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -35,6 +35,7 @@ public enum MappingType { GeoPoint("geo_point", ExprCoreType.UNKNOWN), Binary("binary", ExprCoreType.UNKNOWN), Date("date", ExprCoreType.TIMESTAMP), + DateNanos("date_nanos", ExprCoreType.TIMESTAMP), Object("object", ExprCoreType.STRUCT), Nested("nested", ExprCoreType.ARRAY), Byte("byte", ExprCoreType.BYTE), @@ -137,6 +138,7 @@ public static Map parseMapping(Map i * @param mappingType A mapping type. * @return An instance or inheritor of `OpenSearchDataType`. */ + @SuppressWarnings("unchecked") public static OpenSearchDataType of(MappingType mappingType, Map innerMap) { OpenSearchDataType res = instances.getOrDefault(mappingType.toString(), new OpenSearchDataType(mappingType) @@ -162,6 +164,7 @@ public static OpenSearchDataType of(MappingType mappingType, Map case Binary: return OpenSearchBinaryType.of(); case Ip: return OpenSearchIpType.of(); case Date: + case DateNanos: // Default date formatter is used when "" is passed as the second parameter String format = (String) innerMap.getOrDefault("format", ""); return OpenSearchDateType.of(format); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 8d69b3d855..c74d06d262 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -106,6 +106,7 @@ private static Stream getTestDataWithType() { Arguments.of(MappingType.Double, "double", DOUBLE), Arguments.of(MappingType.Boolean, "boolean", BOOLEAN), Arguments.of(MappingType.Date, "date", TIMESTAMP), + Arguments.of(MappingType.DateNanos, "date", TIMESTAMP), Arguments.of(MappingType.Object, "object", STRUCT), Arguments.of(MappingType.Nested, "nested", ARRAY), Arguments.of(MappingType.GeoPoint, "geo_point",