From b84c6d01d52621052f573e27057f490dee356593 Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Mon, 6 May 2024 11:02:38 +0800 Subject: [PATCH] [VL] enable unix_millis,unix_micros,timestamp_millis,timestamp_micros (#5601) [VL] Enable unix_millis,unix_micros,timestamp_millis,timestamp_micros functions --- .../gluten/utils/CHExpressionUtil.scala | 6 ++- .../ScalarFunctionsValidateSuite.scala | 42 +++++++++++++++++++ docs/velox-backend-support-progress.md | 6 ++- .../expression/ExpressionMappings.scala | 4 ++ .../gluten/expression/ExpressionNames.scala | 4 ++ 5 files changed, 59 insertions(+), 3 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index 3663ef07a7ac..540af65acac5 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -190,6 +190,10 @@ object CHExpressionUtil { REGR_INTERCEPT -> DefaultValidator(), REGR_SXY -> DefaultValidator(), TO_UTC_TIMESTAMP -> DefaultValidator(), - FROM_UTC_TIMESTAMP -> DefaultValidator() + FROM_UTC_TIMESTAMP -> DefaultValidator(), + UNIX_MILLIS -> DefaultValidator(), + UNIX_MICROS -> DefaultValidator(), + TIMESTAMP_MILLIS -> DefaultValidator(), + TIMESTAMP_MICROS -> DefaultValidator() ) } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 84f580b0b781..d4ef3941205f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -826,4 +826,46 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("unix_millis") { + withTempPath { + path => + val t1 = Timestamp.valueOf("2015-07-22 10:00:00.012") + val t2 = Timestamp.valueOf("2014-12-31 23:59:59.012") + val t3 = Timestamp.valueOf("2014-12-31 23:59:59.001") + Seq(t1, t2, t3).toDF("t").write.parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("time") + runQueryAndCompare("select unix_millis(t) from time") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + + test("unix_micros") { + withTempPath { + path => + val t1 = Timestamp.valueOf("2015-07-22 10:00:00.012") + val t2 = Timestamp.valueOf("2014-12-31 23:59:59.012") + val t3 = Timestamp.valueOf("2014-12-31 23:59:59.001") + Seq(t1, t2, t3).toDF("t").write.parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("time") + runQueryAndCompare("select unix_micros(t) from time") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + + test("timestamp_millis") { + runQueryAndCompare("select timestamp_millis(l_orderkey) from lineitem") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + + test("timestamp_micros") { + runQueryAndCompare("select timestamp_micros(l_orderkey) from lineitem") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index 78d122147143..dcd73196158c 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -338,8 +338,8 @@ Gluten supports 199 functions. (Drag to right to see all data types) | second | second | | S | | | | | | | | | | | | | | | | | | | | | session_window | | | | | | | | | | | | | | | | | | | | | | | | timestamp | | | | | | | | | | | | | | | | | | | | | | | -| timestamp_micros | | | | | | | | | | | | | | | | | | | | | | | -| timestamp_millis | | | | | | | | | | | | | | | | | | | | | | | +| timestamp_micros | | timestamp_micros | S | | | | | | | | | | | | | | | | | | | | +| timestamp_millis | | timestamp_millis | S | | | | | | | | | | | | | | | | | | | | | timestamp_seconds | | | | | | | | | | | | | | | | | | | | | | | | to_date | | | S | | | | | | | | | S | S | | | | | | | | | | | to_timestamp | | | | | | | | | | | | | | | | | | | | | | | @@ -347,6 +347,8 @@ Gluten supports 199 functions. (Drag to right to see all data types) | to_utc_timestamp | | | | | | | | | | | | | | | | | | | | | | | | trunc | | | | | | | | | | | | | | | | | | | | | | | | unix_timestamp | | unix_timestamp | | | | | | | | | | | | | | | | | | | | | +| unix_millis | | unix_millis | S | | | | | | | | | | | | | | | | | | | | +| unix_micros | | unix_micros | S | | | | | | | | | | | | | | | | | | | | | weekday | | | S | | | | | | | | | S | | | | | | | | | | | | weekofyear | week,week_of_year | | S | | | | | | | | | | | | | | | | | | | | | window | | | | | | | | | | | | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 6be5b0f9bb58..7b4d6cd159ea 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -189,6 +189,10 @@ object ExpressionMappings { Sig[MakeYMInterval](MAKE_YM_INTERVAL), Sig[ToUTCTimestamp](TO_UTC_TIMESTAMP), Sig[FromUTCTimestamp](FROM_UTC_TIMESTAMP), + Sig[UnixMillis](UNIX_MILLIS), + Sig[UnixMicros](UNIX_MICROS), + Sig[MillisToTimestamp](TIMESTAMP_MILLIS), + Sig[MicrosToTimestamp](TIMESTAMP_MICROS), // JSON functions Sig[GetJsonObject](GET_JSON_OBJECT), Sig[LengthOfJsonArray](JSON_ARRAY_LENGTH), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 44384be72f61..7d516ecd006d 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -209,6 +209,10 @@ object ExpressionNames { final val MAKE_YM_INTERVAL = "make_ym_interval" final val TO_UTC_TIMESTAMP = "to_utc_timestamp" final val FROM_UTC_TIMESTAMP = "from_utc_timestamp" + final val UNIX_MILLIS = "unix_millis" + final val UNIX_MICROS = "unix_micros" + final val TIMESTAMP_MILLIS = "timestamp_millis" + final val TIMESTAMP_MICROS = "timestamp_micros" // JSON functions final val GET_JSON_OBJECT = "get_json_object"