Skip to content

Commit

Permalink
[VL] enable unix_millis,unix_micros,timestamp_millis,timestamp_micros (…
Browse files Browse the repository at this point in the history 
…apache#5601)

[VL] Enable unix_millis,unix_micros,timestamp_millis,timestamp_micros functions
  • Loading branch information
@zhli1142015
zhli1142015 authored May 6, 2024
1 parent 4dce49b commit b84c6d0
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 3 deletions.
6 changes: 5 additions & 1 deletion backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ object CHExpressionUtil {
REGR_INTERCEPT -> DefaultValidator(),
REGR_SXY -> DefaultValidator(),
TO_UTC_TIMESTAMP -> DefaultValidator(),
FROM_UTC_TIMESTAMP -> DefaultValidator()
FROM_UTC_TIMESTAMP -> DefaultValidator(),
UNIX_MILLIS -> DefaultValidator(),
UNIX_MICROS -> DefaultValidator(),
TIMESTAMP_MILLIS -> DefaultValidator(),
TIMESTAMP_MICROS -> DefaultValidator()
)
}
42 changes: 42 additions & 0 deletions backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -826,4 +826,46 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest {
}
}

test("unix_millis") {
withTempPath {
path =>
val t1 = Timestamp.valueOf("2015-07-22 10:00:00.012")
val t2 = Timestamp.valueOf("2014-12-31 23:59:59.012")
val t3 = Timestamp.valueOf("2014-12-31 23:59:59.001")
Seq(t1, t2, t3).toDF("t").write.parquet(path.getCanonicalPath)

spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("time")
runQueryAndCompare("select unix_millis(t) from time") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
}
}

test("unix_micros") {
withTempPath {
path =>
val t1 = Timestamp.valueOf("2015-07-22 10:00:00.012")
val t2 = Timestamp.valueOf("2014-12-31 23:59:59.012")
val t3 = Timestamp.valueOf("2014-12-31 23:59:59.001")
Seq(t1, t2, t3).toDF("t").write.parquet(path.getCanonicalPath)

spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("time")
runQueryAndCompare("select unix_micros(t) from time") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
}
}

test("timestamp_millis") {
runQueryAndCompare("select timestamp_millis(l_orderkey) from lineitem") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
}

test("timestamp_micros") {
runQueryAndCompare("select timestamp_micros(l_orderkey) from lineitem") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
}

}
6 changes: 4 additions & 2 deletions docs/velox-backend-support-progress.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,15 +338,17 @@ Gluten supports 199 functions. (Drag to right to see all data types)
| second | second | | S | | | | | | | | | | | | | | | | | | | |
| session_window | | | | | | | | | | | | | | | | | | | | | | |
| timestamp | | | | | | | | | | | | | | | | | | | | | | |
| timestamp_micros | | | | | | | | | | | | | | | | | | | | | | |
| timestamp_millis | | | | | | | | | | | | | | | | | | | | | | |
| timestamp_micros | | timestamp_micros | S | | | | | | | | | | | | | | | | | | | |
| timestamp_millis | | timestamp_millis | S | | | | | | | | | | | | | | | | | | | |
| timestamp_seconds | | | | | | | | | | | | | | | | | | | | | | |
| to_date | | | S | | | | | | | | | S | S | | | | | | | | | |
| to_timestamp | | | | | | | | | | | | | | | | | | | | | | |
| to_unix_timestamp | to_unixtime | to_unix_timestamp | S | | | | | | | | | | | | | | | | | | | |
| to_utc_timestamp | | | | | | | | | | | | | | | | | | | | | | |
| trunc | | | | | | | | | | | | | | | | | | | | | | |
| unix_timestamp | | unix_timestamp | | | | | | | | | | | | | | | | | | | | |
| unix_millis | | unix_millis | S | | | | | | | | | | | | | | | | | | | |
| unix_micros | | unix_micros | S | | | | | | | | | | | | | | | | | | | |
| weekday | | | S | | | | | | | | | S | | | | | | | | | | |
| weekofyear | week,week_of_year | | S | | | | | | | | | | | | | | | | | | | |
| window | | | | | | | | | | | | | | | | | | | | | | |
Expand Down
4 changes: 4 additions & 0 deletions gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,10 @@ object ExpressionMappings {
Sig[MakeYMInterval](MAKE_YM_INTERVAL),
Sig[ToUTCTimestamp](TO_UTC_TIMESTAMP),
Sig[FromUTCTimestamp](FROM_UTC_TIMESTAMP),
Sig[UnixMillis](UNIX_MILLIS),
Sig[UnixMicros](UNIX_MICROS),
Sig[MillisToTimestamp](TIMESTAMP_MILLIS),
Sig[MicrosToTimestamp](TIMESTAMP_MICROS),
// JSON functions
Sig[GetJsonObject](GET_JSON_OBJECT),
Sig[LengthOfJsonArray](JSON_ARRAY_LENGTH),
Expand Down
4 changes: 4 additions & 0 deletions shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,10 @@ object ExpressionNames {
final val MAKE_YM_INTERVAL = "make_ym_interval"
final val TO_UTC_TIMESTAMP = "to_utc_timestamp"
final val FROM_UTC_TIMESTAMP = "from_utc_timestamp"
final val UNIX_MILLIS = "unix_millis"
final val UNIX_MICROS = "unix_micros"
final val TIMESTAMP_MILLIS = "timestamp_millis"
final val TIMESTAMP_MICROS = "timestamp_micros"

// JSON functions
final val GET_JSON_OBJECT = "get_json_object"
Expand Down

0 comments on commit b84c6d0

Please sign in to comment.