Fix date_format tests

apache · Nov 13, 2023 · bf4f915 · bf4f915
1 parent cd021dd
commit bf4f915
Show file tree

Hide file tree

Showing 8 changed files with 185 additions and 2 deletions.
diff --git a/...-ut/spark32/src/test/scala/io/glutenproject/utils/clickhouse/ClickHouseTestSettings.scala b/...-ut/spark32/src/test/scala/io/glutenproject/utils/clickhouse/ClickHouseTestSettings.scala
@@ -678,6 +678,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("WeekDay")
     .exclude("WeekOfYear")
     .exclude("DateFormat")
+    .exclude("Gluten - DateFormat")
     .exclude("Hour")
     .exclude("Minute")
     .exclude("date add interval")

diff --git a/gluten-ut/spark32/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala
@@ -207,6 +207,8 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("to_unix_timestamp")
     // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS
     .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError")
+    // Replaced by a gluten test to pass timezone through config.
+    .exclude("DateFormat")
   enableSuite[GlutenDecimalExpressionSuite]
   enableSuite[GlutenStringFunctionsSuite]
   enableSuite[GlutenRegexpExpressionsSuite]

diff --git a/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
@@ -16,14 +16,15 @@
  */
 package org.apache.spark.sql.catalyst.expressions
 
-import org.apache.spark.sql.{GlutenTestConstants, GlutenTestsTrait}
+import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST
+import org.apache.spark.sql.GlutenTestsTrait
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{DateType, IntegerType, StringType}
+import org.apache.spark.sql.types.{DateType, IntegerType, StringType, TimestampType}
 import org.apache.spark.unsafe.types.UTF8String
 
 import java.sql.{Date, Timestamp}
@@ -285,4 +286,62 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
     GenerateUnsafeProjection.generate(
       ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil)
   }
+
+  // Modified based on vanilla spark to explicitly set timezone in config.
+  test(GLUTEN_TEST + "DateFormat") {
+    val PST_OPT = Option(PST.getId)
+    val JST_OPT = Option(JST.getId)
+
+    Seq("legacy", "corrected").foreach {
+      legacyParserPolicy =>
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT),
+            null)
+          checkEvaluation(
+            DateFormatClass(
+              Cast(Literal(d), TimestampType, UTC_OPT),
+              Literal.create(null, StringType),
+              UTC_OPT),
+            null)
+
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13")
+        }
+
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5")
+        }
+
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22")
+        }
+    }
+  }
 }
diff --git a/...-ut/spark33/src/test/scala/io/glutenproject/utils/clickhouse/ClickHouseTestSettings.scala b/...-ut/spark33/src/test/scala/io/glutenproject/utils/clickhouse/ClickHouseTestSettings.scala
@@ -721,6 +721,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("WeekDay")
     .exclude("WeekOfYear")
     .exclude("DateFormat")
+    .exclude("Gluten - DateFormat")
     .exclude("Hour")
     .exclude("Minute")
     .exclude("date add interval")

diff --git a/gluten-ut/spark33/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala
@@ -146,6 +146,8 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("to_unix_timestamp")
     // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS
     .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError")
+    // Replaced by a gluten test to pass timezone through config.
+    .exclude("DateFormat")
   enableSuite[GlutenDecimalExpressionSuite]
   enableSuite[GlutenHashExpressionsSuite]
   enableSuite[GlutenIntervalExpressionsSuite]

diff --git a/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
@@ -285,4 +285,62 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
     GenerateUnsafeProjection.generate(
       ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil)
   }
+
+  // Modified based on vanilla spark to explicitly set timezone in config.
+  test(GLUTEN_TEST + "DateFormat") {
+    val PST_OPT = Option(PST.getId)
+    val JST_OPT = Option(JST.getId)
+
+    Seq("legacy", "corrected").foreach {
+      legacyParserPolicy =>
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT),
+            null)
+          checkEvaluation(
+            DateFormatClass(
+              Cast(Literal(d), TimestampType, UTC_OPT),
+              Literal.create(null, StringType),
+              UTC_OPT),
+            null)
+
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13")
+        }
+
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5")
+        }
+
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22")
+        }
+    }
+  }
 }
diff --git a/gluten-ut/spark34/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala
@@ -153,6 +153,8 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("to_unix_timestamp")
     // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS
     .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError")
+    // Replaced by a gluten test to pass timezone through config.
+    .exclude("DateFormat")
   enableSuite[GlutenDecimalExpressionSuite]
   enableSuite[GlutenHashExpressionsSuite]
   enableSuite[GlutenIntervalExpressionsSuite]

diff --git a/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
@@ -285,4 +285,62 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
     GenerateUnsafeProjection.generate(
       ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil)
   }
+
+  // Modified based on vanilla spark to explicitly set timezone in config.
+  test(GLUTEN_TEST + "DateFormat") {
+    val PST_OPT = Option(PST.getId)
+    val JST_OPT = Option(JST.getId)
+
+    Seq("legacy", "corrected").foreach {
+      legacyParserPolicy =>
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT),
+            null)
+          checkEvaluation(
+            DateFormatClass(
+              Cast(Literal(d), TimestampType, UTC_OPT),
+              Literal.create(null, StringType),
+              UTC_OPT),
+            null)
+
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13")
+        }
+
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5")
+        }
+
+        withSQLConf(
+          SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT),
+            "2015")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013")
+          checkEvaluation(
+            DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT),
+            "0")
+          checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22")
+        }
+    }
+  }
 }