fix issue 3627 (#3630)

apache · Nov 7, 2023 · 9c2d057 · 9c2d057
1 parent 5d4cac1
commit 9c2d057
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 12 deletions.
diff --git a/backends-clickhouse/src/test/resources/csv-data/null_string.csv b/backends-clickhouse/src/test/resources/csv-data/null_string.csv
@@ -1,3 +1,4 @@
-\N,1
-,2
-1,3
+\N,1,
+,2,2
+1,,\N
+,,
diff --git a/...lickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala b/...lickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala
@@ -1117,7 +1117,8 @@ class GlutenClickHouseFileFormatSuite
       val schema = StructType.apply(
         Seq(
           StructField.apply("c1", StringType, nullable = true),
-          StructField.apply("c2", ShortType, nullable = true)
+          StructField.apply("c2", ShortType, nullable = true),
+          StructField.apply("c3", StringType, nullable = true)
         ))
 
       val options = new util.HashMap[String, String]()
@@ -1130,9 +1131,10 @@ class GlutenClickHouseFileFormatSuite
         .toDF()
 
       val dataCorrect = new util.ArrayList[Row]()
-      dataCorrect.add(Row(null, 1.toShort))
-      dataCorrect.add(Row(null, 2.toShort))
-      dataCorrect.add(Row("1", 3.toShort))
+      dataCorrect.add(Row(null, 1.toShort, null))
+      dataCorrect.add(Row(null, 2.toShort, "2"))
+      dataCorrect.add(Row("1", null, null))
+      dataCorrect.add(Row(null, null, null))
 
       var expectedAnswer: Seq[Row] = null
       withSQLConf(vanillaSparkConfs(): _*) {
@@ -1152,7 +1154,8 @@ class GlutenClickHouseFileFormatSuite
       val schema = StructType.apply(
         Seq(
           StructField.apply("c1", StringType, nullable = true),
-          StructField.apply("c2", ShortType, nullable = true)
+          StructField.apply("c2", ShortType, nullable = true),
+          StructField.apply("c3", StringType, nullable = true)
         ))
 
       val options = new util.HashMap[String, String]()
@@ -1165,9 +1168,10 @@ class GlutenClickHouseFileFormatSuite
         .toDF()
 
       val dataCorrect = new util.ArrayList[Row]()
-      dataCorrect.add(Row(null, 1.toShort))
-      dataCorrect.add(Row("", 2.toShort))
-      dataCorrect.add(Row("1", 3.toShort))
+      dataCorrect.add(Row(null, 1.toShort, ""))
+      dataCorrect.add(Row("", 2.toShort, "2"))
+      dataCorrect.add(Row("1", null, null))
+      dataCorrect.add(Row("", null, ""))
 
       var expectedAnswer: Seq[Row] = null
       withSQLConf(vanillaSparkConfs(): _*) {

diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
@@ -213,7 +213,7 @@ bool ExcelTextFormatReader::readField(
     bool is_last_file_column,
     const String &)
 {
-    if (isEndOfLine())
+    if (isEndOfLine() && format_settings.csv.empty_as_default)
     {
         column.insertDefault();
         return false;