[fix](stream_load)streamload with double quotes #27942 (#39634)

apache · Sep 23, 2024 · d3f9f9b · d3f9f9b
1 parent 61aa9cb
commit d3f9f9b
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 25 deletions.
diff --git a/be/src/util/slice.h b/be/src/util/slice.h
@@ -168,8 +168,8 @@ struct Slice {
     bool trim_quote() {
         int32_t begin = 0;
         bool change = false;
-        if (size > 2 && ((data[begin] == '"' && data[size - 1] == '"') ||
-                         (data[begin] == '\'' && data[size - 1] == '\''))) {
+        if (size >= 2 && ((data[begin] == '"' && data[size - 1] == '"') ||
+                          (data[begin] == '\'' && data[size - 1] == '\''))) {
             data += 1;
             size -= 2;
             change = true;

diff --git a/be/test/vec/data_types/serde/data_type_serde_text_test.cpp b/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
@@ -486,8 +486,8 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) {
                          "{\"hello "
                          "world\":0.2222222,\"hello2\":null,null:1111.1,\"NULL\":null,\"null\":"
                          "null,\"null\":0.1}"},
-                        {"{ ,.amory:111.2343, \"\":112, dggs:13.14, NULL:12.2222222, :NULL}",
-                         "{\"\":NULL, NULL:12.44}", "{}", "{}", "NULL", "{}", "NULL",
+                        {"{ ,.amory:111.2343, :112, dggs:13.14, NULL:12.2222222, :NULL}",
+                         "{:NULL, NULL:12.44}", "{}", "{}", "NULL", "{}", "NULL",
                          "{NULL:NULL, null:NULL}",
                          "{hello world:0.2222222, hello2:NULL, NULL:1111.1, NULL:NULL, null:NULL, "
                          "null:0.1}"}),

diff --git a/regression-test/data/load_p0/stream_load/csv_with_double_quotes.csv b/regression-test/data/load_p0/stream_load/csv_with_double_quotes.csv
@@ -6,3 +6,4 @@
 "6","7","3","abc","2022-12-01","2022-12-01:09:30:31"
 "7","8","3","abc","2022-12-01","2022-12-01:09:30:31"
 "8","9","3","abc","2022-12-01","2022-12-01:09:30:31"
+"2","","","",2023-07-15,"2023-07-20:05:48:31"
diff --git a/regression-test/data/load_p0/stream_load/test_csv_with_double_quotes.out b/regression-test/data/load_p0/stream_load/test_csv_with_double_quotes.out
@@ -1,5 +1,6 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
+\N	\N	\N	""	2023-07-15	\N
 \N	\N	\N	"abc"	\N	\N
 \N	\N	\N	"abc"	\N	\N
 \N	\N	\N	"abc"	\N	\N
@@ -11,6 +12,7 @@
 
 -- !sql --
 1	2	3	abc	2022-12-01	2022-12-01T09:30:31
+2	\N	\N		2023-07-15	2023-07-20T05:48:31
 2	3	3	abc	2022-12-01	2022-12-01T09:30:31
 3	4	3	abc	2022-12-01	2022-12-01T09:30:31
 4	5	3	abc	2022-12-01	2022-12-01T09:30:31