[fix](OrcWriter) fix be core when upgrading BE without upgrading FE (a…

…pache#40303) bp: apache#40282
eldenmoon · Sep 4, 2024 · b4beec8 · b4beec8
1 parent db06c67
commit b4beec8
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 0 deletions.
diff --git a/be/src/vec/sink/vresult_sink.h b/be/src/vec/sink/vresult_sink.h
@@ -73,6 +73,7 @@ struct ResultFileOptions {
     std::string file_suffix;
     //Bring BOM when exporting to CSV format
     bool with_bom = false;
+    int64_t orc_writer_version = 0;
 
     ResultFileOptions(const TResultFileSinkOptions& t_opt) {
         file_path = t_opt.file_path;
@@ -123,6 +124,9 @@ struct ResultFileOptions {
         if (t_opt.__isset.orc_compression_type) {
             orc_compression_type = t_opt.orc_compression_type;
         }
+        if (t_opt.__isset.orc_writer_version) {
+            orc_writer_version = t_opt.orc_writer_version;
+        }
     }
 };
 

diff --git a/be/src/vec/sink/writer/vfile_result_writer.cpp b/be/src/vec/sink/writer/vfile_result_writer.cpp
@@ -81,6 +81,11 @@ VFileResultWriter::VFileResultWriter(const ResultFileOptions* file_opts,
 Status VFileResultWriter::open(RuntimeState* state, RuntimeProfile* profile) {
     _state = state;
     _init_profile(profile);
+    // check orc writer version
+    if (_file_opts->file_format == TFileFormatType::FORMAT_ORC &&
+        _file_opts->orc_writer_version < 1) {
+        return Status::InternalError("orc writer version is less than 1.");
+    }
     // Delete existing files
     if (_file_opts->delete_existing_files) {
         RETURN_IF_ERROR(_delete_dir());

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -878,6 +878,7 @@ public TResultFileSinkOptions toSinkOptions() {
         if (isOrcFormat()) {
             sinkOptions.setOrcSchema(serializeOrcSchema());
             sinkOptions.setOrcCompressionType(orcCompressionType);
+            sinkOptions.setOrcWriterVersion(1);
         }
         return sinkOptions;
     }

diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift
@@ -133,6 +133,13 @@ struct TResultFileSinkOptions {
     18: optional bool with_bom;
 
     19: optional PlanNodes.TFileCompressType orc_compression_type;
+
+    // Since we have changed the type mapping from Doris to Orc type,
+    // using the Outfile to export Date/Datetime types will cause BE core dump
+    // when only upgrading BE without upgrading FE.
+    // orc_writer_version = 1 means doris FE is higher than version 2.1.5
+    // orc_writer_version = 0 means doris FE is less than or equal to version 2.1.5
+    20: optional i64 orc_writer_version;
 }
 
 struct TMemoryScratchSink {