-
Notifications
You must be signed in to change notification settings - Fork 435
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[GLUTEN-6470][CH]Fix Task not serializable error when inserting merge…
…tree data (#6473) * [GLUTEN-6470][CH]Fix Task not serializable error when inserting mergetree data When inserting mergetree data, it occurs the Task not serializable error in some cases. RC: In the Delta, the options of the `DeltaOptions` is a `CaseInsensitiveMap`, if calling the api `filterKeys()` of the `CaseInsensitiveMap`, it may become not serializable, so lead to this error. Close #6470. * fix ut * add ut
- Loading branch information
Showing
7 changed files
with
143 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
120 changes: 120 additions & 0 deletions
120
.../org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.gluten.execution | ||
|
|
||
| import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings | ||
|
|
||
| import org.apache.spark.SparkConf | ||
| import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper | ||
|
|
||
| // Some sqls' line length exceeds 100 | ||
| // scalastyle:off line.size.limit | ||
|
|
||
| class GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite | ||
| extends GlutenClickHouseTPCHAbstractSuite | ||
| with AdaptiveSparkPlanHelper { | ||
|
|
||
| override protected val needCopyParquetToTablePath = true | ||
|
|
||
| override protected val tablesPath: String = basePath + "/tpch-data" | ||
| override protected val tpchQueries: String = rootPath + "queries/tpch-queries-ch" | ||
| override protected val queriesResults: String = rootPath + "mergetree-queries-output" | ||
|
|
||
| /** Run Gluten + ClickHouse Backend with SortShuffleManager */ | ||
| override protected def sparkConf: SparkConf = { | ||
| super.sparkConf | ||
| .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") | ||
| .set("spark.io.compression.codec", "LZ4") | ||
| .set("spark.sql.shuffle.partitions", "5") | ||
| .set("spark.sql.autoBroadcastJoinThreshold", "10MB") | ||
| .set("spark.sql.adaptive.enabled", "true") | ||
| .set("spark.sql.files.maxPartitionBytes", "20000000") | ||
| .set("spark.memory.offHeap.size", "4G") | ||
| } | ||
|
|
||
| override protected def createTPCHNotNullTables(): Unit = { | ||
| createNotNullTPCHTablesInParquet(tablesPath) | ||
| } | ||
|
|
||
| test("GLUTEN-6470: Fix Task not serializable error when inserting mergetree data") { | ||
|
|
||
| val externalSortKey = s"${CHBackendSettings.getBackendConfigPrefix}.runtime_settings" + | ||
| s".max_bytes_before_external_sort" | ||
| assertResult(3435973836L)(spark.conf.get(externalSortKey).toLong) | ||
|
|
||
| spark.sql(s""" | ||
| |DROP TABLE IF EXISTS lineitem_task_not_serializable; | ||
| |""".stripMargin) | ||
|
|
||
| spark.sql(s""" | ||
| |CREATE TABLE IF NOT EXISTS lineitem_task_not_serializable | ||
| |( | ||
| | l_orderkey bigint, | ||
| | l_partkey bigint, | ||
| | l_suppkey bigint, | ||
| | l_linenumber bigint, | ||
| | l_quantity double, | ||
| | l_extendedprice double, | ||
| | l_discount double, | ||
| | l_tax double, | ||
| | l_returnflag string, | ||
| | l_linestatus string, | ||
| | l_shipdate date, | ||
| | l_commitdate date, | ||
| | l_receiptdate date, | ||
| | l_shipinstruct string, | ||
| | l_shipmode string, | ||
| | l_comment string | ||
| |) | ||
| |USING clickhouse | ||
| |LOCATION '$basePath/lineitem_task_not_serializable' | ||
| |""".stripMargin) | ||
|
|
||
| spark.sql(s""" | ||
| | insert into table lineitem_task_not_serializable | ||
| | select * from lineitem | ||
| |""".stripMargin) | ||
|
|
||
| val sqlStr = | ||
| s""" | ||
| |SELECT | ||
| | l_returnflag, | ||
| | l_linestatus, | ||
| | sum(l_quantity) AS sum_qty, | ||
| | sum(l_extendedprice) AS sum_base_price, | ||
| | sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, | ||
| | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, | ||
| | avg(l_quantity) AS avg_qty, | ||
| | avg(l_extendedprice) AS avg_price, | ||
| | avg(l_discount) AS avg_disc, | ||
| | count(*) AS count_order | ||
| |FROM | ||
| | lineitem_task_not_serializable | ||
| |WHERE | ||
| | l_shipdate <= date'1998-09-02' - interval 1 day | ||
| |GROUP BY | ||
| | l_returnflag, | ||
| | l_linestatus | ||
| |ORDER BY | ||
| | l_returnflag, | ||
| | l_linestatus; | ||
| | | ||
| |""".stripMargin | ||
| runTPCHQueryBySQL(1, sqlStr)(_ => {}) | ||
| } | ||
| } | ||
| // scalastyle:off line.size.limit |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters