From f9d3be47771e2d2eb62e82f400a55ee7a201ffe6 Mon Sep 17 00:00:00 2001 From: xmy Date: Thu, 9 Nov 2023 16:44:31 +0800 Subject: [PATCH] [CH] enable replaceSortAggWithHashAgg --- .../execution/GlutenClickHouseFileFormatSuite.scala | 5 ++--- .../GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala | 2 +- .../io/glutenproject/backendsapi/velox/VeloxBackend.scala | 2 -- .../io/glutenproject/backendsapi/BackendSettingsApi.scala | 2 +- .../src/main/scala/io/glutenproject/utils/FallbackUtil.scala | 4 ++-- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala index 40752eb9c2e5..5ded993f9fc4 100644 --- a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala +++ b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala @@ -196,7 +196,7 @@ class GlutenClickHouseFileFormatSuite ) } - test("read data from csv file format witsh agg") { + test("read data from csv file format with agg") { val filePath = basePath + "/csv_test_agg.csv" val csvFileFormat = "csv" val sql = @@ -214,8 +214,7 @@ class GlutenClickHouseFileFormatSuite case f: FileSourceScanExecTransformer => f } assert(csvFileScan.size == 1) - }, - noFallBack = false + } ) } diff --git a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala index 51b685f0fdf7..60271cbc09cc 100644 --- a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala +++ b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala @@ -323,7 +323,7 @@ class GlutenClickHouseTPCHColumnarShuffleParquetAQESuite | l_returnflag, | l_linestatus |""".stripMargin - runQueryAndCompare(sql, noFallBack = false) { df => } + runQueryAndCompare(sql) { df => } } } } diff --git a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala index beee871bc0ef..693716f83365 100644 --- a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala @@ -278,8 +278,6 @@ object BackendSettings extends BackendSettingsApi { override def removeHashColumnFromColumnarShuffleExchangeExec(): Boolean = true override def rescaleDecimalLiteral(): Boolean = true - override def replaceSortAggWithHashAgg: Boolean = GlutenConfig.getConf.forceToUseHashAgg - /** Get the config prefix for each backend */ override def getBackendConfigPrefix(): String = GlutenConfig.GLUTEN_CONFIG_PREFIX + VeloxBackend.BACKEND_NAME diff --git a/gluten-core/src/main/scala/io/glutenproject/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/io/glutenproject/backendsapi/BackendSettingsApi.scala index feb09e6c0fc0..64e7b9396a45 100644 --- a/gluten-core/src/main/scala/io/glutenproject/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/io/glutenproject/backendsapi/BackendSettingsApi.scala @@ -82,7 +82,7 @@ trait BackendSettingsApi { * Whether to replace sort agg with hash agg., e.g., sort agg will be used in spark's planning for * string type input. */ - def replaceSortAggWithHashAgg: Boolean = false + def replaceSortAggWithHashAgg: Boolean = GlutenConfig.getConf.forceToUseHashAgg /** Get the config prefix for each backend */ def getBackendConfigPrefix: String diff --git a/gluten-core/src/main/scala/io/glutenproject/utils/FallbackUtil.scala b/gluten-core/src/main/scala/io/glutenproject/utils/FallbackUtil.scala index 40c0786599f2..a98dc7ffdb4f 100644 --- a/gluten-core/src/main/scala/io/glutenproject/utils/FallbackUtil.scala +++ b/gluten-core/src/main/scala/io/glutenproject/utils/FallbackUtil.scala @@ -78,9 +78,9 @@ object FallbackUtil extends Logging with AdaptiveSparkPlanHelper { } } - if (!fallbackOperator.isEmpty) { + if (fallbackOperator.nonEmpty) { fallbackOperator.foreach(operator => log.info(s"gluten fallback operator:{$operator}")) } - return fallbackOperator.nonEmpty + fallbackOperator.nonEmpty } }