Skip to content

Commit

Permalink
replace for spark33
Browse files Browse the repository at this point in the history
  • Loading branch information
@marin-ma
marin-ma committed Nov 21, 2023
1 parent e453be9 commit 69ffdff
Show file tree
Hide file tree
Showing 9 changed files with 2,241 additions and 233 deletions.
6 changes: 4 additions & 2 deletions backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHTestApi.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ package io.glutenproject.backendsapi.clickhouse
import io.glutenproject.backendsapi.TestApi

class CHTestApi extends TestApi {
override def getSupportedSQLQueryTests: Set[String] = Set[String]()
override def getSupportedSQLQueryTests(sparkVersion: TestApi.SparkVersion): Set[String] =
Set[String]()

override def getOverwriteSQLQueryTests: Set[String] = Set[String]()
override def getOverwriteSQLQueryTests(sparkVersion: TestApi.SparkVersion): Set[String] =
Set[String]()
}
212 changes: 207 additions & 5 deletions backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/TestApiImpl.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,33 @@
package io.glutenproject.backendsapi.velox

import io.glutenproject.backendsapi.TestApi
import io.glutenproject.backendsapi.TestApi.{SparkVersion, SparkVersion32, SparkVersion33}

class TestApiImpl extends TestApi {

/**
* List of supported cases to run with Velox backend, in lower case. Please add to the supported
* list after enabling a sql test.
*/
override def getSupportedSQLQueryTests: Set[String] = {
override def getSupportedSQLQueryTests(sparkVersion: SparkVersion): Set[String] = {
sparkVersion match {
case SparkVersion32() =>
SUPPORTED_SQL_QUERY_LIST_SPARK32
case SparkVersion33() =>
SUPPORTED_SQL_QUERY_LIST_SPARK33
}
}

override def getOverwriteSQLQueryTests(sparkVersion: SparkVersion): Set[String] = {
sparkVersion match {
case SparkVersion32() =>
OVERWRITE_SQL_QUERY_LIST_SPARK32
case SparkVersion33() =>
OVERWRITE_SQL_QUERY_LIST_SPARK33
}
}

private val SUPPORTED_SQL_QUERY_LIST_SPARK32: Set[String] =
Set(
"bitwise.sql",
"cast.sql",
Expand Down Expand Up @@ -201,13 +220,196 @@ class TestApiImpl extends TestApi {
"udf/udf-union.sql",
"udf/udf-window.sql"
)
}

override def getOverwriteSQLQueryTests: Set[String] = Set[String](
private val OVERWRITE_SQL_QUERY_LIST_SPARK32: Set[String] = Set(
// Velox corr has better computation logic but it fails Spark's precision check.
// Remove -- SPARK-24369 multiple distinct aggregations having the same argument set
"group-by.sql",
// Remove -- SPARK-24369 multiple distinct aggregations having the same argument set
"udf/udf-group-by.sql"
)

private val SUPPORTED_SQL_QUERY_LIST_SPARK33: Set[String] = Set(
"bitwise.sql",
"cast.sql",
"change-column.sql",
"charvarchar.sql",
"columnresolution-negative.sql",
"columnresolution-views.sql",
"columnresolution.sql",
"comments.sql",
"comparator.sql",
"count.sql",
"cross-join.sql",
"csv-functions.sql",
"cte-legacy.sql",
"cte-nested.sql",
"cte-nonlegacy.sql",
"cte.sql",
"current_database_catalog.sql",
"date.sql",
"datetime-formatting-invalid.sql",
// Velox had different handling for some illegal cases.
// "datetime-formatting-legacy.sql",
// "datetime-formatting.sql",
"datetime-legacy.sql",
"datetime-parsing-invalid.sql",
"datetime-parsing-legacy.sql",
"datetime-parsing.sql",
"datetime-special.sql",
"decimalArithmeticOperations.sql",
"describe-part-after-analyze.sql",
"describe-query.sql",
"describe-table-after-alter-table.sql",
"describe-table-column.sql",
"describe.sql",
"except-all.sql",
"except.sql",
"extract.sql",
"group-by-filter.sql",
"group-by-ordinal.sql",
"grouping_set.sql",
"having.sql",
"ignored.sql",
"inline-table.sql",
"inner-join.sql",
"intersect-all.sql",
"interval.sql",
"join-empty-relation.sql",
"join-lateral.sql",
"json-functions.sql",
"like-all.sql",
"like-any.sql",
"limit.sql",
"literals.sql",
"map.sql",
"misc-functions.sql",
"natural-join.sql",
"null-handling.sql",
"null-propagation.sql",
"operators.sql",
"order-by-nulls-ordering.sql",
"order-by-ordinal.sql",
"outer-join.sql",
"parse-schema-string.sql",
"pivot.sql",
"pred-pushdown.sql",
"predicate-functions.sql",
"query_regex_column.sql",
"random.sql",
"regexp-functions.sql",
"show-create-table.sql",
"show-tables.sql",
"show-tblproperties.sql",
"show-views.sql",
"show_columns.sql",
"sql-compatibility-functions.sql",
"string-functions.sql",
"struct.sql",
"subexp-elimination.sql",
"table-aliases.sql",
"table-valued-functions.sql",
"tablesample-negative.sql",
"subquery/exists-subquery/exists-aggregate.sql",
"subquery/exists-subquery/exists-basic.sql",
"subquery/exists-subquery/exists-cte.sql",
"subquery/exists-subquery/exists-having.sql",
"subquery/exists-subquery/exists-joins-and-set-ops.sql",
"subquery/exists-subquery/exists-orderby-limit.sql",
"subquery/exists-subquery/exists-within-and-or.sql",
"subquery/in-subquery/in-basic.sql",
"subquery/in-subquery/in-group-by.sql",
"subquery/in-subquery/in-having.sql",
"subquery/in-subquery/in-joins.sql",
"subquery/in-subquery/in-limit.sql",
"subquery/in-subquery/in-multiple-columns.sql",
"subquery/in-subquery/in-order-by.sql",
"subquery/in-subquery/in-set-operations.sql",
"subquery/in-subquery/in-with-cte.sql",
"subquery/in-subquery/nested-not-in.sql",
"subquery/in-subquery/not-in-group-by.sql",
"subquery/in-subquery/not-in-joins.sql",
"subquery/in-subquery/not-in-unit-tests-multi-column.sql",
"subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql",
"subquery/in-subquery/not-in-unit-tests-single-column.sql",
"subquery/in-subquery/not-in-unit-tests-single-column-literal.sql",
"subquery/in-subquery/simple-in.sql",
"subquery/negative-cases/invalid-correlation.sql",
"subquery/negative-cases/subq-input-typecheck.sql",
"subquery/scalar-subquery/scalar-subquery-predicate.sql",
"subquery/scalar-subquery/scalar-subquery-select.sql",
"subquery/subquery-in-from.sql",
"postgreSQL/aggregates_part1.sql",
"postgreSQL/aggregates_part2.sql",
"postgreSQL/aggregates_part3.sql",
"postgreSQL/aggregates_part4.sql",
"postgreSQL/boolean.sql",
"postgreSQL/case.sql",
"postgreSQL/comments.sql",
"postgreSQL/create_view.sql",
"postgreSQL/date.sql",
"postgreSQL/float4.sql",
"postgreSQL/insert.sql",
"postgreSQL/int2.sql",
"postgreSQL/int4.sql",
"postgreSQL/int8.sql",
"postgreSQL/interval.sql",
"postgreSQL/join.sql",
"postgreSQL/limit.sql",
"postgreSQL/numeric.sql",
"postgreSQL/select.sql",
"postgreSQL/select_distinct.sql",
"postgreSQL/select_having.sql",
"postgreSQL/select_implicit.sql",
"postgreSQL/strings.sql",
"postgreSQL/text.sql",
"postgreSQL/timestamp.sql",
"postgreSQL/union.sql",
"postgreSQL/window_part1.sql",
"postgreSQL/window_part2.sql",
"postgreSQL/window_part3.sql",
"postgreSQL/window_part4.sql",
"postgreSQL/with.sql",
"datetime-special.sql",
"timestamp-ansi.sql",
"timestamp.sql",
"arrayJoin.sql",
"binaryComparison.sql",
"booleanEquality.sql",
"caseWhenCoercion.sql",
"concat.sql",
"dateTimeOperations.sql",
"decimalPrecision.sql",
"division.sql",
"elt.sql",
"ifCoercion.sql",
"implicitTypeCasts.sql",
"inConversion.sql",
"mapZipWith.sql",
"mapconcat.sql",
"promoteStrings.sql",
"stringCastAndExpressions.sql",
"widenSetOperationTypes.sql",
"windowFrameCoercion.sql",
"timestamp-ltz.sql",
"timestamp-ntz.sql",
"timezone.sql",
"transform.sql",
"try_arithmetic.sql",
"try_cast.sql",
"udaf.sql",
"union.sql",
"using-join.sql",
"window.sql",
"udf-union.sql",
"udf-window.sql"
)

private val OVERWRITE_SQL_QUERY_LIST_SPARK33: Set[String] = Set(
// Velox corr has better computation logic but it fails Spark's precision check.
// Remove corr in group-by.sql
// Remove -- SPARK-24369 multiple distinct aggregations having the same argument set
"group-by.sql",
// Remove corr in udf/udf-group-by.sql
// Remove -- SPARK-24369 multiple distinct aggregations having the same argument set
"udf/udf-group-by.sql"
)
}
10 changes: 8 additions & 2 deletions gluten-core/src/main/scala/io/glutenproject/backendsapi/TestApi.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,13 @@
package io.glutenproject.backendsapi

trait TestApi {
def getSupportedSQLQueryTests: Set[String]
def getSupportedSQLQueryTests(sparkVersion: TestApi.SparkVersion): Set[String]

def getOverwriteSQLQueryTests: Set[String]
def getOverwriteSQLQueryTests(sparkVersion: TestApi.SparkVersion): Set[String]
}

object TestApi {
trait SparkVersion
case class SparkVersion32() extends SparkVersion
case class SparkVersion33() extends SparkVersion
}
7 changes: 3 additions & 4 deletions gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package org.apache.spark.sql

import io.glutenproject.GlutenConfig
import io.glutenproject.backendsapi.BackendsApiManager
import io.glutenproject.backendsapi.{BackendsApiManager, TestApi}
import io.glutenproject.utils.{BackendTestUtils, SystemParameters}

import org.apache.spark.SparkConf
Expand Down Expand Up @@ -158,7 +158,6 @@ class GlutenSQLQueryTestSuite

protected val overwriteResourcePath =
getClass.getResource("/").getPath + "../../../src/test/resources/sql-tests"

protected val overwriteInputFilePath = new File(overwriteResourcePath, "inputs").getAbsolutePath
protected val overwriteGoldenFilePath = new File(overwriteResourcePath, "results").getAbsolutePath

Expand Down Expand Up @@ -225,8 +224,8 @@ class GlutenSQLQueryTestSuite

// List of supported cases to run with a certain backend, in lower case.
private val supportedList: Set[String] =
BackendsApiManager.getTestApiInstance.getSupportedSQLQueryTests ++
BackendsApiManager.getTestApiInstance.getOverwriteSQLQueryTests
BackendsApiManager.getTestApiInstance.getSupportedSQLQueryTests(TestApi.SparkVersion32()) ++
BackendsApiManager.getTestApiInstance.getOverwriteSQLQueryTests(TestApi.SparkVersion32())
// Create all the test cases.
listTestCases.foreach(createScalaTestCase)

Expand Down
Loading

0 comments on commit 69ffdff

Please sign in to comment.