Skip to content

Commit

Permalink
Pass the compile
Browse files Browse the repository at this point in the history
  • Loading branch information
@JkSelf
JkSelf committed Oct 25, 2023
1 parent 3fa720d commit 8f49d38
Show file tree
Hide file tree
Showing 12 changed files with 70 additions and 408 deletions.
20 changes: 6 additions & 14 deletions ...ain/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,26 +34,18 @@ import org.apache.spark.util.TaskResources

import com.google.common.base.Preconditions
import org.apache.arrow.c.ArrowSchema
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.mapreduce.TaskAttemptContext

import java.io.IOException

trait VeloxFormatWriterInjects extends GlutenFormatWriterInjectsBase {
def createOutputWriter(
filePath: String,
path: String,
dataSchema: StructType,
context: TaskAttemptContext,
nativeConf: java.util.Map[String, String]): OutputWriter = {
// Create the hdfs path if not existed.
val hdfsSchema = "hdfs://"
if (filePath.startsWith(hdfsSchema)) {
val fs = FileSystem.get(context.getConfiguration)
val hdfsPath = new Path(filePath)
if (!fs.exists(hdfsPath)) {
fs.mkdirs(hdfsPath)
}
}
val originPath = path

val arrowSchema =
SparkArrowUtil.toArrowSchema(dataSchema, SQLConf.get.sessionLocalTimeZone)
Expand All @@ -64,7 +56,7 @@ trait VeloxFormatWriterInjects extends GlutenFormatWriterInjectsBase {
try {
ArrowAbiUtil.exportSchema(allocator, arrowSchema, cSchema)
dsHandle = datasourceJniWrapper.nativeInitDatasource(
filePath,
originPath,
cSchema.memoryAddress(),
NativeMemoryManagers.contextInstance("VeloxWriter").getNativeInstanceHandle,
nativeConf)
Expand All @@ -82,7 +74,7 @@ trait VeloxFormatWriterInjects extends GlutenFormatWriterInjectsBase {
arrowSchema,
allocator,
datasourceJniWrapper,
filePath)
originPath)

new OutputWriter {
override def write(row: InternalRow): Unit = {
Expand All @@ -99,7 +91,7 @@ trait VeloxFormatWriterInjects extends GlutenFormatWriterInjectsBase {

// Do NOT add override keyword for compatibility on spark 3.1.
def path(): String = {
filePath
originPath
}
}
}
Expand Down
226 changes: 5 additions & 221 deletions ...-ut/spark34/src/test/scala/io/glutenproject/utils/clickhouse/ClickHouseTestSettings.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ class ClickHouseTestSettings extends BackendTestSettings {
false // nativeDoValidate failed due to spark conf cleanup
case "GlutenDataSourceV2FunctionSuite" =>
false // nativeDoValidate failed due to spark conf cleanup
case "GlutenDataSourceV2SQLSuite" =>
case "GlutenDataSourceV2SQLSuiteV1Filter" =>
false // nativeDoValidate failed due to spark conf cleanup
case "GlutenDataSourceV2SQLSuiteV2Filter" =>
false // nativeDoValidate failed due to spark conf cleanup
case "GlutenMetadataColumnSuite" => false // nativeDoValidate failed due to spark conf cleanup
case "GlutenQueryCompilationErrorsDSv2Suite" =>
Expand Down Expand Up @@ -443,112 +445,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenUnwrapCastInComparisonEndToEndSuite].exclude("cases when literal is max")
enableSuite[GlutenXPathFunctionsSuite]
enableSuite[QueryTestSuite]
enableSuite[GlutenAnsiCastSuiteWithAnsiModeOff]
.exclude("null cast")
.exclude("cast string to date")
.exclude("cast string to timestamp")
.exclude("cast from boolean")
.exclude("cast from int")
.exclude("cast from long")
.exclude("cast from float")
.exclude("cast from double")
.exclude("cast from timestamp")
.exclude("data type casting")
.exclude("cast and add")
.exclude("from decimal")
.exclude("cast from array")
.exclude("cast from map")
.exclude("cast from struct")
.exclude("cast struct with a timestamp field")
.exclude("cast between string and interval")
.exclude("cast string to boolean")
.exclude("SPARK-20302 cast with same structure")
.exclude("SPARK-22500: cast for struct should not generate codes beyond 64KB")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
.exclude("SPARK-22825 Cast array to string")
.exclude("SPARK-33291: Cast array with null elements to string")
.exclude("SPARK-22973 Cast map to string")
.exclude("SPARK-22981 Cast struct to string")
.exclude("SPARK-33291: Cast struct with null elements to string")
.exclude("SPARK-34667: cast year-month interval to string")
.exclude("SPARK-34668: cast day-time interval to string")
.exclude("SPARK-35698: cast timestamp without time zone to string")
.exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone")
.exclude("SPARK-35716: cast timestamp without time zone to date type")
.exclude("SPARK-35718: cast date type to timestamp without timezone")
.exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone")
.exclude("SPARK-35720: cast string to timestamp without timezone")
.exclude("SPARK-35112: Cast string to day-time interval")
.exclude("SPARK-35111: Cast string to year-month interval")
.exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields")
.exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields")
.exclude("SPARK-35768: Take into account year-month interval fields in cast")
.exclude("SPARK-35735: Take into account day-time interval fields in cast")
.exclude("ANSI mode: Throw exception on casting out-of-range value to byte type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to short type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to int type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to long type")
.exclude("Fast fail for cast string type to decimal type in ansi mode")
.exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z")
.exclude("cast from array III")
.exclude("cast from map II")
.exclude("cast from map III")
.exclude("cast from struct II")
.exclude("cast from struct III")
enableSuite[GlutenAnsiCastSuiteWithAnsiModeOn]
.exclude("null cast")
.exclude("cast string to date")
.exclude("cast string to timestamp")
.exclude("cast from boolean")
.exclude("cast from int")
.exclude("cast from long")
.exclude("cast from float")
.exclude("cast from double")
.exclude("cast from timestamp")
.exclude("data type casting")
.exclude("cast and add")
.exclude("from decimal")
.exclude("cast from array")
.exclude("cast from map")
.exclude("cast from struct")
.exclude("cast struct with a timestamp field")
.exclude("cast between string and interval")
.exclude("cast string to boolean")
.exclude("SPARK-20302 cast with same structure")
.exclude("SPARK-22500: cast for struct should not generate codes beyond 64KB")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
.exclude("SPARK-22825 Cast array to string")
.exclude("SPARK-33291: Cast array with null elements to string")
.exclude("SPARK-22973 Cast map to string")
.exclude("SPARK-22981 Cast struct to string")
.exclude("SPARK-33291: Cast struct with null elements to string")
.exclude("SPARK-34667: cast year-month interval to string")
.exclude("SPARK-34668: cast day-time interval to string")
.exclude("SPARK-35698: cast timestamp without time zone to string")
.exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone")
.exclude("SPARK-35716: cast timestamp without time zone to date type")
.exclude("SPARK-35718: cast date type to timestamp without timezone")
.exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone")
.exclude("SPARK-35720: cast string to timestamp without timezone")
.exclude("SPARK-35112: Cast string to day-time interval")
.exclude("SPARK-35111: Cast string to year-month interval")
.exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields")
.exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields")
.exclude("SPARK-35768: Take into account year-month interval fields in cast")
.exclude("SPARK-35735: Take into account day-time interval fields in cast")
.exclude("ANSI mode: Throw exception on casting out-of-range value to byte type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to short type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to int type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to long type")
.exclude("Fast fail for cast string type to decimal type in ansi mode")
.exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z")
.exclude("cast from array III")
.exclude("cast from map II")
.exclude("cast from map III")
.exclude("cast from struct II")
.exclude("cast from struct III")
enableSuite[GlutenArithmeticExpressionSuite]
.exclude("- (UnaryMinus)")
.exclude("/ (Divide) basic")
Expand Down Expand Up @@ -613,59 +509,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SPARK-36924: Cast IntegralType to DayTimeIntervalType")
.exclude("SPARK-36924: Cast YearMonthIntervalType to IntegralType")
.exclude("SPARK-36924: Cast IntegralType to YearMonthIntervalType")
enableSuite[GlutenCastSuiteWithAnsiModeOn]
.exclude("null cast")
.exclude("cast string to date")
.exclude("cast string to timestamp")
.exclude("cast from boolean")
.exclude("cast from int")
.exclude("cast from long")
.exclude("cast from float")
.exclude("cast from double")
.exclude("cast from timestamp")
.exclude("data type casting")
.exclude("cast and add")
.exclude("from decimal")
.exclude("cast from array")
.exclude("cast from map")
.exclude("cast from struct")
.exclude("cast struct with a timestamp field")
.exclude("cast between string and interval")
.exclude("cast string to boolean")
.exclude("SPARK-20302 cast with same structure")
.exclude("SPARK-22500: cast for struct should not generate codes beyond 64KB")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
.exclude("SPARK-22825 Cast array to string")
.exclude("SPARK-33291: Cast array with null elements to string")
.exclude("SPARK-22973 Cast map to string")
.exclude("SPARK-22981 Cast struct to string")
.exclude("SPARK-33291: Cast struct with null elements to string")
.exclude("SPARK-34667: cast year-month interval to string")
.exclude("SPARK-34668: cast day-time interval to string")
.exclude("SPARK-35698: cast timestamp without time zone to string")
.exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone")
.exclude("SPARK-35716: cast timestamp without time zone to date type")
.exclude("SPARK-35718: cast date type to timestamp without timezone")
.exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone")
.exclude("SPARK-35720: cast string to timestamp without timezone")
.exclude("SPARK-35112: Cast string to day-time interval")
.exclude("SPARK-35111: Cast string to year-month interval")
.exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields")
.exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields")
.exclude("SPARK-35768: Take into account year-month interval fields in cast")
.exclude("SPARK-35735: Take into account day-time interval fields in cast")
.exclude("ANSI mode: Throw exception on casting out-of-range value to byte type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to short type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to int type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to long type")
.exclude("Fast fail for cast string type to decimal type in ansi mode")
.exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z")
.exclude("cast from array III")
.exclude("cast from map II")
.exclude("cast from map III")
.exclude("cast from struct II")
.exclude("cast from struct III")
enableSuite[GlutenCollectionExpressionsSuite]
.exclude("Array and Map Size")
.exclude("MapEntries")
Expand Down Expand Up @@ -910,69 +753,10 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("ParseUrl")
.exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url")
.exclude("Sentences")
enableSuite[GlutenTryCastSuite]
.exclude("null cast")
.exclude("cast string to date")
.exclude("cast string to timestamp")
.exclude("cast from boolean")
.exclude("cast from int")
.exclude("cast from long")
.exclude("cast from float")
.exclude("cast from double")
.exclude("cast from timestamp")
.exclude("data type casting")
.exclude("cast and add")
.exclude("from decimal")
.exclude("cast from array")
.exclude("cast from map")
.exclude("cast from struct")
.exclude("cast struct with a timestamp field")
.exclude("cast between string and interval")
.exclude("cast string to boolean")
.exclude("SPARK-20302 cast with same structure")
.exclude("SPARK-22500: cast for struct should not generate codes beyond 64KB")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
.exclude("SPARK-22825 Cast array to string")
.exclude("SPARK-33291: Cast array with null elements to string")
.exclude("SPARK-22973 Cast map to string")
.exclude("SPARK-22981 Cast struct to string")
.exclude("SPARK-33291: Cast struct with null elements to string")
.exclude("SPARK-34667: cast year-month interval to string")
.exclude("SPARK-34668: cast day-time interval to string")
.exclude("SPARK-35698: cast timestamp without time zone to string")
.exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone")
.exclude("SPARK-35716: cast timestamp without time zone to date type")
.exclude("SPARK-35718: cast date type to timestamp without timezone")
.exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone")
.exclude("SPARK-35720: cast string to timestamp without timezone")
.exclude("SPARK-35112: Cast string to day-time interval")
.exclude("SPARK-35111: Cast string to year-month interval")
.exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields")
.exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields")
.exclude("SPARK-35768: Take into account year-month interval fields in cast")
.exclude("SPARK-35735: Take into account day-time interval fields in cast")
.exclude("ANSI mode: Throw exception on casting out-of-range value to byte type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to short type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to int type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to long type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to decimal type")
.exclude("cast from invalid string to numeric should throw NumberFormatException")
.exclude("Fast fail for cast string type to decimal type in ansi mode")
.exclude("ANSI mode: cast string to boolean with parse error")
.exclude("cast from timestamp II")
.exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z II")
.exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z")
.exclude("cast from map II")
.exclude("cast from struct II")
.exclude("ANSI mode: cast string to timestamp with parse error")
.exclude("ANSI mode: cast string to date with parse error")
.exclude("SPARK-26218: Fix the corner case of codegen when casting float to Integer")
.exclude("SPARK-35720: cast invalid string input to timestamp without time zone")
.exclude("Gluten - SPARK-35698: cast timestamp without time zone to string")
enableSuite[GlutenDataSourceV2DataFrameSessionCatalogSuite]
enableSuite[GlutenDataSourceV2SQLSessionCatalogSuite]
enableSuite[GlutenDataSourceV2SQLSuite]
enableSuite[GlutenDataSourceV2SQLSuiteV1Filter]
enableSuite[GlutenDataSourceV2SQLSuiteV2Filter]
enableSuite[GlutenDataSourceV2Suite]
.exclude("partitioning reporting")
.exclude("SPARK-33267: push down with condition 'in (..., null)' should not throw NPE")
Expand Down
Loading

0 comments on commit 8f49d38

Please sign in to comment.