From 2b8e7a43f972704a4d8ba53a71abaf0d71302522 Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Tue, 28 Nov 2023 10:43:42 +0800 Subject: [PATCH] Remove SupportFormat --- .../execution/BaseDataSource.scala | 4 +-- .../execution/BasicScanExecTransformer.scala | 7 +++-- .../substrait/SupportFormat.scala | 27 ------------------- 3 files changed, 6 insertions(+), 32 deletions(-) delete mode 100644 gluten-core/src/main/scala/io/glutenproject/substrait/SupportFormat.scala diff --git a/gluten-core/src/main/scala/io/glutenproject/execution/BaseDataSource.scala b/gluten-core/src/main/scala/io/glutenproject/execution/BaseDataSource.scala index 10dc23dd0ff1..bd5f4e2940eb 100644 --- a/gluten-core/src/main/scala/io/glutenproject/execution/BaseDataSource.scala +++ b/gluten-core/src/main/scala/io/glutenproject/execution/BaseDataSource.scala @@ -16,12 +16,10 @@ */ package io.glutenproject.execution -import io.glutenproject.substrait.SupportFormat - import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.types.StructType -trait BaseDataSource extends SupportFormat { +trait BaseDataSource { /** Returns the actual schema of this data source scan. */ def getDataSchema: StructType diff --git a/gluten-core/src/main/scala/io/glutenproject/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/io/glutenproject/execution/BasicScanExecTransformer.scala index 7e2d23d09506..bd1ea255f82d 100644 --- a/gluten-core/src/main/scala/io/glutenproject/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/io/glutenproject/execution/BasicScanExecTransformer.scala @@ -27,11 +27,11 @@ import io.glutenproject.substrait.rel.{ReadRelNode, RelBuilder, SplitInfo} import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Expression} import org.apache.spark.sql.vectorized.ColumnarBatch - import com.google.common.collect.Lists - import scala.collection.JavaConverters._ +import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat + trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource { /** Returns the filters that can be pushed down to native file scan */ @@ -39,6 +39,9 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource def outputAttributes(): Seq[Attribute] + /** This can be used to report FileFormat for a file based scan operator. */ + val fileFormat: ReadFileFormat + // TODO: Remove this expensive call when CH support scan custom partition location. def getInputFilePaths: Seq[String] = { // This is a heavy operation, and only the required backend executes the corresponding logic. diff --git a/gluten-core/src/main/scala/io/glutenproject/substrait/SupportFormat.scala b/gluten-core/src/main/scala/io/glutenproject/substrait/SupportFormat.scala deleted file mode 100644 index d0ea8648d494..000000000000 --- a/gluten-core/src/main/scala/io/glutenproject/substrait/SupportFormat.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.glutenproject.substrait - -import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat - -/** - * A mix-in interface for BasicScanExecTransformer. This can be used to report FileFormat for a file - * based scan operator. - */ -trait SupportFormat { - @transient val fileFormat: ReadFileFormat -}