Merge branch 'main' into VLScala-2.13

apache · Aug 6, 2024 · 929bcc7 · 929bcc7
2 parents 958d71e + 43d0ff9
commit 929bcc7
Show file tree

Hide file tree

Showing 96 changed files with 811 additions and 319 deletions.
diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml
@@ -51,7 +51,7 @@ concurrency:
 jobs:
   build-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with dependencies installed
+    container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key

diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml
@@ -30,7 +30,7 @@ concurrency:
 jobs:
   cache-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with dependencies installed
+    container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key

diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <artifactId>gluten-parent</artifactId>
     <groupId>org.apache.gluten</groupId>
-    <version>1.2.0-SNAPSHOT</version>
+    <version>1.3.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
@@ -307,8 +307,7 @@ object CHBackendSettings extends BackendSettingsApi with Logging {
           }
 
           wExpression.windowFunction match {
-            case _: RowNumber | _: AggregateExpression | _: Rank | _: DenseRank | _: PercentRank |
-                _: NTile =>
+            case _: RowNumber | _: AggregateExpression | _: Rank | _: DenseRank | _: NTile =>
               allSupported = allSupported
             case l: Lag =>
               checkLagOrLead(l.third)

diff --git a/...est/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/...est/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
@@ -978,7 +978,7 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr
     compareResultsAgainstVanillaSpark(sql, true, { _ => })
   }
 
-  test("window percent_rank") {
+  ignore("window percent_rank") {
     val sql =
       """
         |select n_regionkey, n_nationkey,
@@ -2435,6 +2435,16 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr
     }
   }
 
+  test("GLUTEN-6669: test cast string to boolean") {
+    withSQLConf(
+      SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> (ConstantFolding.ruleName + "," + NullPropagation.ruleName)) {
+      runQueryAndCompare(
+        "select cast('1' as boolean), cast('t' as boolean), cast('all' as boolean), cast('f' as boolean)",
+        noFallBack = false
+      )(checkGlutenOperatorMatch[ProjectExecTransformer])
+    }
+  }
+
   test("GLUTEN-4032: fix shuffle read coredump after union") {
     val sql =
       """

diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <artifactId>gluten-parent</artifactId>
     <groupId>org.apache.gluten</groupId>
-    <version>1.2.0-SNAPSHOT</version>
+    <version>1.3.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -66,6 +66,7 @@ object VeloxBackendSettings extends BackendSettingsApi {
   val GLUTEN_VELOX_UDF_LIB_PATHS = getBackendConfigPrefix() + ".udfLibraryPaths"
   val GLUTEN_VELOX_DRIVER_UDF_LIB_PATHS = getBackendConfigPrefix() + ".driver.udfLibraryPaths"
   val GLUTEN_VELOX_INTERNAL_UDF_LIB_PATHS = getBackendConfigPrefix() + ".internal.udfLibraryPaths"
+  val GLUTEN_VELOX_UDF_ALLOW_TYPE_CONVERSION = getBackendConfigPrefix() + ".udfAllowTypeConversion"
 
   val MAXIMUM_BATCH_SIZE: Int = 32768
 

diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
@@ -34,7 +34,7 @@ class VeloxMetricsApi extends MetricsApi with Logging {
       relMap: JMap[JLong, JList[JLong]],
       joinParamsMap: JMap[JLong, JoinParams],
       aggParamsMap: JMap[JLong, AggregationParams]): IMetrics => Unit = {
-    MetricsUtil.updateNativeMetrics(child, relMap, joinParamsMap, aggParamsMap)
+    MetricsUtil.genMetricsUpdatingFunction(child, relMap, joinParamsMap, aggParamsMap)
   }
 
   override def genInputIteratorTransformerMetrics(
@@ -356,9 +356,7 @@ class VeloxMetricsApi extends MetricsApi with Logging {
       "numMemoryAllocations" -> SQLMetrics.createMetric(
         sparkContext,
         "number of memory allocations"),
-      "spilledBytes" -> SQLMetrics.createSizeMetric(
-        sparkContext,
-        "total bytes written for spilling"),
+      "spilledBytes" -> SQLMetrics.createSizeMetric(sparkContext, "bytes written for spilling"),
       "spilledRows" -> SQLMetrics.createMetric(sparkContext, "total rows written for spilling"),
       "spilledPartitions" -> SQLMetrics.createMetric(sparkContext, "total spilled partitions"),
       "spilledFiles" -> SQLMetrics.createMetric(sparkContext, "total spilled files")
@@ -445,7 +443,7 @@ class VeloxMetricsApi extends MetricsApi with Logging {
         "number of hash build memory allocations"),
       "hashBuildSpilledBytes" -> SQLMetrics.createSizeMetric(
         sparkContext,
-        "total bytes written for spilling of hash build"),
+        "bytes written for spilling of hash build"),
       "hashBuildSpilledRows" -> SQLMetrics.createMetric(
         sparkContext,
         "total rows written for spilling of hash build"),
@@ -479,7 +477,7 @@ class VeloxMetricsApi extends MetricsApi with Logging {
         "number of hash probe memory allocations"),
       "hashProbeSpilledBytes" -> SQLMetrics.createSizeMetric(
         sparkContext,
-        "total bytes written for spilling of hash probe"),
+        "bytes written for spilling of hash probe"),
       "hashProbeSpilledRows" -> SQLMetrics.createMetric(
         sparkContext,
         "total rows written for spilling of hash probe"),

diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala
@@ -147,21 +147,29 @@ object VeloxColumnarToRowExec {
           val rows = batch.numRows()
           val beforeConvert = System.currentTimeMillis()
           val batchHandle = ColumnarBatches.getNativeHandle(batch)
-          val info =
-            jniWrapper.nativeColumnarToRowConvert(c2rId, batchHandle)
+          var info =
+            jniWrapper.nativeColumnarToRowConvert(c2rId, batchHandle, 0)
 
           convertTime += (System.currentTimeMillis() - beforeConvert)
 
           new Iterator[InternalRow] {
             var rowId = 0
+            var baseLength = 0
             val row = new UnsafeRow(cols)
 
             override def hasNext: Boolean = {
               rowId < rows
             }
 
             override def next: UnsafeRow = {
-              val (offset, length) = (info.offsets(rowId), info.lengths(rowId))
+              if (rowId == baseLength + info.lengths.length) {
+                baseLength += info.lengths.length
+                val before = System.currentTimeMillis()
+                info = jniWrapper.nativeColumnarToRowConvert(c2rId, batchHandle, rowId)
+                convertTime += (System.currentTimeMillis() - before)
+              }
+              val (offset, length) =
+                (info.offsets(rowId - baseLength), info.lengths(rowId - baseLength))
               row.pointTo(null, info.memoryAddress + offset, length)
               rowId += 1
               row