[GLUTEN-7670][CH] Fix enable 'files.per.partition.threshold' bug (#7758)

[CH] Fix enable 'files.per.partition.threshold' bug
apache · Nov 1, 2024 · dc65d77 · dc65d77
1 parent 8681142
commit dc65d77
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 1 deletion.
diff --git a/...pache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala b/...pache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala
@@ -232,7 +232,14 @@ object MergeTreePartsPartitionsUtil extends Logging {
       val splitFiles = selectRanges
         .map {
           part =>
-            MergeTreePartSplit(part.name, part.dirName, part.targetNode, 0, part.marks, part.size)
+            MergeTreePartSplit(
+              part.name,
+              part.dirName,
+              part.targetNode,
+              part.start,
+              part.marks,
+              part.size
+            )
         }
       genInputPartitionSeqByFileCnt(
         engine,

diff --git a/...est/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala b/...est/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala
@@ -1805,6 +1805,29 @@ class GlutenClickHouseMergeTreeWriteSuite
           }
         }
       })
+
+    // GLUTEN-7670: fix enable 'files.per.partition.threshold'
+    withSQLConf(
+      CHConf.runtimeSettings("enabled_driver_filter_mergetree_index") -> "true",
+      CHConf.prefixOf("files.per.partition.threshold") -> "10"
+    ) {
+      runTPCHQueryBySQL(6, sqlStr) {
+        df =>
+          val scanExec = collect(df.queryExecution.executedPlan) {
+            case f: FileSourceScanExecTransformer => f
+          }
+          assertResult(1)(scanExec.size)
+
+          val mergetreeScan = scanExec.head
+          assert(mergetreeScan.nodeName.startsWith("Scan mergetree"))
+
+          val plans = collect(df.queryExecution.executedPlan) {
+            case scanExec: BasicScanExecTransformer => scanExec
+          }
+          assertResult(1)(plans.size)
+          assertResult(1)(plans.head.getSplitInfos(null).size)
+      }
+    }
   }
 
   test("test mergetree with primary keys filter pruning by driver with bucket") {