From 6c9c81b6e3c543e197804884423893a10473bb1a Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 16 Nov 2023 20:00:15 +0800 Subject: [PATCH] [GLUTEN-1902] Separate the implementations for TranslateTransformer (#1903) --- .../clickhouse/CHSparkPlanExecApi.scala | 9 +++++ .../expression/CHExpressionTransformer.scala | 33 +++++++++++++++++++ .../backendsapi/SparkPlanExecApi.scala | 12 +++++++ .../expression/ExpressionConverter.scala | 2 +- .../StringExpressionTransformer.scala | 33 ------------------- 5 files changed, 55 insertions(+), 34 deletions(-) diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 9e0230b3f6a8..60c19a469ddf 100644 --- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -378,6 +378,15 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { CHEqualNullSafeTransformer(substraitExprName, left, right, original) } + override def genStringTranslateTransformer( + substraitExprName: String, + srcExpr: ExpressionTransformer, + matchingExpr: ExpressionTransformer, + replaceExpr: ExpressionTransformer, + original: StringTranslate): ExpressionTransformer = { + CHStringTranslateTransformer(substraitExprName, srcExpr, matchingExpr, replaceExpr, original) + } + override def genStringLocateTransformer( substraitExprName: String, first: ExpressionTransformer, diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/expression/CHExpressionTransformer.scala b/backends-clickhouse/src/main/scala/io/glutenproject/expression/CHExpressionTransformer.scala index 8d6a9f987051..ad667179cc00 100644 --- a/backends-clickhouse/src/main/scala/io/glutenproject/expression/CHExpressionTransformer.scala +++ b/backends-clickhouse/src/main/scala/io/glutenproject/expression/CHExpressionTransformer.scala @@ -335,6 +335,39 @@ case class CHTruncTimestampTransformer( } } +case class CHStringTranslateTransformer( + substraitExprName: String, + srcExpr: ExpressionTransformer, + matchingExpr: ExpressionTransformer, + replaceExpr: ExpressionTransformer, + original: StringTranslate) + extends ExpressionTransformer { + + override def doTransform(args: java.lang.Object): ExpressionNode = { + // In CH, translateUTF8 requires matchingExpr and replaceExpr argument have the same length + val matchingNode = matchingExpr.doTransform(args) + val replaceNode = replaceExpr.doTransform(args) + if ( + !matchingNode.isInstanceOf[StringLiteralNode] || + !replaceNode.isInstanceOf[StringLiteralNode] + ) { + throw new UnsupportedOperationException(s"$original not supported yet.") + } + + val matchingLiteral = matchingNode.asInstanceOf[StringLiteralNode].getValue + val replaceLiteral = replaceNode.asInstanceOf[StringLiteralNode].getValue + if (matchingLiteral.length() != replaceLiteral.length()) { + throw new UnsupportedOperationException(s"$original not supported yet.") + } + + GenericExpressionTransformer( + substraitExprName, + Seq(srcExpr, matchingExpr, replaceExpr), + original) + .doTransform(args) + } +} + case class CHStringLocateTransformer( substraitExprName: String, substrExpr: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala index 3a7bfaa560da..f910e16d9a0b 100644 --- a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala @@ -254,6 +254,18 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(child), original) } + def genStringTranslateTransformer( + substraitExprName: String, + srcExpr: ExpressionTransformer, + matchingExpr: ExpressionTransformer, + replaceExpr: ExpressionTransformer, + original: StringTranslate): ExpressionTransformer = { + GenericExpressionTransformer( + substraitExprName, + Seq(srcExpr, matchingExpr, replaceExpr), + original) + } + def genStringLocateTransformer( substraitExprName: String, first: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala index b097c4788cdc..9cd2b16f14fd 100644 --- a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala @@ -288,7 +288,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { getStructField.ordinal, getStructField) case t: StringTranslate => - StringTranslateTransformer( + BackendsApiManager.getSparkPlanExecApiInstance.genStringTranslateTransformer( substraitExprName, replaceWithExpressionTransformer(t.srcExpr, attributeSeq), replaceWithExpressionTransformer(t.matchingExpr, attributeSeq), diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/StringExpressionTransformer.scala b/gluten-core/src/main/scala/io/glutenproject/expression/StringExpressionTransformer.scala index 709c9f8d8018..93d1a7bedf17 100644 --- a/gluten-core/src/main/scala/io/glutenproject/expression/StringExpressionTransformer.scala +++ b/gluten-core/src/main/scala/io/glutenproject/expression/StringExpressionTransformer.scala @@ -48,39 +48,6 @@ case class String2TrimExpressionTransformer( } } -case class StringTranslateTransformer( - substraitExprName: String, - srcExpr: ExpressionTransformer, - matchingExpr: ExpressionTransformer, - replaceExpr: ExpressionTransformer, - original: StringTranslate) - extends ExpressionTransformer { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - // In CH, translateUTF8 requires matchingExpr and replaceExpr argument have the same length - val matchingNode = matchingExpr.doTransform(args) - val replaceNode = replaceExpr.doTransform(args) - if ( - !matchingNode.isInstanceOf[StringLiteralNode] || - !replaceNode.isInstanceOf[StringLiteralNode] - ) { - throw new UnsupportedOperationException(s"$original not supported yet.") - } - - val matchingLiteral = matchingNode.asInstanceOf[StringLiteralNode].getValue - val replaceLiteral = replaceNode.asInstanceOf[StringLiteralNode].getValue - if (matchingLiteral.length() != replaceLiteral.length()) { - throw new UnsupportedOperationException(s"$original not supported yet.") - } - - GenericExpressionTransformer( - substraitExprName, - Seq(srcExpr, matchingExpr, replaceExpr), - original) - .doTransform(args) - } -} - case class RegExpReplaceTransformer( substraitExprName: String, subject: ExpressionTransformer,