Skip to content

Commit

Permalink
[Feat](Nereids) support string functions fold constant by fe (#40441)
Browse files Browse the repository at this point in the history 
support string functions fold constant by fe:
Concat, Substring, Length, Lower, Upper, Trim, Ltrim, Rtrim, Replace,
Left, Right, Locate,
Instr, Ascii, Bin, ConcatWs, CharacterLength, Initcap, Md5, Md5Sum, 
Field, FindInSet, Repeat, Reverse, Space, SplitByChar, SplitByString,
SplitPart,
SubstringIndex, Strcmp, StrLeft, StrRight, Overlay, ParseUrl, UrlDecode,
AppendTrailingCharIfAbsent, EndsWith, ExtractUrlParameter, Quote,
ReplaceEmpty, Password
  • Loading branch information
@LiBinfeng-01
LiBinfeng-01 authored Sep 11, 2024
1 parent 042bfce commit 81bbd65
Show file tree
Hide file tree
Showing 10 changed files with 1,650 additions and 21 deletions.
3 changes: 3 additions & 0 deletions ...e/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
import com.google.common.collect.Lists;
import org.apache.commons.codec.digest.DigestUtils;

import java.time.DateTimeException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
Expand Down Expand Up @@ -443,6 +444,8 @@ public Expression visitCast(Cast cast, ExpressionRewriteContext context) {
// If cast is from type coercion, we don't use NULL literal and will throw exception.
throw t;
}
} catch (DateTimeException e) {
return new NullLiteral(dataType);
}
}
try {
Expand Down
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExecFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,9 @@
* return type
*/
String returnType();

/**
* hasVarArgsc
*/
boolean varArgs() default false;
}
69 changes: 66 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.expressions;

import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
Expand All @@ -26,6 +27,7 @@
import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform;
import org.apache.doris.nereids.trees.expressions.functions.executable.ExecutableFunctions;
import org.apache.doris.nereids.trees.expressions.functions.executable.NumericArithmetic;
import org.apache.doris.nereids.trees.expressions.functions.executable.StringArithmetic;
import org.apache.doris.nereids.trees.expressions.functions.executable.TimeRoundSeries;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
Expand All @@ -36,6 +38,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;

import java.lang.reflect.Array;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
Expand Down Expand Up @@ -96,10 +99,30 @@ public Expression eval(Expression expression) {
}

private Expression invoke(Expression expression, String fnName, DataType[] args) {
FunctionSignature signature = new FunctionSignature(fnName, args, null);
FunctionSignature signature = new FunctionSignature(fnName, args, null, false);
FunctionInvoker invoker = getFunction(signature);
if (invoker != null) {
try {
if (invoker.getSignature().hasVarArgs()) {
int fixedArgsSize = invoker.getSignature().getArgTypes().length - 1;
int totalSize = expression.children().size();
Class<?>[] parameterTypes = invoker.getMethod().getParameterTypes();
Class<?> parameterType = parameterTypes[parameterTypes.length - 1];
Class<?> componentType = parameterType.getComponentType();
Object varArgs = Array.newInstance(componentType, totalSize - fixedArgsSize);
for (int i = fixedArgsSize; i < totalSize; i++) {
if (!(expression.children().get(i) instanceof NullLiteral)) {
Array.set(varArgs, i - fixedArgsSize, expression.children().get(i));
}
}
Object[] objects = new Object[fixedArgsSize + 1];
for (int i = 0; i < fixedArgsSize; i++) {
objects[i] = expression.children().get(i);
}
objects[fixedArgsSize] = varArgs;

return invoker.invokeVars(objects);
}
return invoker.invoke(expression.children());
} catch (AnalysisException e) {
return expression;
Expand All @@ -114,9 +137,34 @@ private FunctionInvoker getFunction(FunctionSignature signature) {
DataType[] candidateTypes = candidate.getSignature().getArgTypes();
DataType[] expectedTypes = signature.getArgTypes();

if (candidate.getSignature().hasVarArgs()) {
if (candidateTypes.length > expectedTypes.length) {
continue;
}
boolean match = true;
for (int i = 0; i < candidateTypes.length - 1; i++) {
if (!(expectedTypes[i].toCatalogDataType().matchesType(candidateTypes[i].toCatalogDataType()))) {
match = false;
break;
}
}
Type varType = candidateTypes[candidateTypes.length - 1].toCatalogDataType();
for (int i = candidateTypes.length - 1; i < expectedTypes.length; i++) {
if (!(expectedTypes[i].toCatalogDataType().matchesType(varType))) {
match = false;
break;
}
}
if (match) {
return candidate;
} else {
continue;
}
}
if (candidateTypes.length != expectedTypes.length) {
continue;
}

boolean match = true;
for (int i = 0; i < candidateTypes.length; i++) {
if (!(expectedTypes[i].toCatalogDataType().matchesType(candidateTypes[i].toCatalogDataType()))) {
Expand All @@ -143,6 +191,7 @@ private void registerFunctions() {
DateLiteral.class,
DateTimeArithmetic.class,
NumericArithmetic.class,
StringArithmetic.class,
TimeRoundSeries.class
);
for (Class<?> cls : classes) {
Expand Down Expand Up @@ -172,7 +221,7 @@ private void registerFEFunction(ImmutableMultimap.Builder<String, FunctionInvoke
for (int i = 0; i < argTypes.size(); i++) {
array[i] = argTypes.get(i);
}
FunctionSignature signature = new FunctionSignature(name, array, returnType);
FunctionSignature signature = new FunctionSignature(name, array, returnType, annotation.varArgs());
mapBuilder.put(name, new FunctionInvoker(method, signature));
}
}
Expand Down Expand Up @@ -204,6 +253,14 @@ public Literal invoke(List<Expression> args) throws AnalysisException {
throw new AnalysisException(e.getLocalizedMessage());
}
}

public Literal invokeVars(Object[] args) throws AnalysisException {
try {
return (Literal) method.invoke(null, args);
} catch (InvocationTargetException | IllegalAccessException | IllegalArgumentException e) {
throw new AnalysisException(e.getLocalizedMessage());
}
}
}

/**
Expand All @@ -213,11 +270,13 @@ public static class FunctionSignature {
private final String name;
private final DataType[] argTypes;
private final DataType returnType;
private final boolean hasVarArgs;

public FunctionSignature(String name, DataType[] argTypes, DataType returnType) {
public FunctionSignature(String name, DataType[] argTypes, DataType returnType, boolean hasVarArgs) {
this.name = name;
this.argTypes = argTypes;
this.returnType = returnType;
this.hasVarArgs = hasVarArgs;
}

public DataType[] getArgTypes() {
Expand All @@ -231,6 +290,10 @@ public DataType getReturnType() {
public String getName() {
return name;
}

public boolean hasVarArgs() {
return hasVarArgs;
}
}

}
Loading

0 comments on commit 81bbd65

Please sign in to comment.