diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ComplexVariantType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ComplexVariantType.java new file mode 100644 index 000000000000000..e866320207773b9 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ComplexVariantType.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + + +import org.apache.doris.thrift.TTypeDesc; +import org.apache.doris.thrift.TTypeNode; +import org.apache.doris.thrift.TTypeNodeType; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.HashMap; + +public class ComplexVariantType extends Type { + + @SerializedName(value = "fieldMap") + private final HashMap fieldMap = Maps.newHashMap(); + + @SerializedName(value = "fields") + private final ArrayList predefinedFields; + + public ComplexVariantType() { + this.predefinedFields = Lists.newArrayList(); + } + + public ComplexVariantType(ArrayList fields) { + Preconditions.checkNotNull(fields); + this.predefinedFields = fields; + for (int i = 0; i < this.predefinedFields.size(); ++i) { + this.predefinedFields.get(i).setPosition(i); + fieldMap.put(this.predefinedFields.get(i).getName().toLowerCase(), this.predefinedFields.get(i)); + } + } + + public ArrayList getPredefinedFields() { + return predefinedFields; + } + + @Override + protected String toSql(int depth) { + if (depth >= MAX_NESTING_DEPTH) { + return "variant<...>"; + } + ArrayList fieldsSql = Lists.newArrayList(); + for (StructField f : predefinedFields) { + fieldsSql.add(f.toSql(depth + 1)); + } + return String.format("variant<%s>", Joiner.on(",").join(fieldsSql)); + } + + @Override + protected String prettyPrint(int lpad) { + return null; + } + + @Override + public void toThrift(TTypeDesc container) { + // not use ScalarType's toThrift for compatibility, because VariantType is not extends ScalarType previously + TTypeNode node = new TTypeNode(); + container.types.add(node); + node.setType(TTypeNodeType.VARIANT); + // predefined fields + node.setStructFields(new ArrayList<>()); + for (StructField field : predefinedFields) { + field.toThrift(container, node); + } + } + + @Override + public PrimitiveType getPrimitiveType() { + return PrimitiveType.VARIANT; + } + + + @Override + public boolean matchesType(Type t) { + return t.isVariantType() || t.isStringType(); + } + + @Override + public boolean supportSubType(Type subType) { + for (Type supportedType : Type.getComplexVariantSubTypes()) { + if (subType.getPrimitiveType() == supportedType.getPrimitiveType()) { + return true; + } + } + return false; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ComplexVariantType)) { + return false; + } + ComplexVariantType otherVariantType = (ComplexVariantType) other; + return otherVariantType.getPredefinedFields().equals(predefinedFields); + } + + @Override + public int getSlotSize() { + return PrimitiveType.VARIANT.getSlotSize(); + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index 9d2abc75d4478c7..5ce41edc41ed0d9 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -135,6 +135,8 @@ public abstract class Type { private static final ArrayList arraySubTypes; private static final ArrayList mapSubTypes; private static final ArrayList structSubTypes; + + private static final ArrayList complexVariantSubTypes; private static final ArrayList trivialTypes; static { @@ -306,6 +308,29 @@ public abstract class Type { structSubTypes.add(ARRAY); structSubTypes.add(MAP); structSubTypes.add(STRUCT); + + complexVariantSubTypes = Lists.newArrayList(); + complexVariantSubTypes.add(BOOLEAN); + complexVariantSubTypes.addAll(integerTypes); + complexVariantSubTypes.add(FLOAT); + complexVariantSubTypes.add(DOUBLE); + complexVariantSubTypes.add(DECIMALV2); + complexVariantSubTypes.add(DECIMAL32); // same DEFAULT_DECIMALV3 + complexVariantSubTypes.add(DECIMAL64); + complexVariantSubTypes.add(DECIMAL128); + complexVariantSubTypes.add(DECIMAL256); + complexVariantSubTypes.add(TIME); + complexVariantSubTypes.add(TIMEV2); + complexVariantSubTypes.add(DATE); + complexVariantSubTypes.add(DATETIME); + complexVariantSubTypes.add(DATEV2); + complexVariantSubTypes.add(DATETIMEV2); + complexVariantSubTypes.add(IPV4); + complexVariantSubTypes.add(IPV6); + complexVariantSubTypes.add(CHAR); + complexVariantSubTypes.add(VARCHAR); + complexVariantSubTypes.add(STRING); + complexVariantSubTypes.add(NULL); } public static final Set DATE_SUPPORTED_JAVA_TYPE = Sets.newHashSet(LocalDate.class, java.util.Date.class, @@ -375,6 +400,10 @@ public static ArrayList getStructSubTypes() { return structSubTypes; } + public static ArrayList getComplexVariantSubTypes() { + return complexVariantSubTypes; + } + /** * Return true if this is complex type and support subType */ @@ -556,7 +585,7 @@ public boolean isJsonbType() { } public boolean isVariantType() { - return isScalarType(PrimitiveType.VARIANT); + return isScalarType(PrimitiveType.VARIANT) || isComplexVariant(); } // only metric types have the following constraint: @@ -688,6 +717,10 @@ public boolean isStructType() { return this instanceof StructType; } + public boolean isComplexVariant() { + return this instanceof ComplexVariantType; + } + public boolean isAnyType() { return this instanceof AnyType; } @@ -981,6 +1014,13 @@ private boolean exceedsMaxNestingDepth(int d) { } else if (isMapType()) { MapType mapType = (MapType) this; return mapType.getValueType().exceedsMaxNestingDepth(d + 1); + } else if (isComplexVariant()) { + ComplexVariantType complexVariantType = (ComplexVariantType) this; + for (StructField f : complexVariantType.getPredefinedFields()) { + if (f.getType().exceedsMaxNestingDepth(d + 1)) { + return true; + } + } } else { Preconditions.checkState(isScalarType() || isAggStateType()); } diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 53745e23cc69a93..532fb1f36b4dc31 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1000,6 +1000,7 @@ dataType : complex=ARRAY LT dataType GT #complexDataType | complex=MAP LT dataType COMMA dataType GT #complexDataType | complex=STRUCT LT complexColTypeList GT #complexDataType + | VARIANT LT complexColTypeList GT #variantPredefined | AGG_STATE LT functionNameIdentifier LEFT_PAREN dataTypes+=dataTypeWithNullable (COMMA dataTypes+=dataTypeWithNullable)* RIGHT_PAREN GT #aggStateDataType @@ -1047,7 +1048,7 @@ complexColTypeList ; complexColType - : identifier COLON dataType commentSpec? + : qualifiedName COLON dataType commentSpec? ; commentSpec diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 98acd9453dd5f9d..2ba0e68237cc09c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -330,8 +330,9 @@ public void createChildrenColumn(Type type, Column column) { v.setIsAllowNull(((MapType) type).getIsValueContainsNull()); column.addChildrenColumn(k); column.addChildrenColumn(v); - } else if (type.isStructType()) { - ArrayList fields = ((StructType) type).getFields(); + } else if (type.isStructType() || type.isComplexVariant()) { + ArrayList fields = type.isStructType() + ? ((StructType) type).getFields() : ((ComplexVariantType) type).getPredefinedFields(); for (StructField field : fields) { Column c = new Column(field.getName(), field.getType()); c.setIsAllowNull(field.getContainsNull()); @@ -657,7 +658,7 @@ private void toChildrenThrift(Column column, TColumn tColumn) { tColumn.setChildrenColumn(new ArrayList<>()); setChildrenTColumn(k, tColumn); setChildrenTColumn(v, tColumn); - } else if (column.type.isStructType()) { + } else if (column.type.isStructType() || column.type.isComplexVariant()) { List childrenColumns = column.getChildren(); tColumn.setChildrenColumn(new ArrayList<>()); for (Column children : childrenColumns) { @@ -802,7 +803,7 @@ public OlapFile.ColumnPB toPb(Set bfColumns, List indexes) throws builder.addChildrenColumns(k.toPb(Sets.newHashSet(), Lists.newArrayList())); Column v = this.getChildren().get(1); builder.addChildrenColumns(v.toPb(Sets.newHashSet(), Lists.newArrayList())); - } else if (this.type.isStructType()) { + } else if (this.type.isStructType() || this.type.isComplexVariant()) { List childrenColumns = this.getChildren(); for (Column c : childrenColumns) { builder.addChildrenColumns(c.toPb(Sets.newHashSet(), Lists.newArrayList())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 2170b552a5ce6b0..8b73073ce9a295d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -202,6 +202,7 @@ import org.apache.doris.nereids.DorisParser.UpdateContext; import org.apache.doris.nereids.DorisParser.UserIdentifyContext; import org.apache.doris.nereids.DorisParser.UserVariableContext; +import org.apache.doris.nereids.DorisParser.VariantPredefinedContext; import org.apache.doris.nereids.DorisParser.WhereClauseContext; import org.apache.doris.nereids.DorisParser.WindowFrameContext; import org.apache.doris.nereids.DorisParser.WindowSpecContext; @@ -458,6 +459,7 @@ import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.ComplexVariantType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.MapType; @@ -2703,6 +2705,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { ? visitPrimitiveDataType(((PrimitiveDataTypeContext) ctx.type)) : ctx.type instanceof ComplexDataTypeContext ? visitComplexDataType((ComplexDataTypeContext) ctx.type) + : ctx.type instanceof VariantPredefinedContext + ? visitVariantPredefined((VariantPredefinedContext) ctx.type) : visitAggStateDataType((AggStateDataTypeContext) ctx.type); colType = colType.conversion(); boolean isKey = ctx.KEY() != null; @@ -3531,6 +3535,11 @@ public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) { }); } + @Override + public DataType visitVariantPredefined(VariantPredefinedContext ctx) { + return new ComplexVariantType(visitComplexColTypeList(ctx.complexColTypeList())); + } + @Override public DataType visitComplexDataType(ComplexDataTypeContext ctx) { return ParserUtils.withOrigin(ctx, () -> { @@ -3561,7 +3570,7 @@ public StructField visitComplexColType(ComplexColTypeContext ctx) { } else { comment = ""; } - return new StructField(ctx.identifier().getText(), typedVisit(ctx.dataType()), true, comment); + return new StructField(ctx.qualifiedName().getText(), typedVisit(ctx.dataType()), true, comment); } private String parseConstant(ConstantContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 0b2694cc311b4c0..51b9e652b10123c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -466,6 +466,20 @@ private void validateDataType(Type catalogType) { } } } + } else if (catalogType.isComplexVariant()) { + ArrayList predefinedFields = + ((org.apache.doris.catalog.ComplexVariantType) catalogType).getPredefinedFields(); + Set fieldNames = new HashSet<>(); + for (org.apache.doris.catalog.StructField field : predefinedFields) { + Type fieldType = field.getType(); + if (fieldType instanceof ScalarType) { + validateNestedType(catalogType, (ScalarType) fieldType); + if (!fieldNames.add(field.getName())) { + throw new AnalysisException("Duplicate field name " + field.getName() + + " in struct " + catalogType.toSql()); + } + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ComplexVariantType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ComplexVariantType.java new file mode 100644 index 000000000000000..7934118adf78bee --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ComplexVariantType.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.analyzer.ComplexDataType; +import org.apache.doris.nereids.annotation.Developing; +import org.apache.doris.nereids.exceptions.AnalysisException; + +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Variant type in Nereids. + * Why Variant is not complex type? Since it's nested structure is not pre-defined, then using + * primitive type will be easy to handle meta info in FE. + */ +@Developing +public class ComplexVariantType extends DataType implements ComplexDataType { + + // predefined fields + private final List predefinedFields; + private final Supplier> nameToFields; + + // No predefined fields + public ComplexVariantType() { + predefinedFields = Lists.newArrayList(); + nameToFields = Suppliers.memoize(Maps::newTreeMap); + } + + /** + * Contains predefined fields like struct + */ + public ComplexVariantType(List fields) { + this.predefinedFields = ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be null")); + this.nameToFields = Suppliers.memoize(() -> this.predefinedFields.stream().collect(ImmutableMap.toImmutableMap( + StructField::getName, f -> f, (f1, f2) -> { + throw new AnalysisException("The name of the struct field cannot be repeated." + + " same name fields are " + f1 + " and " + f2); + }))); + } + + public Map getNameToFields() { + return nameToFields.get(); + } + + public List getPredefinedFields() { + return predefinedFields; + } + + @Override + public boolean acceptsType(DataType other) { + return other instanceof VariantType || other instanceof ComplexVariantType; + } + + @Override + public Type toCatalogDataType() { + return predefinedFields.isEmpty() ? Type.VARIANT + : new org.apache.doris.catalog.ComplexVariantType(predefinedFields.stream() + .map(StructField::toCatalogDataType) + .collect(Collectors.toCollection(ArrayList::new))); + } + + @Override + public String toSql() { + if (predefinedFields.isEmpty()) { + return "VARIANT"; + } + return "VARIANT<" + predefinedFields.stream().map(StructField::toSql).collect(Collectors.joining(",")) + ">"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return super.equals(o); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode()); + } + + @Override + public int width() { + return 24; + } + + @Override + public String toString() { + return toSql(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index f29dbaceab23130..29760c6d189b968 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -68,6 +68,7 @@ import org.apache.doris.catalog.AnyType; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.BrokerTable; +import org.apache.doris.catalog.ComplexVariantType; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.Env; @@ -287,7 +288,8 @@ public class GsonUtils { .registerSubtype(AnyType.class, AnyType.class.getSimpleName()) .registerSubtype(MultiRowType.class, MultiRowType.class.getSimpleName()) .registerSubtype(TemplateType.class, TemplateType.class.getSimpleName()) - .registerSubtype(VariantType.class, VariantType.class.getSimpleName()); + .registerSubtype(VariantType.class, VariantType.class.getSimpleName()) + .registerSubtype(ComplexVariantType.class, ComplexVariantType.class.getSimpleName()); // runtime adapter for class "Expr" private static final RuntimeTypeAdapterFactory exprAdapterFactory