From 42d859ffc65bc717316998074695c527d44aa79e Mon Sep 17 00:00:00 2001 From: mchades Date: Mon, 4 Sep 2023 20:16:14 +0800 Subject: [PATCH] [#303]feat(common): Implement Partition JSON SerDe (#304) ### What changes were proposed in this pull request? add partitionDTO ### Why are the changes needed? Fix: #303 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? UT added. --- .../dto/rel/ExpressionPartitionDTO.java | 193 ++++++++++++++++++ .../graviton/dto/rel/ListPartitionDTO.java | 95 +++++++++ .../graviton/dto/rel/Partition.java | 40 ++++ .../graviton/dto/rel/RangePartitionDTO.java | 107 ++++++++++ .../graviton/dto/rel/SimplePartitionDTO.java | 58 ++++++ .../dto/requests/TableUpdateRequest.java | 110 +++++----- .../graviton/json/TestDTOJsonSerDe.java | 176 ++++++++++++++++ gradle/libs.versions.toml | 2 +- 8 files changed, 730 insertions(+), 51 deletions(-) create mode 100644 common/src/main/java/com/datastrato/graviton/dto/rel/ExpressionPartitionDTO.java create mode 100644 common/src/main/java/com/datastrato/graviton/dto/rel/ListPartitionDTO.java create mode 100644 common/src/main/java/com/datastrato/graviton/dto/rel/Partition.java create mode 100644 common/src/main/java/com/datastrato/graviton/dto/rel/RangePartitionDTO.java create mode 100644 common/src/main/java/com/datastrato/graviton/dto/rel/SimplePartitionDTO.java diff --git a/common/src/main/java/com/datastrato/graviton/dto/rel/ExpressionPartitionDTO.java b/common/src/main/java/com/datastrato/graviton/dto/rel/ExpressionPartitionDTO.java new file mode 100644 index 0000000000..680ff623f8 --- /dev/null +++ b/common/src/main/java/com/datastrato/graviton/dto/rel/ExpressionPartitionDTO.java @@ -0,0 +1,193 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.dto.rel; + +import static com.datastrato.graviton.dto.rel.ExpressionPartitionDTO.ExpressionType.FIELD; +import static com.datastrato.graviton.dto.rel.ExpressionPartitionDTO.ExpressionType.FUNCTION; +import static com.datastrato.graviton.dto.rel.ExpressionPartitionDTO.ExpressionType.LITERAL; + +import com.datastrato.graviton.json.JsonUtils; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.base.Preconditions; +import io.substrait.type.Type; +import lombok.EqualsAndHashCode; +import org.apache.logging.log4j.util.Strings; + +@EqualsAndHashCode(callSuper = false) +public class ExpressionPartitionDTO implements Partition { + + @JsonProperty("expression") + private final Expression expression; + + @Override + public Strategy strategy() { + return Strategy.EXPRESSION; + } + + @JsonCreator + private ExpressionPartitionDTO( + @JsonProperty("strategy") String strategy, + @JsonProperty("expression") Expression expression) { + Preconditions.checkArgument(expression != null, "expression cannot be null"); + this.expression = expression; + } + + public static class Builder { + private Expression expression; + + public Builder(Expression expression) { + this.expression = expression; + } + + public ExpressionPartitionDTO build() { + return new ExpressionPartitionDTO(Strategy.EXPRESSION.name(), expression); + } + } + + enum ExpressionType { + FIELD, + LITERAL, + FUNCTION, + } + + @JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) + @JsonSubTypes({ + @JsonSubTypes.Type(value = ExpressionPartitionDTO.FieldExpression.class), + @JsonSubTypes.Type(value = ExpressionPartitionDTO.LiteralExpression.class), + @JsonSubTypes.Type(value = ExpressionPartitionDTO.FunctionExpression.class), + }) + public interface Expression { + @JsonProperty("expressionType") + ExpressionType expressionType(); + } + + @EqualsAndHashCode + public static class FieldExpression implements Expression { + + @JsonProperty("fieldName") + private final String[] fieldName; + + @JsonCreator + private FieldExpression( + @JsonProperty("expressionType") String expressionType, + @JsonProperty("fieldName") String[] fieldName) { + Preconditions.checkArgument( + fieldName != null && fieldName.length != 0, "fieldName cannot be null or empty"); + this.fieldName = fieldName; + } + + @Override + public ExpressionType expressionType() { + return FIELD; + } + + public static class Builder { + private String[] fieldName; + + public Builder withFieldName(String[] fieldName) { + this.fieldName = fieldName; + return this; + } + + public FieldExpression build() { + return new FieldExpression(FIELD.name(), fieldName); + } + } + } + + @EqualsAndHashCode + public static class LiteralExpression implements Expression { + + @JsonProperty("type") + @JsonSerialize(using = JsonUtils.TypeSerializer.class) + @JsonDeserialize(using = JsonUtils.TypeDeserializer.class) + private final Type type; + + @JsonProperty("value") + private final String value; + + @JsonCreator + private LiteralExpression( + @JsonProperty("expressionType") String expressionType, + @JsonProperty("type") Type type, + @JsonProperty("value") String value) { + this.type = type; + this.value = value; + } + + @Override + public ExpressionType expressionType() { + return LITERAL; + } + + public static class Builder { + private Type type; + private String value; + + public Builder withType(Type type) { + this.type = type; + return this; + } + + public Builder withValue(String value) { + this.value = value; + return this; + } + + public LiteralExpression build() { + return new LiteralExpression(LITERAL.name(), type, value); + } + } + } + + @EqualsAndHashCode + public static class FunctionExpression implements Expression { + + @JsonProperty("funcName") + private final String funcName; + + @JsonProperty("args") + private final Expression[] args; + + @JsonCreator + private FunctionExpression( + @JsonProperty("expressionType") String expressionType, + @JsonProperty("funcName") String funcName, + @JsonProperty("args") Expression[] args) { + Preconditions.checkArgument(Strings.isNotBlank(funcName), "funcName cannot be null or empty"); + this.funcName = funcName; + this.args = args; + } + + @Override + public ExpressionType expressionType() { + return FUNCTION; + } + + public static class Builder { + private String funcName; + private Expression[] args; + + public Builder withFuncName(String funcName) { + this.funcName = funcName; + return this; + } + + public Builder withArgs(Expression[] args) { + this.args = args; + return this; + } + + public FunctionExpression build() { + return new FunctionExpression(FUNCTION.name(), funcName, args); + } + } + } +} diff --git a/common/src/main/java/com/datastrato/graviton/dto/rel/ListPartitionDTO.java b/common/src/main/java/com/datastrato/graviton/dto/rel/ListPartitionDTO.java new file mode 100644 index 0000000000..3197d9e106 --- /dev/null +++ b/common/src/main/java/com/datastrato/graviton/dto/rel/ListPartitionDTO.java @@ -0,0 +1,95 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.dto.rel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import org.apache.logging.log4j.util.Strings; + +@EqualsAndHashCode(callSuper = false) +public class ListPartitionDTO implements Partition { + @JsonProperty("fieldNames") + private final String[][] fieldNames; + + @JsonProperty("assignments") + private final Assignment[] assignments; + + @JsonCreator + private ListPartitionDTO( + @JsonProperty("strategy") String strategy, + @JsonProperty("fieldNames") String[][] fieldNames, + @JsonProperty("assignments") Assignment[] assignments) { + Preconditions.checkArgument( + fieldNames != null && fieldNames.length != 0, "fieldNames cannot be null or empty"); + + if (assignments != null && assignments.length != 0) { + Preconditions.checkArgument( + Arrays.stream(assignments) + .allMatch( + assignment -> + Arrays.stream(assignment.values) + .allMatch(v -> v.length == fieldNames.length)), + "Assignment values length must be equal to field number"); + } + + this.fieldNames = fieldNames; + this.assignments = assignments; + } + + @Override + public Strategy strategy() { + return Strategy.LIST; + } + + @EqualsAndHashCode + public static class Assignment { + + @JsonProperty("name") + private final String name; + + @JsonProperty("values") + private final String[][] values; + + @JsonCreator + private Assignment( + @JsonProperty("name") String name, @JsonProperty("values") String[][] values) { + Preconditions.checkArgument( + !Strings.isBlank(name), "Assignment name cannot be null or empty"); + Preconditions.checkArgument( + values != null && values.length != 0, "values cannot be null or empty"); + this.name = name; + this.values = values; + } + } + + public static class Builder { + + private String[][] fieldNames; + + private final List assignments = Lists.newArrayList(); + + public Builder() {} + + public Builder withFieldNames(String[][] fieldNames) { + this.fieldNames = fieldNames; + return this; + } + + public Builder withAssignment(String partitionName, String[][] values) { + assignments.add(new Assignment(partitionName, values)); + return this; + } + + public ListPartitionDTO build() { + return new ListPartitionDTO( + Strategy.LIST.name(), fieldNames, assignments.toArray(new Assignment[0])); + } + } +} diff --git a/common/src/main/java/com/datastrato/graviton/dto/rel/Partition.java b/common/src/main/java/com/datastrato/graviton/dto/rel/Partition.java new file mode 100644 index 0000000000..ab3e6f0665 --- /dev/null +++ b/common/src/main/java/com/datastrato/graviton/dto/rel/Partition.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.dto.rel; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "strategy", + include = JsonTypeInfo.As.EXISTING_PROPERTY, + visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type( + value = SimplePartitionDTO.class, + names = {"identity", "year", "month", "day", "hour"}), + @JsonSubTypes.Type(value = ListPartitionDTO.class, name = "list"), + @JsonSubTypes.Type(value = RangePartitionDTO.class, name = "range"), + @JsonSubTypes.Type(value = ExpressionPartitionDTO.class, name = "expression"), +}) +public interface Partition { + + /** @return The strategy of partitioning */ + @JsonProperty("strategy") + Strategy strategy(); + + enum Strategy { + IDENTITY, + YEAR, + MONTH, + DAY, + HOUR, + LIST, + RANGE, + EXPRESSION, + } +} diff --git a/common/src/main/java/com/datastrato/graviton/dto/rel/RangePartitionDTO.java b/common/src/main/java/com/datastrato/graviton/dto/rel/RangePartitionDTO.java new file mode 100644 index 0000000000..65b4347cfb --- /dev/null +++ b/common/src/main/java/com/datastrato/graviton/dto/rel/RangePartitionDTO.java @@ -0,0 +1,107 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.dto.rel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import java.time.LocalDateTime; +import java.util.List; +import lombok.EqualsAndHashCode; +import org.apache.logging.log4j.util.Strings; + +@EqualsAndHashCode(callSuper = false) +public class RangePartitionDTO implements Partition { + + @JsonProperty("fieldName") + private final String[] fieldName; + + @JsonProperty("ranges") + private final Range[] ranges; + + @JsonCreator + private RangePartitionDTO( + @JsonProperty("strategy") String strategy, + @JsonProperty("fieldName") String[] fieldName, + @JsonProperty("ranges") Range[] ranges) { + Preconditions.checkArgument( + fieldName != null && fieldName.length != 0, "fieldName cannot be null or empty"); + + this.fieldName = fieldName; + this.ranges = ranges; + } + + @Override + public Strategy strategy() { + return Strategy.RANGE; + } + + @EqualsAndHashCode + private static class Range { + + @JsonProperty("name") + private final String name; + + @JsonProperty("lower") + private final String lower; + + @JsonProperty("upper") + private final String upper; + + @JsonCreator + private Range( + @JsonProperty("name") String name, + @JsonProperty("lower") String lower, + @JsonProperty("upper") String upper) { + Preconditions.checkArgument(Strings.isNotBlank(name), "Range name cannot be null or empty"); + Preconditions.checkArgument( + validateDatetimeLiteral(lower), + "Range boundary(lower) only supports ISO date-time format literal"); + Preconditions.checkArgument( + validateDatetimeLiteral(upper), + "Range boundary(upper) only supports ISO date-time format literal"); + + this.name = name; + this.lower = lower; + this.upper = upper; + } + + private boolean validateDatetimeLiteral(String literal) { + if (literal == null) { + return true; + } + try { + LocalDateTime.parse(literal); + } catch (Exception e) { + return false; + } + return true; + } + } + + public static class Builder { + + private String[] fieldName; + + private final List ranges = Lists.newArrayList(); + + public Builder() {} + + public Builder withFieldName(String[] fieldName) { + this.fieldName = fieldName; + return this; + } + + public Builder withRange(String partitionName, String lower, String upper) { + ranges.add(new Range(partitionName, lower, upper)); + return this; + } + + public RangePartitionDTO build() { + return new RangePartitionDTO(Strategy.RANGE.name(), fieldName, ranges.toArray(new Range[0])); + } + } +} diff --git a/common/src/main/java/com/datastrato/graviton/dto/rel/SimplePartitionDTO.java b/common/src/main/java/com/datastrato/graviton/dto/rel/SimplePartitionDTO.java new file mode 100644 index 0000000000..46e6eb7195 --- /dev/null +++ b/common/src/main/java/com/datastrato/graviton/dto/rel/SimplePartitionDTO.java @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.dto.rel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = false) +public class SimplePartitionDTO implements Partition { + + private final Strategy strategy; + + @JsonProperty("fieldName") + private final String[] fieldName; + + @JsonCreator + private SimplePartitionDTO( + @JsonProperty(value = "strategy", required = true) String strategy, + @JsonProperty("fieldName") String[] fieldName) { + Preconditions.checkArgument( + fieldName != null && fieldName.length != 0, "fieldName cannot be null or empty"); + + this.strategy = Strategy.valueOf(strategy.toUpperCase()); + this.fieldName = fieldName; + } + + @Override + public Strategy strategy() { + return strategy; + } + + public static class Builder { + + private Strategy strategy; + private String[] fieldName; + + public Builder() {} + + public Builder withFieldName(String[] fieldName) { + this.fieldName = fieldName; + return this; + } + + public Builder withStrategy(Strategy strategy) { + this.strategy = strategy; + return this; + } + + public SimplePartitionDTO build() { + Preconditions.checkArgument(strategy != null, "strategy cannot be null"); + return new SimplePartitionDTO(strategy.name(), fieldName); + } + } +} diff --git a/common/src/main/java/com/datastrato/graviton/dto/requests/TableUpdateRequest.java b/common/src/main/java/com/datastrato/graviton/dto/requests/TableUpdateRequest.java index 5bcdf95172..c584b4812f 100644 --- a/common/src/main/java/com/datastrato/graviton/dto/requests/TableUpdateRequest.java +++ b/common/src/main/java/com/datastrato/graviton/dto/requests/TableUpdateRequest.java @@ -181,8 +181,8 @@ public TableChange tableChange() { class AddTableColumnRequest implements TableUpdateRequest { @Getter - @JsonProperty("name") - private final String[] fieldNames; + @JsonProperty("fieldName") + private final String[] fieldName; @Getter @JsonProperty("type") @@ -203,8 +203,8 @@ class AddTableColumnRequest implements TableUpdateRequest { private final TableChange.ColumnPosition position; public AddTableColumnRequest( - String[] fieldNames, Type dataType, String comment, TableChange.ColumnPosition position) { - this.fieldNames = fieldNames; + String[] fieldName, Type dataType, String comment, TableChange.ColumnPosition position) { + this.fieldName = fieldName; this.dataType = dataType; this.comment = comment; this.position = position; @@ -217,17 +217,17 @@ public AddTableColumnRequest() { @Override public void validate() throws IllegalArgumentException { Preconditions.checkArgument( - fieldNames != null - && fieldNames.length > 0 - && Arrays.stream(fieldNames).allMatch(StringUtils::isNotBlank), - "\"name\" field is required and cannot be empty"); + fieldName != null + && fieldName.length > 0 + && Arrays.stream(fieldName).allMatch(StringUtils::isNotBlank), + "\"fieldName\" field is required and cannot be empty"); Preconditions.checkArgument( dataType != null, "\"type\" field is required and cannot be empty"); } @Override public TableChange tableChange() { - return TableChange.addColumn(fieldNames, dataType, comment, position); + return TableChange.addColumn(fieldName, dataType, comment, position); } } @@ -236,16 +236,16 @@ public TableChange tableChange() { class RenameTableColumnRequest implements TableUpdateRequest { @Getter - @JsonProperty("oldName") - private final String[] oldName; + @JsonProperty("oldFieldName") + private final String[] oldFieldName; @Getter - @JsonProperty("newName") - private final String newName; + @JsonProperty("newFieldName") + private final String newFieldName; - public RenameTableColumnRequest(String[] oldName, String newName) { - this.oldName = oldName; - this.newName = newName; + public RenameTableColumnRequest(String[] oldFieldName, String newFieldName) { + this.oldFieldName = oldFieldName; + this.newFieldName = newFieldName; } public RenameTableColumnRequest() { @@ -255,17 +255,18 @@ public RenameTableColumnRequest() { @Override public void validate() throws IllegalArgumentException { Preconditions.checkArgument( - oldName != null - && oldName.length > 0 - && Arrays.stream(oldName).allMatch(StringUtils::isNotBlank), - "\"oldName\" field is required and cannot be empty"); + oldFieldName != null + && oldFieldName.length > 0 + && Arrays.stream(oldFieldName).allMatch(StringUtils::isNotBlank), + "\"oldFieldName\" field is required and cannot be empty"); Preconditions.checkArgument( - StringUtils.isNotBlank(newName), "\"newName\" field is required and cannot be empty"); + StringUtils.isNotBlank(newFieldName), + "\"newFieldName\" field is required and cannot be empty"); } @Override public TableChange tableChange() { - return TableChange.renameColumn(oldName, newName); + return TableChange.renameColumn(oldFieldName, newFieldName); } } @@ -274,8 +275,8 @@ public TableChange tableChange() { class UpdateTableColumnTypeRequest implements TableUpdateRequest { @Getter - @JsonProperty("name") - private final String[] name; + @JsonProperty("fieldName") + private final String[] fieldName; @Getter @JsonProperty("newType") @@ -283,8 +284,8 @@ class UpdateTableColumnTypeRequest implements TableUpdateRequest { @JsonDeserialize(using = JsonUtils.TypeDeserializer.class) private final Type newType; - public UpdateTableColumnTypeRequest(String[] name, Type newType) { - this.name = name; + public UpdateTableColumnTypeRequest(String[] fieldName, Type newType) { + this.fieldName = fieldName; this.newType = newType; } @@ -295,15 +296,17 @@ public UpdateTableColumnTypeRequest() { @Override public void validate() throws IllegalArgumentException { Preconditions.checkArgument( - name != null && name.length > 0 && Arrays.stream(name).allMatch(StringUtils::isNotBlank), - "\"name\" field is required and cannot be empty"); + fieldName != null + && fieldName.length > 0 + && Arrays.stream(fieldName).allMatch(StringUtils::isNotBlank), + "\"fieldName\" field is required and cannot be empty"); Preconditions.checkArgument( newType != null, "\"newType\" field is required and cannot be empty"); } @Override public TableChange tableChange() { - return TableChange.updateColumnType(name, newType); + return TableChange.updateColumnType(fieldName, newType); } } @@ -312,15 +315,15 @@ public TableChange tableChange() { class UpdateTableColumnCommentRequest implements TableUpdateRequest { @Getter - @JsonProperty("name") - private final String[] name; + @JsonProperty("fieldName") + private final String[] fieldName; @Getter @JsonProperty("newComment") private final String newComment; - public UpdateTableColumnCommentRequest(String[] name, String newComment) { - this.name = name; + public UpdateTableColumnCommentRequest(String[] fieldName, String newComment) { + this.fieldName = fieldName; this.newComment = newComment; } @@ -331,8 +334,10 @@ public UpdateTableColumnCommentRequest() { @Override public void validate() throws IllegalArgumentException { Preconditions.checkArgument( - name != null && name.length > 0 && Arrays.stream(name).allMatch(StringUtils::isNotBlank), - "\"name\" field is required and cannot be empty"); + fieldName != null + && fieldName.length > 0 + && Arrays.stream(fieldName).allMatch(StringUtils::isNotBlank), + "\"fieldName\" field is required and cannot be empty"); Preconditions.checkArgument( StringUtils.isNotBlank(newComment), "\"newComment\" field is required and cannot be empty"); @@ -340,7 +345,7 @@ public void validate() throws IllegalArgumentException { @Override public TableChange tableChange() { - return TableChange.updateColumnComment(name, newComment); + return TableChange.updateColumnComment(fieldName, newComment); } } @@ -349,8 +354,8 @@ public TableChange tableChange() { class UpdateTableColumnPositionRequest implements TableUpdateRequest { @Getter - @JsonProperty("name") - private final String[] name; + @JsonProperty("fieldName") + private final String[] fieldName; @Getter @JsonProperty("newPosition") @@ -358,8 +363,9 @@ class UpdateTableColumnPositionRequest implements TableUpdateRequest { @JsonDeserialize(using = JsonUtils.ColumnPositionDeserializer.class) private final TableChange.ColumnPosition newPosition; - public UpdateTableColumnPositionRequest(String[] name, TableChange.ColumnPosition newPosition) { - this.name = name; + public UpdateTableColumnPositionRequest( + String[] fieldName, TableChange.ColumnPosition newPosition) { + this.fieldName = fieldName; this.newPosition = newPosition; } @@ -370,15 +376,17 @@ public UpdateTableColumnPositionRequest() { @Override public void validate() throws IllegalArgumentException { Preconditions.checkArgument( - name != null && name.length > 0 && Arrays.stream(name).allMatch(StringUtils::isNotBlank), - "\"name\" field is required and cannot be empty"); + fieldName != null + && fieldName.length > 0 + && Arrays.stream(fieldName).allMatch(StringUtils::isNotBlank), + "\"fieldName\" field is required and cannot be empty"); Preconditions.checkArgument( newPosition != null, "\"newPosition\" field is required and cannot be empty"); } @Override public TableChange tableChange() { - return TableChange.updateColumnPosition(name, newPosition); + return TableChange.updateColumnPosition(fieldName, newPosition); } } @@ -387,15 +395,15 @@ public TableChange tableChange() { class DeleteTableColumnRequest implements TableUpdateRequest { @Getter - @JsonProperty("name") - private final String[] name; + @JsonProperty("fieldName") + private final String[] fieldName; @Getter @JsonProperty("ifExists") private final boolean ifExists; - public DeleteTableColumnRequest(String[] name, boolean ifExists) { - this.name = name; + public DeleteTableColumnRequest(String[] fieldName, boolean ifExists) { + this.fieldName = fieldName; this.ifExists = ifExists; } @@ -406,13 +414,15 @@ public DeleteTableColumnRequest() { @Override public void validate() throws IllegalArgumentException { Preconditions.checkArgument( - name != null && name.length > 0 && Arrays.stream(name).allMatch(StringUtils::isNotBlank), - "\"name\" field is required and cannot be empty"); + fieldName != null + && fieldName.length > 0 + && Arrays.stream(fieldName).allMatch(StringUtils::isNotBlank), + "\"fieldName\" field is required and cannot be empty"); } @Override public TableChange tableChange() { - return TableChange.deleteColumn(name, ifExists); + return TableChange.deleteColumn(fieldName, ifExists); } } } diff --git a/common/src/test/java/com/datastrato/graviton/json/TestDTOJsonSerDe.java b/common/src/test/java/com/datastrato/graviton/json/TestDTOJsonSerDe.java index af8e9f955e..224eb63c01 100644 --- a/common/src/test/java/com/datastrato/graviton/json/TestDTOJsonSerDe.java +++ b/common/src/test/java/com/datastrato/graviton/json/TestDTOJsonSerDe.java @@ -9,7 +9,16 @@ import com.datastrato.graviton.dto.CatalogDTO; import com.datastrato.graviton.dto.MetalakeDTO; import com.datastrato.graviton.dto.rel.ColumnDTO; +import com.datastrato.graviton.dto.rel.ExpressionPartitionDTO; +import com.datastrato.graviton.dto.rel.ListPartitionDTO; +import com.datastrato.graviton.dto.rel.Partition; +import com.datastrato.graviton.dto.rel.RangePartitionDTO; +import com.datastrato.graviton.dto.rel.SimplePartitionDTO; import com.datastrato.graviton.dto.rel.TableDTO; +import com.fasterxml.jackson.databind.cfg.EnumFeature; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; +import com.fasterxml.jackson.databind.exc.ValueInstantiationException; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.google.common.collect.ImmutableMap; import io.substrait.type.StringTypeVisitor; import io.substrait.type.TypeCreator; @@ -208,4 +217,171 @@ public void testTableDTOSerDe() throws Exception { String.format(auditJson, withQuotes(creator), withQuotes(now.toString()), null, null)); Assertions.assertEquals(expectedJson, serJson); } + + @Test + public void testPartitionDTOSerDe() throws Exception { + + String[] field1 = new String[] {"dt"}; + String[] field2 = new String[] {"city"}; + + // construct simple partition + Partition identity = + new SimplePartitionDTO.Builder() + .withStrategy(Partition.Strategy.IDENTITY) + .withFieldName(field1) + .build(); + Partition hourPart = + new SimplePartitionDTO.Builder() + .withStrategy(Partition.Strategy.HOUR) + .withFieldName(field1) + .build(); + Partition dayPart = + new SimplePartitionDTO.Builder() + .withStrategy(Partition.Strategy.DAY) + .withFieldName(field1) + .build(); + Partition monthPart = + new SimplePartitionDTO.Builder() + .withStrategy(Partition.Strategy.MONTH) + .withFieldName(field1) + .build(); + Partition yearPart = + new SimplePartitionDTO.Builder() + .withStrategy(Partition.Strategy.YEAR) + .withFieldName(field1) + .build(); + + // construct list partition + String[][] p1Value = {{"2023-04-01", "San Francisco"}, {"2023-04-01", "San Francisco"}}; + String[][] p2Value = {{"2023-04-01", "Houston"}, {"2023-04-01", "Dallas"}}; + Partition listPart = + new ListPartitionDTO.Builder() + .withFieldNames(new String[][] {field1, field2}) + .withAssignment("p202304_California", p1Value) + .withAssignment("p202304_Texas", p2Value) + .build(); + + // construct range partition + Partition rangePart = + new RangePartitionDTO.Builder() + .withFieldName(field1) + .withRange("p20230101", "2023-01-01T00:00:00", "2023-01-02T00:00:00") + .withRange("p20230102", "2023-01-01T00:00:00", null) + .build(); + + // construct expression partition, toYYYYMM(toDate(ts, ‘Asia/Shanghai’)) + ExpressionPartitionDTO.Expression arg1 = + new ExpressionPartitionDTO.FieldExpression.Builder().withFieldName(field1).build(); + ExpressionPartitionDTO.Expression arg2 = + new ExpressionPartitionDTO.LiteralExpression.Builder() + .withType(TypeCreator.REQUIRED.STRING) + .withValue("Asia/Shanghai") + .build(); + ExpressionPartitionDTO.Expression toDateFunc = + new ExpressionPartitionDTO.FunctionExpression.Builder() + .withFuncName("toDate") + .withArgs(new ExpressionPartitionDTO.Expression[] {arg1, arg2}) + .build(); + ExpressionPartitionDTO.Expression monthFunc = + new ExpressionPartitionDTO.FunctionExpression.Builder() + .withFuncName("toYYYYMM") + .withArgs(new ExpressionPartitionDTO.Expression[] {toDateFunc}) + .build(); + Partition expressionPart = new ExpressionPartitionDTO.Builder(monthFunc).build(); + + Partition[] partitions = { + identity, hourPart, dayPart, monthPart, yearPart, listPart, rangePart, expressionPart + }; + String serJson = + JsonMapper.builder() + .configure(EnumFeature.WRITE_ENUMS_TO_LOWERCASE, true) + .build() + .writeValueAsString(partitions); + Partition[] desPartitions = JsonUtils.objectMapper().readValue(serJson, Partition[].class); + + Assertions.assertArrayEquals(partitions, desPartitions); + } + + @Test + public void testPartitionDTOSerDeFail() throws Exception { + // test `strategy` value null + String wrongJson1 = "{\"strategy\": null,\"fieldName\":[\"dt\"]}"; + InvalidTypeIdException invalidTypeIdException = + Assertions.assertThrows( + InvalidTypeIdException.class, + () -> JsonUtils.objectMapper().readValue(wrongJson1, Partition.class)); + Assertions.assertTrue( + invalidTypeIdException.getMessage().contains("missing type id property 'strategy'")); + + // test `fieldName` value empty + String wrongJson2 = "{\"strategy\": \"day\",\"fieldName\":[]}"; + ValueInstantiationException valueInstantiationException = + Assertions.assertThrows( + ValueInstantiationException.class, + () -> JsonUtils.objectMapper().readValue(wrongJson2, Partition.class)); + Assertions.assertTrue( + valueInstantiationException.getMessage().contains("fieldName cannot be null or empty")); + + // test listPartition assignment values missing + String wrongJson3 = + "{\"strategy\": \"list\",\"fieldNames\":[[\"dt\"],[\"city\"]]," + + "\"assignments\":[" + + "{\"name\":\"p202304_California\", " + + "\"values\":[[\"2023-04-01\",\"San Francisco\"], [\"2023-04-01\"]]}]}"; + valueInstantiationException = + Assertions.assertThrows( + ValueInstantiationException.class, + () -> JsonUtils.objectMapper().readValue(wrongJson3, Partition.class)); + Assertions.assertTrue( + valueInstantiationException + .getMessage() + .contains("Assignment values length must be equal to field number")); + + // test rangePartition range name missing + String wrongJson4 = + "{\"strategy\": \"range\",\"fieldName\":[\"dt\"]," + + "\"ranges\":[" + + "{\"lower\":null, \"upper\":\"2023-01-02T00:00:00\"}," + + "{\"lower\":\"2023-01-01T00:00:00\", \"upper\":null}]}"; + valueInstantiationException = + Assertions.assertThrows( + ValueInstantiationException.class, + () -> JsonUtils.objectMapper().readValue(wrongJson4, Partition.class)); + Assertions.assertTrue( + valueInstantiationException.getMessage().contains("Range name cannot be null or empty")); + + // test rangePartition range bound literal format wrong + String wrongJson5 = + "{\"strategy\": \"range\",\"fieldName\":[\"dt\"]," + + "\"ranges\":[" + + "{\"name\":\"p20230101\", \"lower\":\"2023-01-01T00:00:00\", \"upper\":\"2023-01-02\"}," + + "{\"name\":\"p20230102\", \"lower\":\"2023-01-01T00:00:00\", \"upper\":null}]}"; + valueInstantiationException = + Assertions.assertThrows( + ValueInstantiationException.class, + () -> JsonUtils.objectMapper().readValue(wrongJson5, Partition.class)); + Assertions.assertTrue( + valueInstantiationException + .getMessage() + .contains("only supports ISO date-time format literal")); + + // test expressionPartition `expression` value null + String wrongJson6 = "{\"strategy\": \"expression\", \"expression\": null}"; + valueInstantiationException = + Assertions.assertThrows( + ValueInstantiationException.class, + () -> JsonUtils.objectMapper().readValue(wrongJson6, Partition.class)); + Assertions.assertTrue( + valueInstantiationException.getMessage().contains("expression cannot be null")); + + // test expressionPartition with FunctionExpression `funcName` value empty + String wrongJson7 = + "{\"strategy\": \"expression\", \"expression\": {\"funcName\": \"\", \"args\": []}}"; + valueInstantiationException = + Assertions.assertThrows( + ValueInstantiationException.class, + () -> JsonUtils.objectMapper().readValue(wrongJson7, Partition.class)); + Assertions.assertTrue( + valueInstantiationException.getMessage().contains("funcName cannot be null or empty")); + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ad001e9912..25b1b8ddbe 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,7 +6,7 @@ junit = "5.8.1" protoc = "3.17.3" substrait = "0.9.0" -jackson = "2.14.1" +jackson = "2.15.2" guava = "29.0-jre" lombok = "1.18.20" slf4j = "2.0.7"