Skip to content

Commit

Permalink
1
Browse files Browse the repository at this point in the history
  • Loading branch information
@zclllyybb
zclllyybb committed Aug 29, 2024
1 parent 56fca53 commit 3302fc3
Show file tree
Hide file tree
Showing 8 changed files with 130 additions and 154 deletions.
52 changes: 48 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.thrift.TNullableStringLiteral;

import com.google.common.collect.Maps;
Expand All @@ -43,6 +44,14 @@ public class PartitionExprUtil {
private static final Logger LOG = LogManager.getLogger(PartitionExprUtil.class);
private static final PartitionExprUtil partitionExprUtil = new PartitionExprUtil();

/**
* properties should be checked before call this method
*/
public static void checkAndSetAutoPartitionProperty(OlapTable olapTable, Map<String, String> properties) {
String useSimpleAutoPartitionName = properties.get(PropertyAnalyzer.PROPERTIES_USE_SIMPLE_AUTO_PARTITION_NAME);
olapTable.setUseSimpleAutoPartitionName(useSimpleAutoPartitionName);
}

public static FunctionIntervalInfo getFunctionIntervalInfo(ArrayList<Expr> partitionExprs,
PartitionType partitionType) throws AnalysisException {
if (partitionType != PartitionType.RANGE) {
Expand Down Expand Up @@ -90,7 +99,7 @@ public static FunctionIntervalInfo getFunctionIntervalInfo(ArrayList<Expr> parti
return partitionExprUtil.new FunctionIntervalInfo(timeUnit, interval);
}

public static DateLiteral getRangeEnd(DateLiteral beginTime, FunctionIntervalInfo intervalInfo)
private static DateLiteral getRangeEnd(DateLiteral beginTime, FunctionIntervalInfo intervalInfo)
throws AnalysisException {
String timeUnit = intervalInfo.timeUnit;
long interval = intervalInfo.interval;
Expand All @@ -117,6 +126,35 @@ public static DateLiteral getRangeEnd(DateLiteral beginTime, FunctionIntervalInf
return null;
}

private static String getSimpleRangePartitionName(DateLiteral beginTime, FunctionIntervalInfo intervalInfo)
throws AnalysisException {
String timeUnit = intervalInfo.timeUnit;
switch (timeUnit) {
case "year":
return String.format("%04d", beginTime.getYear());
case "quarter":
return String.format("%04d%02d", beginTime.getYear(), beginTime.getMonth());
case "month":
return String.format("%04d%02d", beginTime.getYear(), beginTime.getMonth());
case "week":
return String.format("%04d%02d%02d", beginTime.getYear(), beginTime.getMonth(), beginTime.getDay());
case "day":
return String.format("%04d%02d%02d", beginTime.getYear(), beginTime.getMonth(), beginTime.getDay());
case "hour":
return String.format("%04d%02d%02d%02d", beginTime.getYear(), beginTime.getMonth(), beginTime.getDay(),
beginTime.getHour());
case "minute":
return String.format("%04d%02d%02d%02d%02d", beginTime.getYear(), beginTime.getMonth(),
beginTime.getDay(), beginTime.getHour(), beginTime.getMinute());
case "second":
return String.format("%04d%02d%02d%02d%02d%02d", beginTime.getYear(), beginTime.getMonth(),
beginTime.getDay(), beginTime.getHour(), beginTime.getMinute(), beginTime.getSecond());
default:
break;
}
return null;
}

public static Map<String, AddPartitionClause> getAddPartitionClauseFromPartitionValues(OlapTable olapTable,
ArrayList<List<TNullableStringLiteral>> partitionValues, PartitionInfo partitionInfo)
throws AnalysisException {
Expand Down Expand Up @@ -154,9 +192,15 @@ public static Map<String, AddPartitionClause> getAddPartitionClauseFromPartition
String beginTime = curPartitionValues.get(0); // have check range type size must be 1
Type partitionColumnType = partitionColumn.get(0).getType();
DateLiteral beginDateTime = new DateLiteral(beginTime, partitionColumnType);
partitionName += String.format(DATETIME_NAME_FORMATTER,
beginDateTime.getYear(), beginDateTime.getMonth(), beginDateTime.getDay(),
beginDateTime.getHour(), beginDateTime.getMinute(), beginDateTime.getSecond());

if (olapTable.useSimpleAutoPartitionName()) {
partitionName += getSimpleRangePartitionName(beginDateTime, intervalInfo);
} else {
partitionName += String.format(DATETIME_NAME_FORMATTER, beginDateTime.getYear(),
beginDateTime.getMonth(), beginDateTime.getDay(), beginDateTime.getHour(),
beginDateTime.getMinute(), beginDateTime.getSecond());
}

DateLiteral endDateTime = getRangeEnd(beginDateTime, intervalInfo);
partitionKeyDesc = createPartitionKeyDescWithRange(beginDateTime, endDateTime, partitionColumnType);
} else if (partitionType == PartitionType.LIST) {
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5286,7 +5286,8 @@ public void modifyTableProperties(Database db, OlapTable table, Map<String, Stri
.buildEnableSingleReplicaCompaction()
.buildTimeSeriesCompactionEmptyRowsetsThreshold()
.buildTimeSeriesCompactionLevelThreshold()
.buildTTLSeconds();
.buildTTLSeconds()
.buildUseSimpleAutoPartitionName();

// need to update partition info meta
for (Partition partition : table.getPartitions()) {
Expand Down
15 changes: 15 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ public enum OlapTableState {
@SerializedName(value = "bid", alternate = {"baseIndexId"})
private long baseIndexId = -1;

// to change, need setXXX then tableProperty.buildXXX. then could directly call getter of tableProperty
@SerializedName(value = "tp", alternate = {"tableProperty"})
private TableProperty tableProperty;

Expand Down Expand Up @@ -2085,6 +2086,20 @@ public void setIsInMemory(boolean isInMemory) {
tableProperty.buildInMemory();
}

public Boolean useSimpleAutoPartitionName() {
if (tableProperty != null) {
return tableProperty.useSimpleAutoPartitionName();
}
return false;
}

public void setUseSimpleAutoPartitionName(String useSimpleAutoPartitionName) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_USE_SIMPLE_AUTO_PARTITION_NAME,
useSimpleAutoPartitionName);
tableProperty.buildUseSimpleAutoPartitionName();
}

public Boolean isAutoBucket() {
if (tableProperty != null) {
return tableProperty.isAutoBucket();
Expand Down
17 changes: 17 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@
* TableProperty includes properties to persistent the additional information
* Different properties is recognized by prefix such as dynamic_partition
* If there is different type properties is added, write a method such as buildDynamicProperty to build it.
* modify property:
* 1. call setXXX to olapTable to set TableProperty.properties
* 2. call TableProperty.buildXXX to set specific value of TableProperty
*/
public class TableProperty implements Writable, GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(TableProperty.class);
Expand All @@ -78,6 +81,8 @@ public class TableProperty implements Writable, GsonPostProcessable {
// which columns stored in RowStore column
private List<String> rowStoreColumns;

private Boolean useSimpleAutoPartitionName = false;

/*
* the default storage format of this table.
* DEFAULT: depends on BE's config 'default_rowset_type'
Expand Down Expand Up @@ -189,6 +194,7 @@ public TableProperty resetPropertiesForRestore(boolean reserveDynamicPartitionEn
return this;
}

// make properties of dynamic partition go into dynamicPartitionProperty
public TableProperty buildDynamicProperty() {
executeBuildDynamicProperty();
return this;
Expand All @@ -214,6 +220,12 @@ public TableProperty buildInMemory() {
return this;
}

public TableProperty buildUseSimpleAutoPartitionName() {
useSimpleAutoPartitionName = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_USE_SIMPLE_AUTO_PARTITION_NAME, "false"));
return this;
}

public TableProperty buildTTLSeconds() {
ttlSeconds = Long.parseLong(properties.getOrDefault(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS, "0"));
return this;
Expand Down Expand Up @@ -503,6 +515,7 @@ public TableProperty buildInvertedIndexFileStorageFormat() {
return this;
}

// modify means update with argument. not clean others
public void modifyTableProperties(Map<String, String> modifyProperties) {
properties.putAll(modifyProperties);
removeDuplicateReplicaNumProperty();
Expand Down Expand Up @@ -550,6 +563,10 @@ public boolean isInMemory() {
return isInMemory;
}

public boolean useSimpleAutoPartitionName() {
return useSimpleAutoPartitionName;
}

public boolean isAutoBucket() {
return Boolean.parseBoolean(properties.getOrDefault(PropertyAnalyzer.PROPERTIES_AUTO_BUCKET, "false"));
}
Expand Down
2 changes: 2 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,8 @@ public class PropertyAnalyzer {
public static final long TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE = 5;
public static final long TIME_SERIES_COMPACTION_LEVEL_THRESHOLD_DEFAULT_VALUE = 1;

public static final String PROPERTIES_USE_SIMPLE_AUTO_PARTITION_NAME = "use_simple_auto_partition_name";

public enum RewriteType {
PUT, // always put property
REPLACE, // replace if exists property
Expand Down
12 changes: 6 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.MultiPartitionDesc;
import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.PartitionExprUtil;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.PartitionKeyDesc.PartitionKeyValueType;
import org.apache.doris.analysis.PartitionNames;
Expand Down Expand Up @@ -3050,12 +3051,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
try {
DataProperty dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
Map<String, String> propertiesCheck = new HashMap<>(properties);
propertiesCheck.entrySet().removeIf(entry -> entry.getKey().contains("dynamic_partition"));
if (propertiesCheck != null && !propertiesCheck.isEmpty()) {
// here, all properties should be checked
throw new DdlException("Unknown properties: " + propertiesCheck);
}

// just for remove entries in stmt.getProperties(),
// and then check if there still has unknown properties
olapTable.setStorageMedium(dataProperty.getStorageMedium());
Expand Down Expand Up @@ -3137,6 +3133,10 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
throw new DdlException("Unsupported partition method: " + partitionInfo.getType().name());
}

if (partitionDesc != null && partitionDesc.isAutoCreatePartitions()) {
PartitionExprUtil.checkAndSetAutoPartitionProperty(olapTable, properties);
}

Pair<Boolean, Boolean> result = db.createTableWithLock(olapTable, false, stmt.isSetIfNotExists());
if (!result.first) {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
Expand Down
144 changes: 1 addition & 143 deletions regression-test/suites/ddl_p0/test_create_table_properties.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,32 +18,7 @@
// this suite is for creating table with timestamp datatype in defferent
// case. For example: 'year' and 'Year' datatype should also be valid in definition

suite("test_create_table_properties") {
try {
sql """
create table test_create_table_properties (
`user_id` LARGEINT NOT NULL COMMENT "用户id",
`start_time` DATETIME,
`billing_cycle_id` INT
)
partition by range(`billing_cycle_id`, `start_time`)(
PARTITION p202201_otr VALUES [("202201", '2000-01-01 00:00:00'), ("202201", '2022-01-01 00:00:00')),
PARTITION error_partition VALUES [("999999", '1970-01-01 00:00:00'), ("999999", '1970-01-02 00:00:00'))
)
distributed by hash(`user_id`) buckets 1
properties(
"replication_num"="1",
"abc"="false"
);
"""
assertTrue(false, "should not be able to execute")
}
catch (Exception ex) {
assertTrue(ex.getMessage().contains("Unknown properties"))
} finally {
sql """ DROP TABLE IF EXISTS test_create_table_properties"""
}

suite("test_create_table_properties") {
def cooldown_ttl = "10"
def create_s3_resource = try_sql """
CREATE RESOURCE IF NOT EXISTS "test_create_table_use_resource"
Expand Down Expand Up @@ -84,33 +59,6 @@ suite("test_create_table_properties") {
"""

sql """ DROP TABLE IF EXISTS test_create_table_properties"""

try {
sql """
create table test_create_table_properties (
`user_id` LARGEINT NOT NULL COMMENT "用户id",
`start_time` DATETIME,
`billing_cycle_id` INT
)
partition by range(`billing_cycle_id`, `start_time`)(
PARTITION p202201_otr VALUES [("202201", '2000-01-01 00:00:00'), ("202201", '2022-01-01 00:00:00')),
PARTITION error_partition VALUES [("999999", '1970-01-01 00:00:00'), ("999999", '1970-01-02 00:00:00'))
)
distributed by hash(`user_id`) buckets 1
properties(
"replication_num"="1",
"storage_policy" = "test_create_table_use_policy",
"abc"="false"
);
"""
assertTrue(false, "should not be able to execute")
}
catch (Exception ex) {
assertTrue(ex.getMessage().contains("Unknown properties"))
} finally {
sql """ DROP TABLE IF EXISTS test_create_table_properties"""
}

try {
sql """
create table test_create_table_properties (
Expand All @@ -137,36 +85,6 @@ suite("test_create_table_properties") {
sql """ DROP TABLE IF EXISTS test_create_table_properties"""
}

try {
sql """
create table test_create_table_properties (
`user_id` LARGEINT NOT NULL COMMENT "用户id",
`start_time` DATETIME,
`billing_cycle_id` INT
)
partition by range(`billing_cycle_id`, `start_time`)(
PARTITION p202201_otr VALUES [("202201", '2000-01-01 00:00:00'), ("202201", '2022-01-01 00:00:00')),
PARTITION error_partition VALUES [("999999", '1970-01-01 00:00:00'), ("999999", '1970-01-02 00:00:00'))
)
distributed by hash(`user_id`) buckets 1
properties(
"replication_num"="1",
"storage_policy" = "test_create_table_use_policy",
"dynamic_partition.start" = "-7",
"abc"="false"
);
"""
assertTrue(false, "should not be able to execute")
}
catch (Exception ex) {
assertTrue(ex.getMessage().contains("Unknown properties"))
} finally {
sql """ DROP TABLE IF EXISTS test_create_table_properties"""
}



sql "drop table if exists test_create_table_properties"
sql """
CREATE TABLE IF NOT EXISTS test_create_table_properties (
k1 tinyint NOT NULL,
Expand All @@ -185,66 +103,6 @@ suite("test_create_table_properties") {
"""

sql "drop table if exists test_create_table_properties"
try {
sql """
CREATE TABLE IF NOT EXISTS test_create_table_properties (
k1 tinyint NOT NULL,
k2 smallint NOT NULL,
k3 int NOT NULL,
k4 bigint NOT NULL,
k5 decimal(9, 3) NOT NULL,
k8 double max NOT NULL,
k9 float sum NOT NULL )
AGGREGATE KEY(k1,k2,k3,k4,k5)
PARTITION BY LIST(k1,k2) (
PARTITION p1 VALUES IN (("1","2"),("3","4")),
PARTITION p2 VALUES IN (("5","6"),("7","8")),
PARTITION p3 )
DISTRIBUTED BY HASH(k1) BUCKETS 5
properties(
"replication_num" = "1",
"abc" = "false"
)
"""
assertTrue(false, "should not be able to execute")
}
catch (Exception ex) {
assertTrue(ex.getMessage().contains("Unknown properties"))
} finally {
sql """ DROP TABLE IF EXISTS test_create_table_properties"""
}


try {
sql """
CREATE TABLE IF NOT EXISTS test_create_table_properties (
k1 tinyint NOT NULL,
k2 smallint NOT NULL,
k3 int NOT NULL,
k4 bigint NOT NULL,
k5 decimal(9, 3) NOT NULL,
k8 double max NOT NULL,
k9 float sum NOT NULL )
AGGREGATE KEY(k1,k2,k3,k4,k5)
PARTITION BY LIST(k1,k2) (
PARTITION p1 VALUES IN (("1","2"),("3","4")),
PARTITION p2 VALUES IN (("5","6"),("7","8")),
PARTITION p3 )
DISTRIBUTED BY HASH(k1) BUCKETS 5
properties(
"replication_num" = "1",
"storage_policy" = "test_create_table_use_policy",
"abc" = "false"
)
"""
assertTrue(false, "should not be able to execute")
}
catch (Exception ex) {
assertTrue(ex.getMessage().contains("Unknown properties"))
} finally {
sql """ DROP TABLE IF EXISTS test_create_table_properties"""
}

sql """
CREATE TABLE IF NOT EXISTS test_create_table_properties (
k1 tinyint NOT NULL,
Expand Down
Loading

0 comments on commit 3302fc3

Please sign in to comment.