Skip to content

Commit

Permalink
[Bugfix] Fix mv column type is not changed when do schema change (#34598
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@Lchangliang
Lchangliang authored May 27, 2024
1 parent 28aad01 commit ffa5ac3
Show file tree
Hide file tree
Showing 9 changed files with 582 additions and 57 deletions.
63 changes: 31 additions & 32 deletions fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.doris.analysis.CancelStmt;
import org.apache.doris.analysis.ColumnPosition;
import org.apache.doris.analysis.CreateIndexClause;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.DropColumnClause;
import org.apache.doris.analysis.DropIndexClause;
import org.apache.doris.analysis.IndexDef;
Expand Down Expand Up @@ -672,52 +673,50 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol
}
List<Column> schema = entry.getValue();
for (Column column : schema) {
if (column.getName().equalsIgnoreCase(modColumn.getName())) {
String columnName = column.getName();
if (column.isMaterializedViewColumn()) {
columnName = MaterializedIndexMeta.normalizeName(
CreateMaterializedViewStmt.mvColumnBreaker(columnName));
}
if (columnName.equalsIgnoreCase(modColumn.getName())) {
otherIndexIds.add(entry.getKey());
break;
}
}
}

if (KeysType.AGG_KEYS == olapTable.getKeysType() || KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
for (Long otherIndexId : otherIndexIds) {
List<Column> otherIndexSchema = indexSchemaMap.get(otherIndexId);
for (Long otherIndexId : otherIndexIds) {
List<Column> otherIndexSchema = indexSchemaMap.get(otherIndexId);
for (int i = 0; i < otherIndexSchema.size(); i++) {
modColIndex = -1;
for (int i = 0; i < otherIndexSchema.size(); i++) {
if (otherIndexSchema.get(i).getName().equalsIgnoreCase(modColumn.getName())) {
modColIndex = i;
break;
}
Column otherCol = null;
Column col = otherIndexSchema.get(i);
String columnName = col.getName();
if (col.isMaterializedViewColumn()) {
columnName = MaterializedIndexMeta.normalizeName(
CreateMaterializedViewStmt.mvColumnBreaker(columnName));
}
Preconditions.checkState(modColIndex != -1);
// replace the old column
otherIndexSchema.set(modColIndex, modColumn);
} // end for other indices
} else {
// DUPLICATE data model has a little
for (Long otherIndexId : otherIndexIds) {
List<Column> otherIndexSchema = indexSchemaMap.get(otherIndexId);
modColIndex = -1;
for (int i = 0; i < otherIndexSchema.size(); i++) {
if (otherIndexSchema.get(i).getName().equalsIgnoreCase(modColumn.getName())) {
modColIndex = i;
break;
}
if (!columnName.equalsIgnoreCase(modColumn.getName())) {
continue;
}

modColIndex = i;
otherCol = new Column(modColumn);
otherCol.setName(col.getName());
otherCol.setDefineExpr(col.getDefineExpr());
Preconditions.checkState(modColIndex != -1);
Preconditions.checkState(otherCol != null);
// replace the old column
Column oldCol = otherIndexSchema.get(modColIndex);
Column otherCol = new Column(modColumn);
otherCol.setIsKey(oldCol.isKey());
if (null != oldCol.getAggregationType()) {
if (KeysType.AGG_KEYS != olapTable.getKeysType()
&& KeysType.UNIQUE_KEYS != olapTable.getKeysType()) {
Column oldCol = otherIndexSchema.get(modColIndex);
otherCol.setIsKey(oldCol.isKey());
otherCol.setAggregationType(oldCol.getAggregationType(), oldCol.isAggregationTypeImplicit());
} else {
otherCol.setAggregationType(null, oldCol.isAggregationTypeImplicit());
}
if (typeChanged && !lightSchemaChange) {
otherCol.setName(SHADOW_NAME_PREFIX + otherCol.getName());
}
otherIndexSchema.set(modColIndex, otherCol);
}
}
} // end for other indices
} // end for handling other indices

if (typeChanged && !lightSchemaChange) {
Expand Down
48 changes: 24 additions & 24 deletions fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,30 +430,6 @@ protected void runWaitingTxnJob() throws AlterCancelException {
}

Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE);

Map<String, Expr> defineExprs = Maps.newHashMap();
List<Column> fullSchema = tbl.getBaseSchema(true);
DescriptorTable descTable = new DescriptorTable();
TupleDescriptor destTupleDesc = descTable.createTupleDescriptor();
for (Column column : fullSchema) {
SlotDescriptor destSlotDesc = descTable.addSlotDescriptor(destTupleDesc);
destSlotDesc.setIsMaterialized(true);
destSlotDesc.setColumn(column);
destSlotDesc.setIsNullable(column.isAllowNull());

if (indexColumnMap.containsKey(SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName())) {
Column newColumn = indexColumnMap.get(SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName());
if (newColumn.getType() != column.getType()) {
try {
SlotRef slot = new SlotRef(destSlotDesc);
slot.setCol(column.getName());
defineExprs.put(column.getName(), slot.castTo(newColumn.getType()));
} catch (AnalysisException e) {
throw new AlterCancelException(e.getMessage());
}
}
}
}
for (long partitionId : partitionIndexMap.rowKeySet()) {
Partition partition = tbl.getPartition(partitionId);
Preconditions.checkNotNull(partition, partitionId);
Expand All @@ -467,6 +443,30 @@ protected void runWaitingTxnJob() throws AlterCancelException {
long shadowIdxId = entry.getKey();
MaterializedIndex shadowIdx = entry.getValue();
long originIdxId = indexIdMap.get(shadowIdxId);
Map<String, Expr> defineExprs = Maps.newHashMap();
List<Column> fullSchema = tbl.getSchemaByIndexId(originIdxId, true);
DescriptorTable descTable = new DescriptorTable();
TupleDescriptor destTupleDesc = descTable.createTupleDescriptor();
for (Column column : fullSchema) {
SlotDescriptor destSlotDesc = descTable.addSlotDescriptor(destTupleDesc);
destSlotDesc.setIsMaterialized(true);
destSlotDesc.setColumn(column);
destSlotDesc.setIsNullable(column.isAllowNull());

if (indexColumnMap.containsKey(SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName())) {
Column newColumn = indexColumnMap.get(
SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName());
if (newColumn.getType() != column.getType()) {
try {
SlotRef slot = new SlotRef(destSlotDesc);
slot.setCol(column.getName());
defineExprs.put(column.getName(), slot.castTo(newColumn.getType()));
} catch (AnalysisException e) {
throw new AlterCancelException(e.getMessage());
}
}
}
}
int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash;
int originSchemaHash = tbl.getSchemaHashByIndexId(indexIdMap.get(shadowIdxId));
List<Column> originSchemaColumns = tbl.getSchemaByIndexId(originIdxId, true);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -671,7 +671,7 @@ public static String mvColumnBuilder(Optional<String> functionName, String sourc

public static String mvColumnBreaker(String name) {
if (name.startsWith(MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
// mva_SUM__k2 -> k2
// mva_SUM__`k2` -> `k2`;
return mvColumnBreaker(name.substring(name.indexOf(MATERIALIZED_VIEW_AGGREGATE_NAME_LINK)
+ MATERIALIZED_VIEW_AGGREGATE_NAME_LINK.length()));
} else if (name.startsWith(MATERIALIZED_VIEW_NAME_PREFIX)) {
Expand Down
103 changes: 103 additions & 0 deletions regression-test/data/schema_change_p0/schema_change_modify_mv_column_type.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
tbl_scalar_types_dup DUP_KEYS k1 BIGINT BIGINT Yes true \N true
c_bool BOOLEAN BOOLEAN Yes false \N NONE true
c_tinyint TINYINT TINYINT Yes false \N NONE true
c_smallint SMALLINT SMALLINT Yes false \N NONE true
c_int INT INT Yes false \N NONE true
c_bigint BIGINT BIGINT Yes false \N NONE true
c_largeint LARGEINT LARGEINT Yes false \N NONE true
c_float FLOAT FLOAT Yes false \N NONE true
c_double DOUBLE DOUBLE Yes false \N NONE true
c_decimal DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true
c_decimalv3 DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true
c_date DATE DATEV2 Yes false \N NONE true
c_datetime DATETIME DATETIMEV2(0) Yes false \N NONE true
c_datev2 DATE DATEV2 Yes false \N NONE true
c_datetimev2 DATETIME DATETIMEV2(0) Yes false \N NONE true
c_char CHAR(15) CHAR(15) Yes false \N NONE true
c_varchar VARCHAR(100) VARCHAR(100) Yes false \N NONE true
c_string TEXT TEXT Yes false \N NONE true

mv_tbl_scalar_types_dup_1 DUP_KEYS mv_c_tinyint TINYINT TINYINT Yes true \N true `c_tinyint`
mv_c_bool BOOLEAN BOOLEAN Yes true \N true `c_bool`
mv_k1 BIGINT BIGINT Yes true \N true `k1`
mv_c_smallint SMALLINT SMALLINT Yes false \N NONE true `c_smallint`
mv_c_int INT INT Yes false \N NONE true `c_int`
mv_c_bigint BIGINT BIGINT Yes false \N NONE true `c_bigint`
mv_c_largeint LARGEINT LARGEINT Yes false \N NONE true `c_largeint`
mv_c_float FLOAT FLOAT Yes false \N NONE true `c_float`
mv_c_double DOUBLE DOUBLE Yes false \N NONE true `c_double`
mv_c_decimal DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true `c_decimal`
mv_c_decimalv3 DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true `c_decimalv3`
mv_c_date DATE DATEV2 Yes false \N NONE true `c_date`
mv_c_datetime DATETIME DATETIMEV2(0) Yes false \N NONE true `c_datetime`
mv_c_datev2 DATE DATEV2 Yes false \N NONE true `c_datev2`
mv_c_datetimev2 DATETIME DATETIMEV2(0) Yes false \N NONE true `c_datetimev2`
mv_c_char CHARACTER CHARACTER Yes false \N NONE true `c_char`
mv_c_varchar VARCHAR(65533) VARCHAR(65533) Yes false \N NONE true `c_varchar`
mv_c_string TEXT TEXT Yes false \N NONE true `c_string`

-- !sql --
-2147475406 true 45 23794 -11023 915989078 2115356192 15927.068 1.392557423391501E9 45951348783208518.810 8340516346665031.310 2022-01-26 2022-04-13T11:13:48 2022-01-31 2022-02-16T06:07:21 130.50.6.0 [email protected] Londonderry Alley 61
-2147424303 false -28 -5177 -1409 149417728 553396597 -10123.558 -1.268722910924068E9 67354830622005524.848 52407243294991364.348 2022-06-29 2022-05-06T09:30:02 2023-01-09 2022-03-12T14:26 109.50.92.119 [email protected] Heath Drive 38
-2147413967 true -75 30533 -5435 -727385447 32929830 9577.564 1.334766997510087E9 39973144022098028.800 5886463393340733.108 2022-06-23 2022-05-10T19:13:50 2022-01-17 2022-11-26T22:49:36 157.38.90.25 [email protected] Loeprich Crossing 43
-2147380173 true -79 -5785 9752 1851350218 1121852298 25652.402 -1.618061059513558E9 95821873014545736.897 38923569966532828.626 2022-10-30 2022-05-02T17:06:33 2022-08-11 2022-02-08T10:19:47 217.198.98.239 [email protected] Lawn Lane 78
-2147374459 false -118 -30267 -14606 262497842 -1811881704 8211.805 2.37851933046663E8 37354136531251060.755 63024710145324035.720 2022-10-11 2022-01-17T10:20:18 2022-04-12 2022-10-24T18:14:38 16.243.195.81 [email protected] Annamark Pass 72
-2147369329 false -121 -22859 4733 -378861997 385323541 -22969.846 1.483825622420542E9 50940877800950041.950 87108729227937387.294 2022-06-05 2022-08-18T05:39:56 2022-08-21 2022-12-12T08:43:59 16.27.107.167 [email protected] Village Green Terrace 55
-2147367329 true 84 21471 -29331 1823545950 1200800855 -13832.219 8.01505090724918E8 45495296019797580.477 45196001436348967.557 2022-02-17 2022-05-23T01:44:31 2022-08-01 2022-08-16T10:32:36 84.110.209.128 [email protected] Packers Street 34
-2147339287 true 62 28989 -32018 650184880 -365849435 -21644.414 -7.8648426469503E7 92593387160450273.870 39588697152489527.185 2022-07-23 2023-01-03T11:54:35 2022-08-02 2022-05-19T18:35:36 30.194.6.115 [email protected] Basil Street 79
-2147336695 false 42 -7202 27919 1898713395 1177326785 -302.0104 -1.268944460183375E9 61604656210729534.717 6683002058708470.832 2022-08-20 2022-08-14T01:41:12 2022-11-02 2022-05-15T04:22:07 36.86.77.214 [email protected] Briar Crest Crossing 37
-2147330925 false -122 -21211 -2331 1906695924 -1342280417 5545.3013 -1.286038914681617E9 31911132334645267.930 84364209624711210.131 2022-02-16 2022-03-11T12:05:33 2022-11-24 2022-12-17T19:56:16 6.87.14.74 [email protected] Forest Run Terrace 13

-- !sql --
-2145739104 true 10 -22603 6132 -984517723 138439036 8683.904 1.681202635040786E9 49683339998558535.395 38251259739648714.297 2022-04-26 2022-09-12T00:32:18 2022-11-20 2023-01-09T16:19:06 180.215.212.86 [email protected] Darwin Center 26
-2140012242 false 10 30893 -16192 -175522451 -1382546546 21324.643 2.017216342012696E9 41477187479096470.647 25445001389089818.791 2022-11-06 2022-09-02T12:04:05 2022-05-29 2022-02-04T22:21:46 24.25.69.81 [email protected] Jay Way 9
-2130269306 false 10 30342 -18732 1461226453 -1257020753 -10751.815 3.44246067782915E8 2456538047280540.838 37394928326629689.946 2022-11-28 2022-05-04T20:40:19 2022-08-25 2022-03-18T10:17:35 179.198.200.96 [email protected] Tennyson Street 83
-2122709724 true 10 -8985 -30620 -1375603501 631094037 14711.055 -1.210030062083139E9 96220820029888063.156 42161382030214480.728 2022-05-28 2023-01-03T20:44:27 2022-06-11 2022-07-26T22:49:22 13.249.135.222 [email protected] Riverside Parkway 72
-2117749737 false 10 26335 30644 1841596444 283308539 18848.148 3.5339747538014E8 11924963560520504.166 28287350935413049.601 2022-08-01 2022-04-21T02:28:54 2022-02-27 2022-09-02T17:11:17 183.108.102.1 [email protected] Maple Wood Street 40
-2113239713 false 10 27624 31311 711781944 -1838033894 -12299.482 -1.88263132184351E9 9480201396831049.605 52114965946122870.302 2022-06-11 2022-08-31T08:54:30 2022-03-26 2023-01-08T23:28:27 200.161.156.176 [email protected] Westport Drive 82
-2107773486 false 10 27096 10368 1579374450 1370327646 -15339.031 2.110010890135424E9 54514853031265543.378 38546969634312019.180 2022-12-31 2022-10-07T10:18:27 2022-10-01 2022-07-09T11:41:11 121.120.227.53 [email protected] Sugar Crossing 43
-2107242025 true 10 25215 26566 1292568651 -2126795906 11912.074 -2.140044503516609E9 98695561934257164.368 18845397264645075.775 2022-05-21 2022-09-24T23:00:21 2022-02-12 2022-11-24T19:17:03 141.226.90.50 [email protected] Cody Street 78
-2106969609 true 10 29572 16738 1736115820 -957295886 -13319.206 -1.333603562816737E9 91224478600376111.942 69457425159617037.453 2022-09-06 2022-05-08T19:52:36 2022-04-05 2022-08-17T19:23:31 222.79.139.99 [email protected] Oxford Alley 77
-2102307005 true 10 -23674 24613 -1810828490 -47095409 -14686.167 2.072108685694799E9 39847820962230526.125 584354832299375.156 2022-03-27 2022-02-11T13:46:06 2022-12-25 2022-11-28T09:37:49 213.146.33.250 [email protected] Eagle Crest Terrace 84

-- !sql --
tbl_scalar_types_dup DUP_KEYS k1 BIGINT BIGINT Yes true \N true
c_bool BOOLEAN BOOLEAN Yes false \N NONE true
c_tinyint TINYINT TINYINT Yes false \N NONE true
c_smallint SMALLINT SMALLINT Yes false \N NONE true
c_int BIGINT BIGINT Yes false \N NONE true
c_bigint BIGINT BIGINT Yes false \N NONE true
c_largeint LARGEINT LARGEINT Yes false \N NONE true
c_float FLOAT FLOAT Yes false \N NONE true
c_double DOUBLE DOUBLE Yes false \N NONE true
c_decimal DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true
c_decimalv3 DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true
c_date DATE DATEV2 Yes false \N NONE true
c_datetime DATETIME DATETIMEV2(0) Yes false \N NONE true
c_datev2 DATE DATEV2 Yes false \N NONE true
c_datetimev2 DATETIME DATETIMEV2(0) Yes false \N NONE true
c_char CHAR(15) CHAR(15) Yes false \N NONE true
c_varchar VARCHAR(100) VARCHAR(100) Yes false \N NONE true
c_string TEXT TEXT Yes false \N NONE true

mv_tbl_scalar_types_dup_1 DUP_KEYS mv_c_tinyint TINYINT TINYINT Yes true \N true `c_tinyint`
mv_c_bool BOOLEAN BOOLEAN Yes true \N true `c_bool`
mv_k1 BIGINT BIGINT Yes true \N true `k1`
mv_c_smallint SMALLINT SMALLINT Yes false \N NONE true `c_smallint`
mv_c_int BIGINT BIGINT Yes false \N NONE true `c_int`
mv_c_bigint BIGINT BIGINT Yes false \N NONE true `c_bigint`
mv_c_largeint LARGEINT LARGEINT Yes false \N NONE true `c_largeint`
mv_c_float FLOAT FLOAT Yes false \N NONE true `c_float`
mv_c_double DOUBLE DOUBLE Yes false \N NONE true `c_double`
mv_c_decimal DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true `c_decimal`
mv_c_decimalv3 DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true `c_decimalv3`
mv_c_date DATE DATEV2 Yes false \N NONE true `c_date`
mv_c_datetime DATETIME DATETIMEV2(0) Yes false \N NONE true `c_datetime`
mv_c_datev2 DATE DATEV2 Yes false \N NONE true `c_datev2`
mv_c_datetimev2 DATETIME DATETIMEV2(0) Yes false \N NONE true `c_datetimev2`
mv_c_char CHARACTER CHARACTER Yes false \N NONE true `c_char`
mv_c_varchar VARCHAR(65533) VARCHAR(65533) Yes false \N NONE true `c_varchar`
mv_c_string TEXT TEXT Yes false \N NONE true `c_string`

Loading

0 comments on commit ffa5ac3

Please sign in to comment.