Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Bugfix] Fix mv column type is not changed when do schema change #37596

Merged
merged 1 commit into from
Jul 18, 2024
Merged

[Bugfix] Fix mv column type is not changed when do schema change #37596

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 31 additions & 32 deletions fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.doris.analysis.CancelStmt;
import org.apache.doris.analysis.ColumnPosition;
import org.apache.doris.analysis.CreateIndexClause;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.DropColumnClause;
import org.apache.doris.analysis.DropIndexClause;
import org.apache.doris.analysis.IndexDef;
Expand Down Expand Up @@ -671,52 +672,50 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol
}
List<Column> schema = entry.getValue();
for (Column column : schema) {
if (column.getName().equalsIgnoreCase(modColumn.getName())) {
String columnName = column.getName();
if (column.isMaterializedViewColumn()) {
columnName = MaterializedIndexMeta.normalizeName(
CreateMaterializedViewStmt.mvColumnBreaker(columnName));
}
if (columnName.equalsIgnoreCase(modColumn.getName())) {
otherIndexIds.add(entry.getKey());
break;
}
}
}

if (KeysType.AGG_KEYS == olapTable.getKeysType() || KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
for (Long otherIndexId : otherIndexIds) {
List<Column> otherIndexSchema = indexSchemaMap.get(otherIndexId);
for (Long otherIndexId : otherIndexIds) {
List<Column> otherIndexSchema = indexSchemaMap.get(otherIndexId);
for (int i = 0; i < otherIndexSchema.size(); i++) {
modColIndex = -1;
for (int i = 0; i < otherIndexSchema.size(); i++) {
if (otherIndexSchema.get(i).getName().equalsIgnoreCase(modColumn.getName())) {
modColIndex = i;
break;
}
Column otherCol = null;
Column col = otherIndexSchema.get(i);
String columnName = col.getName();
if (col.isMaterializedViewColumn()) {
columnName = MaterializedIndexMeta.normalizeName(
CreateMaterializedViewStmt.mvColumnBreaker(columnName));
}
Preconditions.checkState(modColIndex != -1);
// replace the old column
otherIndexSchema.set(modColIndex, modColumn);
} // end for other indices
} else {
// DUPLICATE data model has a little
for (Long otherIndexId : otherIndexIds) {
List<Column> otherIndexSchema = indexSchemaMap.get(otherIndexId);
modColIndex = -1;
for (int i = 0; i < otherIndexSchema.size(); i++) {
if (otherIndexSchema.get(i).getName().equalsIgnoreCase(modColumn.getName())) {
modColIndex = i;
break;
}
if (!columnName.equalsIgnoreCase(modColumn.getName())) {
continue;
}

modColIndex = i;
otherCol = new Column(modColumn);
otherCol.setName(col.getName());
otherCol.setDefineExpr(col.getDefineExpr());
Preconditions.checkState(modColIndex != -1);
Preconditions.checkState(otherCol != null);
// replace the old column
Column oldCol = otherIndexSchema.get(modColIndex);
Column otherCol = new Column(modColumn);
otherCol.setIsKey(oldCol.isKey());
if (null != oldCol.getAggregationType()) {
if (KeysType.AGG_KEYS != olapTable.getKeysType()
&& KeysType.UNIQUE_KEYS != olapTable.getKeysType()) {
Column oldCol = otherIndexSchema.get(modColIndex);
otherCol.setIsKey(oldCol.isKey());
otherCol.setAggregationType(oldCol.getAggregationType(), oldCol.isAggregationTypeImplicit());
} else {
otherCol.setAggregationType(null, oldCol.isAggregationTypeImplicit());
}
if (typeChanged && !lightSchemaChange) {
otherCol.setName(SHADOW_NAME_PREFIX + otherCol.getName());
}
otherIndexSchema.set(modColIndex, otherCol);
}
}
} // end for other indices
} // end for handling other indices

if (typeChanged && !lightSchemaChange) {
Expand Down
47 changes: 24 additions & 23 deletions fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,29 +424,6 @@ protected void runWaitingTxnJob() throws AlterCancelException {

Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE);

Map<String, Expr> defineExprs = Maps.newHashMap();
List<Column> fullSchema = tbl.getBaseSchema(true);
DescriptorTable descTable = new DescriptorTable();
TupleDescriptor destTupleDesc = descTable.createTupleDescriptor();
for (Column column : fullSchema) {
SlotDescriptor destSlotDesc = descTable.addSlotDescriptor(destTupleDesc);
destSlotDesc.setIsMaterialized(true);
destSlotDesc.setColumn(column);
destSlotDesc.setIsNullable(column.isAllowNull());

if (indexColumnMap.containsKey(SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName())) {
Column newColumn = indexColumnMap.get(SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName());
if (!newColumn.getType().equals(column.getType())) {
try {
SlotRef slot = new SlotRef(destSlotDesc);
slot.setCol(column.getName());
defineExprs.put(column.getName(), slot.castTo(newColumn.getType()));
} catch (AnalysisException e) {
throw new AlterCancelException(e.getMessage());
}
}
}
}
for (long partitionId : partitionIndexMap.rowKeySet()) {
Partition partition = tbl.getPartition(partitionId);
Preconditions.checkNotNull(partition, partitionId);
Expand All @@ -460,6 +437,30 @@ protected void runWaitingTxnJob() throws AlterCancelException {
long shadowIdxId = entry.getKey();
MaterializedIndex shadowIdx = entry.getValue();
long originIdxId = indexIdMap.get(shadowIdxId);
Map<String, Expr> defineExprs = Maps.newHashMap();
List<Column> fullSchema = tbl.getSchemaByIndexId(originIdxId, true);
DescriptorTable descTable = new DescriptorTable();
TupleDescriptor destTupleDesc = descTable.createTupleDescriptor();
for (Column column : fullSchema) {
SlotDescriptor destSlotDesc = descTable.addSlotDescriptor(destTupleDesc);
destSlotDesc.setIsMaterialized(true);
destSlotDesc.setColumn(column);
destSlotDesc.setIsNullable(column.isAllowNull());

if (indexColumnMap.containsKey(SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName())) {
Column newColumn = indexColumnMap.get(
SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName());
if (!newColumn.getType().equals(column.getType())) {
try {
SlotRef slot = new SlotRef(destSlotDesc);
slot.setCol(column.getName());
defineExprs.put(column.getName(), slot.castTo(newColumn.getType()));
} catch (AnalysisException e) {
throw new AlterCancelException(e.getMessage());
}
}
}
}
int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash;
int originSchemaHash = tbl.getSchemaHashByIndexId(indexIdMap.get(shadowIdxId));
List<Column> originSchemaColumns = tbl.getSchemaByIndexId(originIdxId, true);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -654,7 +654,7 @@ public static String mvColumnBuilder(Optional<String> functionName, String sourc

public static String mvColumnBreaker(String name) {
if (name.startsWith(MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
// mva_SUM__k2 -> k2
// mva_SUM__`k2` -> `k2`;
return mvColumnBreaker(name.substring(name.indexOf(MATERIALIZED_VIEW_AGGREGATE_NAME_LINK)
+ MATERIALIZED_VIEW_AGGREGATE_NAME_LINK.length()));
} else if (name.startsWith(MATERIALIZED_VIEW_NAME_PREFIX)) {
Expand Down
103 changes: 103 additions & 0 deletions regression-test/data/schema_change_p0/schema_change_modify_mv_column_type.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
tbl_scalar_types_dup DUP_KEYS k1 BIGINT BIGINT Yes true \N true
c_bool BOOLEAN BOOLEAN Yes false \N NONE true
c_tinyint TINYINT TINYINT Yes false \N NONE true
c_smallint SMALLINT SMALLINT Yes false \N NONE true
c_int INT INT Yes false \N NONE true
c_bigint BIGINT BIGINT Yes false \N NONE true
c_largeint LARGEINT LARGEINT Yes false \N NONE true
c_float FLOAT FLOAT Yes false \N NONE true
c_double DOUBLE DOUBLE Yes false \N NONE true
c_decimal DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true
c_decimalv3 DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true
c_date DATE DATEV2 Yes false \N NONE true
c_datetime DATETIME DATETIMEV2(0) Yes false \N NONE true
c_datev2 DATE DATEV2 Yes false \N NONE true
c_datetimev2 DATETIME DATETIMEV2(0) Yes false \N NONE true
c_char CHAR(15) CHAR(15) Yes false \N NONE true
c_varchar VARCHAR(100) VARCHAR(100) Yes false \N NONE true
c_string TEXT TEXT Yes false \N NONE true

mv_tbl_scalar_types_dup_1 DUP_KEYS mv_c_tinyint TINYINT TINYINT Yes true \N true `c_tinyint`
mv_c_bool BOOLEAN BOOLEAN Yes true \N true `c_bool`
mv_k1 BIGINT BIGINT Yes true \N true `k1`
mv_c_smallint SMALLINT SMALLINT Yes false \N NONE true `c_smallint`
mv_c_int INT INT Yes false \N NONE true `c_int`
mv_c_bigint BIGINT BIGINT Yes false \N NONE true `c_bigint`
mv_c_largeint LARGEINT LARGEINT Yes false \N NONE true `c_largeint`
mv_c_float FLOAT FLOAT Yes false \N NONE true `c_float`
mv_c_double DOUBLE DOUBLE Yes false \N NONE true `c_double`
mv_c_decimal DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true `c_decimal`
mv_c_decimalv3 DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true `c_decimalv3`
mv_c_date DATE DATEV2 Yes false \N NONE true `c_date`
mv_c_datetime DATETIME DATETIMEV2(0) Yes false \N NONE true `c_datetime`
mv_c_datev2 DATE DATEV2 Yes false \N NONE true `c_datev2`
mv_c_datetimev2 DATETIME DATETIMEV2(0) Yes false \N NONE true `c_datetimev2`
mv_c_char CHARACTER CHARACTER Yes false \N NONE true `c_char`
mv_c_varchar VARCHAR(65533) VARCHAR(65533) Yes false \N NONE true `c_varchar`
mv_c_string TEXT TEXT Yes false \N NONE true `c_string`

-- !sql --
-2147475406 true 45 23794 -11023 915989078 2115356192 15927.068 1.392557423391501E9 45951348783208518.810 8340516346665031.310 2022-01-26 2022-04-13T11:13:48 2022-01-31 2022-02-16T06:07:21 130.50.6.0 [email protected] Londonderry Alley 61
-2147424303 false -28 -5177 -1409 149417728 553396597 -10123.558 -1.268722910924068E9 67354830622005524.848 52407243294991364.348 2022-06-29 2022-05-06T09:30:02 2023-01-09 2022-03-12T14:26 109.50.92.119 [email protected] Heath Drive 38
-2147413967 true -75 30533 -5435 -727385447 32929830 9577.564 1.334766997510087E9 39973144022098028.800 5886463393340733.108 2022-06-23 2022-05-10T19:13:50 2022-01-17 2022-11-26T22:49:36 157.38.90.25 [email protected] Loeprich Crossing 43
-2147380173 true -79 -5785 9752 1851350218 1121852298 25652.402 -1.618061059513558E9 95821873014545736.897 38923569966532828.626 2022-10-30 2022-05-02T17:06:33 2022-08-11 2022-02-08T10:19:47 217.198.98.239 [email protected] Lawn Lane 78
-2147374459 false -118 -30267 -14606 262497842 -1811881704 8211.805 2.37851933046663E8 37354136531251060.755 63024710145324035.720 2022-10-11 2022-01-17T10:20:18 2022-04-12 2022-10-24T18:14:38 16.243.195.81 [email protected] Annamark Pass 72
-2147369329 false -121 -22859 4733 -378861997 385323541 -22969.846 1.483825622420542E9 50940877800950041.950 87108729227937387.294 2022-06-05 2022-08-18T05:39:56 2022-08-21 2022-12-12T08:43:59 16.27.107.167 [email protected] Village Green Terrace 55
-2147367329 true 84 21471 -29331 1823545950 1200800855 -13832.219 8.01505090724918E8 45495296019797580.477 45196001436348967.557 2022-02-17 2022-05-23T01:44:31 2022-08-01 2022-08-16T10:32:36 84.110.209.128 [email protected] Packers Street 34
-2147339287 true 62 28989 -32018 650184880 -365849435 -21644.414 -7.8648426469503E7 92593387160450273.870 39588697152489527.185 2022-07-23 2023-01-03T11:54:35 2022-08-02 2022-05-19T18:35:36 30.194.6.115 [email protected] Basil Street 79
-2147336695 false 42 -7202 27919 1898713395 1177326785 -302.0104 -1.268944460183375E9 61604656210729534.717 6683002058708470.832 2022-08-20 2022-08-14T01:41:12 2022-11-02 2022-05-15T04:22:07 36.86.77.214 [email protected] Briar Crest Crossing 37
-2147330925 false -122 -21211 -2331 1906695924 -1342280417 5545.3013 -1.286038914681617E9 31911132334645267.930 84364209624711210.131 2022-02-16 2022-03-11T12:05:33 2022-11-24 2022-12-17T19:56:16 6.87.14.74 [email protected] Forest Run Terrace 13

-- !sql --
-2145739104 true 10 -22603 6132 -984517723 138439036 8683.904 1.681202635040786E9 49683339998558535.395 38251259739648714.297 2022-04-26 2022-09-12T00:32:18 2022-11-20 2023-01-09T16:19:06 180.215.212.86 [email protected] Darwin Center 26
-2140012242 false 10 30893 -16192 -175522451 -1382546546 21324.643 2.017216342012696E9 41477187479096470.647 25445001389089818.791 2022-11-06 2022-09-02T12:04:05 2022-05-29 2022-02-04T22:21:46 24.25.69.81 [email protected] Jay Way 9
-2130269306 false 10 30342 -18732 1461226453 -1257020753 -10751.815 3.44246067782915E8 2456538047280540.838 37394928326629689.946 2022-11-28 2022-05-04T20:40:19 2022-08-25 2022-03-18T10:17:35 179.198.200.96 [email protected] Tennyson Street 83
-2122709724 true 10 -8985 -30620 -1375603501 631094037 14711.055 -1.210030062083139E9 96220820029888063.156 42161382030214480.728 2022-05-28 2023-01-03T20:44:27 2022-06-11 2022-07-26T22:49:22 13.249.135.222 [email protected] Riverside Parkway 72
-2117749737 false 10 26335 30644 1841596444 283308539 18848.148 3.5339747538014E8 11924963560520504.166 28287350935413049.601 2022-08-01 2022-04-21T02:28:54 2022-02-27 2022-09-02T17:11:17 183.108.102.1 [email protected] Maple Wood Street 40
-2113239713 false 10 27624 31311 711781944 -1838033894 -12299.482 -1.88263132184351E9 9480201396831049.605 52114965946122870.302 2022-06-11 2022-08-31T08:54:30 2022-03-26 2023-01-08T23:28:27 200.161.156.176 [email protected] Westport Drive 82
-2107773486 false 10 27096 10368 1579374450 1370327646 -15339.031 2.110010890135424E9 54514853031265543.378 38546969634312019.180 2022-12-31 2022-10-07T10:18:27 2022-10-01 2022-07-09T11:41:11 121.120.227.53 [email protected] Sugar Crossing 43
-2107242025 true 10 25215 26566 1292568651 -2126795906 11912.074 -2.140044503516609E9 98695561934257164.368 18845397264645075.775 2022-05-21 2022-09-24T23:00:21 2022-02-12 2022-11-24T19:17:03 141.226.90.50 [email protected] Cody Street 78
-2106969609 true 10 29572 16738 1736115820 -957295886 -13319.206 -1.333603562816737E9 91224478600376111.942 69457425159617037.453 2022-09-06 2022-05-08T19:52:36 2022-04-05 2022-08-17T19:23:31 222.79.139.99 [email protected] Oxford Alley 77
-2102307005 true 10 -23674 24613 -1810828490 -47095409 -14686.167 2.072108685694799E9 39847820962230526.125 584354832299375.156 2022-03-27 2022-02-11T13:46:06 2022-12-25 2022-11-28T09:37:49 213.146.33.250 [email protected] Eagle Crest Terrace 84

-- !sql --
tbl_scalar_types_dup DUP_KEYS k1 BIGINT BIGINT Yes true \N true
c_bool BOOLEAN BOOLEAN Yes false \N NONE true
c_tinyint TINYINT TINYINT Yes false \N NONE true
c_smallint SMALLINT SMALLINT Yes false \N NONE true
c_int BIGINT BIGINT Yes false \N NONE true
c_bigint BIGINT BIGINT Yes false \N NONE true
c_largeint LARGEINT LARGEINT Yes false \N NONE true
c_float FLOAT FLOAT Yes false \N NONE true
c_double DOUBLE DOUBLE Yes false \N NONE true
c_decimal DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true
c_decimalv3 DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true
c_date DATE DATEV2 Yes false \N NONE true
c_datetime DATETIME DATETIMEV2(0) Yes false \N NONE true
c_datev2 DATE DATEV2 Yes false \N NONE true
c_datetimev2 DATETIME DATETIMEV2(0) Yes false \N NONE true
c_char CHAR(15) CHAR(15) Yes false \N NONE true
c_varchar VARCHAR(100) VARCHAR(100) Yes false \N NONE true
c_string TEXT TEXT Yes false \N NONE true

mv_tbl_scalar_types_dup_1 DUP_KEYS mv_c_tinyint TINYINT TINYINT Yes true \N true `c_tinyint`
mv_c_bool BOOLEAN BOOLEAN Yes true \N true `c_bool`
mv_k1 BIGINT BIGINT Yes true \N true `k1`
mv_c_smallint SMALLINT SMALLINT Yes false \N NONE true `c_smallint`
mv_c_int BIGINT BIGINT Yes false \N NONE true `c_int`
mv_c_bigint BIGINT BIGINT Yes false \N NONE true `c_bigint`
mv_c_largeint LARGEINT LARGEINT Yes false \N NONE true `c_largeint`
mv_c_float FLOAT FLOAT Yes false \N NONE true `c_float`
mv_c_double DOUBLE DOUBLE Yes false \N NONE true `c_double`
mv_c_decimal DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true `c_decimal`
mv_c_decimalv3 DECIMAL(20, 3) DECIMALV3(20, 3) Yes false \N NONE true `c_decimalv3`
mv_c_date DATE DATEV2 Yes false \N NONE true `c_date`
mv_c_datetime DATETIME DATETIMEV2(0) Yes false \N NONE true `c_datetime`
mv_c_datev2 DATE DATEV2 Yes false \N NONE true `c_datev2`
mv_c_datetimev2 DATETIME DATETIMEV2(0) Yes false \N NONE true `c_datetimev2`
mv_c_char CHARACTER CHARACTER Yes false \N NONE true `c_char`
mv_c_varchar VARCHAR(65533) VARCHAR(65533) Yes false \N NONE true `c_varchar`
mv_c_string TEXT TEXT Yes false \N NONE true `c_string`

Loading
Loading