[feat](Nereids) persist constraint in table (#29767)

apache · Jan 16, 2024 · 4e41e1d · 4e41e1d
1 parent 2916745
commit 4e41e1d
Show file tree

Hide file tree

Showing 22 changed files with 683 additions and 45 deletions.
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
@@ -72,9 +72,11 @@ public final class FeMetaVersion {
     public static final int VERSION_125 = 125;
     // For write/read function nullable mode info
     public static final int VERSION_126 = 126;
+    // For constraints
+    public static final int VERSION_127 = 127;
 
     // note: when increment meta version, should assign the latest version to VERSION_CURRENT
-    public static final int VERSION_CURRENT = VERSION_126;
+    public static final int VERSION_CURRENT = VERSION_127;
 
     // all logs meta version should >= the minimum version, so that we could remove many if clause, for example
     // if (FE_METAVERSION < VERSION_94) ...

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -22,12 +22,14 @@
 import org.apache.doris.common.Config;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.FeMetaVersion;
 import org.apache.doris.common.MetaNotFoundException;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.common.util.QueryableReentrantReadWriteLock;
 import org.apache.doris.common.util.SqlUtils;
 import org.apache.doris.common.util.Util;
+import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.statistics.AnalysisInfo;
 import org.apache.doris.statistics.BaseAnalysisTask;
 import org.apache.doris.statistics.ColumnStatistic;
@@ -50,7 +52,6 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -116,8 +117,8 @@ public abstract class Table extends MetaObject implements Writable, TableIf {
     @SerializedName(value = "comment")
     protected String comment = "";
 
-    @SerializedName(value = "constraints")
-    private HashMap<String, Constraint> constraintsMap = new HashMap<>();
+    @SerializedName(value = "ta")
+    private TableAttributes tableAttributes = new TableAttributes();
 
     // check read lock leaky
     private Map<Long, String> readLockThreads = null;
@@ -334,12 +335,12 @@ public String getQualifiedName() {
     }
 
     public Constraint getConstraint(String name) {
-        return constraintsMap.get(name);
+        return getConstraintsMap().get(name);
     }
 
     @Override
     public Map<String, Constraint> getConstraintsMapUnsafe() {
-        return constraintsMap;
+        return tableAttributes.getConstraintsMap();
     }
 
     public TableType getType() {
@@ -455,9 +456,9 @@ public void write(DataOutput out) throws IOException {
         for (Column column : fullSchema) {
             column.write(out);
         }
-
         Text.writeString(out, comment);
-
+        // write table attributes
+        Text.writeString(out, GsonUtils.GSON.toJson(tableAttributes));
         // write create time
         out.writeLong(createTime);
     }
@@ -488,7 +489,12 @@ public void readFields(DataInput in) throws IOException {
             hasCompoundKey = true;
         }
         comment = Text.readString(in);
+        // table attribute only support after version 127
+        if (FeMetaVersion.VERSION_127 <= Env.getCurrentEnvJournalVersion()) {
+            String json = Text.readString(in);
+            this.tableAttributes = GsonUtils.GSON.fromJson(json, TableAttributes.class);
 
+        }
         // read create time
         this.createTime = in.readLong();
     }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableAttributes.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableAttributes.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.catalog.constraint.Constraint;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * TableAttributes contains additional information about all table
+ */
+public class TableAttributes implements Writable {
+    @SerializedName(value = "constraints")
+    private final Map<String, Constraint> constraintsMap = new HashMap<>();
+
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+        Text.writeString(out, GsonUtils.GSON.toJson(this));
+    }
+
+    public TableAttributes read(DataInput in)  throws IOException {
+        return GsonUtils.GSON.fromJson(Text.readString(in), TableAttributes.class);
+    }
+
+    public Map<String, Constraint> getConstraintsMap() {
+        return constraintsMap;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -183,7 +183,7 @@ default Set<ForeignKeyConstraint> getForeignKeyConstraints() {
         }
     }
 
-    default Map<String, Constraint> getConstraintMap() {
+    default Map<String, Constraint> getConstraintsMap() {
         readLock();
         try {
             return ImmutableMap.copyOf(getConstraintsMapUnsafe());
@@ -236,25 +236,27 @@ default void checkConstraintNotExistence(String name, Constraint primaryKeyConst
         }
     }
 
-    default void addUniqueConstraint(String name, ImmutableList<String> columns) {
+    default Constraint addUniqueConstraint(String name, ImmutableList<String> columns) {
         writeLock();
         try {
             Map<String, Constraint> constraintMap = getConstraintsMapUnsafe();
             UniqueConstraint uniqueConstraint =  new UniqueConstraint(name, ImmutableSet.copyOf(columns));
             checkConstraintNotExistence(name, uniqueConstraint, constraintMap);
             constraintMap.put(name, uniqueConstraint);
+            return uniqueConstraint;
         } finally {
             writeUnlock();
         }
     }
 
-    default void addPrimaryKeyConstraint(String name, ImmutableList<String> columns) {
+    default Constraint addPrimaryKeyConstraint(String name, ImmutableList<String> columns) {
         writeLock();
         try {
             Map<String, Constraint> constraintMap = getConstraintsMapUnsafe();
             PrimaryKeyConstraint primaryKeyConstraint = new PrimaryKeyConstraint(name, ImmutableSet.copyOf(columns));
             checkConstraintNotExistence(name, primaryKeyConstraint, constraintMap);
             constraintMap.put(name, primaryKeyConstraint);
+            return primaryKeyConstraint;
         } finally {
             writeUnlock();
         }
@@ -277,7 +279,7 @@ default void updatePrimaryKeyForForeignKey(PrimaryKeyConstraint requirePrimaryKe
         }
     }
 
-    default void addForeignConstraint(String name, ImmutableList<String> columns,
+    default Constraint addForeignConstraint(String name, ImmutableList<String> columns,
             TableIf referencedTable, ImmutableList<String> referencedColumns) {
         writeLock();
         try {
@@ -289,12 +291,32 @@ default void addForeignConstraint(String name, ImmutableList<String> columns,
                     foreignKeyConstraint.getReferencedColumnNames());
             updatePrimaryKeyForForeignKey(requirePrimaryKey, referencedTable);
             constraintMap.put(name, foreignKeyConstraint);
+            return foreignKeyConstraint;
         } finally {
             writeUnlock();
         }
     }
 
-    default void dropConstraint(String name) {
+    default void replayAddConstraint(Constraint constraint) {
+        if (constraint instanceof UniqueConstraint) {
+            UniqueConstraint uniqueConstraint = (UniqueConstraint) constraint;
+            this.addUniqueConstraint(constraint.getName(),
+                    ImmutableList.copyOf(uniqueConstraint.getUniqueColumnNames()));
+        } else if (constraint instanceof PrimaryKeyConstraint) {
+            PrimaryKeyConstraint primaryKeyConstraint = (PrimaryKeyConstraint) constraint;
+            this.addPrimaryKeyConstraint(primaryKeyConstraint.getName(),
+                    ImmutableList.copyOf(primaryKeyConstraint.getPrimaryKeyNames()));
+        } else if (constraint instanceof ForeignKeyConstraint) {
+            ForeignKeyConstraint foreignKey = (ForeignKeyConstraint) constraint;
+            this.addForeignConstraint(foreignKey.getName(),
+                    ImmutableList.copyOf(foreignKey.getForeignKeyNames()),
+                    foreignKey.getReferencedTable(),
+                    ImmutableList.copyOf(foreignKey.getReferencedColumnNames()));
+        }
+    }
+
+    default Constraint dropConstraint(String name) {
+        Constraint dropConstraint;
         writeLock();
         try {
             Map<String, Constraint> constraintMap = getConstraintsMapUnsafe();
@@ -308,9 +330,11 @@ default void dropConstraint(String name) {
                 ((PrimaryKeyConstraint) constraint).getForeignTables()
                         .forEach(t -> t.dropFKReferringPK(this, (PrimaryKeyConstraint) constraint));
             }
+            dropConstraint = constraint;
         } finally {
             writeUnlock();
         }
+        return dropConstraint;
     }
 
     default void dropFKReferringPK(TableIf table, PrimaryKeyConstraint constraint) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/Constraint.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/Constraint.java
@@ -17,15 +17,25 @@
 
 package org.apache.doris.catalog.constraint;
 
-public abstract class Constraint {
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public abstract class Constraint implements Writable {
     public enum ConstraintType {
         FOREIGN_KEY("FOREIGN KEY"),
         PRIMARY_KEY("PRIMARY KEY"),
         UNIQUE("UNIQUE");
-
+        @SerializedName(value = "tn")
         private final String name;
 
-        private ConstraintType(String stringValue) {
+        ConstraintType(String stringValue) {
             this.name = stringValue;
         }
 
@@ -34,7 +44,9 @@ public String getName() {
         }
     }
 
+    @SerializedName(value = "n")
     private final String name;
+    @SerializedName(value = "ty")
     private final ConstraintType type;
 
 
@@ -50,4 +62,17 @@ public String getName() {
     public ConstraintType getType() {
         return type;
     }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+        Text.writeString(out, GsonUtils.GSON.toJson(this));
+    }
+
+    /**
+     * Read Constraint.
+     **/
+    public static Constraint read(DataInput in) throws IOException {
+        String json = Text.readString(in);
+        return GsonUtils.GSON.fromJson(json, Constraint.class);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/ForeignKeyConstraint.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/ForeignKeyConstraint.java
@@ -24,13 +24,17 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMap.Builder;
 import com.google.common.collect.ImmutableSet;
+import com.google.gson.annotations.SerializedName;
 
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 public class ForeignKeyConstraint extends Constraint {
-    private final ImmutableMap<String, String> foreignToReference;
+    @SerializedName(value = "ftr")
+    private final Map<String, String> foreignToReference;
+    @SerializedName(value = "rt")
     private final TableIdentifier referencedTable;
 
     public ForeignKeyConstraint(String name, List<String> columns,
@@ -50,19 +54,19 @@ public ForeignKeyConstraint(String name, List<String> columns,
         this.foreignToReference = builder.build();
     }
 
-    public ImmutableSet<String> getForeignKeyNames() {
+    public Set<String> getForeignKeyNames() {
         return foreignToReference.keySet();
     }
 
-    public ImmutableSet<String> getReferencedColumnNames() {
+    public Set<String> getReferencedColumnNames() {
         return ImmutableSet.copyOf(foreignToReference.values());
     }
 
     public String getReferencedColumnName(String column) {
         return foreignToReference.get(column);
     }
 
-    public ImmutableMap<String, String> getForeignToReference() {
+    public Map<String, String> getForeignToReference() {
         return foreignToReference;
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/PrimaryKeyConstraint.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/PrimaryKeyConstraint.java
@@ -23,27 +23,30 @@
 import com.google.common.base.Objects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.gson.annotations.SerializedName;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 public class PrimaryKeyConstraint extends Constraint {
-    private final ImmutableSet<String> columns;
+    @SerializedName(value = "cols")
+    private final Set<String> columns;
 
     // record the foreign table which references the primary key
+    @SerializedName(value = "ft")
     private final Set<TableIdentifier> foreignTables = new HashSet<>();
 
     public PrimaryKeyConstraint(String name, Set<String> columns) {
         super(ConstraintType.PRIMARY_KEY, name);
         this.columns = ImmutableSet.copyOf(columns);
     }
 
-    public ImmutableSet<String> getPrimaryKeyNames() {
+    public Set<String> getPrimaryKeyNames() {
         return columns;
     }
 
-    public ImmutableSet<Column> getPrimaryKeys(TableIf table) {
+    public Set<Column> getPrimaryKeys(TableIf table) {
         return columns.stream().map(table::getColumn).collect(ImmutableSet.toImmutableSet());
     }