[Feature] Support paimon Datasource (DataLinkDC#3641)

Co-authored-by: gaoyan1998 <[email protected]>
aiwenmo · Jul 17, 2024 · e866501 · e866501
1 parent 778d482
commit e866501
Show file tree

Hide file tree

Showing 44 changed files with 1,673 additions and 174 deletions.
diff --git a/dinky-admin/src/main/java/org/dinky/controller/DataSourceController.java b/dinky-admin/src/main/java/org/dinky/controller/DataSourceController.java
@@ -31,6 +31,7 @@
 import org.dinky.data.model.QueryData;
 import org.dinky.data.model.Schema;
 import org.dinky.data.model.SqlGeneration;
+import org.dinky.data.model.Table;
 import org.dinky.data.result.Result;
 import org.dinky.metadata.driver.DriverPool;
 import org.dinky.metadata.result.JdbcSelectResult;
@@ -443,4 +444,53 @@ public Result<Void> copyDatabase(@RequestBody DataBaseDTO dataBaseDTO) {
             return Result.failed(Status.COPY_FAILED);
         }
     }
+
+    /**
+     * get columns of table
+     *
+     * @param id         {@link Integer}
+     * @param schemaName {@link String}
+     * @param tableName  {@link String}
+     * @return {@link Result}< {@link List}< {@link Column}>>
+     */
+    @GetMapping("/getTable")
+    @ApiOperation("Get Columns Of Table")
+    @ApiImplicitParams(
+            value = {
+                @ApiImplicitParam(
+                        name = "id",
+                        value = "DataBase Id",
+                        required = true,
+                        dataType = "Integer",
+                        paramType = "path",
+                        dataTypeClass = Integer.class,
+                        example = "1"),
+                @ApiImplicitParam(
+                        name = "schemaName",
+                        value = "Schema Name",
+                        required = true,
+                        dataType = "String",
+                        paramType = "query",
+                        dataTypeClass = String.class,
+                        example = "public"),
+                @ApiImplicitParam(
+                        name = "tableName",
+                        value = "Table Name",
+                        required = true,
+                        dataType = "String",
+                        paramType = "query",
+                        dataTypeClass = String.class,
+                        example = "user")
+            })
+    @SaCheckPermission(
+            value = {
+                PermissionConstants.REGISTRATION_DATA_SOURCE_DETAIL_REFRESH,
+                PermissionConstants.REGISTRATION_DATA_SOURCE_DETAIL_TREE,
+                PermissionConstants.REGISTRATION_DATA_SOURCE_DETAIL_DESC,
+            },
+            mode = SaMode.OR)
+    public Result<Table> getTable(
+            @RequestParam Integer id, @RequestParam String schemaName, @RequestParam String tableName) {
+        return Result.succeed(databaseService.getTable(id, schemaName, tableName));
+    }
 }
diff --git a/dinky-admin/src/main/java/org/dinky/service/DataBaseService.java b/dinky-admin/src/main/java/org/dinky/service/DataBaseService.java
@@ -27,6 +27,7 @@
 import org.dinky.data.model.QueryData;
 import org.dinky.data.model.Schema;
 import org.dinky.data.model.SqlGeneration;
+import org.dinky.data.model.Table;
 import org.dinky.data.result.SqlExplainResult;
 import org.dinky.job.JobResult;
 import org.dinky.metadata.result.JdbcSelectResult;
@@ -213,4 +214,14 @@ public interface DataBaseService extends ISuperService<DataBase> {
      * @return {@link Boolean} true: has relationship, false: no relationship
      */
     boolean hasRelationShip(Integer id);
+
+    /**
+     * get table
+     *
+     * @param id {@link Integer}
+     * @param schemaName {@link String}
+     * @param tableName {@link String}
+     * @return {@link List}< {@link Column}>
+     */
+    Table getTable(Integer id, String schemaName, String tableName);
 }
diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java
@@ -196,8 +196,7 @@ public JdbcSelectResult queryData(QueryData queryData) {
         DataBase dataBase = getById(queryData.getId());
         Asserts.checkNotNull(dataBase, Status.DATASOURCE_NOT_EXIST.getMessage());
         Driver driver = Driver.build(dataBase.getDriverConfig());
-        StringBuilder queryOption = driver.genQueryOption(queryData);
-        return driver.query(queryOption.toString(), null);
+        return driver.query(queryData);
     }
 
     @Override
@@ -389,4 +388,15 @@ public boolean hasRelationShip(Integer id) {
                 .list(new LambdaQueryWrapper<Task>().eq(Task::getAlertGroupId, id))
                 .isEmpty();
     }
+
+    @Override
+    public Table getTable(Integer id, String schemaName, String tableName) {
+        DataBase dataBase = getById(id);
+        Asserts.checkNotNull(dataBase, Status.DATASOURCE_NOT_EXIST.getMessage());
+        try (Driver driver = Driver.build(dataBase.getDriverConfig())) {
+            return driver.getTable(schemaName, tableName);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
 }
diff --git a/...org/dinky/gateway/model/CustomConfig.java → ...va/org/dinky/data/model/CustomConfig.java b/...org/dinky/gateway/model/CustomConfig.java → ...va/org/dinky/data/model/CustomConfig.java
@@ -17,7 +17,7 @@
  *
  */
 
-package org.dinky.gateway.model;
+package org.dinky.data.model;
 
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;

diff --git a/dinky-common/src/main/java/org/dinky/data/model/QueryData.java b/dinky-common/src/main/java/org/dinky/data/model/QueryData.java
@@ -81,13 +81,13 @@ public class Option {
                 dataType = "String",
                 example = "0",
                 notes = "Starting row number for the LIMIT clause")
-        private String limitStart;
+        private int limitStart;
 
         @ApiModelProperty(
                 value = "Ending row for LIMIT",
                 dataType = "String",
                 example = "10",
                 notes = "Ending row number for the LIMIT clause")
-        private String limitEnd;
+        private int limitEnd;
     }
 }
diff --git a/dinky-common/src/main/java/org/dinky/data/model/Table.java b/dinky-common/src/main/java/org/dinky/data/model/Table.java
@@ -32,16 +32,20 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
-import lombok.Setter;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
 
 /**
  * Table
  *
  * @since 2021/7/19 23:27
  */
-@Getter
-@Setter
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
 public class Table implements Serializable, Comparable<Table>, Cloneable {
 
     private static final long serialVersionUID = 4209205512472367171L;
@@ -66,7 +70,9 @@ public class Table implements Serializable, Comparable<Table>, Cloneable {
     /** 驱动类型, @see org.dinky.metadata.enums.DriverType */
     private String driverType;
 
-    public Table() {}
+    public Table(String name) {
+        this.name = name;
+    }
 
     public Table(String name, String schema, List<Column> columns) {
         this.name = name;

diff --git a/dinky-core/pom.xml b/dinky-core/pom.xml
@@ -84,6 +84,11 @@
             <artifactId>dinky-alert-http</artifactId>
             <scope>${scope.runtime}</scope>
         </dependency>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-metadata-paimon</artifactId>
+            <scope>${scope.runtime}</scope>
+        </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
             <artifactId>dinky-metadata-mysql</artifactId>

diff --git a/dinky-core/src/main/java/org/dinky/job/JobConfig.java b/dinky-core/src/main/java/org/dinky/job/JobConfig.java
@@ -22,11 +22,11 @@
 import org.dinky.assertion.Asserts;
 import org.dinky.data.constant.NetConstant;
 import org.dinky.data.enums.GatewayType;
+import org.dinky.data.model.CustomConfig;
 import org.dinky.executor.ExecutorConfig;
 import org.dinky.gateway.config.FlinkConfig;
 import org.dinky.gateway.config.GatewayConfig;
 import org.dinky.gateway.enums.SavePointStrategy;
-import org.dinky.gateway.model.CustomConfig;
 import org.dinky.gateway.model.FlinkClusterConfig;
 
 import org.apache.flink.configuration.Configuration;

diff --git a/dinky-gateway/src/main/java/org/dinky/gateway/config/ClusterConfig.java b/dinky-gateway/src/main/java/org/dinky/gateway/config/ClusterConfig.java
@@ -19,7 +19,7 @@
 
 package org.dinky.gateway.config;
 
-import org.dinky.gateway.model.CustomConfig;
+import org.dinky.data.model.CustomConfig;
 
 import java.util.ArrayList;
 import java.util.HashMap;

diff --git a/dinky-gateway/src/main/java/org/dinky/gateway/config/FlinkConfig.java b/dinky-gateway/src/main/java/org/dinky/gateway/config/FlinkConfig.java
@@ -20,9 +20,9 @@
 package org.dinky.gateway.config;
 
 import org.dinky.assertion.Asserts;
+import org.dinky.data.model.CustomConfig;
 import org.dinky.gateway.enums.ActionType;
 import org.dinky.gateway.enums.SavePointType;
-import org.dinky.gateway.model.CustomConfig;
 
 import java.util.ArrayList;
 import java.util.HashMap;

diff --git a/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java b/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java
@@ -23,6 +23,7 @@
 import org.dinky.constant.CustomerConfigureOptions;
 import org.dinky.context.FlinkUdfPathContextHolder;
 import org.dinky.data.enums.JobStatus;
+import org.dinky.data.model.CustomConfig;
 import org.dinky.data.model.SystemConfiguration;
 import org.dinky.executor.ClusterDescriptorAdapterImpl;
 import org.dinky.gateway.AbstractGateway;
@@ -32,7 +33,6 @@
 import org.dinky.gateway.enums.ActionType;
 import org.dinky.gateway.enums.SavePointType;
 import org.dinky.gateway.exception.GatewayException;
-import org.dinky.gateway.model.CustomConfig;
 import org.dinky.gateway.result.SavePointResult;
 import org.dinky.gateway.result.TestResult;
 import org.dinky.gateway.result.YarnResult;

diff --git a/dinky-gateway/src/test/java/org/dinky/gateway/model/FlinkClusterConfigTest.java b/dinky-gateway/src/test/java/org/dinky/gateway/model/FlinkClusterConfigTest.java
@@ -19,6 +19,8 @@
 
 package org.dinky.gateway.model;
 
+import org.dinky.data.model.CustomConfig;
+
 import java.io.IOException;
 import java.util.List;
 

diff --git a/...-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractDriver.java b/...-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractDriver.java
@@ -21,12 +21,14 @@
 
 import org.dinky.assertion.Asserts;
 import org.dinky.data.model.Column;
+import org.dinky.data.model.QueryData;
 import org.dinky.data.model.Schema;
 import org.dinky.data.model.Table;
 import org.dinky.metadata.config.DriverConfig;
 import org.dinky.metadata.config.IConnectConfig;
 import org.dinky.metadata.convert.ITypeConvert;
 import org.dinky.metadata.query.IDBQuery;
+import org.dinky.metadata.result.JdbcSelectResult;
 
 import java.util.List;
 import java.util.Map;
@@ -45,6 +47,12 @@ public abstract class AbstractDriver<T extends IConnectConfig> implements Driver
 
     public abstract ITypeConvert<T> getTypeConvert();
 
+    @Override
+    public JdbcSelectResult query(QueryData queryData) {
+        StringBuilder queryOption = genQueryOption(queryData);
+        return query(queryOption.toString(), null);
+    }
+
     public boolean canHandle(String type) {
         return Asserts.isEqualsIgnoreCase(getType(), type);
     }

diff --git a/...adata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/...adata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java
@@ -38,7 +38,6 @@
 import org.dinky.metadata.result.JdbcSelectResult;
 import org.dinky.utils.JsonUtils;
 import org.dinky.utils.LogUtil;
-import org.dinky.utils.TextUtil;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -525,8 +524,8 @@ public StringBuilder genQueryOption(QueryData queryData) {
 
         String where = queryData.getOption().getWhere();
         String order = queryData.getOption().getOrder();
-        String limitStart = queryData.getOption().getLimitStart();
-        String limitEnd = queryData.getOption().getLimitEnd();
+        int limitStart = queryData.getOption().getLimitStart();
+        int limitEnd = queryData.getOption().getLimitEnd();
 
         StringBuilder optionBuilder = new StringBuilder()
                 .append("select * from ")
@@ -541,12 +540,6 @@ public StringBuilder genQueryOption(QueryData queryData) {
             optionBuilder.append(" order by ").append(order);
         }
 
-        if (TextUtil.isEmpty(limitStart)) {
-            limitStart = "0";
-        }
-        if (TextUtil.isEmpty(limitEnd)) {
-            limitEnd = "100";
-        }
         optionBuilder.append(" limit ").append(limitStart).append(",").append(limitEnd);
 
         return optionBuilder;

diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/Driver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/Driver.java
@@ -217,6 +217,8 @@ static Driver build(String connector, String url, String username, String passwo
 
     JdbcSelectResult query(String sql, Integer limit);
 
+    JdbcSelectResult query(QueryData queryData);
+
     StringBuilder genQueryOption(QueryData queryData);
 
     JdbcSelectResult executeSql(String sql, Integer limit);

diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/enums/DriverType.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/enums/DriverType.java
@@ -30,6 +30,7 @@
  * @since 2024/2/6
  */
 public enum DriverType {
+    Paimon("Paimon"),
     MYSQL("MySQL"),
     ORACLE("Oracle"),
     POSTGRESQL("PostgreSql"),

diff --git a/dinky-metadata/dinky-metadata-mysql/src/main/java/org/dinky/metadata/driver/MySqlDriver.java b/dinky-metadata/dinky-metadata-mysql/src/main/java/org/dinky/metadata/driver/MySqlDriver.java
@@ -29,7 +29,6 @@
 import org.dinky.metadata.enums.DriverType;
 import org.dinky.metadata.query.IDBQuery;
 import org.dinky.metadata.query.MySqlQuery;
-import org.dinky.utils.TextUtil;
 
 import java.text.MessageFormat;
 import java.util.HashMap;
@@ -170,8 +169,8 @@ public StringBuilder genQueryOption(QueryData queryData) {
 
         String where = queryData.getOption().getWhere();
         String order = queryData.getOption().getOrder();
-        String limitStart = queryData.getOption().getLimitStart();
-        String limitEnd = queryData.getOption().getLimitEnd();
+        int limitStart = queryData.getOption().getLimitStart();
+        int limitEnd = queryData.getOption().getLimitEnd();
 
         StringBuilder optionBuilder = new StringBuilder()
                 .append(String.format("select * from `%s`.`%s`", queryData.getSchemaName(), queryData.getTableName()));
@@ -182,13 +181,6 @@ public StringBuilder genQueryOption(QueryData queryData) {
         if (order != null && !order.isEmpty()) {
             optionBuilder.append(" order by ").append(order);
         }
-
-        if (TextUtil.isEmpty(limitStart)) {
-            limitStart = "0";
-        }
-        if (TextUtil.isEmpty(limitEnd)) {
-            limitEnd = "100";
-        }
         optionBuilder.append(" limit ").append(limitStart).append(",").append(limitEnd);
 
         return optionBuilder;