Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Feature][catalog] Streampark start job ship catalogstore plugin and database crud. #3987

Merged
merged 4 commits into from
Oct 2, 2024
Merged

[Feature][catalog] Streampark start job ship catalogstore plugin and database crud. #3987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a1cee12
[feature][catalog] fixed dev conflict
Mrart Oct 1, 2024
feafb0a
fixed confict1
Mrart Oct 1, 2024
8647612
edit database service to private
Mrart Sep 11, 2024
f2ad3be
Update FlinkCatalogMapper.xml
wolfboys Oct 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 4 additions & 77 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,9 @@
<scala.xml.version>1.3.0</scala.xml.version>
<scalatest.version>3.2.9</scalatest.version>
<scala.binary.flink.version>_${scala.binary.version}</scala.binary.flink.version>

<flink.version>1.14.4</flink.version>
<flink.connector.version>3.2.0-1.18</flink.connector.version>
<flink.elasticserch.connector.version>3.0.1-1.17</flink.elasticserch.connector.version>
<flink.version>1.18.1</flink.version>
<flink.shaded.version>1.8.1</flink.shaded.version>
Mrart marked this conversation as resolved.
Show resolved Hide resolved
<streampark.shaded.version>1.0.0</streampark.shaded.version>
<streampark.flink.shims.version>1.14</streampark.flink.shims.version>
Expand Down Expand Up @@ -138,14 +139,13 @@
<owasp-dependency-check-maven.version>10.0.2</owasp-dependency-check-maven.version>
<build-helper-maven-plugin.version>3.3.0</build-helper-maven-plugin.version>
<streampark.shaded.package>org.apache.streampark.shaded</streampark.shaded.package>
<flink.table.uber.artifact.id>flink-table-uber_${scala.binary.version}</flink.table.uber.artifact.id>
<httpclient5.version>5.1</httpclient5.version>
<lombok.version>1.18.24</lombok.version>
<jupiter.version>5.9.1</jupiter.version>
<mockito.version>3.4.6</mockito.version>
<mockito-scala.version>1.17.14</mockito-scala.version>
<assertj.version>3.23.1</assertj.version>
<testcontainer.version>1.19.3</testcontainer.version>
<testcontainer.version>1.19.8</testcontainer.version>
<curator.version>5.3.0</curator.version>
<snappy.version>1.1.10.1</snappy.version>
<mybatis-plus.version>3.5.3.1</mybatis-plus.version>
Expand Down Expand Up @@ -318,27 +318,6 @@
<version>${mybatis-plus.version}</version>
</dependency>

<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core</artifactId>
<version>${oshi-core.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
</exclusion>
<exclusion>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest</artifactId>
</exclusion>
</exclusions>
</dependency>

<!--jackson-->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
Expand All @@ -351,12 +330,6 @@
<version>${jackson.version}</version>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson.version}</version>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
Expand All @@ -369,52 +342,6 @@
</exclusions>
</dependency>

<!--curator-->
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>log4j-1.2-api</groupId>
<artifactId>org.apache.logging.log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<version>${curator.version}</version>
</dependency>

<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>

<!--apache.commons -->
<dependency>
<groupId>org.apache.commons</groupId>
Expand Down
3 changes: 3 additions & 0 deletions streampark-common/src/main/java/org/apache/streampark/common/enums/CatalogType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@

/** catalog type */
public enum CatalogType {
MYSQL,
PGSQL,
ORACLE,
Mrart marked this conversation as resolved.
Show resolved Hide resolved
JDBC,
HIVE,
PAIMON,
Expand Down
60 changes: 60 additions & 0 deletions streampark-console/streampark-console-service/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,10 @@
<groupId>org.pac4j</groupId>
<artifactId>pac4j-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-web</artifactId>
</exclusion>
</exclusions>

</dependency>
Expand Down Expand Up @@ -377,12 +381,32 @@
<groupId>org.apache.streampark</groupId>
<artifactId>streampark-spark-client-api_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.streampark</groupId>
<artifactId>streampark-flink-kubernetes_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand Down Expand Up @@ -458,6 +482,36 @@
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java</artifactId>
<version>${flink.version}</version>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mysql</artifactId>
<version>${testcontainer.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.streampark</groupId>
<artifactId>streampark-flink-connector-plugin</artifactId>
<version>2.2.0-SNAPSHOT</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-kubernetes</artifactId>
<version>${flink.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -573,6 +627,12 @@
<version>${project.version}</version>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</dependency>
<dependency>
<groupId>org.apache.streampark</groupId>
<artifactId>streampark-flink-connector-plugin</artifactId>
<version>${project.version}</version>
<outputDirectory>${project.build.directory}/plugins</outputDirectory>
</dependency>
</artifactItems>
</configuration>
<executions>
Expand Down
5 changes: 5 additions & 0 deletions streampark-console/streampark-console-service/src/main/assembly/assembly.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@
<outputDirectory>lib</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>${project.build.directory}/plugins</directory>
<outputDirectory>plugins</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>${project.build.directory}/../src/main/assembly/conf</directory>
<outputDirectory>conf</outputDirectory>
Expand Down
45 changes: 40 additions & 5 deletions streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,25 @@ insert into `t_menu` values (150502, 150500, 'member add', null, null, 'member:a
insert into `t_menu` values (150503, 150500, 'member update', null, null, 'member:update', null, '1', 1, null, now(), now());
insert into `t_menu` values (150504, 150500, 'member delete', null, null, 'member:delete', null, '1', 1, null, now(), now());

insert into `t_menu` values (150601, 150600, 'catalog view', null, null, 'catalog:view', null, '1', 1, null, now(), now());
insert into `t_menu` values (150602, 150600, 'catalog create', null, null, 'catalog:create', null, '1', 1, null, now(), now());
insert into `t_menu` values (150603, 150600, 'catalog update', null, null, 'catalog:update', null, '1', 1, null, now(), now());
insert into `t_menu` values (150604, 150600, 'catalog delete', null, null, 'catalog:delete', null, '1', 1, null, now(), now());

insert into `t_menu` values (150605, 150600, 'database view', null, null, 'database:view', null, '1', 1, null, now(), now());
insert into `t_menu` values (150606, 150600, 'database create', null, null, 'database:create', null, '1', 1, null, now(), now());
insert into `t_menu` values (150607, 150600, 'database delete', null, null, 'database:delete', null, '1', 1, null, now(), now());

insert into `t_menu` values (150608, 150600, 'table view', null, null, 'table:view', null, '1', 1, null, now(), now());
insert into `t_menu` values (150609, 150600, 'table create', null, null, 'table:create', null, '1', 1, null, now(), now());
insert into `t_menu` values (150610, 150600, 'table update', null, null, 'table:update', null, '1', 1, null, now(), now());
insert into `t_menu` values (150611, 150600, 'table view', null, null, 'table:column:add', null, '1', 1, null, now(), now());
insert into `t_menu` values (150612, 150600, 'table column list', null, null, 'table:column:list', null, '1', 1, null, now(), now());
insert into `t_menu` values (150613, 150600, 'table column drop', null, null, 'table:column:drop', null, '1', 1, null, now(), now());
insert into `t_menu` values (150614, 150600, 'table option add', null, null, 'option:add', null, '1', 1, null, now(), now());
insert into `t_menu` values (150615, 150600, 'table option remove', null, null, 'option:remove', null, '1', 1, null, now(), now());

-- -------
-- ----------------------------
-- Records of t_role
-- ----------------------------
Expand Down Expand Up @@ -296,11 +315,27 @@ insert into `t_role_menu` values (100102, 100002, 140401);
insert into `t_role_menu` values (100103, 100002, 140402);
insert into `t_role_menu` values (100104, 100002, 140403);
insert into `t_role_menu` values (100105, 100002, 150000);
insert into `t_role_menu` values (100106, 100002, 150500);
insert into `t_role_menu` values (100107, 100002, 150501);
insert into `t_role_menu` values (100108, 100002, 150502);
insert into `t_role_menu` values (100109, 100002, 150503);
insert into `t_role_menu` values (100110, 100002, 150504);
insert into `t_role_menu` values (100107, 100002, 150601);
insert into `t_role_menu` values (100108, 100002, 150602);
insert into `t_role_menu` values (100109, 100002, 150603);
insert into `t_role_menu` values (100110, 100002, 150604);
insert into `t_role_menu` values (100111, 100002, 150601);
insert into `t_role_menu` values (100112, 100002, 150602);
insert into `t_role_menu` values (100113, 100002, 150603);
insert into `t_role_menu` values (100114, 100002, 150604);
insert into `t_role_menu` values (100115, 100002, 150605);
insert into `t_role_menu` values (100116, 100002, 150606);
insert into `t_role_menu` values (100117, 100002, 150607);
insert into `t_role_menu` values (100118, 100002, 150608);
insert into `t_role_menu` values (100119, 100002, 150609);
insert into `t_role_menu` values (100120, 100002, 150610);
insert into `t_role_menu` values (100121, 100002, 150611);
insert into `t_role_menu` values (100122, 100002, 150612);
insert into `t_role_menu` values (100123, 100002, 150613);
insert into `t_role_menu` values (100124, 100002, 150614);
insert into `t_role_menu` values (100125, 100002, 150615);
insert into `t_role_menu` values (100126, 100002, 150600);
insert into `t_role_menu` values (100127, 100001, 150600);

-- ----------------------------
-- Records of t_setting
Expand Down
32 changes: 32 additions & 0 deletions streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,22 @@ insert into "public"."t_menu" values (130504, 130500, 'link delete', null, null,
insert into "public"."t_menu" values (130601, 130600, 'add yarn queue', null, null, 'yarnQueue:create', '', '1', '0', null, now(), now());
insert into "public"."t_menu" values (130602, 130600, 'edit yarn queue', null, null, 'yarnQueue:update', '', '1', '0', null, now(), now());
insert into "public"."t_menu" values (130603, 130600, 'delete yarn queue', null, null, 'yarnQueue:delete', '', '1', '0', null, now(), now());
insert into "public"."t_menu" values (130701, 130700, 'catalog view', null, null, 'catalog:view', '', '1', '0', null, now(), now());
insert into "public"."t_menu" values (130702, 130700, 'catalog create', null, null, 'catalog:create', '', '1', '0', null, now(), now());
insert into "public"."t_menu" values (130703, 130700, 'catalog delete', null, null, 'catalog:delete', '', '1', '0', null, now(), now());
insert into "public"."t_menu" values (130704, 130700, 'catalog update', null, null, 'catalog:update', '', '1', '0', null, now(), now());
insert into "public"."t_menu" values (150605, 150600, 'database view', null, null, 'database:view', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150606, 150600, 'database create', null, null, 'database:create', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150607, 150600, 'database delete', null, null, 'database:delete', null, '1', 1, null, now(), now());

insert into "public"."t_menu" values (150608, 150600, 'table view', null, null, 'table:view', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150609, 150600, 'table create', null, null, 'table:create', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150610, 150600, 'table update', null, null, 'table:update', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150611, 150600, 'table view', null, null, 'table:column:add', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150612, 150600, 'table column list', null, null, 'table:column:list', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150613, 150600, 'table column drop', null, null, 'table:column:drop', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150614, 150600, 'table option add', null, null, 'option:add', null, '1', 1, null, now(), now());
insert into "public"."t_menu" values (150615, 150600, 'table option remove', null, null, 'option:remove', null, '1', 1, null, now(), now());
-- ----------------------------
-- Records of t_role
-- ----------------------------
Expand Down Expand Up @@ -258,6 +273,23 @@ insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 130600);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 130601);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 130602);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 130603);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 130701);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 130702);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 130703);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 130704);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150605);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150606);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150607);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150608);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150609);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150610);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150611);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150612);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150613);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150614);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150615);
insert into "public"."t_role_menu" (role_id, menu_id) values (100002, 150600);
insert into "public"."t_role_menu" (role_id, menu_id) values (100001, 150600);

-- ----------------------------
-- Records of t_setting
Expand Down
16 changes: 16 additions & 0 deletions ...mpark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,22 @@ create table `t_spark_log` (
primary key (`id`) using btree
) engine=innodb auto_increment=100000 default charset=utf8mb4 collate=utf8mb4_general_ci;

-- ----------------------------
-- table structure for t_flink_catalog
-- ----------------------------
drop table if exists `t_flink_catalog`;
CREATE TABLE `t_flink_catalog` (
`id` BIGINT AUTO_INCREMENT PRIMARY KEY,
`team_id` BIGINT NOT NULL,
`user_id` BIGINT DEFAULT NULL,
`catalog_type` VARCHAR(255) NOT NULL,
`catalog_name` VARCHAR(255) NOT NULL,
`configuration` TEXT,
`create_time` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
`update_time` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uniq_catalog_name (`catalog_name`)
) ENGINE=InnoDB auto_increment=100000 default charset=utf8mb4 collate=utf8mb4_general_ci;

-- ----------------------------
-- Table structure for jdbc registry
-- ----------------------------
Expand Down
Loading
Loading