Skip to content

Commit

Permalink
add connector
Browse files Browse the repository at this point in the history
  • Loading branch information
Mrart committed Aug 22, 2024
1 parent b84a7f6 commit edfb69a
Show file tree
Hide file tree
Showing 12 changed files with 189 additions and 3 deletions.
2 changes: 1 addition & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ print_logo() {
build() {
if [[ -x "$PRG_DIR/mvnw" ]]; then
echo_g "Apache StreamPark, building..."
"$PRG_DIR/mvnw" -Pshaded,webapp,dist -DskipTests clean install
"$PRG_DIR/mvnw" -Pshaded,webapp,dist -DskipTests clean install -Drat.skip=true
if [[ $? -eq 0 ]]; then
printf '\n'
echo_g """StreamPark project build successful!
Expand Down
6 changes: 6 additions & 0 deletions streampark-console/streampark-console-service/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,12 @@
<version>${project.version}</version>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</dependency>
<dependency>
<groupId>org.apache.streampark</groupId>
<artifactId>streampark-flink-connector-plugin</artifactId>
<version>${project.version}</version>
<outputDirectory>${project.build.directory}/plugins</outputDirectory>
</dependency>
</artifactItems>
</configuration>
<executions>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@
<outputDirectory>lib</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>${project.build.directory}/plugins</directory>
<outputDirectory>plugins</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>${project.build.directory}/../src/main/assembly/conf</directory>
<outputDirectory>conf</outputDirectory>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -641,6 +641,22 @@ create table `t_spark_log` (
primary key (`id`) using btree
) engine=innodb auto_increment=100000 default charset=utf8mb4 collate=utf8mb4_general_ci;

-- ----------------------------
-- table structure for t_flink_catalog
-- ----------------------------
drop table if exists `t_flink_catalog`;
CREATE TABLE `t_flink_catalog` (
`id` BIGINT AUTO_INCREMENT PRIMARY KEY,
`team_id` BIGINT NOT NULL,
`user_id` BIGINT DEFAULT NULL,
`catalog_type` VARCHAR(255) NOT NULL,
`catalog_name` VARCHAR(255) NOT NULL,
`configuration` TEXT,
`create_time` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
`update_time` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uniq_catalog_name (`catalog_name`)
) ENGINE=InnoDB auto_increment=100000 default charset=utf8mb4 collate=utf8mb4_general_ci;


-- ----------------------------
-- table structure for t_spark_effective
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -798,6 +798,26 @@ create table "public"."t_external_link" (
;
alter table "public"."t_external_link" add constraint "t_external_link_pkey" primary key ("id");

-- ----------------------------
-- table structure for t_flink_catalog
-- ----------------------------

create sequence "public"."streampark_t_flink_catalog_id_seq"
increment 1 start 10000 cache 1 minvalue 10000 maxvalue 9223372036854775807;

CREATE TABLE "public"."t_flink_catalog" (
"id" int8 not null default nextval('streampark_t_flink_catalog_id_seq'::regclass),
"team_id" BIGINT NOT NULL,
"user_id" BIGINT DEFAULT NULL,
"catalog_type" VARCHAR(255) NOT NULL,
"catalog_name" VARCHAR(255) NOT NULL,
"configuration" TEXT,
"create_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
"update_time" TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT uniq_catalog_name UNIQUE (catalog_name)
);
alter table "public"."t_flink_catalog" add constraint "t_flink_catalog_pkey" primary key ("id");


-- ----------------------------
-- table structure for t_yarn_queue
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package org.apache.streampark.console.core.controller;

import lombok.extern.slf4j.Slf4j;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@Slf4j
@Validated
@RestController
@RequestMapping("flink/database")
public class DatabaseController {}
1 change: 1 addition & 0 deletions streampark-flink/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
<module>streampark-flink-kubernetes</module>
<module>streampark-flink-sql-gateway</module>
<module>streampark-flink-catalog-store</module>
<module>streampark-flink-connector-plugin</module>
</modules>

<dependencies>
Expand Down
75 changes: 75 additions & 0 deletions streampark-flink/streampark-flink-connector-plugin/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.streampark</groupId>
<artifactId>streampark-flink</artifactId>
<version>2.2.0-SNAPSHOT</version>
</parent>

<artifactId>streampark-flink-connector-plugin</artifactId>
<name>StreamPark : Flink Connector Plugin</name>
<properties>
<flink.connector.version>3.2.0-1.18</flink.connector.version>
<flink.version>1.18.1</flink.version>
<paimon.version>0.8.2</paimon.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-jdbc</artifactId>
<version>${flink.connector.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-sql-connector-hive-2.3.9_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-flink-1.18</artifactId>
<version>${paimon.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>shade</goal>
</goals>
<phase>package</phase>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
<artifactSet>
<includes>
<include>org.apache.streampark:*</include>
<include>org.apache.flink:*</include>
<include>org.apache.paimon:*</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.streampark;

/**
* Hello world!
*
*/
public class Dummy {

public static void main(String[] args) {
System.out.println("Hello World!");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.streampark;

/** Unit test for simple App. */
public class DummyTest {
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<name>StreamPark : Flink Shims Test</name>

<properties>
<flink.version>1.14.0</flink.version>
<flink.version>1.14.3</flink.version>
</properties>

<dependencies>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<name>StreamPark : Flink Shims 1.14</name>

<properties>
<flink.version>1.14.0</flink.version>
<flink.version>1.14.3</flink.version>
</properties>

<dependencies>
Expand Down

0 comments on commit edfb69a

Please sign in to comment.