Refactor: Remove flink-kubenretes-api shaded module and generate code…

… directly from CRD yaml file. #2881
apache · Aug 1, 2023 · 4c8c3df · 4c8c3df
1 parent 71b086d
commit 4c8c3df
Show file tree

Hide file tree

Showing 51 changed files with 10,799 additions and 499 deletions.
diff --git a/pom.xml b/pom.xml
@@ -124,6 +124,7 @@
         <enumeratum.version>1.6.1</enumeratum.version>
         <assertj.version>3.23.1</assertj.version>
         <zio.version>2.0.15</zio.version>
+        <zio-logging.version>2.1.13</zio-logging.version>
         <pprint.version>0.8.1</pprint.version>
 
         <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version>
@@ -525,7 +526,9 @@
                         </resourceIncludes>
                         <resourceExcludes>
                             **/.asf.yaml,
-                            **/.github/**
+                            **/.github/**,
+                            **/crd/**.yml,
+                            **/crd/**.yaml
                         </resourceExcludes>
                         <excludes>
                         </excludes>

diff --git a/streampark-common/pom.xml b/streampark-common/pom.xml
@@ -129,6 +129,12 @@
         </dependency>
 
         <!-- ZIO -->
+        <dependency>
+            <groupId>dev.zio</groupId>
+            <artifactId>zio-logging_${scala.binary.version}</artifactId>
+            <version>${zio-logging.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>dev.zio</groupId>
             <artifactId>zio-streams_${scala.binary.version}</artifactId>

diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/zio/package.scala b/streampark-common/src/main/scala/org/apache/streampark/common/zio/package.scala
@@ -24,7 +24,7 @@ package object zio {
   /** Similar to python's pprint, format print any type of instance. */
   @inline def toPrettyString(value: Any): String = value match {
     case v: String => v
-    case v => pprint.apply(v, height = 2000).render
+    case v => pprint.apply(v, height = 2000, showFieldNames = true).render
   }
 
   implicit class PrettyStringOps(value: Any) {

diff --git a/...nsole-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java b/...nsole-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java
@@ -26,13 +26,15 @@
 import org.apache.streampark.common.fs.FsOperator;
 import org.apache.streampark.common.util.SystemPropertyUtils;
 import org.apache.streampark.common.util.Utils;
+import org.apache.streampark.common.zio.ZIOExt;
 import org.apache.streampark.console.base.util.WebUtils;
 import org.apache.streampark.console.core.entity.FlinkEnv;
 import org.apache.streampark.console.core.service.SettingService;
 
 import org.apache.commons.lang3.StringUtils;
 
 import lombok.extern.slf4j.Slf4j;
+import org.apache.streampark.flink.kubernetes.v2.httpfs.EmbeddedFileServer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
@@ -98,6 +100,8 @@ public void run(ApplicationArguments args) throws Exception {
     overrideSystemProp(ConfigConst.KEY_HADOOP_USER_NAME(), hadoopUserName);
     // initialize local file system resources
     storageInitialize(LFS);
+    // Launch the embedded http file server.
+    ZIOExt.unsafeRun(EmbeddedFileServer.launch());
   }
 
   private void initInternalConfig(Environment springEnv) {

diff --git a/streampark-flink/streampark-flink-kubernetes-v2/pom.xml b/streampark-flink/streampark-flink-kubernetes-v2/pom.xml
@@ -16,108 +16,25 @@
   ~ limitations under the License.
   -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
-
     <parent>
         <groupId>org.apache.streampark</groupId>
         <artifactId>streampark-flink</artifactId>
         <version>2.2.0-SNAPSHOT</version>
     </parent>
 
-    <artifactId>streampark-flink-kubernetes-v2_${scala.binary.version}</artifactId>
+    <artifactId>streampark-flink-kubernetes-v2</artifactId>
     <name>StreamPark : Flink Kubernetes Integration V2</name>
+    <packaging>pom</packaging>
 
     <properties>
-        <zio-http.version>3.0.0-RC2</zio-http.version>
         <fabric8.version>6.8.0</fabric8.version>
-        <sttp-client.version>3.8.13</sttp-client.version>
-        <os-lib.version>0.9.1</os-lib.version>
-        <upickle.version>3.0.0</upickle.version>
-        <jackson-dataformat-yaml.version>2.14.2</jackson-dataformat-yaml.version>
-        <scalatest.skiptests>true</scalatest.skiptests>
     </properties>
 
-    <dependencies>
-        <!-- Streampark modules -->
-        <dependency>
-            <groupId>org.apache.streampark</groupId>
-            <artifactId>streampark-common_${scala.binary.version}</artifactId>
-            <version>${project.version}</version>
-            <scope>provided</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.streampark</groupId>
-            <artifactId>streampark-shaded-flink-kubernetes-operator-api</artifactId>
-            <version>${streampark.shaded.version}</version>
-        </dependency>
-
-        <!-- K8s Client -->
-        <dependency>
-            <groupId>io.fabric8</groupId>
-            <artifactId>kubernetes-client</artifactId>
-            <version>${fabric8.version}</version>
-        </dependency>
-
-        <!-- ZIO -->
-        <dependency>
-            <groupId>dev.zio</groupId>
-            <artifactId>zio-http_${scala.binary.version}</artifactId>
-            <version>${zio-http.version}</version>
-        </dependency>
-
-        <!-- Http client -->
-        <dependency>
-            <groupId>com.softwaremill.sttp.client3</groupId>
-            <artifactId>zio_${scala.binary.version}</artifactId>
-            <version>${sttp-client.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>com.softwaremill.sttp.client3</groupId>
-            <artifactId>zio-json_${scala.binary.version}</artifactId>
-            <version>${sttp-client.version}</version>
-        </dependency>
-
-        <!-- Tools -->
-        <dependency>
-            <groupId>com.lihaoyi</groupId>
-            <artifactId>os-lib_${scala.binary.version}</artifactId>
-            <version>${os-lib.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>com.lihaoyi</groupId>
-            <artifactId>upickle_${scala.binary.version}</artifactId>
-            <version>${upickle.version}</version>
-        </dependency>
-
-        <!-- yaml parser -->
-        <dependency>
-            <groupId>com.fasterxml.jackson.dataformat</groupId>
-            <artifactId>jackson-dataformat-yaml</artifactId>
-            <version>${jackson-dataformat-yaml.version}</version>
-        </dependency>
-
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>build-helper-maven-plugin</artifactId>
-            </plugin>
-        </plugins>
-    </build>
-
-    <profiles>
-        <profile>
-            <id>apache-release</id>
-            <properties>
-                <maven.deploy.skip>true</maven.deploy.skip>
-            </properties>
-        </profile>
-    </profiles>
+    <modules>
+        <module>streampark-flink-kubernetes-crd</module>
+        <module>streampark-flink-kubernetes-engine</module>
+    </modules>
 
 </project>
diff --git a/...2/src/main/scala/org/apache/streampark/flink/kubernetes/v2/model/FlinkDeploymentDef.scala b/...2/src/main/scala/org/apache/streampark/flink/kubernetes/v2/model/FlinkDeploymentDef.scala