-
Notifications
You must be signed in to change notification settings - Fork 1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Feat][Flink-K8s-V2] Refactor the lifecycle control of Flink K8s appl…
…ication-mode jobs (#3037) * [Feat] Refactor Flink resource build pipeline for fink-k8s-v2 module. #2882 * [Feat] Adaptation of the submission client for flink-k8s-v2. #2882 * [Feat][flink-k8s-v2] Migrate ENABLE_V2 to streampark-common module. #2882 * [Feat][flink-k8s-v2] Disable Flink job tracking watcher at flink-k8s-v1. #2882 * [Feat][flink-k8s-v2] Adaptation of Flink job canceling and triggering savepoint operations . #2882 * [Feat][flink-k8s-v2] Untrack flink when it is deleted. #2882
- Loading branch information
Showing
27 changed files
with
749 additions
and
168 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
streampark-common/src/main/scala/org/apache/streampark/common/zio/ZIOJavaUtil.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.streampark.common.zio | ||
|
|
||
| import zio.{FiberFailure, IO, UIO} | ||
|
|
||
| /** Util for running ZIO effects in Java. */ | ||
| object ZIOJavaUtil { | ||
|
|
||
| @throws[FiberFailure] | ||
| def runIO[E, A](zio: IO[E, A]): A = ZIOExt.unsafeRun(zio) | ||
|
|
||
| def runUIO[A](uio: UIO[A]): A = ZIOExt.unsafeRun(uio) | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
...nt-api/src/main/scala/org/apache/streampark/flink/client/bean/KubernetesSubmitParam.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.streampark.flink.client.bean | ||
|
|
||
| import org.apache.streampark.common.enums.FlinkK8sRestExposedType | ||
|
|
||
| import javax.annotation.Nullable | ||
|
|
||
| import java.util | ||
| import java.util.{Map => JMap} | ||
|
|
||
| /** | ||
| * TODO Need to display more K8s submission parameters in the front-end UI. | ||
| * | ||
| * It will eventually be converted to | ||
| * [[org.apache.streampark.flink.kubernetes.v2.model.FlinkDeploymentDef]] | ||
| * | ||
| * The logic of conversion is located at： | ||
| * [[org.apache.streampark.flink.client.impl.KubernetesApplicationClientV2#genFlinkDeployDef]] | ||
| */ | ||
| // todo split into Application mode and SessionJob mode | ||
| case class KubernetesSubmitParam( | ||
| clusterId: String, | ||
| kubernetesNamespace: String, | ||
| baseImage: Option[String] = None, | ||
| imagePullPolicy: Option[String] = None, | ||
| serviceAccount: Option[String] = None, | ||
| podTemplate: Option[String] = None, | ||
| jobManagerCpu: Option[Double] = None, | ||
| jobManagerMemory: Option[String] = None, | ||
| jobManagerEphemeralStorage: Option[String] = None, | ||
| jobManagerPodTemplate: Option[String] = None, | ||
| taskManagerCpu: Option[Double] = None, | ||
| taskManagerMemory: Option[String] = None, | ||
| taskManagerEphemeralStorage: Option[String] = None, | ||
| taskManagerPodTemplate: Option[String] = None, | ||
| logConfiguration: JMap[String, String] = new util.HashMap[String, String](), | ||
| flinkRestExposedType: Option[FlinkK8sRestExposedType] = None | ||
| ) | ||
|
|
||
| object KubernetesSubmitParam { | ||
|
|
||
| /** | ||
| * Compatible with streampark old native k8s submission parameters. | ||
| * | ||
| * @param clusterId | ||
| * flink cluster id in k8s cluster. | ||
| * @param kubernetesNamespace | ||
| * k8s namespace. | ||
| * @param flinkRestExposedType | ||
| * flink rest-service exposed type on k8s cluster. | ||
| */ | ||
| def apply( | ||
| clusterId: String, | ||
| kubernetesNamespace: String, | ||
| baseImage: String, | ||
| @Nullable flinkRestExposedType: FlinkK8sRestExposedType): KubernetesSubmitParam = | ||
| KubernetesSubmitParam( | ||
| clusterId = clusterId, | ||
| kubernetesNamespace = kubernetesNamespace, | ||
| baseImage = Some(baseImage), | ||
| flinkRestExposedType = Option(flinkRestExposedType)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.