diff --git a/Makefile b/Makefile index 2e0d8195..3c9091ee 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | kubectl apply -f - + $(KUSTOMIZE) build config/crd | kubectl apply --server-side=true --force-conflicts -f - .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. diff --git a/README.md b/README.md index 07ebc19b..4a16afe9 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The CDAP Operator is still under active development and has not been extensively ### Build and Run Locally -You can checkout the CDAP Operator source code, build and run locally. To build the CDAP Operator, you need to setup your environment for the [Go](https://golang.org/doc/install) language. Also, you should have a Kubernetes cluster +You can checkout the CDAP Operator source code, build and run locally. To build the CDAP Operator, you need to setup your environment for the [Go](https://golang.org/doc/install) language. Also, you should have a Kubernetes cluster 1. Checkout CDAP Operator source ``` @@ -39,7 +39,7 @@ You can checkout the CDAP Operator source code, build and run locally. To build ``` kubectl apply -f config/samples/cdap_v1alpha1_cdapmaster.yaml ``` - + ### Build Controller Docker Image and Deploy in Kubernetes You can also build a docker image containing the CDAP controller and deploy it to Kubernetes. @@ -47,7 +47,7 @@ You can also build a docker image containing the CDAP controller and deploy it t 1. Build the docker image ``` IMG=cdap-controller:latest make docker-build - ``` + ``` You can change the target image name and tag by setting the `IMG` environment variable. 1. Push the docker image ``` @@ -62,6 +62,111 @@ You can also build a docker image containing the CDAP controller and deploy it t A step by step guide of running CDAP in Kubernetes using CDAP operator can be found in the [blog post](https://link.medium.com/hpPbiUYT9X). +### Using the Admission Controller + +The CDAP operator can be configured to optionally run a webhook server for a [mutating admission controller](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/). The mutating admission controller allows the operator to change the following fields in CDAP pods: +1. Add [init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) +1. Add [Node Selectors](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/) +1. Add [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) + +These mutations can be defined using the `MutationConfigs` field in CDAPMaster. + +#### Prerequisites + +Kubernetes requires that the webhook server uses TLS to authenticate with the kube API server. For this you will need to ensure the TLS certificates are present in the `/tmp/k8s-webhook-server/serving-certs` directory in the `cdap-controller` pod. To simplify the management of TLS certificates, you can use [cert-manager](https://github.com/cert-manager/cert-manager). The following steps assume you are in the root directory of the Git repository and have already deployed the CDAP operator stateful set. +1. [Deploy cert-manager](https://cert-manager.io/docs/installation/#default-static-install) in the cluster. +```bash +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml +``` +You should see 3 pods running for cert-manager. +```bash +kubectl get pods -n cert-manager +NAME READY STATUS RESTARTS AGE +cert-manager-655c4cf99d-rbzwr 1/1 Running 0 2m +cert-manager-cainjector-845856c584-csbsw 1/1 Running 0 2m +cert-manager-webhook-57876b9fd-68vgc 1/1 Running 0 2m +``` +2. Deploy a kubernetes service for the webhook server. +```bash +# set the namespace in which CDAPMaster is deployed. +export CDAP_NAMESPACE=default +sed -e 's@{CDAP_NAMESPACE}@'"$CDAP_NAMESPACE"'@g' <"./webhooks/templates/webhook-service.yaml" | kubectl apply -f - +``` +3. Deploy the cert-manager self-signed issuer. +```bash +sed -e 's@{CDAP_NAMESPACE}@'"$CDAP_NAMESPACE"'@g' <"./webhooks/templates/issuer.yaml" | kubectl apply -f - +``` +4. Deploy the Certificate resource. +```bash +sed -e 's@{CDAP_NAMESPACE}@'"$CDAP_NAMESPACE"'@g' <"./webhooks/templates/certificate.yaml" | kubectl apply -f - +``` +Wait for the certificate to be ready. +```bash +kubectl get Certificates +NAME READY SECRET AGE +cdap-webhook-cert True cdap-webhook-server-cert 1d +``` +5. Add the following fields in the CDAP operator stateful set spec: +```yaml +# Filename: cdap-controller.yaml +spec: + containers: + - command: + - /manager + args: ["--enable-webhook", "true"] +... + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true +... + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: cdap-webhook-server-cert + +``` +6. Deploy the mutating webhook resource: +```bash +sed -e 's@{CDAP_NAMESPACE}@'"$CDAP_NAMESPACE"'@g' <"./webhooks/templates/webhook.yaml" | kubectl apply -f - +``` +The webhook is now configured and it will intercept requests to create new pods made by CDAP. + +#### Example use case: Isolate pods that execute user code in Google Kubernetes Engine. + +Assuming task workers are enabled, the pods that execute user code in CDAP are task workers and preview runners. Let us call these pods as "worker pods". To isolate these worker pods in a dedicated node pool with the help of the admission controller, you follow these steps: +1. Create a node pool for running only worker pods. +```bash +gcloud container node-pools create worker-pool \ + --cluster cdap-cluster --project my-gcp-projet --location us-east1 +``` +2. Add a taint to the new node pool. This will prevent pods from being scheduled on the node pool unless they specify the corresponding toleration. +```bash +gcloud beta container node-pools update worker-pool \ +--node-taints="worker-pods-only=true:NoExecute" \ +--cluster cdap-cluster --project my-gcp-projet --location us-east1 +``` +3. Add the following configuration to the CDAPMaster: +```yaml +# Filename: cdapmaster.yaml +spec: +... + mutationConfigs: + - labelSelector: + matchExpressions: + - {key: cdap.twill.app, operator: In, values: [task.worker, preview.runner]} + podMutations: + nodeSelectors: + cloud.google.com/gke-nodepool: worker-pool + tolerations: + - effect: NoExecute + key: worker-pods-only + operator: Equal + tolerationSeconds: 3600 + value: "true" +``` +Now whenever CDAP launches preview runner of task worker pods, the admission controller will mutate the pod specifications before they are deployed to ensure the pods get scheduled only on the node pool "worker-pool". ### Running Unit Tests 1. Install [kubebuilder](https://book-v1.book.kubebuilder.io/quick_start.html). diff --git a/api/v1alpha1/cdapmaster_types.go b/api/v1alpha1/cdapmaster_types.go index f2a062f9..271304a7 100644 --- a/api/v1alpha1/cdapmaster_types.go +++ b/api/v1alpha1/cdapmaster_types.go @@ -120,6 +120,10 @@ type CDAPMasterSpec struct { // AdditionalVolumeMounts defines a list of additional volume mounts for all services. // For information on supported volume mount types, see https://kubernetes.io/docs/concepts/storage/volumes/. AdditionalVolumeMounts []corev1.VolumeMount `json:"additionalVolumeMounts,omitempty"` + // MutationConfigs specifies mutations that can be applied to resources with the "cdap.instance" label. + // Mutations can include adding init containers, tolerations and node selectors to pods. To use mutations, + // the admission control webhook should be enabled in the cdap operator. + MutationConfigs []MutationConfig `json:"mutationConfigs,omitempty"` } // CDAPServiceSpec defines the base set of specifications applicable to all master services. @@ -342,6 +346,27 @@ type SecurityContext struct { ReadOnlyRootFilesystem *bool `json:"readOnlyRootFilesystem,omitempty"` } +// MutationConfig defines mutations that can be applied to resources with the "cdap.instance" label and that +// satisfy a label selector. +type MutationConfig struct { + // LabelSelector selects resources to apply the mutation to. + LabelSelector metav1.LabelSelector `json:"labelSelector,omitempty"` + // PodMutations specifies mutations that are be applied to pods which satisfy specified label selectors. + // Pod mutations can include adding init containers, tolerations and node selectors etc. + PodMutations PodMutationConfig `json:"podMutations,omitempty"` +} + +// PodMutationConfig specifies a mutation along with the selector to identify target pods. +type PodMutationConfig struct { + // InitContainersBefore specifies a list of init containers that will be added before the + // the init containers specified in the pod spec. + InitContainersBefore []corev1.Container `json:"initContainersBefore,omitempty"` + // NodeSelectors specifies additional node selectors that will be added to the pod. + NodeSelector *map[string]string `json:"nodeSelectors,omitempty"` + // Tolerations specifies additional tolerations that will be added to the pod. + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` +} + func init() { SchemeBuilder.Register(&CDAPMaster{}, &CDAPMasterList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 80266c80..f1336def 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -273,6 +273,13 @@ func (in *CDAPMasterSpec) DeepCopyInto(out *CDAPMasterSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.MutationConfigs != nil { + in, out := &in.MutationConfigs, &out.MutationConfigs + *out = make([]MutationConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CDAPMasterSpec. @@ -522,6 +529,63 @@ func (in *MetricsSpec) DeepCopy() *MetricsSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MutationConfig) DeepCopyInto(out *MutationConfig) { + *out = *in + in.LabelSelector.DeepCopyInto(&out.LabelSelector) + in.PodMutations.DeepCopyInto(&out.PodMutations) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MutationConfig. +func (in *MutationConfig) DeepCopy() *MutationConfig { + if in == nil { + return nil + } + out := new(MutationConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodMutationConfig) DeepCopyInto(out *PodMutationConfig) { + *out = *in + if in.InitContainersBefore != nil { + in, out := &in.InitContainersBefore, &out.InitContainersBefore + *out = make([]v1.Container, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodMutationConfig. +func (in *PodMutationConfig) DeepCopy() *PodMutationConfig { + if in == nil { + return nil + } + out := new(PodMutationConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PreviewSpec) DeepCopyInto(out *PreviewSpec) { *out = *in diff --git a/config/crd/bases/cdap.cdap.io_cdapmasters.yaml b/config/crd/bases/cdap.cdap.io_cdapmasters.yaml index 0524af01..f4c14231 100644 --- a/config/crd/bases/cdap.cdap.io_cdapmasters.yaml +++ b/config/crd/bases/cdap.cdap.io_cdapmasters.yaml @@ -21912,6 +21912,1448 @@ spec: size used by the service. type: string type: object + mutationConfigs: + description: MutationConfigs specifies mutations that can be applied + to resources with the "cdap.instance" label. Mutations can include + adding init containers, tolerations and node selectors to pods. + To use mutations, the admission control webhook should be enabled + in the cdap operator. + items: + description: MutationConfig defines mutations that can be applied + to resources with the "cdap.instance" label and that satisfy a + label selector. + properties: + labelSelector: + description: LabelSelector selects resources to apply the mutation + to. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + podMutations: + description: PodMutations specifies mutations that are be applied + to pods which satisfy specified label selectors. Pod mutations + can include adding init containers, tolerations and node selectors + etc. + properties: + initContainersBefore: + description: InitContainersBefore specifies a list of init + containers that will be added before the the init containers + specified in the pod spec. + items: + description: A single application container that you want + to run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The container + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the + reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of + whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within + a shell. The container image''s ENTRYPOINT is used + if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. + If a variable cannot be resolved, the reference + in the input string will be unchanged. Double $$ + are reduced to a single $, which allows for escaping + the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of + whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set + in the container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined + environment variables in the container and + any service environment variables. If a variable + cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: + supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the + container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in + the pod's namespace + properties: + key: + description: The key of the secret to + select from. Must be a valid secret + key. + type: string + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment + variables in the container. The keys defined within + a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container + is starting. When a key exists in multiple sources, + the value associated with the last source will take + precedence. Values defined by an Env with a duplicate + key will take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config + management to default or override container images + in workload controllers like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag + is specified, or IfNotPresent otherwise. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should + take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT + supported as a LifecycleHandler and kept + for the backward compatibility. There are + no validation of this field and lifecycle + hooks will fail in runtime when tcp handler + is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness/startup + probe failure, preemption, resource contention, + etc. The handler is not called if the container + crashes or exits. The Pod''s termination grace + period countdown begins before the PreStop hook + is executed. Regardless of the outcome of the + handler, the container will eventually terminate + within the Pod''s termination grace period (unless + delayed by finalizers). Other management of + the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT + supported as a LifecycleHandler and kept + for the backward compatibility. There are + no validation of this field and lifecycle + hooks will fail in runtime when tcp handler + is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for + the probe to be considered failed after having + succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum value + is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for + the probe to be considered successful after + having failed. Defaults to 1. Must be 1 for + liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the + pod needs to terminate gracefully upon probe + failure. The grace period is the duration in + seconds after the processes running in the pod + are sent a termination signal and the time when + the processes are forcibly halted with a kill + signal. Set this value longer than the expected + cleanup time for your process. If this value + is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a + DNS_LABEL. Each container in a pod must have a unique + name (DNS_LABEL). Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Not specifying a port here DOES NOT prevent that + port from being exposed. Any port which is listening + on the default "0.0.0.0" address inside a container + will be accessible from the network. Modifying this + array with strategic merge patch may corrupt the + data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: Number of port to expose on the + pod's IP address. This must be a valid port + number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: Number of port to expose on the + host. If specified, this must be a valid port + number, 0 < x < 65536. If HostNetwork is specified, + this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port + in a pod must have a unique name. Name for + the port that can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be UDP, + TCP, or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service + readiness. Container will be removed from service + endpoints if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for + the probe to be considered failed after having + succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum value + is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for + the probe to be considered successful after + having failed. Defaults to 1. Must be 1 for + liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the + pod needs to terminate gracefully upon probe + failure. The grace period is the duration in + seconds after the processes running in the pod + are sent a termination signal and the time when + the processes are forcibly halted with a kill + signal. Set this value longer than the expected + cleanup time for your process. If this value + is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'SecurityContext defines the security + options the container should be run with. If set, + the fields of SecurityContext override the equivalent + fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges than + its parent process. This bool directly controls + if the no_new_privs flag will be set on the + container process. AllowPrivilegeEscalation + is true always when the container is: 1) run + as Privileged 2) has CAP_SYS_ADMIN Note that + this field cannot be set when spec.os.name is + windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when + running containers. Defaults to the default + set of capabilities granted by the container + runtime. Note that this field cannot be set + when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults to + false. Note that this field cannot be set when + spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc + mount to use for the containers. The default + is DefaultProcMount which uses the container + runtime defaults for readonly paths and masked + paths. This requires the ProcMountType feature + flag to be enabled. Note that this field cannot + be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. Note that + this field cannot be set when spec.os.name is + windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of + the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot be set + when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must + run as a non-root user. If true, the Kubelet + will validate the image at runtime to ensure + that it does not run as UID 0 (root) and fail + to start the container if it does. If unset + or false, no such validation will be performed. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of + the container process. Defaults to user specified + in image metadata if unspecified. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. Note that + this field cannot be set when spec.os.name is + windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot be set + when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this + container. If seccomp options are provided at + both the pod & container level, the container + options override the pod options. Note that + this field cannot be set when spec.os.name is + windows. + properties: + localhostProfile: + description: localhostProfile indicates a + profile defined in a file on the node should + be used. The profile must be preconfigured + on the node to work. Must be a descending + path, relative to the kubelet's configured + seccomp profile location. Must only be set + if type is "Localhost". + type: string + type: + description: "type indicates which kind of + seccomp profile will be applied. Valid options + are: \n Localhost - a profile defined in + a file on the node should be used. RuntimeDefault + - the container runtime default profile + should be used. Unconfined - no profile + should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied + to all containers. If unspecified, the options + from the PodSecurityContext will be used. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot be set + when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the + GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. + This field is alpha-level and will only + be honored by components that enable the + WindowsHostProcessContainers feature flag. + Setting this field without the feature flag + will result in errors when validating the + Pod. All of a Pod's containers must have + the same effective HostProcess value (it + is not allowed to have a mix of HostProcess + containers and non-HostProcess containers). In + addition, if HostProcess is true then HostNetwork + must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run + the entrypoint of the container process. + Defaults to the user specified in image + metadata if unspecified. May also be set + in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod + has successfully initialized. If specified, no other + probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, + just as if the livenessProbe failed. This can be + used to provide different probe parameters at the + beginning of a Pod''s lifecycle, when it might take + a long time to load data or warm a cache, than during + steady-state operation. This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for + the probe to be considered failed after having + succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum value + is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for + the probe to be considered successful after + having failed. Defaults to 1. Must be 1 for + liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the + pod needs to terminate gracefully upon probe + failure. The grace period is the duration in + seconds after the processes running in the pod + are sent a termination signal and the time when + the processes are forcibly halted with a kill + signal. Set this value longer than the expected + cleanup time for your process. If this value + is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate + a buffer for stdin in the container runtime. If + this is not set, reads from stdin in the container + will always result in EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach sessions. + If stdinOnce is set to true, stdin is opened on + container start, is empty until the first client + attaches to stdin, and then remains open and accepts + data until the client disconnects, at which time + stdin is closed and remains closed until the container + is restarted. If this flag is false, a container + processes that reads from stdin will never receive + an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to + which the container''s termination message will + be written is mounted into the container''s filesystem. + Message written is intended to be brief final status, + such as an assertion failure message. Will be truncated + by the node if greater than 4096 bytes. The total + message length across all containers will be limited + to 12kb. Defaults to /dev/termination-log. Cannot + be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. FallbackToLogsOnError + will use the last chunk of container log output + if the termination message file is empty and the + container exited with an error. The log output is + limited to 2048 bytes or 80 lines, whichever is + smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. + items: + description: volumeDevice describes a mapping of + a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of + the container that the device will be mapped + to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting of + a Volume within a container. + properties: + mountPath: + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. + type: string + mountPropagation: + description: mountPropagation determines how + mounts are propagated from the host to container + and the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults + to false. + type: boolean + subPath: + description: Path within the volume from which + the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume + from which the container's volume should be + mounted. Behaves similarly to SubPath but + environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not + specified, the container runtime's default will + be used, which might be configured in the container + image. Cannot be updated. + type: string + required: + - name + type: object + type: array + nodeSelectors: + additionalProperties: + type: string + description: NodeSelectors specifies additional node selectors + that will be added to the pod. + type: object + tolerations: + description: Tolerations specifies additional tolerations + that will be added to the pod. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple + using the matching operator . + properties: + effect: + description: Effect indicates the taint effect to + match. Empty means match all taint effects. When + specified, allowed values are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If + the key is empty, operator must be Exists; this + combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints + of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period + of time the toleration (which must be of effect + NoExecute, otherwise this field is ignored) tolerates + the taint. By default, it is not set, which means + tolerate the taint forever (do not evict). Zero + and negative values will be treated as 0 (evict + immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value + should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + type: object + type: array preview: description: Preview is specification for the CDAP preview service. properties: diff --git a/go.mod b/go.mod index ad8def5c..e0b624e3 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/zapr v1.2.3 // indirect diff --git a/go.sum b/go.sum index 021752da..7a31b713 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= diff --git a/main.go b/main.go index fc499a2a..b353dd46 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ package main import ( "flag" + "fmt" "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -30,9 +31,11 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/webhook" cdapv1alpha1 "cdap.io/cdap-operator/api/v1alpha1" "cdap.io/cdap-operator/controllers" + cdapwebhooks "cdap.io/cdap-operator/webhooks" //+kubebuilder:scaffold:imports ) @@ -52,11 +55,17 @@ func main() { var metricsAddr string var enableLeaderElection bool var probeAddr string + var enableWebhook bool + var webhookPort int flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&enableWebhook, "enable-webhook", false, + "Enable the admission controller webhook server. "+ + "Enabling this will allow the operator to mutate CDAP pods based on the mutation configuration in the CR.") + flag.IntVar(&webhookPort, "webhook-server-port", 9443, "The port on which the webhook server will listen.") opts := zap.Options{ Development: true, } @@ -68,7 +77,7 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, - Port: 9443, + Port: webhookPort, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "548f2421.cdap.io", @@ -85,6 +94,12 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "CDAPMaster") os.Exit(1) } + + if enableWebhook { + setupLog.Info(fmt.Sprintf("Starting webhook server at port %d", webhookPort)) + mgr.GetWebhookServer().Register("/mutate-v1-pod", &webhook.Admission{Handler: cdapwebhooks.NewPodMutator(mgr.GetClient())}) + } + //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/webhooks/cdap_webhook.go b/webhooks/cdap_webhook.go new file mode 100644 index 00000000..6005186d --- /dev/null +++ b/webhooks/cdap_webhook.go @@ -0,0 +1,127 @@ +package webhooks + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + + v1alpha1 "cdap.io/cdap-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + labelInstanceKey = "cdap.instance" + // cdapMasterNamespaceKey is the label added by CDAP on pods launched by CDAP. + cdapMasterNamespaceKey = "cdap.k8s.namespace" + // customResourceNamespaceKey is the label added by controller-reconciler on resources + // managed by the CDAP operator. + customResourceNamespaceKey = "custom-resource-namespace" +) + +type PodMutator struct { + Client client.Client + decoder *admission.Decoder +} + +func NewPodMutator(client client.Client) *PodMutator { + return &PodMutator{ + Client: client, + } +} + +func (s *PodMutator) Handle(ctx context.Context, req admission.Request) admission.Response { + pod := &corev1.Pod{} + err := s.decoder.Decode(req, pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + // Apply pod mutations. + if err := s.mutatePod(ctx, pod); err != nil { + log.Printf("Error while getting cdap master: %v", err) + if errors.IsNotFound(err) { + return admission.Denied(err.Error()) + } + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("Error while getting cdap master: %v", err)) + + } + marshaledPod, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} + +func (s *PodMutator) mutatePod(ctx context.Context, pod *corev1.Pod) error { + log.Printf("Got admission request for pod name: %s", pod.Name) + + cdapMasterName := pod.ObjectMeta.Labels[labelInstanceKey] + cdapMaster, err := s.cdapMaster(ctx, cdapMasterName, cdapMasterNamespace(pod)) + if err != nil { + if errors.IsNotFound(err) { + return fmt.Errorf("pod label %q refers to a non-existent CDAPMaster object %q: %w", labelInstanceKey, cdapMasterName, err) + } + return err + } + + mutationConfigs := cdapMaster.Spec.MutationConfigs + for _, mc := range mutationConfigs { + selector, err := metav1.LabelSelectorAsSelector(&mc.LabelSelector) + if err != nil { + log.Printf("Ignoring failure to parse label selector: %v", err) + continue + } + if !selector.Matches(labels.Set(pod.Labels)) { + continue + } + + if mc.PodMutations.NodeSelector != nil { + if pod.Spec.NodeSelector == nil { + pod.Spec.NodeSelector = map[string]string{} + } + for key, value := range *mc.PodMutations.NodeSelector { + pod.Spec.NodeSelector[key] = value + } + } + + pod.Spec.Tolerations = append(pod.Spec.Tolerations, mc.PodMutations.Tolerations...) + + ics := append([]corev1.Container{}, mc.PodMutations.InitContainersBefore...) + pod.Spec.InitContainers = append(ics, pod.Spec.InitContainers...) + } + return nil +} + +func (s *PodMutator) InjectDecoder(d *admission.Decoder) error { + s.decoder = d + return nil +} + +func (s *PodMutator) cdapMaster(ctx context.Context, cdapMasterName, namespace string) (*v1alpha1.CDAPMaster, error) { + log.Printf("Fetching CDAP CR with name %q.", cdapMasterName) + var cdapMaster v1alpha1.CDAPMaster + err := s.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: cdapMasterName}, &cdapMaster) + if err != nil { + return nil, err + } + return &cdapMaster, nil +} + +func cdapMasterNamespace(pod *corev1.Pod) string { + // First look for the cdap root namespace labels. + if ns, ok := pod.Labels[customResourceNamespaceKey]; ok { + return ns + } + if ns, ok := pod.Labels[cdapMasterNamespaceKey]; ok { + return ns + } + // Assume that the pod is in the same namespace as the CR. + return pod.Namespace +} diff --git a/webhooks/cdap_webhook_test.go b/webhooks/cdap_webhook_test.go new file mode 100644 index 00000000..b29e0f15 --- /dev/null +++ b/webhooks/cdap_webhook_test.go @@ -0,0 +1,229 @@ +package webhooks + +import ( + "context" + "fmt" + "testing" + + "cdap.io/cdap-operator/api/v1alpha1" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestMutatePod(t *testing.T) { + ctx := context.Background() + + cdapMaster := &v1alpha1.CDAPMaster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cdap-instance-1", + Namespace: "cdap-namespace", + }, + Spec: v1alpha1.CDAPMasterSpec{ + MutationConfigs: []v1alpha1.MutationConfig{ + { + LabelSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "cdap.twill.app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"app-1", "app-2", "app-3"}, + }, + }, + }, + PodMutations: v1alpha1.PodMutationConfig{ + NodeSelector: &map[string]string{ + "secure-nodepool": "true", + }, + InitContainersBefore: []v1.Container{ + { + Name: "init-container-1", + }, + }, + Tolerations: []v1.Toleration{ + { + Key: "toleration-key", + Value: "toleration-value", + }, + }, + }, + }, + }, + }, + } + + testCases := []struct { + description string + pod v1.Pod + wantErr error + wantPod v1.Pod + }{ + { + description: "successful", + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "cdap-namespace", + Labels: map[string]string{ + "cdap.twill.app": "app-2", + "cdap.instance": "cdap-instance-1", + }, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "original-init-container", + }, + }, + NodeSelector: map[string]string{ + "container-scanner": "true", + }, + }, + }, + wantPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "cdap-namespace", + Labels: map[string]string{ + "cdap.twill.app": "app-2", + "cdap.instance": "cdap-instance-1", + }, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "init-container-1", + }, + { + Name: "original-init-container", + }, + }, + Tolerations: []v1.Toleration{ + { + Key: "toleration-key", + Value: "toleration-value", + }, + }, + NodeSelector: map[string]string{ + "secure-nodepool": "true", + "container-scanner": "true", + }, + }, + }, + }, + { + description: "pod_in_other_namespace", + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "default", + Labels: map[string]string{ + "cdap.twill.app": "app-1", + "cdap.instance": "cdap-instance-1", + "cdap.k8s.namespace": "cdap-namespace", + }, + }, + }, + wantPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "default", + Labels: map[string]string{ + "cdap.twill.app": "app-1", + "cdap.instance": "cdap-instance-1", + "cdap.k8s.namespace": "cdap-namespace", + }, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "init-container-1", + }, + }, + Tolerations: []v1.Toleration{ + { + Key: "toleration-key", + Value: "toleration-value", + }, + }, + NodeSelector: map[string]string{ + "secure-nodepool": "true", + }, + }, + }, + }, + { + description: "non_existent_cdap_master", + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "default", + Labels: map[string]string{ + "cdap.twill.app": "app-1", + "cdap.instance": "cdap-instance-1", + }, + }, + }, + wantErr: cmpopts.AnyError, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + client, err := newFakeClient() + if err != nil { + t.Fatalf("Failed to create fake client: %v", err) + } + + if err := client.Create(ctx, cdapMaster.DeepCopy()); err != nil { + t.Fatalf("Failed to create CDAP CR: %v", err) + } + + webhook := PodMutator{ + Client: client, + } + + pod := tc.pod.DeepCopy() + err = webhook.mutatePod(ctx, pod) + if !cmp.Equal(err, tc.wantErr, cmpopts.EquateErrors()) { + t.Fatalf("mutatePod(%+v) returned unexpected error: want %v, got %v", tc.pod, tc.wantErr, err) + } + if err != nil { + t.Logf("%v", errors.IsNotFound(err)) + return + } + if diff := cmp.Diff(tc.wantPod, *pod); diff != "" { + t.Errorf("mutatePod(%+v) returned unexpected diff:(-want +got):\n%s", pod, diff) + } + }) + } +} + +// newFakeClient returns a fake kubernetes client for unit test. +func newFakeClient() (client.WithWatch, error) { + scheme, err := newScheme() + if err != nil { + return nil, err + } + return fake.NewClientBuilder().WithScheme(scheme).Build(), nil +} + +func newScheme() (*runtime.Scheme, error) { + sch := runtime.NewScheme() + if err := scheme.AddToScheme(sch); err != nil { + return nil, fmt.Errorf("failed to add scheme: %v", err) + } + + if err := v1alpha1.AddToScheme(sch); err != nil { + return nil, fmt.Errorf("failed to add cdap to scheme: %v", err) + } + + metav1.AddToGroupVersion(sch, v1alpha1.GroupVersion) + return sch, nil +} diff --git a/webhooks/templates/certificate.yaml b/webhooks/templates/certificate.yaml new file mode 100644 index 00000000..47c7e783 --- /dev/null +++ b/webhooks/templates/certificate.yaml @@ -0,0 +1,23 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cdap-webhook-cert + namespace: {CDAP_NAMESPACE} +spec: + isCA: true + commonName: selfsigned-ca + secretName: cdap-webhook-server-cert + duration: 2160h # 90d + renewBefore: 360h # 15d + privateKey: + algorithm: ECDSA + size: 256 + rotationPolicy: Always + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io + dnsNames: + - cdap-webhook-server + - cdap-webhook-server.{CDAP_NAMESPACE}.svc.cluster.local + - cdap-webhook-server.{CDAP_NAMESPACE}.svc diff --git a/webhooks/templates/issuer.yaml b/webhooks/templates/issuer.yaml new file mode 100644 index 00000000..3dc7e4bf --- /dev/null +++ b/webhooks/templates/issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: {CDAP_NAMESPACE} +spec: + selfSigned: {} diff --git a/webhooks/templates/webhook-service.yaml b/webhooks/templates/webhook-service.yaml new file mode 100644 index 00000000..76c6e9cf --- /dev/null +++ b/webhooks/templates/webhook-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: cdap-webhook-server + namespace: {CDAP_NAMESPACE} +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + control-plane: cdap-controller diff --git a/webhooks/templates/webhook.yaml b/webhooks/templates/webhook.yaml new file mode 100644 index 00000000..7a1ab6fb --- /dev/null +++ b/webhooks/templates/webhook.yaml @@ -0,0 +1,26 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: cdap-webhook + annotations: + cert-manager.io/inject-ca-from: {CDAP_NAMESPACE}/cdap-webhook-cert +webhooks: + - name: cdap-webhook-server.{CDAP_NAMESPACE}.svc.cluster.local + admissionReviewVersions: + - "v1" + sideEffects: "None" + timeoutSeconds: 30 + objectSelector: + matchExpressions: + - { key: "cdap.instance", operator: Exists } + clientConfig: + service: + name: cdap-webhook-server + namespace: {CDAP_NAMESPACE} + path: "/mutate-v1-pod" + caBundle: "" + rules: + - operations: ["CREATE"] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"]