diff --git a/docs/examples/kafka/autoscaler/compute/kafka-broker-autoscaler.yaml b/docs/examples/kafka/autoscaler/compute/kafka-broker-autoscaler.yaml new file mode 100644 index 000000000..1e2a722a5 --- /dev/null +++ b/docs/examples/kafka/autoscaler/compute/kafka-broker-autoscaler.yaml @@ -0,0 +1,24 @@ +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-broker-autoscaler + namespace: demo +spec: + databaseRef: + name: kafka-prod + opsRequestOptions: + timeout: 5m + apply: IfReady + compute: + broker: + trigger: "On" + podLifeTimeThreshold: 5m + resourceDiffPercentage: 20 + minAllowed: + cpu: 600m + memory: 1.5Gi + maxAllowed: + cpu: 1 + memory: 2Gi + controlledResources: ["cpu", "memory"] + containerControlledValues: "RequestsAndLimits" \ No newline at end of file diff --git a/docs/examples/kafka/autoscaler/compute/kafka-combined-autoscaler.yaml b/docs/examples/kafka/autoscaler/compute/kafka-combined-autoscaler.yaml new file mode 100644 index 000000000..b2c2e663b --- /dev/null +++ b/docs/examples/kafka/autoscaler/compute/kafka-combined-autoscaler.yaml @@ -0,0 +1,24 @@ +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-combined-autoscaler + namespace: demo +spec: + databaseRef: + name: kafka-dev + opsRequestOptions: + timeout: 5m + apply: IfReady + compute: + node: + trigger: "On" + podLifeTimeThreshold: 5m + resourceDiffPercentage: 20 + minAllowed: + cpu: 600m + memory: 1.5Gi + maxAllowed: + cpu: 1 + memory: 2Gi + controlledResources: ["cpu", "memory"] + containerControlledValues: "RequestsAndLimits" \ No newline at end of file diff --git a/docs/examples/kafka/autoscaler/compute/kafka-controller-autoscaler.yaml b/docs/examples/kafka/autoscaler/compute/kafka-controller-autoscaler.yaml new file mode 100644 index 000000000..e6e499905 --- /dev/null +++ b/docs/examples/kafka/autoscaler/compute/kafka-controller-autoscaler.yaml @@ -0,0 +1,24 @@ +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-controller-autoscaler + namespace: demo +spec: + databaseRef: + name: kafka-prod + opsRequestOptions: + timeout: 5m + apply: IfReady + compute: + controller: + trigger: "On" + podLifeTimeThreshold: 5m + resourceDiffPercentage: 20 + minAllowed: + cpu: 600m + memory: 1.5Gi + maxAllowed: + cpu: 1 + memory: 2Gi + controlledResources: ["cpu", "memory"] + containerControlledValues: "RequestsAndLimits" \ No newline at end of file diff --git a/docs/examples/kafka/autoscaler/kafka-combined.yaml b/docs/examples/kafka/autoscaler/kafka-combined.yaml new file mode 100644 index 000000000..d674b750b --- /dev/null +++ b/docs/examples/kafka/autoscaler/kafka-combined.yaml @@ -0,0 +1,27 @@ +apiVersion: kubedb.com/v1 +kind: Kafka +metadata: + name: kafka-dev + namespace: demo +spec: + replicas: 2 + version: 3.6.1 + podTemplate: + spec: + containers: + - name: kafka + resources: + limits: + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +# storageClassName: standard + storageType: Durable + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/examples/kafka/autoscaler/kafka-topology.yaml b/docs/examples/kafka/autoscaler/kafka-topology.yaml new file mode 100644 index 000000000..9b2fd9855 --- /dev/null +++ b/docs/examples/kafka/autoscaler/kafka-topology.yaml @@ -0,0 +1,48 @@ +apiVersion: kubedb.com/v1 +kind: Kafka +metadata: + name: kafka-prod + namespace: demo +spec: + version: 3.6.1 + topology: + broker: + replicas: 2 + podTemplate: + spec: + containers: + - name: kafka + resources: + limits: + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + controller: + replicas: 2 + podTemplate: + spec: + containers: + - name: kafka + resources: + limits: + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + storageType: Durable + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/examples/kafka/autoscaler/storage/kafka-storage-autoscaler-combined.yaml b/docs/examples/kafka/autoscaler/storage/kafka-storage-autoscaler-combined.yaml new file mode 100644 index 000000000..8860129c7 --- /dev/null +++ b/docs/examples/kafka/autoscaler/storage/kafka-storage-autoscaler-combined.yaml @@ -0,0 +1,14 @@ +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-storage-autoscaler-combined + namespace: demo +spec: + databaseRef: + name: kafka-dev + storage: + node: + expansionMode: "Online" + trigger: "On" + usageThreshold: 60 + scalingThreshold: 50 \ No newline at end of file diff --git a/docs/examples/kafka/autoscaler/storage/kafka-storage-autoscaler-topology.yaml b/docs/examples/kafka/autoscaler/storage/kafka-storage-autoscaler-topology.yaml new file mode 100644 index 000000000..3800820d6 --- /dev/null +++ b/docs/examples/kafka/autoscaler/storage/kafka-storage-autoscaler-topology.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-storage-autoscaler-topology + namespace: demo +spec: + databaseRef: + name: kafka-prod + storage: + broker: + expansionMode: "Online" + trigger: "On" + usageThreshold: 60 + scalingThreshold: 100 + controller: + expansionMode: "Online" + trigger: "On" + usageThreshold: 60 + scalingThreshold: 100 \ No newline at end of file diff --git a/docs/guides/kafka/autoscaler/compute/combined.md b/docs/guides/kafka/autoscaler/compute/combined.md index 6e8f80e8d..782241096 100644 --- a/docs/guides/kafka/autoscaler/compute/combined.md +++ b/docs/guides/kafka/autoscaler/compute/combined.md @@ -12,7 +12,7 @@ section_menu_id: guides > New to KubeDB? Please start [here](/docs/README.md). -# Autoscaling the Compute Resource of a Kafka Replicaset Database +# Autoscaling the Compute Resource of a Kafka Combined Cluster This guide will show you how to use `KubeDB` to autoscale compute resources i.e. cpu and memory of a Kafka combined cluster. @@ -37,94 +37,95 @@ $ kubectl create ns demo namespace/demo created ``` -> **Note:** YAML files used in this tutorial are stored in [docs/examples/kafka](/docs/examples/mongodb) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. +> **Note:** YAML files used in this tutorial are stored in [docs/examples/kafka](/docs/examples/kafka) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. ## Autoscaling of Combined Cluster -Here, we are going to deploy a `Kafka` Combined Cluser using a supported version by `KubeDB` operator. Then we are going to apply `KafkaAutoscaler` to set up autoscaling. +Here, we are going to deploy a `Kafka` Combined Cluster using a supported version by `KubeDB` operator. Then we are going to apply `KafkaAutoscaler` to set up autoscaling. -#### Deploy Kafka Replicaset +#### Deploy Kafka Combined Cluster -In this section, we are going to deploy a Kafka Replicaset database with version `3.6.1`. Then, in the next section we will set up autoscaling for this database using `KafkaAutoscaler` CRD. Below is the YAML of the `Kafka` CR that we are going to create, +In this section, we are going to deploy a Kafka Topology database with version `3.6.1`. Then, in the next section we will set up autoscaling for this database using `KafkaAutoscaler` CRD. Below is the YAML of the `Kafka` CR that we are going to create, ```yaml apiVersion: kubedb.com/v1 kind: Kafka metadata: - name: mg-rs + name: kafka-dev namespace: demo spec: - version: "4.4.26" - replicaSet: - name: "replicaset" - replicas: 3 - storageType: Durable + replicas: 2 + version: 3.6.1 + podTemplate: + spec: + containers: + - name: kafka + resources: + limits: + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi storage: + accessModes: + - ReadWriteOnce resources: requests: storage: 1Gi - podTemplate: - spec: - containers: - - name: mongo - resources: - requests: - cpu: "200m" - memory: "300Mi" - limits: - cpu: "200m" - memory: "300Mi" + storageClassName: standard + storageType: Durable deletionPolicy: WipeOut - ``` Let's create the `Kafka` CRO we have shown above, ```bash -$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/mongodb/autoscaling/compute/mg-rs.yaml -mongodb.kubedb.com/mg-rs created +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaler/kafka-combined.yaml +kafka.kubedb.com/kafka-dev created ``` -Now, wait until `mg-rs` has status `Ready`. i.e, +Now, wait until `kafka-dev` has status `Ready`. i.e, ```bash -$ kubectl get mg -n demo -NAME VERSION STATUS AGE -mg-rs 4.4.26 Ready 2m53s +$ kubectl get kf -n demo -w +NAME TYPE VERSION STATUS AGE +kafka-dev kubedb.com/v1 3.6.1 Provisioning 0s +kafka-dev kubedb.com/v1 3.6.1 Provisioning 24s +. +. +kafka-dev kubedb.com/v1 3.6.1 Ready 92s ``` Let's check the Pod containers resources, ```bash -$ kubectl get pod -n demo mg-rs-0 -o json | jq '.spec.containers[].resources' +$ kubectl get pod -n demo kafka-dev-0 -o json | jq '.spec.containers[].resources' { "limits": { - "cpu": "200m", - "memory": "300Mi" + "memory": "1Gi" }, "requests": { - "cpu": "200m", - "memory": "300Mi" + "cpu": "500m", + "memory": "1Gi" } } ``` Let's check the Kafka resources, ```bash -$ kubectl get mongodb -n demo mg-rs -o json | jq '.spec.podTemplate.spec.resources' +$ kubectl get kafka -n demo kafka-dev -o json | jq '.spec.podTemplate.spec.containers[].resources' { "limits": { - "cpu": "200m", - "memory": "300Mi" + "memory": "1Gi" }, "requests": { - "cpu": "200m", - "memory": "300Mi" + "cpu": "500m", + "memory": "1Gi" } } ``` -You can see from the above outputs that the resources are same as the one we have assigned while deploying the mongodb. +You can see from the above outputs that the resources are same as the one we have assigned while deploying the kafka. We are now ready to apply the `KafkaAutoscaler` CRO to set up autoscaling for this database. @@ -134,343 +135,264 @@ Here, we are going to set up compute resource autoscaling using a KafkaAutoscale #### Create KafkaAutoscaler Object -In order to set up compute resource autoscaling for this replicaset database, we have to create a `KafkaAutoscaler` CRO with our desired configuration. Below is the YAML of the `KafkaAutoscaler` object that we are going to create, +In order to set up compute resource autoscaling for this combined cluster, we have to create a `KafkaAutoscaler` CRO with our desired configuration. Below is the YAML of the `KafkaAutoscaler` object that we are going to create, ```yaml apiVersion: autoscaling.kubedb.com/v1alpha1 kind: KafkaAutoscaler metadata: - name: mg-as-rs + name: kf-combined-autoscaler namespace: demo spec: databaseRef: - name: mg-rs + name: kafka-dev opsRequestOptions: - timeout: 3m + timeout: 5m apply: IfReady compute: - replicaSet: + node: trigger: "On" podLifeTimeThreshold: 5m resourceDiffPercentage: 20 minAllowed: - cpu: 400m - memory: 400Mi + cpu: 600m + memory: 1.5Gi maxAllowed: cpu: 1 - memory: 1Gi + memory: 2Gi controlledResources: ["cpu", "memory"] containerControlledValues: "RequestsAndLimits" ``` Here, -- `spec.databaseRef.name` specifies that we are performing compute resource scaling operation on `mg-rs` database. -- `spec.compute.replicaSet.trigger` specifies that compute autoscaling is enabled for this database. -- `spec.compute.replicaSet.podLifeTimeThreshold` specifies the minimum lifetime for at least one of the pod to initiate a vertical scaling. -- `spec.compute.replicaset.resourceDiffPercentage` specifies the minimum resource difference in percentage. The default is 10%. +- `spec.databaseRef.name` specifies that we are performing compute resource scaling operation on `kafka-dev` cluster. +- `spec.compute.node.trigger` specifies that compute autoscaling is enabled for this cluster. +- `spec.compute.node.podLifeTimeThreshold` specifies the minimum lifetime for at least one of the pod to initiate a vertical scaling. +- `spec.compute.node.resourceDiffPercentage` specifies the minimum resource difference in percentage. The default is 10%. If the difference between current & recommended resource is less than ResourceDiffPercentage, Autoscaler Operator will ignore the updating. -- `spec.compute.replicaSet.minAllowed` specifies the minimum allowed resources for the database. -- `spec.compute.replicaSet.maxAllowed` specifies the maximum allowed resources for the database. -- `spec.compute.replicaSet.controlledResources` specifies the resources that are controlled by the autoscaler. -- `spec.compute.replicaSet.containerControlledValues` specifies which resource values should be controlled. The default is "RequestsAndLimits". -- `spec.opsRequestOptions` contains the options to pass to the created OpsRequest. It has 3 fields. Know more about them here : [readinessCriteria](/docs/guides/mongodb/concepts/opsrequest.md#specreadinesscriteria), [timeout](/docs/guides/mongodb/concepts/opsrequest.md#spectimeout), [apply](/docs/guides/mongodb/concepts/opsrequest.md#specapply). - -If it was an `InMemory database`, we could also autoscaler the inMemory resources using Kafka compute autoscaler, like below. - -#### Autoscale inMemory database -To autoscale inMemory databases, you need to specify the `spec.compute.replicaSet.inMemoryStorage` section. - -```yaml - ... - inMemoryStorage: - usageThresholdPercentage: 80 - scalingFactorPercentage: 30 - ... -``` -It has two fields inside it. -- `usageThresholdPercentage`. If db uses more than usageThresholdPercentage of the total memory, memoryStorage should be increased. Default usage threshold is 70%. -- `scalingFactorPercentage`. If db uses more than usageThresholdPercentage of the total memory, memoryStorage should be increased by this given scaling percentage. Default scaling percentage is 50%. - -> Note: To inform you, We use `db.serverStatus().inMemory.cache["bytes currently in the cache"]` & `db.serverStatus().inMemory.cache["maximum bytes configured"]` to calculate the used & maximum inMemory storage respectively. +- `spec.compute.node.minAllowed` specifies the minimum allowed resources for the cluster. +- `spec.compute.node.maxAllowed` specifies the maximum allowed resources for the cluster. +- `spec.compute.node.controlledResources` specifies the resources that are controlled by the autoscaler. +- `spec.compute.node.containerControlledValues` specifies which resource values should be controlled. The default is "RequestsAndLimits". +- `spec.opsRequestOptions` contains the options to pass to the created OpsRequest. It has 2 fields. + - `timeout` specifies the timeout for the OpsRequest. + - `apply` specifies when the OpsRequest should be applied. The default is "IfReady". Let's create the `KafkaAutoscaler` CR we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/mongodb/autoscaling/compute/mg-as-rs.yaml -mongodbautoscaler.autoscaling.kubedb.com/mg-as-rs created +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaler/compute/kafka-combined-autoscaler.yaml +kafkaautoscaler.autoscaling.kubedb.com/kf-combined-autoscaler created ``` #### Verify Autoscaling is set up successfully -Let's check that the `mongodbautoscaler` resource is created successfully, +Let's check that the `kafkaautoscaler` resource is created successfully, ```bash -$ kubectl get mongodbautoscaler -n demo -NAME AGE -mg-as-rs 102s - -$ kubectl describe mongodbautoscaler mg-as-rs -n demo -Name: mg-as-rs +$ kubectl describe kafkaautoscaler kf-combined-autoscaler -n demo +Name: kf-combined-autoscaler Namespace: demo Labels: Annotations: API Version: autoscaling.kubedb.com/v1alpha1 Kind: KafkaAutoscaler Metadata: - Creation Timestamp: 2022-10-27T06:56:34Z + Creation Timestamp: 2024-08-27T05:55:51Z Generation: 1 - Managed Fields: - API Version: autoscaling.kubedb.com/v1alpha1 - Fields Type: FieldsV1 - fieldsV1: - f:metadata: - f:annotations: - .: - f:kubectl.kubernetes.io/last-applied-configuration: - f:spec: - .: - f:compute: - .: - f:replicaSet: - .: - f:containerControlledValues: - f:controlledResources: - f:maxAllowed: - .: - f:cpu: - f:memory: - f:minAllowed: - .: - f:cpu: - f:memory: - f:podLifeTimeThreshold: - f:resourceDiffPercentage: - f:trigger: - f:databaseRef: - f:opsRequestOptions: - .: - f:apply: - f:timeout: - Manager: kubectl-client-side-apply - Operation: Update - Time: 2022-10-27T06:56:34Z - API Version: autoscaling.kubedb.com/v1alpha1 - Fields Type: FieldsV1 - fieldsV1: - f:status: - .: - f:checkpoints: - f:conditions: - f:vpas: - Manager: kubedb-autoscaler - Operation: Update - Subresource: status - Time: 2022-10-27T07:01:05Z - Resource Version: 640314 - UID: ab03414a-67a2-4da4-8960-6e67ae56b503 + Owner References: + API Version: kubedb.com/v1 + Block Owner Deletion: true + Controller: true + Kind: Kafka + Name: kafka-dev + UID: a0153c7f-1e1e-4070-a318-c7c1153b810a + Resource Version: 1104655 + UID: 817602cc-f851-4fc5-b2c1-1d191462ac56 Spec: Compute: - Replica Set: + Node: Container Controlled Values: RequestsAndLimits Controlled Resources: cpu memory Max Allowed: Cpu: 1 - Memory: 1Gi + Memory: 2Gi Min Allowed: - Cpu: 400m - Memory: 400Mi + Cpu: 600m + Memory: 1536Mi Pod Life Time Threshold: 5m0s Resource Diff Percentage: 20 Trigger: On Database Ref: - Name: mg-rs + Name: kafka-dev Ops Request Options: Apply: IfReady - Timeout: 3m0s + Timeout: 5m0s Status: Checkpoints: - Cpu Histogram: - Bucket Weights: - Index: 2 - Weight: 10000 - Index: 3 - Weight: 5000 - Reference Timestamp: 2022-10-27T00:00:00Z - Total Weight: 0.3673624107285783 - First Sample Start: 2022-10-27T07:00:42Z - Last Sample Start: 2022-10-27T07:00:55Z - Last Update Time: 2022-10-27T07:01:00Z - Memory Histogram: - Reference Timestamp: 2022-10-28T00:00:00Z - Ref: - Container Name: mongodb - Vpa Object Name: mg-rs - Total Samples Count: 3 - Version: v3 Cpu Histogram: Bucket Weights: Index: 0 + Weight: 4610 + Index: 1 Weight: 10000 - Reference Timestamp: 2022-10-27T00:00:00Z - Total Weight: 0.3673624107285783 - First Sample Start: 2022-10-27T07:00:42Z - Last Sample Start: 2022-10-27T07:00:55Z - Last Update Time: 2022-10-27T07:01:00Z + Reference Timestamp: 2024-08-27T05:55:00Z + Total Weight: 0.35081120875606336 + First Sample Start: 2024-08-27T05:55:44Z + Last Sample Start: 2024-08-27T05:56:49Z + Last Update Time: 2024-08-27T05:57:10Z Memory Histogram: - Reference Timestamp: 2022-10-28T00:00:00Z + Reference Timestamp: 2024-08-27T06:00:00Z Ref: - Container Name: replication-mode-detector - Vpa Object Name: mg-rs + Container Name: kafka + Vpa Object Name: kafka-dev Total Samples Count: 3 Version: v3 Conditions: - Last Transition Time: 2022-10-27T07:01:05Z - Message: Successfully created mongoDBOpsRequest demo/mops-mg-rs-cxhsy1 + Last Transition Time: 2024-08-27T05:56:32Z + Message: Successfully created kafkaOpsRequest demo/kfops-kafka-dev-z8d3l5 Observed Generation: 1 Reason: CreateOpsRequest Status: True Type: CreateOpsRequest Vpas: Conditions: - Last Transition Time: 2022-10-27T07:01:00Z + Last Transition Time: 2024-08-27T05:56:10Z Status: True Type: RecommendationProvided Recommendation: Container Recommendations: - Container Name: mongodb + Container Name: kafka Lower Bound: - Cpu: 400m - Memory: 400Mi + Cpu: 600m + Memory: 1536Mi Target: - Cpu: 400m - Memory: 400Mi + Cpu: 600m + Memory: 1536Mi Uncapped Target: - Cpu: 49m - Memory: 262144k + Cpu: 100m + Memory: 511772986 Upper Bound: Cpu: 1 - Memory: 1Gi - Vpa Name: mg-rs + Memory: 2Gi + Vpa Name: kafka-dev Events: ``` -So, the `mongodbautoscaler` resource is created successfully. +So, the `kafkaautoscaler` resource is created successfully. -you can see in the `Status.VPAs.Recommendation` section, that recommendation has been generated for our database. Our autoscaler operator continuously watches the recommendation generated and creates an `mongodbopsrequest` based on the recommendations, if the database pods are needed to scaled up or down. +you can see in the `Status.VPAs.Recommendation` section, that recommendation has been generated for our database. Our autoscaler operator continuously watches the recommendation generated and creates an `kafkaopsrequest` based on the recommendations, if the database pods resources are needed to scaled up or down. -Let's watch the `mongodbopsrequest` in the demo namespace to see if any `mongodbopsrequest` object is created. After some time you'll see that a `mongodbopsrequest` will be created based on the recommendation. +Let's watch the `kafkaopsrequest` in the demo namespace to see if any `kafkaopsrequest` object is created. After some time you'll see that a `kafkaopsrequest` will be created based on the recommendation. ```bash -$ watch kubectl get mongodbopsrequest -n demo -Every 2.0s: kubectl get mongodbopsrequest -n demo -NAME TYPE STATUS AGE -mops-mg-rs-cxhsy1 VerticalScaling Progressing 10s +$ watch kubectl get kafkaopsrequest -n demo +Every 2.0s: kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-dev-z8d3l5 VerticalScaling Progressing 10s ``` Let's wait for the ops request to become successful. ```bash -$ watch kubectl get mongodbopsrequest -n demo -Every 2.0s: kubectl get mongodbopsrequest -n demo -NAME TYPE STATUS AGE -mops-mg-rs-cxhsy1 VerticalScaling Successful 68s +$ kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-dev-z8d3l5 VerticalScaling Successful 3m2s ``` -We can see from the above output that the `KafkaOpsRequest` has succeeded. If we describe the `KafkaOpsRequest` we will get an overview of the steps that were followed to scale the database. +We can see from the above output that the `KafkaOpsRequest` has succeeded. If we describe the `KafkaOpsRequest` we will get an overview of the steps that were followed to scale the cluster. ```bash -$ kubectl describe mongodbopsrequest -n demo mops-mg-rs-cxhsy1 -Name: mops-mg-rs-cxhsy1 +$ kubectl describe kafkaopsrequests -n demo kfops-kafka-dev-z8d3l5 +Name: kfops-kafka-dev-z8d3l5 Namespace: demo -Labels: +Labels: app.kubernetes.io/component=database + app.kubernetes.io/instance=kafka-dev + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=kafkas.kubedb.com Annotations: API Version: ops.kubedb.com/v1alpha1 Kind: KafkaOpsRequest Metadata: - Creation Timestamp: 2022-10-27T07:01:05Z + Creation Timestamp: 2024-08-27T05:56:32Z Generation: 1 - Managed Fields: - API Version: ops.kubedb.com/v1alpha1 - Fields Type: FieldsV1 - fieldsV1: - f:metadata: - f:ownerReferences: - .: - k:{"uid":"ab03414a-67a2-4da4-8960-6e67ae56b503"}: - f:spec: - .: - f:apply: - f:databaseRef: - f:timeout: - f:type: - f:verticalScaling: - .: - f:replicaSet: - .: - f:limits: - .: - f:cpu: - f:memory: - f:requests: - .: - f:cpu: - f:memory: - Manager: kubedb-autoscaler - Operation: Update - Time: 2022-10-27T07:01:05Z - API Version: ops.kubedb.com/v1alpha1 - Fields Type: FieldsV1 - fieldsV1: - f:status: - .: - f:conditions: - f:observedGeneration: - f:phase: - Manager: kubedb-ops-manager - Operation: Update - Subresource: status - Time: 2022-10-27T07:02:31Z Owner References: API Version: autoscaling.kubedb.com/v1alpha1 Block Owner Deletion: true Controller: true Kind: KafkaAutoscaler - Name: mg-as-rs - UID: ab03414a-67a2-4da4-8960-6e67ae56b503 - Resource Version: 640598 - UID: f7c6db00-dd0e-4850-8bad-5f0855ce3850 + Name: kf-combined-autoscaler + UID: 817602cc-f851-4fc5-b2c1-1d191462ac56 + Resource Version: 1104871 + UID: 8b7615c6-d38b-4d5a-b733-6aa93cd41a29 Spec: Apply: IfReady Database Ref: - Name: mg-rs - Timeout: 3m0s + Name: kafka-dev + Timeout: 5m0s Type: VerticalScaling Vertical Scaling: - Replica Set: - Limits: - Cpu: 400m - Memory: 400Mi - Requests: - Cpu: 400m - Memory: 400Mi + Node: + Resources: + Limits: + Memory: 1536Mi + Requests: + Cpu: 600m + Memory: 1536Mi Status: Conditions: - Last Transition Time: 2022-10-27T07:01:05Z - Message: Kafka ops request is vertically scaling database + Last Transition Time: 2024-08-27T05:56:32Z + Message: Kafka ops-request has started to vertically scaling the kafka nodes Observed Generation: 1 Reason: VerticalScaling Status: True Type: VerticalScaling - Last Transition Time: 2022-10-27T07:02:30Z - Message: Successfully Vertically Scaled Replicaset Resources + Last Transition Time: 2024-08-27T05:56:35Z + Message: Successfully updated PetSets Resources + Observed Generation: 1 + Reason: UpdatePetSets + Status: True + Type: UpdatePetSets + Last Transition Time: 2024-08-27T05:56:40Z + Message: get pod; ConditionStatus:True; PodName:kafka-dev-0 + Observed Generation: 1 + Status: True + Type: GetPod--kafka-dev-0 + Last Transition Time: 2024-08-27T05:56:40Z + Message: evict pod; ConditionStatus:True; PodName:kafka-dev-0 + Observed Generation: 1 + Status: True + Type: EvictPod--kafka-dev-0 + Last Transition Time: 2024-08-27T05:57:10Z + Message: check pod running; ConditionStatus:True; PodName:kafka-dev-0 + Observed Generation: 1 + Status: True + Type: CheckPodRunning--kafka-dev-0 + Last Transition Time: 2024-08-27T05:57:15Z + Message: get pod; ConditionStatus:True; PodName:kafka-dev-1 Observed Generation: 1 - Reason: UpdateReplicaSetResources Status: True - Type: UpdateReplicaSetResources - Last Transition Time: 2022-10-27T07:02:31Z - Message: Successfully Vertically Scaled Database + Type: GetPod--kafka-dev-1 + Last Transition Time: 2024-08-27T05:57:16Z + Message: evict pod; ConditionStatus:True; PodName:kafka-dev-1 + Observed Generation: 1 + Status: True + Type: EvictPod--kafka-dev-1 + Last Transition Time: 2024-08-27T05:57:25Z + Message: check pod running; ConditionStatus:True; PodName:kafka-dev-1 + Observed Generation: 1 + Status: True + Type: CheckPodRunning--kafka-dev-1 + Last Transition Time: 2024-08-27T05:57:30Z + Message: Successfully Restarted Pods With Resources + Observed Generation: 1 + Reason: RestartPods + Status: True + Type: RestartPods + Last Transition Time: 2024-08-27T05:57:30Z + Message: Successfully completed the vertical scaling for kafka Observed Generation: 1 Reason: Successful Status: True @@ -478,58 +400,70 @@ Status: Observed Generation: 1 Phase: Successful Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal PauseDatabase 4m9s KubeDB Ops-manager Operator Pausing Kafka demo/mg-rs - Normal PauseDatabase 4m9s KubeDB Ops-manager Operator Successfully paused Kafka demo/mg-rs - Normal Starting 4m9s KubeDB Ops-manager Operator Updating Resources of PetSet: mg-rs - Normal UpdateReplicaSetResources 4m9s KubeDB Ops-manager Operator Successfully updated replicaset Resources - Normal Starting 4m9s KubeDB Ops-manager Operator Updating Resources of PetSet: mg-rs - Normal UpdateReplicaSetResources 4m9s KubeDB Ops-manager Operator Successfully updated replicaset Resources - Normal UpdateReplicaSetResources 2m44s KubeDB Ops-manager Operator Successfully Vertically Scaled Replicaset Resources - Normal ResumeDatabase 2m43s KubeDB Ops-manager Operator Resuming Kafka demo/mg-rs - Normal ResumeDatabase 2m43s KubeDB Ops-manager Operator Successfully resumed Kafka demo/mg-rs - Normal Successful 2m43s KubeDB Ops-manager Operator Successfully Vertically Scaled Database - Normal UpdateReplicaSetResources 2m43s KubeDB Ops-manager Operator Successfully Vertically Scaled Replicaset Resources - + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Starting 4m33s KubeDB Ops-manager Operator Start processing for KafkaOpsRequest: demo/kfops-kafka-dev-z8d3l5 + Normal Starting 4m33s KubeDB Ops-manager Operator Pausing Kafka databse: demo/kafka-dev + Normal Successful 4m33s KubeDB Ops-manager Operator Successfully paused Kafka database: demo/kafka-dev for KafkaOpsRequest: kfops-kafka-dev-z8d3l5 + Normal UpdatePetSets 4m30s KubeDB Ops-manager Operator Successfully updated PetSets Resources + Warning get pod; ConditionStatus:True; PodName:kafka-dev-0 4m25s KubeDB Ops-manager Operator get pod; ConditionStatus:True; PodName:kafka-dev-0 + Warning evict pod; ConditionStatus:True; PodName:kafka-dev-0 4m25s KubeDB Ops-manager Operator evict pod; ConditionStatus:True; PodName:kafka-dev-0 + Warning check pod running; ConditionStatus:False; PodName:kafka-dev-0 4m19s KubeDB Ops-manager Operator check pod running; ConditionStatus:False; PodName:kafka-dev-0 + Warning check pod running; ConditionStatus:True; PodName:kafka-dev-0 3m55s KubeDB Ops-manager Operator check pod running; ConditionStatus:True; PodName:kafka-dev-0 + Warning get pod; ConditionStatus:True; PodName:kafka-dev-1 3m50s KubeDB Ops-manager Operator get pod; ConditionStatus:True; PodName:kafka-dev-1 + Warning evict pod; ConditionStatus:True; PodName:kafka-dev-1 3m49s KubeDB Ops-manager Operator evict pod; ConditionStatus:True; PodName:kafka-dev-1 + Warning check pod running; ConditionStatus:False; PodName:kafka-dev-1 3m45s KubeDB Ops-manager Operator check pod running; ConditionStatus:False; PodName:kafka-dev-1 + Warning check pod running; ConditionStatus:True; PodName:kafka-dev-1 3m40s KubeDB Ops-manager Operator check pod running; ConditionStatus:True; PodName:kafka-dev-1 + Normal RestartPods 3m35s KubeDB Ops-manager Operator Successfully Restarted Pods With Resources + Normal Starting 3m35s KubeDB Ops-manager Operator Resuming Kafka database: demo/kafka-dev + Normal Successful 3m35s KubeDB Ops-manager Operator Successfully resumed Kafka database: demo/kafka-dev for KafkaOpsRequest: kfops-kafka-dev-z8d3l5 ``` -Now, we are going to verify from the Pod, and the Kafka yaml whether the resources of the replicaset database has updated to meet up the desired state, Let's check, +Now, we are going to verify from the Pod, and the Kafka yaml whether the resources of the topology database has updated to meet up the desired state, Let's check, ```bash -$ kubectl get pod -n demo mg-rs-0 -o json | jq '.spec.containers[].resources' +$ kubectl get pod -n demo kafka-dev-0 -o json | jq '.spec.containers[].resources' { "limits": { - "cpu": "400m", - "memory": "400Mi" + "memory": "1536Mi" }, "requests": { - "cpu": "400m", - "memory": "400Mi" + "cpu": "600m", + "memory": "1536Mi" } } -$ kubectl get mongodb -n demo mg-rs -o json | jq '.spec.podTemplate.spec.resources' + +$ kubectl get kafka -n demo kafka-dev -o json | jq '.spec.podTemplate.spec.containers[].resources' { "limits": { - "cpu": "400m", - "memory": "400Mi" + "memory": "1536Mi" }, "requests": { - "cpu": "400m", - "memory": "400Mi" + "cpu": "600m", + "memory": "1536Mi" } } ``` -The above output verifies that we have successfully auto scaled the resources of the Kafka replicaset database. +The above output verifies that we have successfully auto scaled the resources of the Kafka combined cluster. ## Cleaning Up To clean up the Kubernetes resources created by this tutorial, run: ```bash -kubectl delete mg -n demo mg-rs -kubectl delete mongodbautoscaler -n demo mg-as-rs -``` \ No newline at end of file +kubectl delete kafkaopsrequest -n demo kfops-kafka-dev-z8d3l5 +kubectl delete kafkaautoscaler -n demo kf-combined-autoscaler +kubectl delete kf -n demo kafka-dev +kubectl delete ns demo +``` +## Next Steps + +- Detail concepts of [Kafka object](/docs/guides/kafka/concepts/kafka.md). +- Different Kafka topology clustering modes [here](/docs/guides/kafka/clustering/_index.md). +- Monitor your Kafka database with KubeDB using [out-of-the-box Prometheus operator](/docs/guides/kafka/monitoring/using-prometheus-operator.md). + +[//]: # (- Monitor your Kafka database with KubeDB using [out-of-the-box builtin-Prometheus](/docs/guides/kafka/monitoring/using-builtin-prometheus.md).) +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/kafka/autoscaler/compute/overview.md b/docs/guides/kafka/autoscaler/compute/overview.md index 529d7d4cc..fd2bdfa5a 100644 --- a/docs/guides/kafka/autoscaler/compute/overview.md +++ b/docs/guides/kafka/autoscaler/compute/overview.md @@ -20,7 +20,7 @@ This guide will give an overview on how KubeDB Autoscaler operator autoscales th - You should be familiar with the following `KubeDB` concepts: - [Kafka](/docs/guides/mongodb/concepts/mongodb.md) - - [KafkaAutoscaler](/docs/guides/kafka/concepts/autoscaler.md) + - [KafkaAutoscaler](/docs/guides/kafka/concepts/kafkaautoscaler.md) - [KafkaOpsRequest](/docs/guides/kafka/concepts/kafkaopsrequest.md) ## How Compute Autoscaling Works diff --git a/docs/guides/kafka/autoscaler/compute/topology.md b/docs/guides/kafka/autoscaler/compute/topology.md new file mode 100644 index 000000000..b2e1d35f4 --- /dev/null +++ b/docs/guides/kafka/autoscaler/compute/topology.md @@ -0,0 +1,852 @@ +--- +title: Kafka Topology Autoscaling +menu: + docs_{{ .version }}: + identifier: kf-auto-scaling-topology + name: Topology Cluster + parent: kf-compute-auto-scaling + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Autoscaling the Compute Resource of a Kafka Topology Cluster + +This guide will show you how to use `KubeDB` to autoscale compute resources i.e. cpu and memory of a Kafka topology cluster. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. + +- Install `KubeDB` Provisioner, Ops-manager and Autoscaler operator in your cluster following the steps [here](/docs/setup/README.md). + +- Install `Metrics Server` from [here](https://github.com/kubernetes-sigs/metrics-server#installation) + +- You should be familiar with the following `KubeDB` concepts: + - [Kafka](/docs/guides/kafka/concepts/kafka.md) + - [KafkaAutoscaler](/docs/guides/kafka/concepts/kafkaautoscaler.md) + - [KafkaOpsRequest](/docs/guides/kafka/concepts/kafkaopsrequest.md) + - [Compute Resource Autoscaling Overview](/docs/guides/kafka/autoscaler/compute/overview.md) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/examples/kafka](/docs/examples/kafka) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + +## Autoscaling of Topology Cluster + +Here, we are going to deploy a `Kafka` Topology Cluster using a supported version by `KubeDB` operator. Then we are going to apply `KafkaAutoscaler` to set up autoscaling. + +#### Deploy Kafka Topology Cluster + +In this section, we are going to deploy a Kafka Topology cluster with version `3.6.1`. Then, in the next section we will set up autoscaling for this database using `KafkaAutoscaler` CRD. Below is the YAML of the `Kafka` CR that we are going to create, + +```yaml +apiVersion: kubedb.com/v1 +kind: Kafka +metadata: + name: kafka-prod + namespace: demo +spec: + version: 3.6.1 + topology: + broker: + replicas: 2 + podTemplate: + spec: + containers: + - name: kafka + resources: + limits: + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + controller: + replicas: 2 + podTemplate: + spec: + containers: + - name: kafka + resources: + limits: + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + storageType: Durable + deletionPolicy: WipeOut +``` + +Let's create the `Kafka` CRO we have shown above, + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaler/kafka-topology.yaml +kafka.kubedb.com/kafka-prod created +``` + +Now, wait until `kafka-prod` has status `Ready`. i.e, + +```bash +$ kubectl get kf -n demo -w +NAME TYPE VERSION STATUS AGE +kafka-prod kubedb.com/v1 3.6.1 Provisioning 0s +kafka-prod kubedb.com/v1 3.6.1 Provisioning 24s +. +. +kafka-prod kubedb.com/v1 3.6.1 Ready 118s +``` + +## Kafka Topology Autoscaler(Broker) + +Let's check the Broker Pod containers resources, + +```bash +$ kubectl get pod -n demo kafka-prod-broker-0 -o json | jq '.spec.containers[].resources' +{ + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } +} +``` + +Let's check the Kafka resources for broker, +```bash +$ kubectl get kafka -n demo kafka-prod -o json | jq '.spec.topology.broker.podTemplate.spec.containers[].resources' +{ + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } +} +``` + +You can see from the above outputs that the resources for broker are same as the one we have assigned while deploying the kafka. + +We are now ready to apply the `KafkaAutoscaler` CRO to set up autoscaling for this broker nodes. + +### Compute Resource Autoscaling + +Here, we are going to set up compute resource autoscaling using a KafkaAutoscaler Object. + +#### Create KafkaAutoscaler Object + +In order to set up compute resource autoscaling for this topology cluster, we have to create a `KafkaAutoscaler` CRO with our desired configuration. Below is the YAML of the `KafkaAutoscaler` object that we are going to create, + +```yaml +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-broker-autoscaler + namespace: demo +spec: + databaseRef: + name: kafka-prod + opsRequestOptions: + timeout: 5m + apply: IfReady + compute: + broker: + trigger: "On" + podLifeTimeThreshold: 5m + resourceDiffPercentage: 20 + minAllowed: + cpu: 600m + memory: 1.5Gi + maxAllowed: + cpu: 1 + memory: 2Gi + controlledResources: ["cpu", "memory"] + containerControlledValues: "RequestsAndLimits" +``` + +Here, + +- `spec.databaseRef.name` specifies that we are performing compute resource scaling operation on `kafka-prod` cluster. +- `spec.compute.broker.trigger` specifies that compute autoscaling is enabled for this node. +- `spec.compute.broker.podLifeTimeThreshold` specifies the minimum lifetime for at least one of the pod to initiate a vertical scaling. +- `spec.compute.broker.resourceDiffPercentage` specifies the minimum resource difference in percentage. The default is 10%. + If the difference between current & recommended resource is less than ResourceDiffPercentage, Autoscaler Operator will ignore the updating. +- `spec.compute.broker.minAllowed` specifies the minimum allowed resources for the cluster. +- `spec.compute.broker.maxAllowed` specifies the maximum allowed resources for the cluster. +- `spec.compute.broker.controlledResources` specifies the resources that are controlled by the autoscaler. +- `spec.compute.broker.containerControlledValues` specifies which resource values should be controlled. The default is "RequestsAndLimits". +- `spec.opsRequestOptions` contains the options to pass to the created OpsRequest. It has 2 fields. + - `timeout` specifies the timeout for the OpsRequest. + - `apply` specifies when the OpsRequest should be applied. The default is "IfReady". + +Let's create the `KafkaAutoscaler` CR we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaler/compute/kafka-broker-autoscaler.yaml +kafkaautoscaler.autoscaling.kubedb.com/kf-broker-autoscaler created +``` + +#### Verify Autoscaling is set up successfully + +Let's check that the `kafkaautoscaler` resource is created successfully, + +```bash +$ kubectl describe kafkaautoscaler kf-broker-autoscaler -n demo +$ kubectl describe kafkaautoscaler kf-broker-autoscaler -n demo +Name: kf-broker-autoscaler +Namespace: demo +Labels: +Annotations: +API Version: autoscaling.kubedb.com/v1alpha1 +Kind: KafkaAutoscaler +Metadata: + Creation Timestamp: 2024-08-27T06:17:07Z + Generation: 1 + Owner References: + API Version: kubedb.com/v1 + Block Owner Deletion: true + Controller: true + Kind: Kafka + Name: kafka-prod + UID: 7cee41e0-259c-4a5e-856a-e8ca90056120 + Resource Version: 1113275 + UID: 7e3be99f-cd4d-440a-a477-8e8994840ebb +Spec: + Compute: + Broker: + Container Controlled Values: RequestsAndLimits + Controlled Resources: + cpu + memory + Max Allowed: + Cpu: 1 + Memory: 2Gi + Min Allowed: + Cpu: 600m + Memory: 1536Mi + Pod Life Time Threshold: 5m0s + Resource Diff Percentage: 20 + Trigger: On + Database Ref: + Name: kafka-prod + Ops Request Options: + Apply: IfReady + Timeout: 5m0s +Status: + Checkpoints: + Cpu Histogram: + Bucket Weights: + Index: 1 + Weight: 10000 + Index: 2 + Weight: 2485 + Index: 30 + Weight: 1923 + Reference Timestamp: 2024-08-27T06:20:00Z + Total Weight: 0.8587070656303101 + First Sample Start: 2024-08-27T06:20:45Z + Last Sample Start: 2024-08-27T06:23:53Z + Last Update Time: 2024-08-27T06:24:10Z + Memory Histogram: + Bucket Weights: + Index: 20 + Weight: 9682 + Index: 21 + Weight: 10000 + Reference Timestamp: 2024-08-27T06:25:00Z + Total Weight: 1.9636285054518687 + Ref: + Container Name: kafka + Vpa Object Name: kafka-prod-broker + Total Samples Count: 6 + Version: v3 + Conditions: + Last Transition Time: 2024-08-27T06:21:32Z + Message: Successfully created kafkaOpsRequest demo/kfops-kafka-prod-broker-f6qbth + Observed Generation: 1 + Reason: CreateOpsRequest + Status: True + Type: CreateOpsRequest + Vpas: + Conditions: + Last Transition Time: 2024-08-27T06:21:10Z + Status: True + Type: RecommendationProvided + Recommendation: + Container Recommendations: + Container Name: kafka + Lower Bound: + Cpu: 600m + Memory: 1536Mi + Target: + Cpu: 813m + Memory: 1536Mi + Uncapped Target: + Cpu: 813m + Memory: 442809964 + Upper Bound: + Cpu: 1 + Memory: 2Gi + Vpa Name: kafka-prod-broker +Events: +``` +So, the `kafkaautoscaler` resource is created successfully. + +you can see in the `Status.VPAs.Recommendation` section, that recommendation has been generated for our database. Our autoscaler operator continuously watches the recommendation generated and creates an `kafkaopsrequest` based on the recommendations, if the database pods resources are needed to scaled up or down. + +Let's watch the `kafkaopsrequest` in the demo namespace to see if any `kafkaopsrequest` object is created. After some time you'll see that a `kafkaopsrequest` will be created based on the recommendation. + +```bash +$ watch kubectl get kafkaopsrequest -n demo +Every 2.0s: kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-prod-broker-f6qbth VerticalScaling Progressing 10s +``` + +Let's wait for the ops request to become successful. + +```bash +$ kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-prod-broker-f6qbth VerticalScaling Successful 3m2s +``` + +We can see from the above output that the `KafkaOpsRequest` has succeeded. If we describe the `KafkaOpsRequest` we will get an overview of the steps that were followed to scale the cluster. + +```bash +$ kubectl describe kafkaopsrequests -n demo kfops-kafka-prod-broker-f6qbth +Name: kfops-kafka-prod-broker-f6qbth +Namespace: demo +Labels: app.kubernetes.io/component=database + app.kubernetes.io/instance=kafka-prod + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=kafkas.kubedb.com +Annotations: +API Version: ops.kubedb.com/v1alpha1 +Kind: KafkaOpsRequest +Metadata: + Creation Timestamp: 2024-08-27T06:21:32Z + Generation: 1 + Owner References: + API Version: autoscaling.kubedb.com/v1alpha1 + Block Owner Deletion: true + Controller: true + Kind: KafkaAutoscaler + Name: kf-broker-autoscaler + UID: 7e3be99f-cd4d-440a-a477-8e8994840ebb + Resource Version: 1113011 + UID: a040a45b-135c-454a-8ddd-d4bd5000ffba +Spec: + Apply: IfReady + Database Ref: + Name: kafka-prod + Timeout: 5m0s + Type: VerticalScaling + Vertical Scaling: + Broker: + Resources: + Limits: + Memory: 1536Mi + Requests: + Cpu: 813m + Memory: 1536Mi +Status: + Conditions: + Last Transition Time: 2024-08-27T06:21:32Z + Message: Kafka ops-request has started to vertically scaling the kafka nodes + Observed Generation: 1 + Reason: VerticalScaling + Status: True + Type: VerticalScaling + Last Transition Time: 2024-08-27T06:21:35Z + Message: Successfully updated PetSets Resources + Observed Generation: 1 + Reason: UpdatePetSets + Status: True + Type: UpdatePetSets + Last Transition Time: 2024-08-27T06:21:40Z + Message: get pod; ConditionStatus:True; PodName:kafka-prod-broker-0 + Observed Generation: 1 + Status: True + Type: GetPod--kafka-prod-broker-0 + Last Transition Time: 2024-08-27T06:21:41Z + Message: evict pod; ConditionStatus:True; PodName:kafka-prod-broker-0 + Observed Generation: 1 + Status: True + Type: EvictPod--kafka-prod-broker-0 + Last Transition Time: 2024-08-27T06:21:55Z + Message: check pod running; ConditionStatus:True; PodName:kafka-prod-broker-0 + Observed Generation: 1 + Status: True + Type: CheckPodRunning--kafka-prod-broker-0 + Last Transition Time: 2024-08-27T06:22:00Z + Message: get pod; ConditionStatus:True; PodName:kafka-prod-broker-1 + Observed Generation: 1 + Status: True + Type: GetPod--kafka-prod-broker-1 + Last Transition Time: 2024-08-27T06:22:01Z + Message: evict pod; ConditionStatus:True; PodName:kafka-prod-broker-1 + Observed Generation: 1 + Status: True + Type: EvictPod--kafka-prod-broker-1 + Last Transition Time: 2024-08-27T06:22:21Z + Message: check pod running; ConditionStatus:True; PodName:kafka-prod-broker-1 + Observed Generation: 1 + Status: True + Type: CheckPodRunning--kafka-prod-broker-1 + Last Transition Time: 2024-08-27T06:22:25Z + Message: Successfully Restarted Pods With Resources + Observed Generation: 1 + Reason: RestartPods + Status: True + Type: RestartPods + Last Transition Time: 2024-08-27T06:22:26Z + Message: Successfully completed the vertical scaling for kafka + Observed Generation: 1 + Reason: Successful + Status: True + Type: Successful + Observed Generation: 1 + Phase: Successful +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Starting 4m55s KubeDB Ops-manager Operator Start processing for KafkaOpsRequest: demo/kfops-kafka-prod-broker-f6qbth + Normal Starting 4m55s KubeDB Ops-manager Operator Pausing Kafka databse: demo/kafka-prod + Normal Successful 4m55s KubeDB Ops-manager Operator Successfully paused Kafka database: demo/kafka-prod for KafkaOpsRequest: kfops-kafka-prod-broker-f6qbth + Normal UpdatePetSets 4m52s KubeDB Ops-manager Operator Successfully updated PetSets Resources + Warning get pod; ConditionStatus:True; PodName:kafka-prod-broker-0 4m47s KubeDB Ops-manager Operator get pod; ConditionStatus:True; PodName:kafka-prod-broker-0 + Warning evict pod; ConditionStatus:True; PodName:kafka-prod-broker-0 4m46s KubeDB Ops-manager Operator evict pod; ConditionStatus:True; PodName:kafka-prod-broker-0 + Warning check pod running; ConditionStatus:False; PodName:kafka-prod-broker-0 4m42s KubeDB Ops-manager Operator check pod running; ConditionStatus:False; PodName:kafka-prod-broker-0 + Warning check pod running; ConditionStatus:True; PodName:kafka-prod-broker-0 4m32s KubeDB Ops-manager Operator check pod running; ConditionStatus:True; PodName:kafka-prod-broker-0 + Warning get pod; ConditionStatus:True; PodName:kafka-prod-broker-1 4m27s KubeDB Ops-manager Operator get pod; ConditionStatus:True; PodName:kafka-prod-broker-1 + Warning evict pod; ConditionStatus:True; PodName:kafka-prod-broker-1 4m26s KubeDB Ops-manager Operator evict pod; ConditionStatus:True; PodName:kafka-prod-broker-1 + Warning check pod running; ConditionStatus:False; PodName:kafka-prod-broker-1 4m22s KubeDB Ops-manager Operator check pod running; ConditionStatus:False; PodName:kafka-prod-broker-1 + Warning check pod running; ConditionStatus:True; PodName:kafka-prod-broker-1 4m7s KubeDB Ops-manager Operator check pod running; ConditionStatus:True; PodName:kafka-prod-broker-1 + Normal RestartPods 4m2s KubeDB Ops-manager Operator Successfully Restarted Pods With Resources + Normal Starting 4m1s KubeDB Ops-manager Operator Resuming Kafka database: demo/kafka-prod + Normal Successful 4m1s KubeDB Ops-manager Operator Successfully resumed Kafka database: demo/kafka-prod for KafkaOpsRequest: kfops-kafka-prod-broker-f6qbth +``` + +Now, we are going to verify from the Pod, and the Kafka yaml whether the resources of the broker node has updated to meet up the desired state, Let's check, + +```bash +$ kubectl get pod -n demo kafka-prod-broker-0 -o json | jq '.spec.containers[].resources' +{ + "limits": { + "memory": "1536Mi" + }, + "requests": { + "cpu": "600m", + "memory": "1536Mi" + } +} + + +$ kubectl get kafka -n demo kafka-prod -o json | jq '.spec.topology.broker.podTemplate.spec.containers[].resources' +{ + "limits": { + "memory": "1536Mi" + }, + "requests": { + "cpu": "600m", + "memory": "1536Mi" + } +} +``` + +## Kafka Topology Autoscaler(Controller) + +Let's check the Controller Pod containers resources, + +```bash +$ kubectl get pod -n demo kafka-prod-controller-0 -o json | jq '.spec.containers[].resources' +{ + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } +} +``` + +Let's check the Kafka resources for broker, +```bash +$ kubectl get kafka -n demo kafka-prod -o json | jq '.spec.topology.controller.podTemplate.spec.containers[].resources' +{ + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } +} +``` + +You can see from the above outputs that the resources for controller are same as the one we have assigned while deploying the kafka. + +We are now ready to apply the `KafkaAutoscaler` CRO to set up autoscaling for this broker nodes. + +### Compute Resource Autoscaling + +Here, we are going to set up compute resource autoscaling using a KafkaAutoscaler Object. + +#### Create KafkaAutoscaler Object + +In order to set up compute resource autoscaling for this topology cluster, we have to create a `KafkaAutoscaler` CRO with our desired configuration. Below is the YAML of the `KafkaAutoscaler` object that we are going to create, + +```yaml +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-controller-autoscaler + namespace: demo +spec: + databaseRef: + name: kafka-prod + opsRequestOptions: + timeout: 5m + apply: IfReady + compute: + controller: + trigger: "On" + podLifeTimeThreshold: 5m + resourceDiffPercentage: 20 + minAllowed: + cpu: 600m + memory: 1.5Gi + maxAllowed: + cpu: 1 + memory: 2Gi + controlledResources: ["cpu", "memory"] + containerControlledValues: "RequestsAndLimits" +``` + +Here, + +- `spec.databaseRef.name` specifies that we are performing compute resource scaling operation on `kafka-prod` cluster. +- `spec.compute.controller.trigger` specifies that compute autoscaling is enabled for this node. +- `spec.compute.controller.podLifeTimeThreshold` specifies the minimum lifetime for at least one of the pod to initiate a vertical scaling. +- `spec.compute.controller.resourceDiffPercentage` specifies the minimum resource difference in percentage. The default is 10%. + If the difference between current & recommended resource is less than ResourceDiffPercentage, Autoscaler Operator will ignore the updating. +- `spec.compute.controller.minAllowed` specifies the minimum allowed resources for the cluster. +- `spec.compute.controller.maxAllowed` specifies the maximum allowed resources for the cluster. +- `spec.compute.controller.controlledResources` specifies the resources that are controlled by the autoscaler. +- `spec.compute.controller.containerControlledValues` specifies which resource values should be controlled. The default is "RequestsAndLimits". +- `spec.opsRequestOptions` contains the options to pass to the created OpsRequest. It has 2 fields. + - `timeout` specifies the timeout for the OpsRequest. + - `apply` specifies when the OpsRequest should be applied. The default is "IfReady". + +Let's create the `KafkaAutoscaler` CR we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaler/compute/kafka-controller-autoscaler.yaml +kafkaautoscaler.autoscaling.kubedb.com/kf-controller-autoscaler created +``` + +#### Verify Autoscaling is set up successfully + +Let's check that the `kafkaautoscaler` resource is created successfully, + +```bash +$ kubectl describe kafkaautoscaler kf-controller-autoscaler -n demo +Name: kf-controller-autoscaler +Namespace: demo +Labels: +Annotations: +API Version: autoscaling.kubedb.com/v1alpha1 +Kind: KafkaAutoscaler +Metadata: + Creation Timestamp: 2024-08-27T06:29:45Z + Generation: 1 + Owner References: + API Version: kubedb.com/v1 + Block Owner Deletion: true + Controller: true + Kind: Kafka + Name: kafka-prod + UID: 7cee41e0-259c-4a5e-856a-e8ca90056120 + Resource Version: 1116548 + UID: 49461872-3628-4bc2-8692-f147bc55aa49 +Spec: + Compute: + Controller: + Container Controlled Values: RequestsAndLimits + Controlled Resources: + cpu + memory + Max Allowed: + Cpu: 1 + Memory: 2Gi + Min Allowed: + Cpu: 600m + Memory: 1536Mi + Pod Life Time Threshold: 5m0s + Resource Diff Percentage: 20 + Trigger: On + Database Ref: + Name: kafka-prod + Ops Request Options: + Apply: IfReady + Timeout: 5m0s +Status: + Checkpoints: + Cpu Histogram: + Bucket Weights: + Index: 1 + Weight: 10000 + Index: 3 + Weight: 4666 + Reference Timestamp: 2024-08-27T06:30:00Z + Total Weight: 0.3085514112801626 + First Sample Start: 2024-08-27T06:29:52Z + Last Sample Start: 2024-08-27T06:30:49Z + Last Update Time: 2024-08-27T06:31:11Z + Memory Histogram: + Reference Timestamp: 2024-08-27T06:35:00Z + Ref: + Container Name: kafka + Vpa Object Name: kafka-prod-controller + Total Samples Count: 3 + Version: v3 + Conditions: + Last Transition Time: 2024-08-27T06:30:32Z + Message: Successfully created kafkaOpsRequest demo/kfops-kafka-prod-controller-3vlvzr + Observed Generation: 1 + Reason: CreateOpsRequest + Status: True + Type: CreateOpsRequest + Vpas: + Conditions: + Last Transition Time: 2024-08-27T06:30:11Z + Status: True + Type: RecommendationProvided + Recommendation: + Container Recommendations: + Container Name: kafka + Lower Bound: + Cpu: 600m + Memory: 1536Mi + Target: + Cpu: 600m + Memory: 1536Mi + Uncapped Target: + Cpu: 100m + Memory: 297164212 + Upper Bound: + Cpu: 1 + Memory: 2Gi + Vpa Name: kafka-prod-controller +Events: +``` +So, the `kafkaautoscaler` resource is created successfully. + +you can see in the `Status.VPAs.Recommendation` section, that recommendation has been generated for our controller cluster. Our autoscaler operator continuously watches the recommendation generated and creates an `kafkaopsrequest` based on the recommendations, if the controller node pods resources are needed to scaled up or down. + +Let's watch the `kafkaopsrequest` in the demo namespace to see if any `kafkaopsrequest` object is created. After some time you'll see that a `kafkaopsrequest` will be created based on the recommendation. + +```bash +$ watch kubectl get kafkaopsrequest -n demo +Every 2.0s: kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-prod-controller-3vlvzr VerticalScaling Progressing 10s +``` + +Let's wait for the ops request to become successful. + +```bash +$ kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-prod-controller-3vlvzr VerticalScaling Successful 3m2s +``` + +We can see from the above output that the `KafkaOpsRequest` has succeeded. If we describe the `KafkaOpsRequest` we will get an overview of the steps that were followed to scale the cluster. + +```bash +$ kubectl describe kafkaopsrequests -n demo kfops-kafka-prod-controller-3vlvzr +Name: kfops-kafka-prod-controller-3vlvzr +Namespace: demo +Labels: app.kubernetes.io/component=database + app.kubernetes.io/instance=kafka-prod + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=kafkas.kubedb.com +Annotations: +API Version: ops.kubedb.com/v1alpha1 +Kind: KafkaOpsRequest +Metadata: + Creation Timestamp: 2024-08-27T06:30:32Z + Generation: 1 + Owner References: + API Version: autoscaling.kubedb.com/v1alpha1 + Block Owner Deletion: true + Controller: true + Kind: KafkaAutoscaler + Name: kf-controller-autoscaler + UID: 49461872-3628-4bc2-8692-f147bc55aa49 + Resource Version: 1117285 + UID: 22228813-bf11-4d8a-9bea-53a1995fe4d0 +Spec: + Apply: IfReady + Database Ref: + Name: kafka-prod + Timeout: 5m0s + Type: VerticalScaling + Vertical Scaling: + Controller: + Resources: + Limits: + Memory: 1536Mi + Requests: + Cpu: 600m + Memory: 1536Mi +Status: + Conditions: + Last Transition Time: 2024-08-27T06:30:32Z + Message: Kafka ops-request has started to vertically scaling the kafka nodes + Observed Generation: 1 + Reason: VerticalScaling + Status: True + Type: VerticalScaling + Last Transition Time: 2024-08-27T06:30:35Z + Message: Successfully updated PetSets Resources + Observed Generation: 1 + Reason: UpdatePetSets + Status: True + Type: UpdatePetSets + Last Transition Time: 2024-08-27T06:30:40Z + Message: get pod; ConditionStatus:True; PodName:kafka-prod-controller-0 + Observed Generation: 1 + Status: True + Type: GetPod--kafka-prod-controller-0 + Last Transition Time: 2024-08-27T06:30:40Z + Message: evict pod; ConditionStatus:True; PodName:kafka-prod-controller-0 + Observed Generation: 1 + Status: True + Type: EvictPod--kafka-prod-controller-0 + Last Transition Time: 2024-08-27T06:31:11Z + Message: check pod running; ConditionStatus:True; PodName:kafka-prod-controller-0 + Observed Generation: 1 + Status: True + Type: CheckPodRunning--kafka-prod-controller-0 + Last Transition Time: 2024-08-27T06:31:15Z + Message: get pod; ConditionStatus:True; PodName:kafka-prod-controller-1 + Observed Generation: 1 + Status: True + Type: GetPod--kafka-prod-controller-1 + Last Transition Time: 2024-08-27T06:31:16Z + Message: evict pod; ConditionStatus:True; PodName:kafka-prod-controller-1 + Observed Generation: 1 + Status: True + Type: EvictPod--kafka-prod-controller-1 + Last Transition Time: 2024-08-27T06:31:30Z + Message: check pod running; ConditionStatus:True; PodName:kafka-prod-controller-1 + Observed Generation: 1 + Status: True + Type: CheckPodRunning--kafka-prod-controller-1 + Last Transition Time: 2024-08-27T06:31:35Z + Message: Successfully Restarted Pods With Resources + Observed Generation: 1 + Reason: RestartPods + Status: True + Type: RestartPods + Last Transition Time: 2024-08-27T06:31:36Z + Message: Successfully completed the vertical scaling for kafka + Observed Generation: 1 + Reason: Successful + Status: True + Type: Successful + Observed Generation: 1 + Phase: Successful +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Starting 2m33s KubeDB Ops-manager Operator Start processing for KafkaOpsRequest: demo/kfops-kafka-prod-controller-3vlvzr + Normal Starting 2m33s KubeDB Ops-manager Operator Pausing Kafka databse: demo/kafka-prod + Normal Successful 2m33s KubeDB Ops-manager Operator Successfully paused Kafka database: demo/kafka-prod for KafkaOpsRequest: kfops-kafka-prod-controller-3vlvzr + Normal UpdatePetSets 2m30s KubeDB Ops-manager Operator Successfully updated PetSets Resources + Warning get pod; ConditionStatus:True; PodName:kafka-prod-controller-0 2m25s KubeDB Ops-manager Operator get pod; ConditionStatus:True; PodName:kafka-prod-controller-0 + Warning evict pod; ConditionStatus:True; PodName:kafka-prod-controller-0 2m25s KubeDB Ops-manager Operator evict pod; ConditionStatus:True; PodName:kafka-prod-controller-0 + Warning check pod running; ConditionStatus:False; PodName:kafka-prod-controller-0 2m20s KubeDB Ops-manager Operator check pod running; ConditionStatus:False; PodName:kafka-prod-controller-0 + Warning check pod running; ConditionStatus:True; PodName:kafka-prod-controller-0 115s KubeDB Ops-manager Operator check pod running; ConditionStatus:True; PodName:kafka-prod-controller-0 + Warning get pod; ConditionStatus:True; PodName:kafka-prod-controller-1 110s KubeDB Ops-manager Operator get pod; ConditionStatus:True; PodName:kafka-prod-controller-1 + Warning evict pod; ConditionStatus:True; PodName:kafka-prod-controller-1 109s KubeDB Ops-manager Operator evict pod; ConditionStatus:True; PodName:kafka-prod-controller-1 + Warning check pod running; ConditionStatus:False; PodName:kafka-prod-controller-1 105s KubeDB Ops-manager Operator check pod running; ConditionStatus:False; PodName:kafka-prod-controller-1 + Warning check pod running; ConditionStatus:True; PodName:kafka-prod-controller-1 95s KubeDB Ops-manager Operator check pod running; ConditionStatus:True; PodName:kafka-prod-controller-1 + Normal RestartPods 90s KubeDB Ops-manager Operator Successfully Restarted Pods With Resources + Normal Starting 90s KubeDB Ops-manager Operator Resuming Kafka database: demo/kafka-prod + Normal Successful 90s KubeDB Ops-manager Operator Successfully resumed Kafka database: demo/kafka-prod for KafkaOpsRequest: kfops-kafka-prod-controller-3vlvzr +``` + +Now, we are going to verify from the Pod, and the Kafka yaml whether the resources of the controller node has updated to meet up the desired state, Let's check, + +```bash +$ kubectl get pod -n demo kafka-prod-controller-0 -o json | jq '.spec.containers[].resources' +{ + "limits": { + "memory": "1536Mi" + }, + "requests": { + "cpu": "600m", + "memory": "1536Mi" + } +} + + +$ kubectl get kafka -n demo kafka-prod -o json | jq '.spec.topology.controller.podTemplate.spec.containers[].resources' +{ + "limits": { + "memory": "1536Mi" + }, + "requests": { + "cpu": "600m", + "memory": "1536Mi" + } +} +``` + +The above output verifies that we have successfully auto scaled the resources of the Kafka topology cluster for broker and controller. You can create a similar `KafkaAutoscaler` object with both broker and controller resources to auto scale the resources of the Kafka topology cluster. + +## Cleaning Up + +To clean up the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete kafkaopsrequest -n demo kfops-kafka-prod-broker-f6qbth kfops-kafka-prod-controller-3vlvzr +kubectl delete kafkaautoscaler -n demo kf-broker-autoscaler kf-controller-autoscaler +kubectl delete kf -n demo kafka-prod +kubectl delete ns demo +``` +## Next Steps + +- Detail concepts of [Kafka object](/docs/guides/kafka/concepts/kafka.md). +- Different Kafka topology clustering modes [here](/docs/guides/kafka/clustering/_index.md). +- Monitor your Kafka database with KubeDB using [out-of-the-box Prometheus operator](/docs/guides/kafka/monitoring/using-prometheus-operator.md). + +[//]: # (- Monitor your Kafka database with KubeDB using [out-of-the-box builtin-Prometheus](/docs/guides/kafka/monitoring/using-builtin-prometheus.md).) +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/kafka/autoscaler/storage/_index.md b/docs/guides/kafka/autoscaler/storage/_index.md new file mode 100644 index 000000000..00a2e315f --- /dev/null +++ b/docs/guides/kafka/autoscaler/storage/_index.md @@ -0,0 +1,10 @@ +--- +title: Storage Autoscaling +menu: + docs_{{ .version }}: + identifier: kf-storage-auto-scaling + name: Storage Autoscaling + parent: kf-auto-scaling + weight: 46 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/kafka/autoscaler/storage/kafka-combined.md b/docs/guides/kafka/autoscaler/storage/kafka-combined.md new file mode 100644 index 000000000..d885db2ff --- /dev/null +++ b/docs/guides/kafka/autoscaler/storage/kafka-combined.md @@ -0,0 +1,469 @@ +--- +title: Kafka Combined Autoscaling +menu: + docs_{{ .version }}: + identifier: kf-storage-auto-scaling-combined + name: Combined Cluster + parent: kf-storage-auto-scaling + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Storage Autoscaling of a Kafka Combined Cluster + +This guide will show you how to use `KubeDB` to autoscale the storage of a Kafka Combined cluster. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. + +- Install `KubeDB` Provisioner, Ops-manager and Autoscaler operator in your cluster following the steps [here](/docs/setup/README.md). + +- Install `Metrics Server` from [here](https://github.com/kubernetes-sigs/metrics-server#installation) + +- Install Prometheus from [here](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) + +- You must have a `StorageClass` that supports volume expansion. + +- You should be familiar with the following `KubeDB` concepts: + - [Kafka](/docs/guides/kafka/concepts/kafka.md) + - [KafkaAutoscaler](/docs/guides/kafka/concepts/kafkaautoscaler.md) + - [KafkaOpsRequest](/docs/guides/kafka/concepts/kafkaopsrequest.md) + - [Storage Autoscaling Overview](/docs/guides/kafka/autoscaler/storage/overview.md) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/examples/kafka](/docs/examples/kafka) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + +## Storage Autoscaling of Combined Cluster + +At first verify that your cluster has a storage class, that supports volume expansion. Let's check, + +```bash +$ kubectl get storageclass +NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE +standard (default) kubernetes.io/gce-pd Delete Immediate true 2m49s +``` + +We can see from the output the `standard` storage class has `ALLOWVOLUMEEXPANSION` field as true. So, this storage class supports volume expansion. We can use it. + +Now, we are going to deploy a `Kafka` combined using a supported version by `KubeDB` operator. Then we are going to apply `KafkaAutoscaler` to set up autoscaling. + +#### Deploy Kafka combined + +In this section, we are going to deploy a Kafka combined cluster with version `4.4.26`. Then, in the next section we will set up autoscaling for this cluster using `KafkaAutoscaler` CRD. Below is the YAML of the `Kafka` CR that we are going to create, + +```yaml +apiVersion: kubedb.com/v1 +kind: Kafka +metadata: + name: kafka-dev + namespace: demo +spec: + replicas: 2 + version: 3.6.1 + podTemplate: + spec: + containers: + - name: kafka + resources: + limits: + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + storageType: Durable + deletionPolicy: WipeOut +``` + +Let's create the `Kafka` CRO we have shown above, + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaler/kafka-combined.yaml +kafka.kubedb.com/kafka-dev created +``` + +Now, wait until `kafka-dev` has status `Ready`. i.e, + +```bash +$ kubectl get kf -n demo -w +NAME TYPE VERSION STATUS AGE +kafka-dev kubedb.com/v1 3.6.1 Provisioning 0s +kafka-dev kubedb.com/v1 3.6.1 Provisioning 24s +. +. +kafka-dev kubedb.com/v1 3.6.1 Ready 92s +``` + +Let's check volume size from petset, and from the persistent volume, + +```bash +$ kubectl get petset -n demo kafka-dev -o json | jq '.spec.volumeClaimTemplates[].spec.resources.requests.storage' +"1Gi" + +$ kubectl get pv -n demo +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-129be4b9-f7e8-489e-8bc5-cd420e680f51 1Gi RWO Delete Bound demo/kafka-dev-data-kafka-dev-0 longhorn 40s +pvc-f068d245-718b-4561-b452-f3130bb260f6 1Gi RWO Delete Bound demo/kafka-dev-data-kafka-dev-1 longhorn 35s +``` + +You can see the petset has 1GB storage, and the capacity of all the persistent volume is also 1GB. + +We are now ready to apply the `KafkaAutoscaler` CRO to set up storage autoscaling for this cluster. + +### Storage Autoscaling + +Here, we are going to set up storage autoscaling using a KafkaAutoscaler Object. + +#### Create KafkaAutoscaler Object + +In order to set up vertical autoscaling for this combined cluster, we have to create a `KafkaAutoscaler` CRO with our desired configuration. Below is the YAML of the `KafkaAutoscaler` object that we are going to create, + +```yaml +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-storage-autoscaler-combined + namespace: demo +spec: + databaseRef: + name: kafka-dev + storage: + node: + expansionMode: "Online" + trigger: "On" + usageThreshold: 60 + scalingThreshold: 50 +``` + +Here, + +- `spec.clusterRef.name` specifies that we are performing vertical scaling operation on `kafka-dev` cluster. +- `spec.storage.node.trigger` specifies that storage autoscaling is enabled for this cluster. +- `spec.storage.node.usageThreshold` specifies storage usage threshold, if storage usage exceeds `60%` then storage autoscaling will be triggered. +- `spec.storage.node.scalingThreshold` specifies the scaling threshold. Storage will be scaled to `50%` of the current amount. +- It has another field `spec.storage.node.expansionMode` to set the opsRequest volumeExpansionMode, which support two values: `Online` & `Offline`. Default value is `Online`. + +Let's create the `KafkaAutoscaler` CR we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaling/storage/kafka-storage-autoscaler-combined.yaml +kafkaautoscaler.autoscaling.kubedb.com/kf-storage-autoscaler-combined created +``` + +#### Storage Autoscaling is set up successfully + +Let's check that the `kafkaautoscaler` resource is created successfully, + +```bash +NAME AGE +kf-storage-autoscaler-combined 8s + + +$ kubectl describe kafkaautoscaler -n demo kf-storage-autoscaler-combined +Name: kf-storage-autoscaler-combined +Namespace: demo +Labels: +Annotations: +API Version: autoscaling.kubedb.com/v1alpha1 +Kind: KafkaAutoscaler +Metadata: + Creation Timestamp: 2024-08-27T06:56:57Z + Generation: 1 + Owner References: + API Version: kubedb.com/v1 + Block Owner Deletion: true + Controller: true + Kind: Kafka + Name: kafka-dev + UID: a1d1b2f9-ef72-4ef6-8652-f39ee548c744 + Resource Version: 1123501 + UID: 83c7a7b6-aaf2-4776-8337-114bd1800d7c +Spec: + Database Ref: + Name: kafka-dev + Ops Request Options: + Apply: IfReady + Storage: + Node: + Expansion Mode: Online + Scaling Rules: + Applies Upto: + Threshold: 50pc + Scaling Threshold: 50 + Trigger: On + Usage Threshold: 60 +Events: +``` +So, the `kafkaautoscaler` resource is created successfully. + +Now, for this demo, we are going to manually fill up the persistent volume to exceed the `usageThreshold` using `dd` command to see if storage autoscaling is working or not. + +Let's exec into the cluster pod and fill the cluster volume using the following commands: + +```bash + $ kubectl exec -it -n demo kafka-dev-0 -- bash +kafka@kafka-dev-0:~$ df -h /var/log/kafka +Filesystem Size Used Avail Use% Mounted on +/dev/standard/pvc-129be4b9-f7e8-489e-8bc5-cd420e680f51 974M 168K 958M 1% /var/log/kafka +kafka@kafka-dev-0:~$ dd if=/dev/zero of=/var/log/kafka/file.img bs=600M count=1 +1+0 records in +1+0 records out +629145600 bytes (629 MB, 600 MiB) copied, 7.44144 s, 84.5 MB/s +kafka@kafka-dev-0:~$ df -h /var/log/kafka +Filesystem Size Used Avail Use% Mounted on +/dev/standard/pvc-129be4b9-f7e8-489e-8bc5-cd420e680f51 974M 601M 358M 63% /var/log/kafka +``` + +So, from the above output we can see that the storage usage is 83%, which exceeded the `usageThreshold` 60%. + +Let's watch the `kafkaopsrequest` in the demo namespace to see if any `kafkaopsrequest` object is created. After some time you'll see that a `kafkaopsrequest` of type `VolumeExpansion` will be created based on the `scalingThreshold`. + +```bash +$ watch kubectl get kafkaopsrequest -n demo +Every 2.0s: kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-dev-sa4thn VolumeExpansion Progressing 10s +``` + +Let's wait for the ops request to become successful. + +```bash +$ kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-dev-sa4thn VolumeExpansion Successful 97s +``` + +We can see from the above output that the `KafkaOpsRequest` has succeeded. If we describe the `KafkaOpsRequest` we will get an overview of the steps that were followed to expand the volume of the cluster. + +```bash +$ kubectl describe kafkaopsrequests -n demo kfops-kafka-dev-sa4thn +Name: kfops-kafka-dev-sa4thn +Namespace: demo +Labels: app.kubernetes.io/component=database + app.kubernetes.io/instance=kafka-dev + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=kafkas.kubedb.com +Annotations: +API Version: ops.kubedb.com/v1alpha1 +Kind: KafkaOpsRequest +Metadata: + Creation Timestamp: 2024-08-27T08:12:33Z + Generation: 1 + Owner References: + API Version: autoscaling.kubedb.com/v1alpha1 + Block Owner Deletion: true + Controller: true + Kind: KafkaAutoscaler + Name: kf-storage-autoscaler-combined + UID: a0ce73df-0d42-483a-9c47-ca58e57ea614 + Resource Version: 1135462 + UID: 78b52373-75f9-40a1-8528-3d0cd9beb4c5 +Spec: + Apply: IfReady + Database Ref: + Name: kafka-dev + Type: VolumeExpansion + Volume Expansion: + Mode: Online + Node: 1531054080 +Status: + Conditions: + Last Transition Time: 2024-08-27T08:12:33Z + Message: Kafka ops-request has started to expand volume of kafka nodes. + Observed Generation: 1 + Reason: VolumeExpansion + Status: True + Type: VolumeExpansion + Last Transition Time: 2024-08-27T08:12:41Z + Message: get pet set; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: GetPetSet + Last Transition Time: 2024-08-27T08:12:41Z + Message: is petset deleted; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: IsPetsetDeleted + Last Transition Time: 2024-08-27T08:12:51Z + Message: successfully deleted the petSets with orphan propagation policy + Observed Generation: 1 + Reason: OrphanPetSetPods + Status: True + Type: OrphanPetSetPods + Last Transition Time: 2024-08-27T08:12:56Z + Message: get pvc; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: GetPvc + Last Transition Time: 2024-08-27T08:12:56Z + Message: is pvc patched; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: IsPvcPatched + Last Transition Time: 2024-08-27T08:18:16Z + Message: compare storage; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: CompareStorage + Last Transition Time: 2024-08-27T08:18:21Z + Message: successfully updated combined node PVC sizes + Observed Generation: 1 + Reason: UpdateCombinedNodePVCs + Status: True + Type: UpdateCombinedNodePVCs + Last Transition Time: 2024-08-27T08:18:27Z + Message: successfully reconciled the Kafka resources + Observed Generation: 1 + Reason: UpdatePetSets + Status: True + Type: UpdatePetSets + Last Transition Time: 2024-08-27T08:18:32Z + Message: PetSet is recreated + Observed Generation: 1 + Reason: ReadyPetSets + Status: True + Type: ReadyPetSets + Last Transition Time: 2024-08-27T08:18:32Z + Message: Successfully completed volumeExpansion for kafka + Observed Generation: 1 + Reason: Successful + Status: True + Type: Successful + Observed Generation: 1 + Phase: Successful +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Starting 6m19s KubeDB Ops-manager Operator Start processing for KafkaOpsRequest: demo/kfops-kafka-dev-sa4thn + Normal Starting 6m19s KubeDB Ops-manager Operator Pausing Kafka databse: demo/kafka-dev + Normal Successful 6m19s KubeDB Ops-manager Operator Successfully paused Kafka database: demo/kafka-dev for KafkaOpsRequest: kfops-kafka-dev-sa4thn + Warning get pet set; ConditionStatus:True 6m11s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Warning is petset deleted; ConditionStatus:True 6m11s KubeDB Ops-manager Operator is petset deleted; ConditionStatus:True + Warning get pet set; ConditionStatus:True 6m6s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Normal OrphanPetSetPods 6m1s KubeDB Ops-manager Operator successfully deleted the petSets with orphan propagation policy + Warning get pvc; ConditionStatus:True 5m56s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning is pvc patched; ConditionStatus:True 5m56s KubeDB Ops-manager Operator is pvc patched; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m51s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:False 5m51s KubeDB Ops-manager Operator compare storage; ConditionStatus:False + Warning get pvc; ConditionStatus:True 5m46s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m41s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m36s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m31s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m26s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m21s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m16s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m11s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m6s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m1s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m56s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m51s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m46s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m41s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m36s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m31s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m26s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m21s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m16s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m11s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m6s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m1s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m56s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m51s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m46s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m41s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m36s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m31s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m26s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m21s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:True 3m21s KubeDB Ops-manager Operator compare storage; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m16s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning is pvc patched; ConditionStatus:True 3m16s KubeDB Ops-manager Operator is pvc patched; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m11s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:False 3m11s KubeDB Ops-manager Operator compare storage; ConditionStatus:False + Warning get pvc; ConditionStatus:True 3m6s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m1s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m56s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m51s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m46s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m41s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m36s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m31s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m26s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m21s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m16s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m11s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m6s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m1s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 116s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 111s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 106s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 101s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 96s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 91s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 86s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 81s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 76s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 71s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 66s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 61s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 56s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 51s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 46s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 41s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 36s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:True 36s KubeDB Ops-manager Operator compare storage; ConditionStatus:True + Normal UpdateCombinedNodePVCs 31s KubeDB Ops-manager Operator successfully updated combined node PVC sizes + Normal UpdatePetSets 25s KubeDB Ops-manager Operator successfully reconciled the Kafka resources + Warning get pet set; ConditionStatus:True 20s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Normal ReadyPetSets 20s KubeDB Ops-manager Operator PetSet is recreated + Normal Starting 20s KubeDB Ops-manager Operator Resuming Kafka database: demo/kafka-dev + Normal Successful 20s KubeDB Ops-manager Operator Successfully resumed Kafka database: demo/kafka-dev for KafkaOpsRequest: kfops-kafka-dev-sa4thn +``` + +Now, we are going to verify from the `Petset`, and the `Persistent Volume` whether the volume of the combined cluster has expanded to meet the desired state, Let's check, + +```bash +$ kubectl get petset -n demo kafka-dev -o json | jq '.spec.volumeClaimTemplates[].spec.resources.requests.storage' +"1531054080" +$ kubectl get pv -n demo +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-129be4b9-f7e8-489e-8bc5-cd420e680f51 1462Mi RWO Delete Bound demo/kafka-dev-data-kafka-dev-0 longhorn 30m5s +pvc-f068d245-718b-4561-b452-f3130bb260f6 1462Mi RWO Delete Bound demo/kafka-dev-data-kafka-dev-1 longhorn 30m1s +``` + +The above output verifies that we have successfully autoscaled the volume of the Kafka combined cluster. + +## Cleaning Up + +To clean up the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete kafkaopsrequests -n demo kfops-kafka-dev-sa4thn +kubectl delete kafkautoscaler -n demo kf-storage-autoscaler-combined +kubectl delete kf -n demo kafka-dev +``` + +## Next Steps + +- Detail concepts of [Kafka object](/docs/guides/kafka/concepts/kafka.md). +- Different Kafka topology clustering modes [here](/docs/guides/kafka/clustering/_index.md). +- Monitor your Kafka database with KubeDB using [out-of-the-box Prometheus operator](/docs/guides/kafka/monitoring/using-prometheus-operator.md). + +[//]: # (- Monitor your Kafka database with KubeDB using [out-of-the-box builtin-Prometheus](/docs/guides/kafka/monitoring/using-builtin-prometheus.md).) +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/kafka/autoscaler/storage/kafka-topology.md b/docs/guides/kafka/autoscaler/storage/kafka-topology.md new file mode 100644 index 000000000..d9f8f5858 --- /dev/null +++ b/docs/guides/kafka/autoscaler/storage/kafka-topology.md @@ -0,0 +1,684 @@ +--- +title: Kafka Topology Autoscaling +menu: + docs_{{ .version }}: + identifier: kf-storage-auto-scaling-topology + name: Topology Cluster + parent: kf-storage-auto-scaling + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Storage Autoscaling of a Kafka Topology Cluster + +This guide will show you how to use `KubeDB` to autoscale the storage of a Kafka Topology cluster. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. + +- Install `KubeDB` Provisioner, Ops-manager and Autoscaler operator in your cluster following the steps [here](/docs/setup/README.md). + +- Install `Metrics Server` from [here](https://github.com/kubernetes-sigs/metrics-server#installation) + +- Install Prometheus from [here](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) + +- You must have a `StorageClass` that supports volume expansion. + +- You should be familiar with the following `KubeDB` concepts: + - [Kafka](/docs/guides/kafka/concepts/kafka.md) + - [KafkaAutoscaler](/docs/guides/kafka/concepts/kafkaautoscaler.md) + - [KafkaOpsRequest](/docs/guides/kafka/concepts/kafkaopsrequest.md) + - [Storage Autoscaling Overview](/docs/guides/kafka/autoscaler/storage/overview.md) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/examples/kafka](/docs/examples/kafka) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + +## Storage Autoscaling of Topology Cluster + +At first verify that your cluster has a storage class, that supports volume expansion. Let's check, + +```bash +$ kubectl get storageclass +NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE +standard (default) kubernetes.io/gce-pd Delete Immediate true 2m49s +``` + +We can see from the output the `standard` storage class has `ALLOWVOLUMEEXPANSION` field as true. So, this storage class supports volume expansion. We can use it. + +Now, we are going to deploy a `Kafka` topology using a supported version by `KubeDB` operator. Then we are going to apply `KafkaAutoscaler` to set up autoscaling. + +#### Deploy Kafka topology + +In this section, we are going to deploy a Kafka topology cluster with version `4.4.26`. Then, in the next section we will set up autoscaling for this cluster using `KafkaAutoscaler` CRD. Below is the YAML of the `Kafka` CR that we are going to create, + +```yaml +apiVersion: kubedb.com/v1 +kind: Kafka +metadata: + name: kafka-prod + namespace: demo +spec: + version: 3.6.1 + topology: + broker: + replicas: 2 + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + controller: + replicas: 2 + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + storageType: Durable + deletionPolicy: WipeOut +``` + +Let's create the `Kafka` CRO we have shown above, + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaler/kafka-topology.yaml +kafka.kubedb.com/kafka-prod created +``` + +Now, wait until `kafka-dev` has status `Ready`. i.e, + +```bash +$ kubectl get kf -n demo -w +NAME TYPE VERSION STATUS AGE +kafka-prod kubedb.com/v1 3.6.1 Provisioning 0s +kafka-prod kubedb.com/v1 3.6.1 Provisioning 24s +. +. +kafka-prod kubedb.com/v1 3.6.1 Ready 119s +``` + +Let's check volume size from petset, and from the persistent volume, + +```bash +$ kubectl get petset -n demo kafka-prod-broker -o json | jq '.spec.volumeClaimTemplates[].spec.resources.requests.storage' +"1Gi" +$ kubectl get petset -n demo kafka-prod-controller -o json | jq '.spec.volumeClaimTemplates[].spec.resources.requests.storage' +"1Gi" +$ kubectl get pv -n demo +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-128d9138-64da-4021-8a7c-7ca80823e842 1Gi RWO Delete Bound demo/kafka-prod-data-kafka-prod-controller-1 longhorn 33s +pvc-27fe9102-2e7d-41e0-b77d-729a82c64e21 1Gi RWO Delete Bound demo/kafka-prod-data-kafka-prod-broker-0 longhorn 51s +pvc-3bb98ba1-9cea-46ad-857f-fc843c265d57 1Gi RWO Delete Bound demo/kafka-prod-data-kafka-prod-controller-0 longhorn 50s +pvc-68f86aac-33d1-423a-bc56-8a905b546db2 1Gi RWO Delete Bound demo/kafka-prod-data-kafka-prod-broker-1 longhorn 32s +``` + +You can see the petset for both broker and controller has 1GB storage, and the capacity of all the persistent volume is also 1GB. + +We are now ready to apply the `KafkaAutoscaler` CRO to set up storage autoscaling for this cluster(broker and controller). + +### Storage Autoscaling + +Here, we are going to set up storage autoscaling using a KafkaAutoscaler Object. + +#### Create KafkaAutoscaler Object + +In order to set up vertical autoscaling for this topology cluster, we have to create a `KafkaAutoscaler` CRO with our desired configuration. Below is the YAML of the `KafkaAutoscaler` object that we are going to create, + +```yaml +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: KafkaAutoscaler +metadata: + name: kf-storage-autoscaler-topology + namespace: demo +spec: + databaseRef: + name: kafka-prod + storage: + broker: + expansionMode: "Online" + trigger: "On" + usageThreshold: 60 + scalingThreshold: 100 + controller: + expansionMode: "Online" + trigger: "On" + usageThreshold: 60 + scalingThreshold: 100 +``` + +Here, + +- `spec.clusterRef.name` specifies that we are performing vertical scaling operation on `kafka-prod` cluster. +- `spec.storage.broker.trigger/spec.storage.controller.trigger` specifies that storage autoscaling is enabled for broker and controller of topology cluster. +- `spec.storage.broker.usageThreshold/spec.storage.controller.usageThreshold` specifies storage usage threshold, if storage usage exceeds `60%` then storage autoscaling will be triggered. +- `spec.storage.broker.scalingThreshold/spec.storage.broker.scalingThreshold` specifies the scaling threshold. Storage will be scaled to `100%` of the current amount. +- It has another field `spec.storage.broker.expansionMode/spec.storage.controller.expansionMode` to set the opsRequest volumeExpansionMode, which support two values: `Online` & `Offline`. Default value is `Online`. + +Let's create the `KafkaAutoscaler` CR we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/autoscaling/storage/kafka-storage-autoscaler-topology.yaml +kafkaautoscaler.autoscaling.kubedb.com/kf-storage-autoscaler-topology created +``` + +#### Storage Autoscaling is set up successfully + +Let's check that the `kafkaautoscaler` resource is created successfully, + +```bash +NAME AGE +kf-storage-autoscaler-topology 8s + +$ kubectl describe kafkaautoscaler -n demo kf-storage-autoscaler-topology +Name: kf-storage-autoscaler-topology +Namespace: demo +Labels: +Annotations: +API Version: autoscaling.kubedb.com/v1alpha1 +Kind: KafkaAutoscaler +Metadata: + Creation Timestamp: 2024-08-27T08:54:35Z + Generation: 1 + Owner References: + API Version: kubedb.com/v1 + Block Owner Deletion: true + Controller: true + Kind: Kafka + Name: kafka-prod + UID: 1ae37155-dd92-4547-8aba-589140d1d2cf + Resource Version: 1142604 + UID: bca444d0-d860-4588-9b51-412c614c4771 +Spec: + Database Ref: + Name: kafka-prod + Ops Request Options: + Apply: IfReady + Storage: + Broker: + Expansion Mode: Online + Scaling Rules: + Applies Upto: + Threshold: 100pc + Scaling Threshold: 100 + Trigger: On + Usage Threshold: 60 + Controller: + Expansion Mode: Online + Scaling Rules: + Applies Upto: + Threshold: 100pc + Scaling Threshold: 100 + Trigger: On + Usage Threshold: 60 +Events: +``` +So, the `kafkaautoscaler` resource is created successfully. + +Now, for this demo, we are going to manually fill up the persistent volume to exceed the `usageThreshold` using `dd` command to see if storage autoscaling is working or not. + +We are autoscaling volume for both broker and controller. So we need to fill up the persistent volume for both broker and controller. + +1. Let's exec into the broker pod and fill the cluster volume using the following commands: + +```bash +$ kubectl exec -it -n demo kafka-prod-broker-0 -- bash +kafka@kafka-prod-broker-0:~$ df -h /var/log/kafka +Filesystem Size Used Avail Use% Mounted on +/dev/standard/pvc-27fe9102-2e7d-41e0-b77d-729a82c64e21 974M 256K 958M 1% /var/log/kafka +kafka@kafka-prod-broker-0:~$ dd if=/dev/zero of=/var/log/kafka/file.img bs=600M count=1 +1+0 records in +1+0 records out +629145600 bytes (629 MB, 600 MiB) copied, 5.58851 s, 113 MB/s +kafka@kafka-prod-broker-0:~$ df -h /var/log/kafka +Filesystem Size Used Avail Use% Mounted on +/dev/standard/pvc-27fe9102-2e7d-41e0-b77d-729a82c64e21 974M 601M 358M 63% /var/log/kafka +``` + +2. Let's exec into the controller pod and fill the cluster volume using the following commands: + +```bash +$ kubectl exec -it -n demo kafka-prod-controller-0 -- bash +kafka@kafka-prod-controller-0:~$ df -h /var/log/kafka +Filesystem Size Used Avail Use% Mounted on +/dev/standard/pvc-3bb98ba1-9cea-46ad-857f-fc843c265d57 974M 192K 958M 1% /var/log/kafka +kafka@kafka-prod-controller-0:~$ dd if=/dev/zero of=/var/log/kafka/file.img bs=600M count=1 +1+0 records in +1+0 records out +629145600 bytes (629 MB, 600 MiB) copied, 3.39618 s, 185 MB/s +kafka@kafka-prod-controller-0:~$ df -h /var/log/kafka +Filesystem Size Used Avail Use% Mounted on +/dev/standard/pvc-3bb98ba1-9cea-46ad-857f-fc843c265d57 974M 601M 358M 63% /var/log/kafka +``` + +So, from the above output we can see that the storage usage is 63% for both nodes, which exceeded the `usageThreshold` 60%. + +There will be two `KafkaOpsRequest` created for both broker and controller to expand the volume of the cluster for both nodes. +Let's watch the `kafkaopsrequest` in the demo namespace to see if any `kafkaopsrequest` object is created. After some time you'll see that a `kafkaopsrequest` of type `VolumeExpansion` will be created based on the `scalingThreshold`. + +```bash +$ watch kubectl get kafkaopsrequest -n demo +Every 2.0s: kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-prod-7qwpbn VolumeExpansion Progressing 10s +``` + +Let's wait for the ops request to become successful. + +```bash +$ kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-prod-7qwpbn VolumeExpansion Successful 2m37s +``` + +We can see from the above output that the `KafkaOpsRequest` has succeeded. If we describe the `KafkaOpsRequest` we will get an overview of the steps that were followed to expand the volume of the cluster. + +```bash +$ kubectl describe kafkaopsrequests -n demo kfops-kafka-prod-7qwpbn +Name: kfops-kafka-prod-7qwpbn +Namespace: demo +Labels: app.kubernetes.io/component=database + app.kubernetes.io/instance=kafka-prod + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=kafkas.kubedb.com +Annotations: +API Version: ops.kubedb.com/v1alpha1 +Kind: KafkaOpsRequest +Metadata: + Creation Timestamp: 2024-08-27T08:59:43Z + Generation: 1 + Owner References: + API Version: autoscaling.kubedb.com/v1alpha1 + Block Owner Deletion: true + Controller: true + Kind: KafkaAutoscaler + Name: kf-storage-autoscaler-topology + UID: bca444d0-d860-4588-9b51-412c614c4771 + Resource Version: 1144249 + UID: 2a9bd422-c6ce-47c9-bfd6-ba7f79774c17 +Spec: + Apply: IfReady + Database Ref: + Name: kafka-prod + Type: VolumeExpansion + Volume Expansion: + Broker: 2041405440 + Mode: Online +Status: + Conditions: + Last Transition Time: 2024-08-27T08:59:43Z + Message: Kafka ops-request has started to expand volume of kafka nodes. + Observed Generation: 1 + Reason: VolumeExpansion + Status: True + Type: VolumeExpansion + Last Transition Time: 2024-08-27T08:59:51Z + Message: get pet set; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: GetPetSet + Last Transition Time: 2024-08-27T08:59:51Z + Message: is petset deleted; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: IsPetsetDeleted + Last Transition Time: 2024-08-27T09:00:01Z + Message: successfully deleted the petSets with orphan propagation policy + Observed Generation: 1 + Reason: OrphanPetSetPods + Status: True + Type: OrphanPetSetPods + Last Transition Time: 2024-08-27T09:00:06Z + Message: get pvc; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: GetPvc + Last Transition Time: 2024-08-27T09:00:06Z + Message: is pvc patched; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: IsPvcPatched + Last Transition Time: 2024-08-27T09:03:51Z + Message: compare storage; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: CompareStorage + Last Transition Time: 2024-08-27T09:03:56Z + Message: successfully updated broker node PVC sizes + Observed Generation: 1 + Reason: UpdateBrokerNodePVCs + Status: True + Type: UpdateBrokerNodePVCs + Last Transition Time: 2024-08-27T09:04:03Z + Message: successfully reconciled the Kafka resources + Observed Generation: 1 + Reason: UpdatePetSets + Status: True + Type: UpdatePetSets + Last Transition Time: 2024-08-27T09:04:08Z + Message: PetSet is recreated + Observed Generation: 1 + Reason: ReadyPetSets + Status: True + Type: ReadyPetSets + Last Transition Time: 2024-08-27T09:04:08Z + Message: Successfully completed volumeExpansion for kafka + Observed Generation: 1 + Reason: Successful + Status: True + Type: Successful + Observed Generation: 1 + Phase: Successful +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Starting 6m6s KubeDB Ops-manager Operator Start processing for KafkaOpsRequest: demo/kfops-kafka-prod-7qwpbn + Normal Starting 6m6s KubeDB Ops-manager Operator Pausing Kafka databse: demo/kafka-prod + Normal Successful 6m6s KubeDB Ops-manager Operator Successfully paused Kafka database: demo/kafka-prod for KafkaOpsRequest: kfops-kafka-prod-7qwpbn + Warning get pet set; ConditionStatus:True 5m58s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Warning is petset deleted; ConditionStatus:True 5m58s KubeDB Ops-manager Operator is petset deleted; ConditionStatus:True + Warning get pet set; ConditionStatus:True 5m53s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Normal OrphanPetSetPods 5m48s KubeDB Ops-manager Operator successfully deleted the petSets with orphan propagation policy + Warning get pvc; ConditionStatus:True 5m43s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning is pvc patched; ConditionStatus:True 5m43s KubeDB Ops-manager Operator is pvc patched; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m38s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:False 5m38s KubeDB Ops-manager Operator compare storage; ConditionStatus:False + Warning get pvc; ConditionStatus:True 5m33s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m28s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m23s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m18s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m13s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m8s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m3s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m58s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m53s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m48s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m43s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m38s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m33s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m28s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m23s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m18s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m13s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m8s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m3s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m58s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:True 3m58s KubeDB Ops-manager Operator compare storage; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m53s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning is pvc patched; ConditionStatus:True 3m53s KubeDB Ops-manager Operator is pvc patched; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m48s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:False 3m48s KubeDB Ops-manager Operator compare storage; ConditionStatus:False + Warning get pvc; ConditionStatus:True 3m43s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m38s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m33s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m28s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m23s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m18s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m13s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m8s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m3s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m58s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m53s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m48s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m43s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m38s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m33s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m28s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m23s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m18s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m13s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m8s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 2m3s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 118s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:True 118s KubeDB Ops-manager Operator compare storage; ConditionStatus:True + Normal UpdateBrokerNodePVCs 113s KubeDB Ops-manager Operator successfully updated broker node PVC sizes + Normal UpdatePetSets 106s KubeDB Ops-manager Operator successfully reconciled the Kafka resources + Warning get pet set; ConditionStatus:True 101s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Normal ReadyPetSets 101s KubeDB Ops-manager Operator PetSet is recreated + Normal Starting 101s KubeDB Ops-manager Operator Resuming Kafka database: demo/kafka-prod + Normal Successful 101s KubeDB Ops-manager Operator Successfully resumed Kafka database: demo/kafka-prod for KafkaOpsRequest: kfops-kafka-prod-7qwpbn +``` + +After a few minutes, another `KafkaOpsRequest` of type `VolumeExpansion` will be created for the controller node. + +```bash +$ kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-prod-7qwpbn VolumeExpansion Successful 2m47s +kfops-kafka-prod-sa4thn VolumeExpansion Progressing 10s +``` + +Let's wait for the ops request to become successful. + +```bash +$ kubectl get kafkaopsrequest -n demo +NAME TYPE STATUS AGE +kfops-kafka-prod-7qwpbn VolumeExpansion Successful 4m47s +kfops-kafka-prod-sa4thn VolumeExpansion Successful 2m10s +``` + +We can see from the above output that the `KafkaOpsRequest` `kfops-kafka-prod-sa4thn` has also succeeded. If we describe the `KafkaOpsRequest` we will get an overview of the steps that were followed to expand the volume of the cluster. + +```bash +$ kubectl describe kafkaopsrequests -n demo kfops-kafka-prod-2ta9m6 +Name: kfops-kafka-prod-2ta9m6 +Namespace: demo +Labels: app.kubernetes.io/component=database + app.kubernetes.io/instance=kafka-prod + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=kafkas.kubedb.com +Annotations: +API Version: ops.kubedb.com/v1alpha1 +Kind: KafkaOpsRequest +Metadata: + Creation Timestamp: 2024-08-27T09:04:43Z + Generation: 1 + Owner References: + API Version: autoscaling.kubedb.com/v1alpha1 + Block Owner Deletion: true + Controller: true + Kind: KafkaAutoscaler + Name: kf-storage-autoscaler-topology + UID: bca444d0-d860-4588-9b51-412c614c4771 + Resource Version: 1145309 + UID: c965e481-8dbd-4b1d-8a9a-40239753cbf0 +Spec: + Apply: IfReady + Database Ref: + Name: kafka-prod + Type: VolumeExpansion + Volume Expansion: + Controller: 2041405440 + Mode: Online +Status: + Conditions: + Last Transition Time: 2024-08-27T09:04:43Z + Message: Kafka ops-request has started to expand volume of kafka nodes. + Observed Generation: 1 + Reason: VolumeExpansion + Status: True + Type: VolumeExpansion + Last Transition Time: 2024-08-27T09:04:51Z + Message: get pet set; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: GetPetSet + Last Transition Time: 2024-08-27T09:04:51Z + Message: is petset deleted; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: IsPetsetDeleted + Last Transition Time: 2024-08-27T09:05:01Z + Message: successfully deleted the petSets with orphan propagation policy + Observed Generation: 1 + Reason: OrphanPetSetPods + Status: True + Type: OrphanPetSetPods + Last Transition Time: 2024-08-27T09:05:06Z + Message: get pvc; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: GetPvc + Last Transition Time: 2024-08-27T09:05:06Z + Message: is pvc patched; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: IsPvcPatched + Last Transition Time: 2024-08-27T09:09:36Z + Message: compare storage; ConditionStatus:True + Observed Generation: 1 + Status: True + Type: CompareStorage + Last Transition Time: 2024-08-27T09:09:41Z + Message: successfully updated controller node PVC sizes + Observed Generation: 1 + Reason: UpdateControllerNodePVCs + Status: True + Type: UpdateControllerNodePVCs + Last Transition Time: 2024-08-27T09:09:47Z + Message: successfully reconciled the Kafka resources + Observed Generation: 1 + Reason: UpdatePetSets + Status: True + Type: UpdatePetSets + Last Transition Time: 2024-08-27T09:09:53Z + Message: PetSet is recreated + Observed Generation: 1 + Reason: ReadyPetSets + Status: True + Type: ReadyPetSets + Last Transition Time: 2024-08-27T09:09:53Z + Message: Successfully completed volumeExpansion for kafka + Observed Generation: 1 + Reason: Successful + Status: True + Type: Successful + Observed Generation: 1 + Phase: Successful +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Starting 8m17s KubeDB Ops-manager Operator Start processing for KafkaOpsRequest: demo/kfops-kafka-prod-2ta9m6 + Normal Starting 8m17s KubeDB Ops-manager Operator Pausing Kafka databse: demo/kafka-prod + Normal Successful 8m17s KubeDB Ops-manager Operator Successfully paused Kafka database: demo/kafka-prod for KafkaOpsRequest: kfops-kafka-prod-2ta9m6 + Warning get pet set; ConditionStatus:True 8m9s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Warning is petset deleted; ConditionStatus:True 8m9s KubeDB Ops-manager Operator is petset deleted; ConditionStatus:True + Warning get pet set; ConditionStatus:True 8m4s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Normal OrphanPetSetPods 7m59s KubeDB Ops-manager Operator successfully deleted the petSets with orphan propagation policy + Warning get pvc; ConditionStatus:True 7m54s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning is pvc patched; ConditionStatus:True 7m54s KubeDB Ops-manager Operator is pvc patched; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m49s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:False 7m49s KubeDB Ops-manager Operator compare storage; ConditionStatus:False + Warning get pvc; ConditionStatus:True 7m44s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m39s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m34s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m29s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m24s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m19s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m14s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m9s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 7m4s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m59s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m54s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m49s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m44s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m39s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m34s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m29s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m24s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m19s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m14s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m9s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 6m4s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:True 6m4s KubeDB Ops-manager Operator compare storage; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m59s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning is pvc patched; ConditionStatus:True 5m59s KubeDB Ops-manager Operator is pvc patched; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m54s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:False 5m54s KubeDB Ops-manager Operator compare storage; ConditionStatus:False + Warning get pvc; ConditionStatus:True 5m49s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m44s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m39s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m34s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m29s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m24s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m19s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m14s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m9s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 5m4s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m59s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m54s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m49s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m44s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m39s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m34s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m29s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m24s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m19s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m14s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m9s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 4m4s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m59s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m54s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m49s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m44s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m39s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m34s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m29s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning get pvc; ConditionStatus:True 3m24s KubeDB Ops-manager Operator get pvc; ConditionStatus:True + Warning compare storage; ConditionStatus:True 3m24s KubeDB Ops-manager Operator compare storage; ConditionStatus:True + Normal UpdateControllerNodePVCs 3m19s KubeDB Ops-manager Operator successfully updated controller node PVC sizes + Normal UpdatePetSets 3m12s KubeDB Ops-manager Operator successfully reconciled the Kafka resources + Warning get pet set; ConditionStatus:True 3m7s KubeDB Ops-manager Operator get pet set; ConditionStatus:True + Normal ReadyPetSets 3m7s KubeDB Ops-manager Operator PetSet is recreated + Normal Starting 3m7s KubeDB Ops-manager Operator Resuming Kafka database: demo/kafka-prod + Normal Successful 3m7s KubeDB Ops-manager Operator Successfully resumed Kafka database: demo/kafka-prod for KafkaOpsRequest: kfops-kafka-prod-2ta9m6 +``` + +Now, we are going to verify from the `Petset`, and the `Persistent Volume` whether the volume of the topology cluster has expanded to meet the desired state, Let's check, + +```bash +$ kubectl get petset -n demo kafka-prod-broker -o json | jq '.spec.volumeClaimTemplates[].spec.resources.requests.storage' +"2041405440" +$ kubectl get petset -n demo kafka-prod-controller -o json | jq '.spec.volumeClaimTemplates[].spec.resources.requests.storage' +"2041405440" +$ kubectl get pv -n demo +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-128d9138-64da-4021-8a7c-7ca80823e842 1948Mi RWO Delete Bound demo/kafka-prod-data-kafka-prod-controller-1 longhorn 33s +pvc-27fe9102-2e7d-41e0-b77d-729a82c64e21 1948Mi RWO Delete Bound demo/kafka-prod-data-kafka-prod-broker-0 longhorn 51s +pvc-3bb98ba1-9cea-46ad-857f-fc843c265d57 1948Mi RWO Delete Bound demo/kafka-prod-data-kafka-prod-controller-0 longhorn 50s +pvc-68f86aac-33d1-423a-bc56-8a905b546db2 1948Mi RWO Delete Bound demo/kafka-prod-data-kafka-prod-broker-1 longhorn 32s +``` + +The above output verifies that we have successfully autoscaled the volume of the Kafka topology cluster for both broker and controller. + +## Cleaning Up + +To clean up the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete kafkaopsrequests -n demo kfops-kafka-prod-7qwpbn kfops-kafka-prod-sa4thn +kubectl delete kafkautoscaler -n demo kf-storage-autoscaler-topology +kubectl delete kf -n demo kafka-prod +``` + +## Next Steps + +- Detail concepts of [Kafka object](/docs/guides/kafka/concepts/kafka.md). +- Different Kafka topology clustering modes [here](/docs/guides/kafka/clustering/_index.md). +- Monitor your Kafka database with KubeDB using [out-of-the-box Prometheus operator](/docs/guides/kafka/monitoring/using-prometheus-operator.md). + +[//]: # (- Monitor your Kafka database with KubeDB using [out-of-the-box builtin-Prometheus](/docs/guides/kafka/monitoring/using-builtin-prometheus.md).) +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/kafka/autoscaler/storage/overview.md b/docs/guides/kafka/autoscaler/storage/overview.md new file mode 100644 index 000000000..0110bcb4c --- /dev/null +++ b/docs/guides/kafka/autoscaler/storage/overview.md @@ -0,0 +1,57 @@ +--- +title: Kafka Storage Autoscaling Overview +menu: + docs_{{ .version }}: + identifier: mg-storage-auto-scaling-overview + name: Overview + parent: mg-storage-auto-scaling + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Kafka Vertical Autoscaling + +This guide will give an overview on how KubeDB Autoscaler operator autoscales the database storage using `kafkaautoscaler` crd. + +## Before You Begin + +- You should be familiar with the following `KubeDB` concepts: + - [Kafka](/docs/guides/kafka/concepts/kafka.md) + - [KafkaAutoscaler](/docs/guides/kafka/concepts/kafkaautoscaler.md) + - [KafkaOpsRequest](/docs/guides/kafka/concepts/kafkaopsrequest.md) + +## How Storage Autoscaling Works + +The following diagram shows how KubeDB Autoscaler operator autoscales the resources of `Kafka` cluster components. Open the image in a new tab to see the enlarged version. + +
+  Storage Auto Scaling process of Kafka +
Fig: Storage Auto Scaling process of Kafka
+
+ + +The Auto Scaling process consists of the following steps: + +1. At first, a user creates a `Kafka` Custom Resource (CR). + +2. `KubeDB` Provisioner operator watches the `Kafka` CR. + +3. When the operator finds a `Kafka` CR, it creates required number of `PetSets` and related necessary stuff like secrets, services, etc. + +- Each PetSet creates a Persistent Volume according to the Volume Claim Template provided in the petset configuration. + +4. Then, in order to set up storage autoscaling of the various components (ie. Combined, Broker, Controller.) of the `Kafka` cluster, the user creates a `KafkaAutoscaler` CRO with desired configuration. + +5. `KubeDB` Autoscaler operator watches the `KafkaAutoscaler` CRO. + +6. `KubeDB` Autoscaler operator continuously watches persistent volumes of the clusters to check if it exceeds the specified usage threshold. +- If the usage exceeds the specified usage threshold, then `KubeDB` Autoscaler operator creates a `KafkaOpsRequest` to expand the storage of the database. + +7. `KubeDB` Ops-manager operator watches the `KafkaOpsRequest` CRO. + +8. Then the `KubeDB` Ops-manager operator will expand the storage of the cluster component as specified on the `KafkaOpsRequest` CRO. + +In the next docs, we are going to show a step by step guide on Autoscaling storage of various Kafka cluster components using `KafkaAutoscaler` CRD.