Skip to content

Commit

Permalink
Merge pull request #2 from aidanmelen/kafka-topic
Browse files Browse the repository at this point in the history 
Kafka topic
  • Loading branch information
@aidanmelen
aidanmelen authored Jul 31, 2022
2 parents b2072aa + 5acb616 commit 517bba0
Show file tree
Hide file tree
Showing 54 changed files with 43,677 additions and 7,059 deletions.
2 changes: 0 additions & 2 deletions .github/workflows/pre-commit.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,9 @@ on:
pull_request:
branches:
- main
- master
push:
branches:
- main
- master
workflow_dispatch:

env:
Expand Down
13 changes: 13 additions & 0 deletions .make.docs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
help This help.
build Build docker dev container
run Run docker dev container
setup Setup project
lint Lint with pre-commit
lint-all Lint with pre-commit
tests Tests with Terratest
test-confluent-operator Test the confluent_operator example
test-confluent-platform Test the confluent_platform example
test-confluent-platform-singlenode Test the confluent_platform_singlenode example
test-complete Test the complete example
test-kafka-topics Test the kafka_topics example
clean Clean project
18 changes: 8 additions & 10 deletions .terraform-docs.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ content: |-
terraform apply
```
## Troubleshooting
```
Error: Failed to determine GroupVersionResource for manifest
```
This happens when you attempt to apply the Confluent Platform Terraform without first applying the Confluent for Kubernetes CRDs. Please see [Prerequisites](#prerequisites) for more information.
## Tests
Expand All @@ -40,16 +47,7 @@ content: |-
## Makefile Targets
```
help This help.
build Build docker dev container
run Run docker dev container
install Install project
lint Lint with pre-commit
lint-all Lint with pre-commit
tests Test with Terratest
test-confluent-operator Test confluent-operator Example
test-confluent-platform Test confluent-platform Example
clean Clean project
{{ include ".make.docs" }}
```
{{ .Requirements }}
Expand Down
45 changes: 15 additions & 30 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,47 +15,39 @@ build: ## Build docker dev container
run: ## Run docker dev container
docker run -it --rm -v "$$(pwd)":/workspaces/$(NAME) -v ~/.kube:/root/.kube -v ~/.cache/pre-commit:/root/.cache/pre-commit -v ~/.terraform.d/plugins:/root/.terraform.d/plugins --workdir /workspaces/$(NAME) $(NAME) /bin/bash

install-cfk-crds:
# download the cfk helm chart
curl -O https://confluent-for-kubernetes.s3-us-west-1.amazonaws.com/confluent-for-kubernetes-2.4.0.tar.gz
mkdir -p confluent-for-kubernetes-2.4.0
tar -xzf confluent-for-kubernetes-2.4.0.tar.gz --strip-components=1 -C confluent-for-kubernetes-2.4.0

# install the cfk crds
apply-cfk-crds:
kubectl config set-cluster docker-desktop
kubectl apply -f confluent-for-kubernetes-2.4.0/helm/confluent-for-kubernetes/crds/
kubectl apply -f ./crds/2.4.0

setup: install-cfk-crds ## Setup project
setup: apply-cfk-crds ## Setup project
# terraform
terraform init
cd examples/confluent_operator && terraform init
cd examples/confluent_platform && terraform init
cd examples/confluent_platform_singlenode && terraform init
cd examples/complete && terraform init
cd examples/kafka_topics && terraform init

# pre-commit
git init
git add -A
pre-commit install

# terratest
rm -rf go.mod*
go get github.com/gruntwork-io/terratest/modules/terraform
go mod init test/terraform_confluent_operator_test.go
go mod tidy -go=1.16 && go mod tidy -go=1.17

render-terraform-docs-code:
# render terraform-docs code examples
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/confluent_operator"\n version = ">= 0.3.0"\n/g' examples/confluent_operator/main.tf > examples/confluent_operator/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.3.0"\n/g' examples/confluent_platform/main.tf > examples/confluent_platform/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.3.0"\n/g' examples/confluent_platform_singlenode/main.tf > examples/confluent_platform_singlenode/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.3.0"/g' examples/complete/main.tf > examples/complete/.main.tf.docs
docs:
./bin/render-docs.sh

lint: ## Lint with pre-commit
lint: docs ## Lint with pre-commit
git add -A
pre-commit run
git add -A

lint-all: render-terraform-docs-code ## Lint with pre-commit
lint-all: docs ## Lint with pre-commit
git add -A
pre-commit run --all-files
git add -A
Expand All @@ -81,24 +73,17 @@ test-confluent-platform: test-setup test-confluent-platform test-clean ## Test t

test-confluent-platform-singlenode: test-setup test-confluent-platform-singlenode test-clean ## Test the confluent_platform_singlenode example

test-complete: # Test the complete example
test-complete: ## Test the complete example
go test test/terraform_complete_test.go -timeout 20m -v |& tee test/terraform_complete_test.log

uninstall-cfk-crds:
# download the cfk helm chart
curl -O https://confluent-for-kubernetes.s3-us-west-1.amazonaws.com/confluent-for-kubernetes-2.4.0.tar.gz
mkdir confluent-for-kubernetes-2.4.0
tar -xzf confluent-for-kubernetes-2.4.0.tar.gz --strip-components=1 -C confluent-for-kubernetes-2.4.0
test-kafka-topics: ## Test the kafka_topics example
go test test/terraform_kafka_topics_test.go -timeout 20m -v |& tee test/terraform_kafka_topics_test.log

# install the cfk crds
delete-cfk-crds:
kubectl config set-cluster docker-desktop
kubectl delete -f confluent-for-kubernetes-2.4.0/helm/confluent-for-kubernetes/crds/

# clean up the cfk downloads
rm confluent-for-kubernetes-2.4.0.tar.gz
rm -rf confluent-for-kubernetes-2.4.0
kubectl delete -f ./crds/2.4.0

clean: uninstall-cfk-crds ## Clean project
clean: delete-cfk-crds ## Clean project
@rm -f .terraform.lock.hcl
@rm -f examples/confluent_operator/.terraform.lock.hcl
@rm -f examples/confluent_platform/.terraform.lock.hcl
Expand Down
72 changes: 48 additions & 24 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,38 @@ A Terraform module for running [Confluent for Kubernetes (CFK)](https://docs.con
```hcl
module "confluent_platform" {
source = "aidanmelen/confluent-platform/kubernetes"
version = ">= 0.3.0"
version = ">= 0.4.0"
namespace = var.namespace
# confluent operator
confluent_operator = {
create_namespace = true
name = "confluent-operator"
chart_version = "0.517.12"
}
# confluent platform
/*
zookeeper = { ... }
kafka = { ... }
connect = { ... }
ksqldb = { ... }
controlcenter = { ... }
schemaregistry = { ... }
kafkarestproxy = { ... }
*/
zookeeper = {
"spec" = {
"replicas" = "3"
}
}
kafka = {
"spec" = {
"replicas" = "3"
}
}
create_connect = false
create_ksqldb = false
create_controlcenter = false
create_schemaregistry = false
create_kafkarestproxy = false
kafka_topics = {
"my-topic" = {}
"my-other-topic" = { "spec" = { "configs" = { "cleanup.policy" = "compact" } } }
}
}
```

Expand All @@ -59,6 +70,13 @@ terraform init
terraform apply
```

## Troubleshooting

```
Error: Failed to determine GroupVersionResource for manifest
```

This happens when you attempt to apply the Confluent Platform Terraform without first applying the Confluent for Kubernetes CRDs. Please see [Prerequisites](#prerequisites) for more information.

## Tests

Expand All @@ -70,16 +88,19 @@ Run Terratest using the [Makefile](./Makefile) targets:
## Makefile Targets

```
help This help.
build Build docker dev container
run Run docker dev container
install Install project
lint Lint with pre-commit
lint-all Lint with pre-commit
tests Test with Terratest
test-confluent-operator Test confluent-operator Example
test-confluent-platform Test confluent-platform Example
clean Clean project
help This help.
build Build docker dev container
run Run docker dev container
setup Setup project
lint Lint with pre-commit
lint-all Lint with pre-commit
tests Tests with Terratest
test-confluent-operator Test the confluent_operator example
test-confluent-platform Test the confluent_platform example
test-confluent-platform-singlenode Test the confluent_platform_singlenode example
test-complete Test the complete example
test-kafka-topics Test the kafka_topics example
clean Clean project
```

## Requirements
Expand All @@ -100,17 +121,18 @@ clean Clean project
|------|--------|---------|
| <a name="module_confluent_operator"></a> [confluent\_operator](#module\_confluent\_operator) | ./modules/confluent_operator | n/a |
| <a name="module_confluent_platform_override_values"></a> [confluent\_platform\_override\_values](#module\_confluent\_platform\_override\_values) | Invicton-Labs/deepmerge/null | 0.1.5 |
| <a name="module_kafka_topics"></a> [kafka\_topics](#module\_kafka\_topics) | ./modules/kafka_topic | n/a |
## Resources

| Name | Type |
|------|------|
| [kubernetes_manifest.component](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
| [kubernetes_manifest.components](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
## Inputs

| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_confluent_operator"></a> [confluent\_operator](#input\_confluent\_operator) | Controls if the Confluent Operator resources should be created. | `any` | `null` | no |
| <a name="input_confluent_operator_app_version"></a> [confluent\_operator\_app\_version](#input\_confluent\_operator\_app\_version) | The default Confluent Operator app version. This may be overriden by component override values. This version must be compatible with the`confluent_platform_version`. Please see confluent docs for more information: https://docs.confluent.io/platform/current/installation/versions-interoperability.html#confluent-operator | `string` | `"2.4.0"` | no |
| <a name="input_confluent_operator_app_version"></a> [confluent\_operator\_app\_version](#input\_confluent\_operator\_app\_version) | The default Confluent Operator app version. This may be overriden by component override values. This version must be compatible with the `confluent_platform_version`. Please see confluent docs for more information: https://docs.confluent.io/platform/current/installation/versions-interoperability.html#confluent-operator | `string` | `"2.4.0"` | no |
| <a name="input_confluent_platform_version"></a> [confluent\_platform\_version](#input\_confluent\_platform\_version) | The default Confluent Platform app version. This may be overriden by component override values. This version must be compatible with the `confluent_operator_app_version`. Please see confluent docs for more information: https://docs.confluent.io/platform/current/installation/versions-interoperability.html#confluent-operator | `string` | `"7.2.0"` | no |
| <a name="input_connect"></a> [connect](#input\_connect) | The Connect override values. | `any` | `{}` | no |
| <a name="input_controlcenter"></a> [controlcenter](#input\_controlcenter) | The ControlCenter override values. | `any` | `{}` | no |
Expand All @@ -125,6 +147,7 @@ clean Clean project
| <a name="input_create_zookeeper"></a> [create\_zookeeper](#input\_create\_zookeeper) | Controls if the Zookeeper component of the Confluent Platform should be created. | `bool` | `true` | no |
| <a name="input_delete_timeout"></a> [delete\_timeout](#input\_delete\_timeout) | The delete timeout for each Conlfuent Platform component. | `string` | `"10m"` | no |
| <a name="input_kafka"></a> [kafka](#input\_kafka) | The Kafka override values. | `any` | `{}` | no |
| <a name="input_kafka_topics"></a> [kafka\_topics](#input\_kafka\_topics) | The Kafka Topics to create. | `any` | `[]` | no |
| <a name="input_kafkarestproxy"></a> [kafkarestproxy](#input\_kafkarestproxy) | The KafkaRestProxy override values. | `any` | `{}` | no |
| <a name="input_ksqldb"></a> [ksqldb](#input\_ksqldb) | The KsqlDB override values. | `any` | `{}` | no |
| <a name="input_namespace"></a> [namespace](#input\_namespace) | The namespace to release Confluent Platform into. When `confluent_operator` is specified, this will also ensure the Confluent Operator is released into the same namespace. | `string` | `"confluent"` | no |
Expand All @@ -139,6 +162,7 @@ clean Clean project
| <a name="output_connect_manifest"></a> [connect\_manifest](#output\_connect\_manifest) | The Connect manifest. |
| <a name="output_controlcenter_manifest"></a> [controlcenter\_manifest](#output\_controlcenter\_manifest) | The ControlCenter manifest. |
| <a name="output_kafka_manifest"></a> [kafka\_manifest](#output\_kafka\_manifest) | The Kafka manifest. |
| <a name="output_kafka_topics"></a> [kafka\_topics](#output\_kafka\_topics) | The Kafka Topic manifests |
| <a name="output_kafkarestproxy_manifest"></a> [kafkarestproxy\_manifest](#output\_kafkarestproxy\_manifest) | The KafkaRestProxy manifest. |
| <a name="output_ksqldb_manifest"></a> [ksqldb\_manifest](#output\_ksqldb\_manifest) | The KsqlDB manifest. |
| <a name="output_namespace"></a> [namespace](#output\_namespace) | The default namespace for the Confluent Platform. |
Expand Down
20 changes: 20 additions & 0 deletions bin/render-docs.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

# render terraform-docs code examples
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/confluent_operator"\n version = ">= 0.4.0"\n/g' examples/confluent_operator/main.tf > examples/confluent_operator/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.4.0"\n/g' examples/confluent_platform/main.tf > examples/confluent_platform/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.4.0"\n/g' examples/confluent_platform_singlenode/main.tf > examples/confluent_platform_singlenode/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.4.0"/g' examples/complete/main.tf > examples/complete/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.4.0"/g' examples/kafka_topics/main.tf > examples/kafka_topics/.main.tf.docs

# render Makefile targets examples
make > .make.docs

# remove terminal color codes
sed -i 's/\x1b\[[0-9;]*m//g' .make.docs

# remove make header (first line)
sed -i '1d' .make.docs

# remove make footer (last line)
sed -i '$d' .make.docs
18 changes: 0 additions & 18 deletions confluent_operator.tf
View file

This file was deleted.

Loading

0 comments on commit 517bba0

Please sign in to comment.