Skip to content

Commit

Permalink
Merge pull request #5 from aidanmelen/schema-submodule
Browse files Browse the repository at this point in the history 
schema submodule
  • Loading branch information
@aidanmelen
aidanmelen authored Aug 4, 2022
2 parents 3c32f45 + acbb2c8 commit 4aba36d
Show file tree
Hide file tree
Showing 65 changed files with 45,185 additions and 37,226 deletions.
1 change: 1 addition & 0 deletions .make.docs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,6 @@ test-confluent-platform Test the confluent_platform example
test-confluent-platform-singlenode Test the confluent_platform_singlenode example
test-complete Test the complete example
test-kafka-topic Test the kafka_topic example
test-schema Test the schema example
test-connector Test the connector example
clean Clean project
11 changes: 10 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@ setup: apply-cfk-crds ## Setup project
cd modules/confluent_operator && terraform init
cd modules/kafka_topic && terraform init
cd modules/connector && terraform init
cd modules/schema && terraform init
cd examples/confluent_operator && terraform init
cd examples/confluent_platform && terraform init
cd examples/confluent_platform_singlenode && terraform init
cd examples/complete && terraform init
cd examples/kafka_topic && terraform init
cd examples/schema && terraform init
cd examples/connector && terraform init

# pre-commit
Expand All @@ -56,7 +58,7 @@ lint-all: docs ## Lint all files with pre-commit
pre-commit run --all-files
git add -A

tests: test-confluent-operator test-confluent-platform test-confluent-platform-singlenode test-complete test-kafka-topic test-connector ## Tests with Terratest
tests: test-confluent-operator test-confluent-platform test-confluent-platform-singlenode test-complete test-kafka-topic test-schema test-connector ## Tests with Terratest

test-confluent-operator: ## Test the confluent_operator example
go test test/terraform_confluent_operator_test.go -timeout 5m -v |& tee test/terraform_confluent_operator_test.log
Expand Down Expand Up @@ -84,6 +86,9 @@ test-complete: ## Test the complete example
test-kafka-topic: ## Test the kafka_topic example
go test test/terraform_kafka_topic_test.go -timeout 10m -v |& tee test/terraform_kafka_topic_test.log

test-schema: ## Test the schema example
go test test/terraform_schema_test.go -timeout 10m -v |& tee test/terraform_schema_test.log

test-connector: ## Test the connector example
go test test/terraform_connector_test.go -timeout 10m -v |& tee test/terraform_connector_test.log

Expand All @@ -96,22 +101,26 @@ clean: delete-cfk-crds ## Clean project
@rm -f modules/confluent_operator/.terraform.lock.hcl
@rm -f modules/kafka_topic/.terraform.lock.hcl
@rm -f modules/connector/.terraform.lock.hcl
@rm -f modules/schema/.terraform.lock.hcl
@rm -f examples/confluent_operator/.terraform.lock.hcl
@rm -f examples/confluent_platform/.terraform.lock.hcl
@rm -f examples/confluent_platform_singlenode/.terraform.lock.hcl
@rm -f examples/complete/.terraform.lock.hcl
@rm -rf examples/kafka_topic/.terraform.lock.hcl
@rm -rf examples/schema/.terraform.lock.hcl
@rm -rf examples/connector/.terraform.lock.hcl

@rm -rf .terraform
@rm -rf modules/confluent_operator/.terraform
@rm -rf modules/kafka_topic/.terraform
@rm -rf modules/connector/.terraform
@rm -rf modules/schema/.terraform
@rm -rf examples/confluent_operator/.terraform
@rm -rf examples/confluent_platform/.terraform
@rm -rf examples/confluent_platform_singlenode/.terraform
@rm -rf examples/complete/.terraform
@rm -rf examples/kafka_topic/.terraform
@rm -rf examples/schema/.terraform
@rm -rf examples/connector/.terraform

@rm -f go.mod
Expand Down
41 changes: 32 additions & 9 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ Similar to the [values file for Helm](https://helm.sh/docs/chart_template_guide/
```hcl
module "confluent_platform" {
source = "aidanmelen/confluent-platform/kubernetes"
version = ">= 0.7.0"
version = ">= 0.8.0"
namespace = var.namespace
Expand All @@ -47,24 +47,40 @@ module "confluent_platform" {
chart_version = "0.517.12"
}
zookeeper = { "spec" = { "replicas" = "3" } } # override default value
kafka = { "spec" = { "replicas" = "3" } } # override default value
# hcl value overrides
zookeeper = { "spec" = { "replicas" = "3" } }
# yaml inline value overrides
kafka = yamldecode(<<-EOF
spec:
replicas: 3
EOF
)
# yaml file value overrides
connect = yamldecode(file("${path.module}/values/connect.yaml"))
create_connect = true # create with default values
create_ksqldb = false
create_controlcenter = var.create_controlcenter
create_schemaregistry = false
create_schemaregistry = true # create with default values
create_kafkarestproxy = false
kafka_topics = {
"my-topic" = {}
"pageviews" = {}
"my-other-topic" = {
"values" = { "spec" = { "configs" = { "cleanup.policy" = "compact" } } }
}
}
schemas = {
"pageviews-value" = {
"data" = file("${path.module}/schemas/pageviews_schema.avro")
}
}
connectors = {
"my-connector" = {
"pageviews-source = "aidanmelen/confluent-platform/kubernetes"
version = ">= 0.8.0"
"values" = yamldecode(file("${path.module}/values/connector.yaml"))
}
}
Expand Down Expand Up @@ -111,6 +127,7 @@ test-confluent-platform Test the confluent_platform example
test-confluent-platform-singlenode Test the confluent_platform_singlenode example
test-complete Test the complete example
test-kafka-topic Test the kafka_topic example
test-schema Test the schema example
test-connector Test the connector example
clean Clean project
```
Expand All @@ -135,6 +152,7 @@ clean Clean project
| <a name="module_confluent_platform_override_values"></a> [confluent\_platform\_override\_values](#module\_confluent\_platform\_override\_values) | Invicton-Labs/deepmerge/null | 0.1.5 |
| <a name="module_connectors"></a> [connectors](#module\_connectors) | ./modules/connector | n/a |
| <a name="module_kafka_topics"></a> [kafka\_topics](#module\_kafka\_topics) | ./modules/kafka_topic | n/a |
| <a name="module_schemas"></a> [schemas](#module\_schemas) | ./modules/schema | n/a |
## Resources

| Name | Type |
Expand All @@ -149,7 +167,7 @@ clean Clean project
| <a name="input_confluent_operator_app_version"></a> [confluent\_operator\_app\_version](#input\_confluent\_operator\_app\_version) | The default Confluent Operator app version. This may be overriden by component override values. This version must be compatible with the `confluent_platform_version`. Please see confluent docs for more information: https://docs.confluent.io/platform/current/installation/versions-interoperability.html#confluent-operator | `string` | `"2.4.0"` | no |
| <a name="input_confluent_platform_version"></a> [confluent\_platform\_version](#input\_confluent\_platform\_version) | The default Confluent Platform app version. This may be overriden by component override values. This version must be compatible with the `confluent_operator_app_version`. Please see confluent docs for more information: https://docs.confluent.io/platform/current/installation/versions-interoperability.html#confluent-operator | `string` | `"7.2.0"` | no |
| <a name="input_connect"></a> [connect](#input\_connect) | The Connect override values. | `any` | `{}` | no |
| <a name="input_connectors"></a> [connectors](#input\_connectors) | A map of Connectors to create. The key is the connector name and the value are the override values. | `any` | `{}` | no |
| <a name="input_connectors"></a> [connectors](#input\_connectors) | A map of Connectors to create. The key is the connector name. The value map is the input for the `connector` submodule. | `any` | `{}` | no |
| <a name="input_controlcenter"></a> [controlcenter](#input\_controlcenter) | The ControlCenter override values. | `any` | `{}` | no |
| <a name="input_create"></a> [create](#input\_create) | Controls if the Confluent Platform and Operator resources should be created (affects all resources). | `bool` | `true` | no |
| <a name="input_create_connect"></a> [create\_connect](#input\_create\_connect) | Controls if the Connect component of the Confluent Platform should be created. | `bool` | `true` | no |
Expand All @@ -162,11 +180,12 @@ clean Clean project
| <a name="input_create_zookeeper"></a> [create\_zookeeper](#input\_create\_zookeeper) | Controls if the Zookeeper component of the Confluent Platform should be created. | `bool` | `true` | no |
| <a name="input_delete_timeout"></a> [delete\_timeout](#input\_delete\_timeout) | The delete timeout for each Confluent Platform component. | `string` | `"10m"` | no |
| <a name="input_kafka"></a> [kafka](#input\_kafka) | The Kafka override values. | `any` | `{}` | no |
| <a name="input_kafka_topics"></a> [kafka\_topics](#input\_kafka\_topics) | A map of Kafka Topics to create. The key is the topic name and the value are the override values. | `any` | `{}` | no |
| <a name="input_kafka_topics"></a> [kafka\_topics](#input\_kafka\_topics) | A map of Kafka Topics to create. The key is the topic name. The value map is the input for the `kafka_topic` submodule. | `any` | `{}` | no |
| <a name="input_kafkarestproxy"></a> [kafkarestproxy](#input\_kafkarestproxy) | The KafkaRestProxy override values. | `any` | `{}` | no |
| <a name="input_ksqldb"></a> [ksqldb](#input\_ksqldb) | The KsqlDB override values. | `any` | `{}` | no |
| <a name="input_namespace"></a> [namespace](#input\_namespace) | The namespace to release Confluent Platform into. When `confluent_operator` is specified, this will also ensure the Confluent Operator is released into the same namespace. | `string` | `"confluent"` | no |
| <a name="input_schemaregistry"></a> [schemaregistry](#input\_schemaregistry) | The SchemaRegistry override values. | `any` | `{}` | no |
| <a name="input_schemas"></a> [schemas](#input\_schemas) | A map of Schemas to create. The key is the schema name. The value map is the input for the `schema` submodule. | `any` | `{}` | no |
| <a name="input_update_timeout"></a> [update\_timeout](#input\_update\_timeout) | The update timeout for each Confluent Platform component. | `string` | `"1h"` | no |
| <a name="input_zookeeper"></a> [zookeeper](#input\_zookeeper) | The Zookeeper override values. | `any` | `{}` | no |
## Outputs
Expand All @@ -191,8 +210,12 @@ clean Clean project
| <a name="output_ksqldb_manifest"></a> [ksqldb\_manifest](#output\_ksqldb\_manifest) | The KsqlDB manifest. |
| <a name="output_ksqldb_object"></a> [ksqldb\_object](#output\_ksqldb\_object) | The KsqlDB object. |
| <a name="output_namespace"></a> [namespace](#output\_namespace) | The default namespace for the Confluent Platform. |
| <a name="output_schema_config_map"></a> [schema\_config\_map](#output\_schema\_config\_map) | Map of attribute maps for all the Schema ConfigMap created. |
| <a name="output_schema_manifests"></a> [schema\_manifests](#output\_schema\_manifests) | Map of attribute maps for all the Schema manifests created. |
| <a name="output_schema_objects"></a> [schema\_objects](#output\_schema\_objects) | Map of attribute maps for all the Schema objects created. |
| <a name="output_schemaregistry_manifest"></a> [schemaregistry\_manifest](#output\_schemaregistry\_manifest) | The SchemaRegistry manifest. |
| <a name="output_schemaregistry_object"></a> [schemaregistry\_object](#output\_schemaregistry\_object) | The SchemaRegistry object. |
| <a name="output_schemas"></a> [schemas](#output\_schemas) | Map of attribute maps for all Schema submodules created. |
| <a name="output_zookeeper_manifest"></a> [zookeeper\_manifest](#output\_zookeeper\_manifest) | The Zookeeper manifest. |
| <a name="output_zookeeper_object"></a> [zookeeper\_object](#output\_zookeeper\_object) | The Zookeeper object. |
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
Expand Down
4 changes: 0 additions & 4 deletions TODO.txt
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
Implement the remaining submodules:
- cluster_link
- confluent_cluster_role_binding
- connector
- schema

Create examples:

- hybrid with AWS MSK
- mirrormaker2 with Strimzi
- [security/production-secure-deploy](https://github.com/confluentinc/confluent-kubernetes-examples/tree/master/security/production-secure-deploy)
- [schemas](https://github.com/confluentinc/confluent-kubernetes-examples/tree/master/schemas)
- [connectors](https://github.com/confluentinc/confluent-kubernetes-examples/tree/master/connector) https://github.com/confluentinc/confluent-kubernetes-examples/blob/master/hybrid/ccloud-connect-confluent-hub/kafka-connect.yaml#L2-L19
18 changes: 7 additions & 11 deletions bin/render-docs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,13 @@
# render terraform-docs code examples

## examples
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/confluent_operator"\n version = ">= 0.7.0"\n/g' examples/confluent_operator/main.tf > examples/confluent_operator/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.7.0"\n/g' examples/confluent_platform/main.tf > examples/confluent_platform/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.7.0"\n/g' examples/confluent_platform_singlenode/main.tf > examples/confluent_platform_singlenode/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.7.0"/g' examples/complete/main.tf > examples/complete/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/kafka_topic"\n version = ">= 0.7.0"/g' examples/kafka_topic/main.tf > examples/kafka_topic/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/connector"\n version = ">= 0.7.0"/1' examples/connector/main.tf > examples/connector/.main.tf.docs

## submodules
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/confluent_operator"\n version = ">= 0.7.0"\n/g' examples/confluent_operator/main.tf > modules/confluent_operator/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/kafka_topic"\n version = ">= 0.7.0"/g' examples/kafka_topic/main.tf > modules/kafka_topic/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/connector"\n version = ">= 0.7.0"/1' examples/connector/main.tf > modules/connector/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/confluent_operator"\n version = ">= 0.8.0"\n/g' examples/confluent_operator/main.tf > examples/confluent_operator/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.8.0"\n/g' examples/confluent_platform/main.tf > examples/confluent_platform/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.8.0"\n/g' examples/confluent_platform_singlenode/main.tf > examples/confluent_platform_singlenode/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes"\n version = ">= 0.8.0"/g' examples/complete/main.tf > examples/complete/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/kafka_topic"\n version = ">= 0.8.0"/g' examples/kafka_topic/main.tf > examples/kafka_topic/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/schema"\n version = ">= 0.8.0"\n/g' examples/schema/main.tf > examples/schema/.main.tf.docs
sed -z 's/source[^\r\n]*/source = "aidanmelen\/confluent-platform\/kubernetes\/\/modules\/connector"\n version = ">= 0.8.0"/1' examples/connector/main.tf > examples/connector/.main.tf.docs

# render Makefile targets examples
make > .make.docs
Expand Down
30 changes: 23 additions & 7 deletions examples/complete/.main.tf.docs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module "confluent_platform" {
source = "aidanmelen/confluent-platform/kubernetes"
version = ">= 0.7.0"
version = ">= 0.8.0"

namespace = var.namespace

Expand All @@ -10,24 +10,40 @@ module "confluent_platform" {
chart_version = "0.517.12"
}

zookeeper = { "spec" = { "replicas" = "3" } } # override default value
kafka = { "spec" = { "replicas" = "3" } } # override default value
# hcl value overrides
zookeeper = { "spec" = { "replicas" = "3" } }

# yaml inline value overrides
kafka = yamldecode(<<-EOF
spec:
replicas: 3
EOF
)

# yaml file value overrides
connect = yamldecode(file("${path.module}/values/connect.yaml"))

create_connect = true # create with default values
create_ksqldb = false
create_controlcenter = var.create_controlcenter
create_schemaregistry = false
create_schemaregistry = true # create with default values
create_kafkarestproxy = false

kafka_topics = {
"my-topic" = {}
"pageviews" = {}
"my-other-topic" = {
"values" = { "spec" = { "configs" = { "cleanup.policy" = "compact" } } }
}
}

schemas = {
"pageviews-value" = {
"data" = file("${path.module}/schemas/pageviews_schema.avro")
}
}

connectors = {
"my-connector" = {
"pageviews-source = "aidanmelen/confluent-platform/kubernetes"
version = ">= 0.8.0"
"values" = yamldecode(file("${path.module}/values/connector.yaml"))
}
}
Expand Down
Loading

0 comments on commit 4aba36d

Please sign in to comment.