fixed missing spec.image.application in hybrid_aws_msk example

aidanmelen · Oct 19, 2022 · 080a357 · 080a357
1 parent 51a148a
commit 080a357
Show file tree

Hide file tree

Showing 9 changed files with 97 additions and 49 deletions.
diff --git a/.make.docs b/.make.docs
@@ -12,4 +12,5 @@ test-complete                       Test the complete example
 test-kafka-topic                    Test the kafka_topic example
 test-schema                         Test the schema example
 test-connector                      Test the connector example
+release                             Tag remote triggering Terraform Registry release
 clean                               Clean project
diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
-NAME = terraform-kubernetes-confluent
-
+NAME := terraform-kubernetes-confluent
+VERSION := 0.9.3
 SHELL := /bin/bash
 
 .PHONY: help all
@@ -102,6 +102,10 @@ delete-cfk-crds:
 	kubectl config set-cluster docker-desktop
 	kubectl delete -f ./crds/2.4.0
 
+release: ## Tag remote triggering Terraform Registry release
+	git tag v${VERSION}
+	git push --tag
+
 clean: delete-cfk-crds ## Clean project
 	@rm -f .terraform.lock.hcl
 	@rm -f modules/confluent_operator/.terraform.lock.hcl

diff --git a/README.md b/README.md
@@ -99,6 +99,7 @@ test-complete                       Test the complete example
 test-kafka-topic                    Test the kafka_topic example
 test-schema                         Test the schema example
 test-connector                      Test the connector example
+release                             Tag remote triggering Terraform Registry release
 clean                               Clean project
 ```
 

diff --git a/examples/hybrid_aws_msk/aws/data.tf b/examples/hybrid_aws_msk/aws/data.tf
@@ -1,5 +1,3 @@
-data "aws_caller_identity" "current" {}
-
 data "aws_eks_cluster_auth" "eks" {
   name = var.name
 }
diff --git a/examples/hybrid_aws_msk/aws/iam.tf b/examples/hybrid_aws_msk/aws/iam.tf
@@ -28,7 +28,7 @@ resource "aws_iam_policy" "aws_msk_cluster_full_access" {
                   "kafka-cluster:ReadData"
               ],
               "Resource": [
-                  "arn:aws:kafka:${var.aws_region}:${data.aws_caller_identity.current.account_id}:topic/${var.name}/*"
+                  "${replace(module.msk_cluster.arn, ":cluster/", ":topic/")}/*"
               ]
           },
           {
@@ -38,7 +38,7 @@ resource "aws_iam_policy" "aws_msk_cluster_full_access" {
                   "kafka-cluster:DescribeGroup"
               ],
               "Resource": [
-                  "arn:aws:kafka:${var.aws_region}:${data.aws_caller_identity.current.account_id}:group/${var.name}/*"
+                  "${replace(module.msk_cluster.arn, ":cluster/", ":group/")}/*"
               ]
           }
       ]

diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md
@@ -50,16 +50,19 @@ module "confluent_platform" {
     spec:
       image:
         application: aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0
-      tls:
-        autoGeneratedCerts: true
       configOverrides:
         server:
           - "security.protocol=SASL_SSL"
+          - "admin.security.protocol=SASL_SSL"
+          - "producer.security.protocol=SASL_SSL"
+          - "consumer.security.protocol=SASL_SSL"
           - "sasl.mechanism=AWS_MSK_IAM"
           - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
           - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      tls:
+        autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
@@ -75,9 +78,8 @@ module "confluent_platform" {
 
   ksqldb = yamldecode(<<-EOF
     spec:
-      # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud
-      # tls:
-      #   autoGeneratedCerts: true
+      image:
+        application: aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0
       configOverrides:
         server:
           - "security.protocol=SASL_SSL"
@@ -86,6 +88,9 @@ module "confluent_platform" {
           - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud
+      # tls:
+      #   autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
@@ -99,15 +104,21 @@ module "confluent_platform" {
     EOF
   )
 
+  # ~~~ NOT WORKING ~~~
   controlcenter = yamldecode(<<-EOF
     spec:
-      tls:
-        autoGeneratedCerts: true
+      image:
+        application: aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0
       configOverrides:
         server:
-          - "security.protocol=SSL"
+          - "security.protocol=SASL_SSL"
+          - "sasl.mechanism=AWS_MSK_IAM"
+          - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
+          - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      tls:
+        autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
@@ -133,18 +144,20 @@ module "confluent_platform" {
 
   schemaregistry = yamldecode(<<-EOF
     spec:
+      image:
+        application: aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0
       configOverrides:
         server:
           - "security.protocol=SASL_SSL"
           - "kafkastore.security.protocol=SASL_SSL"
+          - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}"
           - "kafkastore.sasl.mechanism=AWS_MSK_IAM"
           - "kafkastore.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
           - "kafkastore.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
-          - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}"
-      tls:
-        autoGeneratedCerts: true
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      tls:
+        autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
@@ -156,13 +169,18 @@ module "confluent_platform" {
 
   kafkarestproxy = yamldecode(<<-EOF
     spec:
+      image:
+        application: aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0
       configOverrides:
         server:
-          - "security.protocol=SSL"
-      tls:
-        autoGeneratedCerts: true
+          - "security.protocol=SASL_SSL"
+          - "sasl.mechanism=AWS_MSK_IAM"
+          - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
+          - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      tls:
+        autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}

diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.connect b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.connect
@@ -1,5 +1,12 @@
 # https://github.com/aws/aws-msk-iam-auth
 ARG TAG=7.2.0
 FROM confluentinc/cp-server-connect:${TAG}
+
+# https://docs.confluent.io/platform/current/installation/docker/development.html#extend-cp-images
+# RUN confluent-hub install --no-prompt hpgrahsl/kafka-connect-mongodb:1.1.0 \
+#  && confluent-hub install --no-prompt microsoft/kafka-connect-iothub:0.6 \
+#  && confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.1.0
+
+# this must be run after confluent-hub cmds
 ENV CLASSPATH=/usr/share/java/aws-msk-iam-auth-1.1.4-all.jar
 RUN curl -sSL -o /usr/share/java/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Makefile b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Makefile
@@ -1,4 +1,5 @@
 SHELL := /bin/bash
+TAG := 7.2.2
 
 .PHONY: help all
 
@@ -8,21 +9,21 @@ help: ## This help.
 .DEFAULT_GOAL := help
 
 build: ## Build docker images
-	docker build -f Dockerfile.connect . -t cp-server-connect-with-aws-msk-iam-auth
-	docker build -f Dockerfile.ksqldb . -t cp-ksqldb-server-with-aws-msk-iam-auth
-	docker build -f Dockerfile.controlcenter . -t cp-enterprise-control-center-with-aws-msk-iam-auth
-	docker build -f Dockerfile.schemaregistry . -t cp-schema-registry-with-aws-msk-iam-auth
-	docker build -f Dockerfile.kafkarestproxy . -t cp-kafka-rest-with-aws-msk-iam-auth
+	docker build -f Dockerfile.connect . -t cp-server-connect-with-aws-msk-iam-auth --build-arg TAG=$(TAG)
+	docker build -f Dockerfile.ksqldb . -t cp-ksqldb-server-with-aws-msk-iam-auth --build-arg TAG=$(TAG)
+	docker build -f Dockerfile.controlcenter . -t cp-enterprise-control-center-with-aws-msk-iam-auth --build-arg TAG=$(TAG)
+	docker build -f Dockerfile.schemaregistry . -t cp-schema-registry-with-aws-msk-iam-auth --build-arg TAG=$(TAG)
+	docker build -f Dockerfile.kafkarestproxy . -t cp-kafka-rest-with-aws-msk-iam-auth --build-arg TAG=$(TAG)
 
-	docker tag cp-server-connect-with-aws-msk-iam-auth aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0
-	docker tag cp-ksqldb-server-with-aws-msk-iam-auth aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0
-	docker tag cp-enterprise-control-center-with-aws-msk-iam-auth aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0
-	docker tag cp-schema-registry-with-aws-msk-iam-auth aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0
-	docker tag cp-kafka-rest-with-aws-msk-iam-auth aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0
+	docker tag cp-server-connect-with-aws-msk-iam-auth aidanmelen/cp-server-connect-with-aws-msk-iam-auth:$(TAG)
+	docker tag cp-ksqldb-server-with-aws-msk-iam-auth aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:$(TAG)
+	docker tag cp-enterprise-control-center-with-aws-msk-iam-auth aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:$(TAG)
+	docker tag cp-schema-registry-with-aws-msk-iam-auth aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:$(TAG)
+	docker tag cp-kafka-rest-with-aws-msk-iam-auth aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:$(TAG)
 
 push: build ## Push docker images
-	docker push aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0
-	docker push aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0
-	docker push aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0
-	docker push aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0
-	docker push aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0
+	docker push aidanmelen/cp-server-connect-with-aws-msk-iam-auth:$(TAG)
+	docker push aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:$(TAG)
+	docker push aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:$(TAG)
+	docker push aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:$(TAG)
+	docker push aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:$(TAG)
diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf
@@ -37,16 +37,19 @@ module "confluent_platform" {
     spec:
       image:
         application: aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0
-      tls:
-        autoGeneratedCerts: true
       configOverrides:
         server:
           - "security.protocol=SASL_SSL"
+          - "admin.security.protocol=SASL_SSL"
+          - "producer.security.protocol=SASL_SSL"
+          - "consumer.security.protocol=SASL_SSL"
           - "sasl.mechanism=AWS_MSK_IAM"
           - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
           - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      tls:
+        autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
@@ -62,9 +65,8 @@ module "confluent_platform" {
 
   ksqldb = yamldecode(<<-EOF
     spec:
-      # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud
-      # tls:
-      #   autoGeneratedCerts: true
+      image:
+        application: aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0
       configOverrides:
         server:
           - "security.protocol=SASL_SSL"
@@ -73,6 +75,9 @@ module "confluent_platform" {
           - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud
+      # tls:
+      #   autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
@@ -86,15 +91,21 @@ module "confluent_platform" {
     EOF
   )
 
+  # ~~~ NOT WORKING ~~~
   controlcenter = yamldecode(<<-EOF
     spec:
-      tls:
-        autoGeneratedCerts: true
+      image:
+        application: aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0
       configOverrides:
         server:
-          - "security.protocol=SSL"
+          - "security.protocol=SASL_SSL"
+          - "sasl.mechanism=AWS_MSK_IAM"
+          - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
+          - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      tls:
+        autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
@@ -120,18 +131,20 @@ module "confluent_platform" {
 
   schemaregistry = yamldecode(<<-EOF
     spec:
+      image:
+        application: aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0
       configOverrides:
         server:
           - "security.protocol=SASL_SSL"
           - "kafkastore.security.protocol=SASL_SSL"
+          - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}"
           - "kafkastore.sasl.mechanism=AWS_MSK_IAM"
           - "kafkastore.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
           - "kafkastore.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
-          - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}"
-      tls:
-        autoGeneratedCerts: true
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      tls:
+        autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
@@ -143,13 +156,18 @@ module "confluent_platform" {
 
   kafkarestproxy = yamldecode(<<-EOF
     spec:
+      image:
+        application: aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0
       configOverrides:
         server:
-          - "security.protocol=SSL"
-      tls:
-        autoGeneratedCerts: true
+          - "security.protocol=SASL_SSL"
+          - "sasl.mechanism=AWS_MSK_IAM"
+          - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
+          - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
       podTemplate:
         serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
+      tls:
+        autoGeneratedCerts: true
       dependencies:
         kafka:
           bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}