From 080a357a9bcce012cf625f0c23521a7717bebbb2 Mon Sep 17 00:00:00 2001 From: aidanmelen Date: Wed, 19 Oct 2022 02:36:01 +0000 Subject: [PATCH] fixed missing spec.image.application in hybrid_aws_msk example --- .make.docs | 1 + Makefile | 8 +++- README.md | 1 + examples/hybrid_aws_msk/aws/data.tf | 2 - examples/hybrid_aws_msk/aws/iam.tf | 4 +- .../README.md | 46 +++++++++++++------ .../images/Dockerfile.connect | 7 +++ .../images/Makefile | 31 +++++++------ .../main.tf | 46 +++++++++++++------ 9 files changed, 97 insertions(+), 49 deletions(-) diff --git a/.make.docs b/.make.docs index 7eea69b..604deef 100644 --- a/.make.docs +++ b/.make.docs @@ -12,4 +12,5 @@ test-complete Test the complete example test-kafka-topic Test the kafka_topic example test-schema Test the schema example test-connector Test the connector example +release Tag remote triggering Terraform Registry release clean Clean project diff --git a/Makefile b/Makefile index 3584db2..25f5827 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -NAME = terraform-kubernetes-confluent - +NAME := terraform-kubernetes-confluent +VERSION := 0.9.3 SHELL := /bin/bash .PHONY: help all @@ -102,6 +102,10 @@ delete-cfk-crds: kubectl config set-cluster docker-desktop kubectl delete -f ./crds/2.4.0 +release: ## Tag remote triggering Terraform Registry release + git tag v${VERSION} + git push --tag + clean: delete-cfk-crds ## Clean project @rm -f .terraform.lock.hcl @rm -f modules/confluent_operator/.terraform.lock.hcl diff --git a/README.md b/README.md index f27e17d..462b9d6 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ test-complete Test the complete example test-kafka-topic Test the kafka_topic example test-schema Test the schema example test-connector Test the connector example +release Tag remote triggering Terraform Registry release clean Clean project ``` diff --git a/examples/hybrid_aws_msk/aws/data.tf b/examples/hybrid_aws_msk/aws/data.tf index 5ffceb2..6c13b37 100644 --- a/examples/hybrid_aws_msk/aws/data.tf +++ b/examples/hybrid_aws_msk/aws/data.tf @@ -1,5 +1,3 @@ -data "aws_caller_identity" "current" {} - data "aws_eks_cluster_auth" "eks" { name = var.name } diff --git a/examples/hybrid_aws_msk/aws/iam.tf b/examples/hybrid_aws_msk/aws/iam.tf index 5ac1d4e..b08dbe3 100644 --- a/examples/hybrid_aws_msk/aws/iam.tf +++ b/examples/hybrid_aws_msk/aws/iam.tf @@ -28,7 +28,7 @@ resource "aws_iam_policy" "aws_msk_cluster_full_access" { "kafka-cluster:ReadData" ], "Resource": [ - "arn:aws:kafka:${var.aws_region}:${data.aws_caller_identity.current.account_id}:topic/${var.name}/*" + "${replace(module.msk_cluster.arn, ":cluster/", ":topic/")}/*" ] }, { @@ -38,7 +38,7 @@ resource "aws_iam_policy" "aws_msk_cluster_full_access" { "kafka-cluster:DescribeGroup" ], "Resource": [ - "arn:aws:kafka:${var.aws_region}:${data.aws_caller_identity.current.account_id}:group/${var.name}/*" + "${replace(module.msk_cluster.arn, ":cluster/", ":group/")}/*" ] } ] diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md index 7295594..488fd42 100644 --- a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md +++ b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md @@ -50,16 +50,19 @@ module "confluent_platform" { spec: image: application: aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0 - tls: - autoGeneratedCerts: true configOverrides: server: - "security.protocol=SASL_SSL" + - "admin.security.protocol=SASL_SSL" + - "producer.security.protocol=SASL_SSL" + - "consumer.security.protocol=SASL_SSL" - "sasl.mechanism=AWS_MSK_IAM" - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + tls: + autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} @@ -75,9 +78,8 @@ module "confluent_platform" { ksqldb = yamldecode(<<-EOF spec: - # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud - # tls: - # autoGeneratedCerts: true + image: + application: aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0 configOverrides: server: - "security.protocol=SASL_SSL" @@ -86,6 +88,9 @@ module "confluent_platform" { - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud + # tls: + # autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} @@ -99,15 +104,21 @@ module "confluent_platform" { EOF ) + # ~~~ NOT WORKING ~~~ controlcenter = yamldecode(<<-EOF spec: - tls: - autoGeneratedCerts: true + image: + application: aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0 configOverrides: server: - - "security.protocol=SSL" + - "security.protocol=SASL_SSL" + - "sasl.mechanism=AWS_MSK_IAM" + - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" + - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + tls: + autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} @@ -133,18 +144,20 @@ module "confluent_platform" { schemaregistry = yamldecode(<<-EOF spec: + image: + application: aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0 configOverrides: server: - "security.protocol=SASL_SSL" - "kafkastore.security.protocol=SASL_SSL" + - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}" - "kafkastore.sasl.mechanism=AWS_MSK_IAM" - "kafkastore.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" - "kafkastore.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" - - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}" - tls: - autoGeneratedCerts: true podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + tls: + autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} @@ -156,13 +169,18 @@ module "confluent_platform" { kafkarestproxy = yamldecode(<<-EOF spec: + image: + application: aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0 configOverrides: server: - - "security.protocol=SSL" - tls: - autoGeneratedCerts: true + - "security.protocol=SASL_SSL" + - "sasl.mechanism=AWS_MSK_IAM" + - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" + - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + tls: + autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.connect b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.connect index e88d355..964d49d 100644 --- a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.connect +++ b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.connect @@ -1,5 +1,12 @@ # https://github.com/aws/aws-msk-iam-auth ARG TAG=7.2.0 FROM confluentinc/cp-server-connect:${TAG} + +# https://docs.confluent.io/platform/current/installation/docker/development.html#extend-cp-images +# RUN confluent-hub install --no-prompt hpgrahsl/kafka-connect-mongodb:1.1.0 \ +# && confluent-hub install --no-prompt microsoft/kafka-connect-iothub:0.6 \ +# && confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.1.0 + +# this must be run after confluent-hub cmds ENV CLASSPATH=/usr/share/java/aws-msk-iam-auth-1.1.4-all.jar RUN curl -sSL -o /usr/share/java/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar" diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Makefile b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Makefile index 49fcadc..b8e0f75 100644 --- a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Makefile +++ b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Makefile @@ -1,4 +1,5 @@ SHELL := /bin/bash +TAG := 7.2.2 .PHONY: help all @@ -8,21 +9,21 @@ help: ## This help. .DEFAULT_GOAL := help build: ## Build docker images - docker build -f Dockerfile.connect . -t cp-server-connect-with-aws-msk-iam-auth - docker build -f Dockerfile.ksqldb . -t cp-ksqldb-server-with-aws-msk-iam-auth - docker build -f Dockerfile.controlcenter . -t cp-enterprise-control-center-with-aws-msk-iam-auth - docker build -f Dockerfile.schemaregistry . -t cp-schema-registry-with-aws-msk-iam-auth - docker build -f Dockerfile.kafkarestproxy . -t cp-kafka-rest-with-aws-msk-iam-auth + docker build -f Dockerfile.connect . -t cp-server-connect-with-aws-msk-iam-auth --build-arg TAG=$(TAG) + docker build -f Dockerfile.ksqldb . -t cp-ksqldb-server-with-aws-msk-iam-auth --build-arg TAG=$(TAG) + docker build -f Dockerfile.controlcenter . -t cp-enterprise-control-center-with-aws-msk-iam-auth --build-arg TAG=$(TAG) + docker build -f Dockerfile.schemaregistry . -t cp-schema-registry-with-aws-msk-iam-auth --build-arg TAG=$(TAG) + docker build -f Dockerfile.kafkarestproxy . -t cp-kafka-rest-with-aws-msk-iam-auth --build-arg TAG=$(TAG) - docker tag cp-server-connect-with-aws-msk-iam-auth aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0 - docker tag cp-ksqldb-server-with-aws-msk-iam-auth aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0 - docker tag cp-enterprise-control-center-with-aws-msk-iam-auth aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0 - docker tag cp-schema-registry-with-aws-msk-iam-auth aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0 - docker tag cp-kafka-rest-with-aws-msk-iam-auth aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0 + docker tag cp-server-connect-with-aws-msk-iam-auth aidanmelen/cp-server-connect-with-aws-msk-iam-auth:$(TAG) + docker tag cp-ksqldb-server-with-aws-msk-iam-auth aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:$(TAG) + docker tag cp-enterprise-control-center-with-aws-msk-iam-auth aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:$(TAG) + docker tag cp-schema-registry-with-aws-msk-iam-auth aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:$(TAG) + docker tag cp-kafka-rest-with-aws-msk-iam-auth aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:$(TAG) push: build ## Push docker images - docker push aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0 - docker push aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0 - docker push aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0 - docker push aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0 - docker push aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0 + docker push aidanmelen/cp-server-connect-with-aws-msk-iam-auth:$(TAG) + docker push aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:$(TAG) + docker push aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:$(TAG) + docker push aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:$(TAG) + docker push aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:$(TAG) diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf index 6f51eff..06536b5 100644 --- a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf +++ b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf @@ -37,16 +37,19 @@ module "confluent_platform" { spec: image: application: aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0 - tls: - autoGeneratedCerts: true configOverrides: server: - "security.protocol=SASL_SSL" + - "admin.security.protocol=SASL_SSL" + - "producer.security.protocol=SASL_SSL" + - "consumer.security.protocol=SASL_SSL" - "sasl.mechanism=AWS_MSK_IAM" - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + tls: + autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} @@ -62,9 +65,8 @@ module "confluent_platform" { ksqldb = yamldecode(<<-EOF spec: - # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud - # tls: - # autoGeneratedCerts: true + image: + application: aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0 configOverrides: server: - "security.protocol=SASL_SSL" @@ -73,6 +75,9 @@ module "confluent_platform" { - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud + # tls: + # autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} @@ -86,15 +91,21 @@ module "confluent_platform" { EOF ) + # ~~~ NOT WORKING ~~~ controlcenter = yamldecode(<<-EOF spec: - tls: - autoGeneratedCerts: true + image: + application: aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0 configOverrides: server: - - "security.protocol=SSL" + - "security.protocol=SASL_SSL" + - "sasl.mechanism=AWS_MSK_IAM" + - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" + - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + tls: + autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} @@ -120,18 +131,20 @@ module "confluent_platform" { schemaregistry = yamldecode(<<-EOF spec: + image: + application: aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0 configOverrides: server: - "security.protocol=SASL_SSL" - "kafkastore.security.protocol=SASL_SSL" + - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}" - "kafkastore.sasl.mechanism=AWS_MSK_IAM" - "kafkastore.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" - "kafkastore.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" - - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}" - tls: - autoGeneratedCerts: true podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + tls: + autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} @@ -143,13 +156,18 @@ module "confluent_platform" { kafkarestproxy = yamldecode(<<-EOF spec: + image: + application: aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0 configOverrides: server: - - "security.protocol=SSL" - tls: - autoGeneratedCerts: true + - "security.protocol=SASL_SSL" + - "sasl.mechanism=AWS_MSK_IAM" + - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" + - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" podTemplate: serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name} + tls: + autoGeneratedCerts: true dependencies: kafka: bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}