diff --git a/examples/hybrid_aws_msk/aws/iam.tf b/examples/hybrid_aws_msk/aws/iam.tf index 62d5da5..5ac1d4e 100644 --- a/examples/hybrid_aws_msk/aws/iam.tf +++ b/examples/hybrid_aws_msk/aws/iam.tf @@ -47,13 +47,14 @@ resource "aws_iam_policy" "aws_msk_cluster_full_access" { } module "iam_eks_confluent_platform_role" { - source = "terraform-aws-modules/iam/aws//modules/iam-eks-role" - version = "5.3.0" + source = "terraform-aws-modules/iam/aws//modules/iam-eks-role" + version = "5.3.0" + depends_on = [module.eks] role_name = "confluent-platform" cluster_service_accounts = { - confluent_platform = ["${module.confluent_operator.namespace}:confluent-platform"] + (var.name) = ["${module.confluent_operator.namespace}:confluent-platform"] } role_policy_arns = { diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md index 62aa281..01c0404 100644 --- a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md +++ b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/README.md @@ -34,11 +34,14 @@ module "confluent_platform" { create_zookeeper = false create_kafka = false - # TODO implement aws msk iam auth for the following components - create_controlcenter = var.create_controlcenter - create_ksqldb = false - create_schemaregistry = false - create_kafkarestproxy = false + # Confluent Platform components working with AWS MSK with SASL/IAM + create_connect = true + create_schemaregistry = true + create_ksqldb = true + create_kafkarestproxy = true + + # NotImplemented + create_controlcenter = false # Configuring a Kafka client to use AWS IAM # https://github.com/aws/aws-msk-iam-auth#configuring-a-kafka-client-to-use-aws-iam @@ -63,91 +66,107 @@ module "confluent_platform" { tls: enabled: true ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true + EOF + ) + + ksqldb = yamldecode(<<-EOF + spec: + # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud + # tls: + # autoGeneratedCerts: true + configOverrides: + server: + - "security.protocol=SASL_SSL" + - "sasl.mechanism=AWS_MSK_IAM" + - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" + - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" + dependencies: + kafka: + bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} + tls: + enabled: true + ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true EOF ) - # ksqldb = yamldecode(<<-EOF - # spec: - # # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud - # # tls: - # # autoGeneratedCerts: true - # configOverrides: - # server: - # - "security.protocol=SSL" - # dependencies: - # kafka: - # bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} - # tls: - # enabled: true - # ignoreTrustStoreConfig: true - # EOF - # ) - - # controlcenter = yamldecode(<<-EOF - # spec: - # tls: - # autoGeneratedCerts: true - # configOverrides: - # server: - # - "security.protocol=SSL" - # dependencies: - # kafka: - # bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} - # tls: - # enabled: true - # ignoreTrustStoreConfig: true - # schemaRegistry: - # url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 - # tls: - # enabled: true - # ksqldb: - # - name: ksql-dev - # url: http://ksqldb.${var.namespace}.svc.cluster.local:8088 - # tls: - # enabled: true - # connect: - # - name: connect-dev - # url: https://connect.${var.namespace}.svc.cluster.local:8083 - # tls: - # enabled: true - # EOF - # ) - - # schemaregistry = yamldecode(<<-EOF - # spec: - # configOverrides: - # server: - # - "security.protocol=SSL" - # tls: - # autoGeneratedCerts: true - # dependencies: - # kafka: - # bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} - # tls: - # enabled: true - # ignoreTrustStoreConfig: true - # EOF - # ) - - # kafkarestproxy = yamldecode(<<-EOF - # spec: - # configOverrides: - # server: - # - "security.protocol=SSL" - # tls: - # autoGeneratedCerts: true - # dependencies: - # kafka: - # bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} - # tls: - # enabled: true - # ignoreTrustStoreConfig: true - # schemaRegistry: - # url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 - # tls: - # enabled: true - # EOF - # ) + controlcenter = yamldecode(<<-EOF + spec: + tls: + autoGeneratedCerts: true + configOverrides: + server: + - "security.protocol=SSL" + dependencies: + kafka: + bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} + tls: + enabled: true + ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true + ksqldb: + - name: ksql-dev + url: http://ksqldb.${var.namespace}.svc.cluster.local:8088 + tls: + enabled: true + connect: + - name: connect-dev + url: https://connect.${var.namespace}.svc.cluster.local:8083 + tls: + enabled: true + EOF + ) + + schemaregistry = yamldecode(<<-EOF + spec: + configOverrides: + server: + - "security.protocol=SASL_SSL" + - "kafkastore.security.protocol=SASL_SSL" + - "kafkastore.sasl.mechanism=AWS_MSK_IAM" + - "kafkastore.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" + - "kafkastore.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" + - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}" + tls: + autoGeneratedCerts: true + dependencies: + kafka: + bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} + tls: + enabled: true + ignoreTrustStoreConfig: true + EOF + ) + + kafkarestproxy = yamldecode(<<-EOF + spec: + configOverrides: + server: + - "security.protocol=SSL" + tls: + autoGeneratedCerts: true + dependencies: + kafka: + bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} + tls: + enabled: true + ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true + EOF + ) } ``` @@ -170,7 +189,6 @@ module "confluent_platform" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [aws\_region](#input\_aws\_region) | The AWS region name. | `string` | `"us-west-2"` | no | -| [create\_controlcenter](#input\_create\_controlcenter) | Controls if the ControlCenter component of the Confluent Platform should be created. | `bool` | `false` | no | | [name](#input\_name) | The project name. | `string` | `"hybrid-aws-msk"` | no | | [namespace](#input\_namespace) | The namespace to release the Confluent Platform into. | `string` | `"confluent"` | no | ## Outputs diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.ksqldb b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.ksqldb index 88739a5..bd7c861 100644 --- a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.ksqldb +++ b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/images/Dockerfile.ksqldb @@ -1,5 +1,4 @@ # https://github.com/aws/aws-msk-iam-auth ARG TAG=7.2.0 FROM confluentinc/cp-ksqldb-server:${TAG} -ENV CLASSPATH=/usr/share/java/aws-msk-iam-auth-1.1.4-all.jar -RUN curl -sSL -o /usr/share/java/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar" +RUN curl -sSL -o /usr/share/java/ksqldb-server/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar" diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf index 792b680..23b8a4c 100644 --- a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf +++ b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf @@ -21,11 +21,14 @@ module "confluent_platform" { create_zookeeper = false create_kafka = false - # TODO implement aws msk iam auth for the following components - create_controlcenter = var.create_controlcenter - create_ksqldb = false - create_schemaregistry = false - create_kafkarestproxy = false + # Confluent Platform components working with AWS MSK with SASL/IAM + create_connect = true + create_schemaregistry = true + create_ksqldb = true + create_kafkarestproxy = true + + # NotImplemented + create_controlcenter = false # Configuring a Kafka client to use AWS IAM # https://github.com/aws/aws-msk-iam-auth#configuring-a-kafka-client-to-use-aws-iam @@ -50,89 +53,105 @@ module "confluent_platform" { tls: enabled: true ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true EOF ) - # ksqldb = yamldecode(<<-EOF - # spec: - # # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud - # # tls: - # # autoGeneratedCerts: true - # configOverrides: - # server: - # - "security.protocol=SSL" - # dependencies: - # kafka: - # bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} - # tls: - # enabled: true - # ignoreTrustStoreConfig: true - # EOF - # ) + ksqldb = yamldecode(<<-EOF + spec: + # https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud + # tls: + # autoGeneratedCerts: true + configOverrides: + server: + - "security.protocol=SASL_SSL" + - "sasl.mechanism=AWS_MSK_IAM" + - "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" + - "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" + dependencies: + kafka: + bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} + tls: + enabled: true + ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true + EOF + ) - # controlcenter = yamldecode(<<-EOF - # spec: - # tls: - # autoGeneratedCerts: true - # configOverrides: - # server: - # - "security.protocol=SSL" - # dependencies: - # kafka: - # bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} - # tls: - # enabled: true - # ignoreTrustStoreConfig: true - # schemaRegistry: - # url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 - # tls: - # enabled: true - # ksqldb: - # - name: ksql-dev - # url: http://ksqldb.${var.namespace}.svc.cluster.local:8088 - # tls: - # enabled: true - # connect: - # - name: connect-dev - # url: https://connect.${var.namespace}.svc.cluster.local:8083 - # tls: - # enabled: true - # EOF - # ) + controlcenter = yamldecode(<<-EOF + spec: + tls: + autoGeneratedCerts: true + configOverrides: + server: + - "security.protocol=SSL" + dependencies: + kafka: + bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} + tls: + enabled: true + ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true + ksqldb: + - name: ksql-dev + url: http://ksqldb.${var.namespace}.svc.cluster.local:8088 + tls: + enabled: true + connect: + - name: connect-dev + url: https://connect.${var.namespace}.svc.cluster.local:8083 + tls: + enabled: true + EOF + ) - # schemaregistry = yamldecode(<<-EOF - # spec: - # configOverrides: - # server: - # - "security.protocol=SSL" - # tls: - # autoGeneratedCerts: true - # dependencies: - # kafka: - # bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} - # tls: - # enabled: true - # ignoreTrustStoreConfig: true - # EOF - # ) + schemaregistry = yamldecode(<<-EOF + spec: + configOverrides: + server: + - "security.protocol=SASL_SSL" + - "kafkastore.security.protocol=SASL_SSL" + - "kafkastore.sasl.mechanism=AWS_MSK_IAM" + - "kafkastore.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;" + - "kafkastore.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler" + - "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}" + tls: + autoGeneratedCerts: true + dependencies: + kafka: + bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} + tls: + enabled: true + ignoreTrustStoreConfig: true + EOF + ) - # kafkarestproxy = yamldecode(<<-EOF - # spec: - # configOverrides: - # server: - # - "security.protocol=SSL" - # tls: - # autoGeneratedCerts: true - # dependencies: - # kafka: - # bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} - # tls: - # enabled: true - # ignoreTrustStoreConfig: true - # schemaRegistry: - # url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 - # tls: - # enabled: true - # EOF - # ) + kafkarestproxy = yamldecode(<<-EOF + spec: + configOverrides: + server: + - "security.protocol=SSL" + tls: + autoGeneratedCerts: true + dependencies: + kafka: + bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam} + tls: + enabled: true + ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true + EOF + ) } diff --git a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/variables.tf b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/variables.tf index 0646db8..db50188 100644 --- a/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/variables.tf +++ b/examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/variables.tf @@ -15,9 +15,3 @@ variable "namespace" { type = string default = "confluent" } - -variable "create_controlcenter" { - description = "Controls if the ControlCenter component of the Confluent Platform should be created." - type = bool - default = false -} diff --git a/examples/hybrid_aws_msk/confluent_platform_tls_only/README.md b/examples/hybrid_aws_msk/confluent_platform_tls_only/README.md index 75f64d6..81e7679 100644 --- a/examples/hybrid_aws_msk/confluent_platform_tls_only/README.md +++ b/examples/hybrid_aws_msk/confluent_platform_tls_only/README.md @@ -41,6 +41,10 @@ module "confluent_platform" { tls: enabled: true ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true EOF ) @@ -58,6 +62,10 @@ module "confluent_platform" { tls: enabled: true ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true EOF ) diff --git a/examples/hybrid_aws_msk/confluent_platform_tls_only/main.tf b/examples/hybrid_aws_msk/confluent_platform_tls_only/main.tf index 09c0def..e755926 100644 --- a/examples/hybrid_aws_msk/confluent_platform_tls_only/main.tf +++ b/examples/hybrid_aws_msk/confluent_platform_tls_only/main.tf @@ -28,6 +28,10 @@ module "confluent_platform" { tls: enabled: true ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true EOF ) @@ -45,6 +49,10 @@ module "confluent_platform" { tls: enabled: true ignoreTrustStoreConfig: true + schemaRegistry: + url: https://schemaregistry.${var.namespace}.svc.cluster.local:8081 + tls: + enabled: true EOF )