Skip to content

Commit

Permalink
updated configOverrides for hybrid-aws-msk sasl/iam example
Browse files Browse the repository at this point in the history
  • Loading branch information
aidanmelen committed Nov 13, 2022
1 parent 4d45979 commit 7a9ca2c
Show file tree
Hide file tree
Showing 12 changed files with 110 additions and 51 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
NAME := confluent
HOSTNAME := aidanmelen
PROVIDER := kubernetes
VERSION := 0.9.3
VERSION := 0.9.4
SHELL := /bin/bash

.PHONY: help all
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ Run Terratest using the [Makefile](https://github.com/aidanmelen/terraform-aws-s
### Results

```
Terratest Suite (Module v0.9.3) (Terraform v1.2.6)
Terratest Suite (Module v0.9.4) (Terraform v1.2.6)
--- PASS: TestTerraformCompleteExample (175.48s)
--- PASS: TestTerraformConfluentOperatorExample (22.24s)
--- PASS: TestTerraformConfluentPlatformExample (236.10s)
Expand Down
2 changes: 1 addition & 1 deletion examples/complete/.main.tf.docs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module "confluent_platform" {
source = "aidanmelen/confluent/kubernetes"
version = ">= 0.9.3"
version = ">= 0.9.4"

namespace = var.namespace

Expand Down
2 changes: 1 addition & 1 deletion examples/complete/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ kubectl apply -f ./crds/2.4.0
```hcl
module "confluent_platform" {
source = "aidanmelen/confluent/kubernetes"
version = ">= 0.9.3"
version = ">= 0.9.4"
namespace = var.namespace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,31 +34,38 @@ module "confluent_platform" {
create_zookeeper = false
create_kafka = false
# Confluent Platform components working with AWS MSK with SASL/IAM
# Confluent Platform components working with AWS MSK and SASL/IAM
create_connect = true
create_schemaregistry = true
create_ksqldb = true
create_kafkarestproxy = true
# NotImplemented
create_controlcenter = false
create_controlcenter = var.create_controlcenter
# Configuring a Kafka client to use AWS IAM
# https://github.com/aws/aws-msk-iam-auth#configuring-a-kafka-client-to-use-aws-iam
connect = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "admin.security.protocol=SASL_SSL"
- "producer.security.protocol=SASL_SSL"
- "consumer.security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "admin.security.protocol=SASL_SSL"
- "admin.sasl.mechanism=AWS_MSK_IAM"
- "admin.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "admin.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "producer.security.protocol=SASL_SSL"
- "producer.sasl.mechanism=AWS_MSK_IAM"
- "producer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "producer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "consumer.security.protocol=SASL_SSL"
- "consumer.sasl.mechanism=AWS_MSK_IAM"
- "consumer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "consumer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
tls:
Expand All @@ -79,18 +86,23 @@ module "confluent_platform" {
ksqldb = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "ksql.streams.security.protocol=SASL_SSL"
- "ksql.streams.sasl.mechanism=AWS_MSK_IAM"
- "ksql.streams.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "ksql.streams.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
# https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud
# tls:
# autoGeneratedCerts: true
tls:
# https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud
autoGeneratedCerts: false
secretRef: ${kubernetes_secret_v1.ca_pair_sslcerts.metadata[0].name}
dependencies:
kafka:
bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
Expand All @@ -104,17 +116,23 @@ module "confluent_platform" {
EOF
)
# ~~~ NOT WORKING ~~~
# MSK does not support the Confluent Metrics Reporter; therefore, Grafana dashboards are a better alternative.
# https://grafana.com/grafana/dashboards/11773-confluent-open-source/
# https://grafana.com/grafana/dashboards/16082-msk-overview/
controlcenter = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "confluent.controlcenter.streams.security.protocol=SASL_SSL"
- "confluent.controlcenter.streams.sasl.mechanism=AWS_MSK_IAM"
- "confluent.controlcenter.streams.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "confluent.controlcenter.streams.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
tls:
Expand All @@ -131,7 +149,7 @@ module "confluent_platform" {
enabled: true
ksqldb:
- name: ksql-dev
url: http://ksqldb.${var.namespace}.svc.cluster.local:8088
url: https://ksqldb.${var.namespace}.svc.cluster.local:8088
tls:
enabled: true
connect:
Expand All @@ -145,15 +163,18 @@ module "confluent_platform" {
schemaregistry = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "kafkastore.security.protocol=SASL_SSL"
- "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}"
- "kafkastore.sasl.mechanism=AWS_MSK_IAM"
- "kafkastore.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "kafkastore.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
tls:
Expand All @@ -170,13 +191,17 @@ module "confluent_platform" {
kafkarestproxy = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "client.security.protocol=SASL_SSL"
- "client.sasl.mechanism=AWS_MSK_IAM"
- "client.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "client.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
tls:
Expand Down Expand Up @@ -215,6 +240,7 @@ module "confluent_platform" {
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | The AWS region name. | `string` | `"us-west-2"` | no |
| <a name="input_create_controlcenter"></a> [create\_controlcenter](#input\_create\_controlcenter) | Controls if the ControlCenter component of the Confluent Platform should be created. | `bool` | `true` | no |
| <a name="input_name"></a> [name](#input\_name) | The project name. | `string` | `"hybrid-aws-msk"` | no |
| <a name="input_namespace"></a> [namespace](#input\_namespace) | The namespace to release the Confluent Platform into. | `string` | `"confluent"` | no |
## Outputs
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
# https://github.com/aws/aws-msk-iam-auth
ARG TAG=7.2.0
FROM confluentinc/cp-server-connect:${TAG}
USER root
RUN curl -sSL -o /usr/share/java/kafka/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
USER appuser

# https://docs.confluent.io/platform/current/installation/docker/development.html#extend-cp-images
# RUN confluent-hub install --no-prompt hpgrahsl/kafka-connect-mongodb:1.1.0 \
# && confluent-hub install --no-prompt microsoft/kafka-connect-iothub:0.6 \
# && confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.1.0

# this must be run after confluent-hub cmds
ENV CLASSPATH=/usr/share/java/aws-msk-iam-auth-1.1.4-all.jar
RUN curl -sSL -o /usr/share/java/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# https://github.com/aws/aws-msk-iam-auth
ARG TAG=7.2.0
FROM confluentinc/cp-enterprise-control-center:${TAG}
ENV CLASSPATH=/usr/share/java/aws-msk-iam-auth-1.1.4-all.jar
RUN curl -sSL -o /usr/share/java/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
USER root
RUN curl -sSL -o /usr/share/java/confluent-control-center/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
USER appuser
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# https://github.com/aws/aws-msk-iam-auth
ARG TAG=7.2.0
FROM confluentinc/cp-kafka-rest:${TAG}
ENV CLASSPATH=/usr/share/java/aws-msk-iam-auth-1.1.4-all.jar
RUN curl -sSL -o /usr/share/java/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
USER root
RUN curl -sSL -o /usr/share/java/kafka-rest-lib/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
USER appuser
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# https://github.com/aws/aws-msk-iam-auth
ARG TAG=7.2.0
FROM confluentinc/cp-schema-registry:${TAG}
ENV CLASSPATH=/usr/share/java/aws-msk-iam-auth-1.1.4-all.jar
RUN curl -sSL -o /usr/share/java/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
USER root
RUN curl -sSL -o /usr/share/java/schema-registry/aws-msk-iam-auth-1.1.4-all.jar "https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.4/aws-msk-iam-auth-1.1.4-all.jar"
USER appuser
61 changes: 43 additions & 18 deletions examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -21,31 +21,38 @@ module "confluent_platform" {
create_zookeeper = false
create_kafka = false

# Confluent Platform components working with AWS MSK with SASL/IAM
# Confluent Platform components working with AWS MSK and SASL/IAM
create_connect = true
create_schemaregistry = true
create_ksqldb = true
create_kafkarestproxy = true

# NotImplemented
create_controlcenter = false
create_controlcenter = var.create_controlcenter

# Configuring a Kafka client to use AWS IAM
# https://github.com/aws/aws-msk-iam-auth#configuring-a-kafka-client-to-use-aws-iam

connect = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-server-connect-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "admin.security.protocol=SASL_SSL"
- "producer.security.protocol=SASL_SSL"
- "consumer.security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "admin.security.protocol=SASL_SSL"
- "admin.sasl.mechanism=AWS_MSK_IAM"
- "admin.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "admin.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "producer.security.protocol=SASL_SSL"
- "producer.sasl.mechanism=AWS_MSK_IAM"
- "producer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "producer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "consumer.security.protocol=SASL_SSL"
- "consumer.sasl.mechanism=AWS_MSK_IAM"
- "consumer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "consumer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
tls:
Expand All @@ -66,18 +73,23 @@ module "confluent_platform" {
ksqldb = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-ksqldb-server-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "ksql.streams.security.protocol=SASL_SSL"
- "ksql.streams.sasl.mechanism=AWS_MSK_IAM"
- "ksql.streams.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "ksql.streams.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
# https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud
# tls:
# autoGeneratedCerts: true
tls:
# https://docs.confluent.io/operator/current/co-troubleshooting.html#issue-ksqldb-cannot-use-auto-generated-certificates-for-ccloud
autoGeneratedCerts: false
secretRef: ${kubernetes_secret_v1.ca_pair_sslcerts.metadata[0].name}
dependencies:
kafka:
bootstrapEndpoint: ${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}
Expand All @@ -91,17 +103,23 @@ module "confluent_platform" {
EOF
)

# ~~~ NOT WORKING ~~~
# MSK does not support the Confluent Metrics Reporter; therefore, Grafana dashboards are a better alternative.
# https://grafana.com/grafana/dashboards/11773-confluent-open-source/
# https://grafana.com/grafana/dashboards/16082-msk-overview/
controlcenter = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-enterprise-control-center-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "confluent.controlcenter.streams.security.protocol=SASL_SSL"
- "confluent.controlcenter.streams.sasl.mechanism=AWS_MSK_IAM"
- "confluent.controlcenter.streams.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "confluent.controlcenter.streams.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
tls:
Expand All @@ -118,7 +136,7 @@ module "confluent_platform" {
enabled: true
ksqldb:
- name: ksql-dev
url: http://ksqldb.${var.namespace}.svc.cluster.local:8088
url: https://ksqldb.${var.namespace}.svc.cluster.local:8088
tls:
enabled: true
connect:
Expand All @@ -132,15 +150,18 @@ module "confluent_platform" {
schemaregistry = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-schema-registry-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "kafkastore.security.protocol=SASL_SSL"
- "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}"
- "kafkastore.sasl.mechanism=AWS_MSK_IAM"
- "kafkastore.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "kafkastore.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "kafkastore.bootstrap.servers=${data.aws_msk_cluster.msk.bootstrap_brokers_sasl_iam}"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
tls:
Expand All @@ -157,13 +178,17 @@ module "confluent_platform" {
kafkarestproxy = yamldecode(<<-EOF
spec:
image:
application: aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.0
application: aidanmelen/cp-kafka-rest-with-aws-msk-iam-auth:7.2.2
configOverrides:
server:
- "security.protocol=SASL_SSL"
- "sasl.mechanism=AWS_MSK_IAM"
- "sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
- "client.security.protocol=SASL_SSL"
- "client.sasl.mechanism=AWS_MSK_IAM"
- "client.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;"
- "client.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler"
podTemplate:
serviceAccountName: ${kubernetes_service_account_v1.confluent_platform.metadata[0].name}
tls:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,9 @@ variable "namespace" {
type = string
default = "confluent"
}

variable "create_controlcenter" {
description = "Controls if the ControlCenter component of the Confluent Platform should be created."
type = bool
default = true
}
Loading

0 comments on commit 7a9ca2c

Please sign in to comment.