Merge pull request #9 from aidanmelen/hybrid-aws-msk-secure

added hybrid-aws-msk examples for tls and iam
aidanmelen · Aug 14, 2022 · 38474d6 · 38474d6
2 parents 7b7bf1e + 9a4463f
commit 38474d6
Show file tree

Hide file tree

Showing 51 changed files with 7,199 additions and 8,883 deletions.
diff --git a/.terraform-version b/.terraform-version
@@ -0,0 +1 @@
+1.2.6
diff --git a/Makefile b/Makefile
@@ -38,6 +38,8 @@ setup: apply-cfk-crds ## Setup project
 	cd examples/connector && terraform init
 	cd examples/hybrid_aws_msk/aws && terraform init
 	cd examples/hybrid_aws_msk/confluent_platform && terraform init
+	cd examples/hybrid_aws_msk/confluent_platform_tls_only && terraform init
+	cd examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure && terraform init
 
 	# pre-commit
 	git init
@@ -75,27 +77,26 @@ test-clean:
 	cd examples/confluent_operator && terraform destroy --auto-approve
 
 _test-confluent-platform:
-	# This test can take longer because controlcenter takes a while to become healthy
-	go test test/terraform_confluent_platform_test.go -timeout 10m -v |& tee test/terraform_confluent_platform_test.log
+	go test test/terraform_confluent_platform_test.go -timeout 30m -v |& tee test/terraform_confluent_platform_test.log
 
 _test-confluent-platform-singlenode:
-	go test test/terraform_confluent_platform_singlenode_test.go -timeout 10m -v |& tee test/terraform_confluent_platform_singlenode_test.log
+	go test test/terraform_confluent_platform_singlenode_test.go -timeout 30m -v |& tee test/terraform_confluent_platform_singlenode_test.log
 
 test-confluent-platform: test-setup _test-confluent-platform test-clean ## Test the confluent_platform example
 
 test-confluent-platform-singlenode: test-setup _test-confluent-platform-singlenode test-clean ## Test the confluent_platform_singlenode example
 
 test-complete: ## Test the complete example
-	go test test/terraform_complete_test.go -timeout 10m -v |& tee test/terraform_complete_test.log
+	go test test/terraform_complete_test.go -timeout 30m -v |& tee test/terraform_complete_test.log
 
 test-kafka-topic: ## Test the kafka_topic example
-	go test test/terraform_kafka_topic_test.go -timeout 10m -v |& tee test/terraform_kafka_topic_test.log
+	go test test/terraform_kafka_topic_test.go -timeout 30m -v |& tee test/terraform_kafka_topic_test.log
 
 test-schema: ## Test the schema example
-	go test test/terraform_schema_test.go -timeout 10m -v |& tee test/terraform_schema_test.log
+	go test test/terraform_schema_test.go -timeout 30m -v |& tee test/terraform_schema_test.log
 
 test-connector: ## Test the connector example
-	go test test/terraform_connector_test.go -timeout 10m -v |& tee test/terraform_connector_test.log
+	go test test/terraform_connector_test.go -timeout 30m -v |& tee test/terraform_connector_test.log
 
 delete-cfk-crds:
 	kubectl config set-cluster docker-desktop
@@ -119,6 +120,8 @@ clean: delete-cfk-crds ## Clean project
 	@rm -f examples/connector/.terraform.lock.hcl
 	@rm -f examples/hybrid_aws_msk/aws.terraform.lock.hcl
 	@rm -f examples/hybrid_aws_msk/confluent_platform.terraform.lock.hcl
+	@rm -f examples/hybrid_aws_msk/confluent_platform_tls_only.terraform.lock.hcl
+	@rm -f examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure.terraform.lock.hcl
 
 	@rm -rf .terraform
 	@rm -rf modules/confluent_operator/.terraform
@@ -137,6 +140,8 @@ clean: delete-cfk-crds ## Clean project
 	@rm -rf examples/connector/.terraform
 	@rm -rf examples/hybrid_aws_msk/aws.terraform
 	@rm -rf examples/hybrid_aws_msk/confluent_platform.terraform
+	@rm -rf examples/hybrid_aws_msk/confluent_platform_tls_only.terraform
+	@rm -rf examples/hybrid_aws_msk/confluent_platform_sasl_iam_secure.terraform
 
 	@rm -f go.mod
 	@rm -f go.sum

diff --git a/examples/autogenerated_tls_only/README.md b/examples/autogenerated_tls_only/README.md
@@ -6,6 +6,8 @@ Deploy the Confluent Platform with with full TLS network encryption. This Terraf
 
 This example assumes you have a Kubernetes cluster running locally on Docker Desktop. Please see [Docker's official documentation](https://docs.docker.com/desktop/kubernetes/) for more information.
 
+Keep in mind that you computer may not be able to handle the compute overhead added when enabling TLS. You may need to run this example on a Kubernetes server with more CPU and memory.
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Example

diff --git a/examples/complete/.main.tf.docs b/examples/complete/.main.tf.docs
@@ -47,7 +47,15 @@ module "confluent_platform" {
 
   connectors = {
     "pageviews-source" = {
-      "values" = yamldecode(file("${path.module}/values/connector.yaml"))
+      "values" = yamldecode(
+        templatefile(
+          "${path.module}/values/connector.yaml",
+          {
+            "datagen_source_connector_max_interval" : var.datagen_source_connector_max_interval,
+            "datagen_source_connector_iterations" : var.datagen_source_connector_iterations
+          }
+        )
+      )
     }
   }
 }
diff --git a/examples/complete/README.md b/examples/complete/README.md
@@ -69,7 +69,15 @@ module "confluent_platform" {
 
   connectors = {
     "pageviews-source" = {
-      "values" = yamldecode(file("${path.module}/values/connector.yaml"))
+      "values" = yamldecode(
+        templatefile(
+          "${path.module}/values/connector.yaml",
+          {
+            "datagen_source_connector_max_interval" : var.datagen_source_connector_max_interval,
+            "datagen_source_connector_iterations" : var.datagen_source_connector_iterations
+          }
+        )
+      )
     }
   }
 }
@@ -92,6 +100,8 @@ module "confluent_platform" {
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_create_controlcenter"></a> [create\_controlcenter](#input\_create\_controlcenter) | Controls if the ControlCenter component of the Confluent Platform should be created. | `bool` | `true` | no |
+| <a name="input_datagen_source_connector_iterations"></a> [datagen\_source\_connector\_iterations](#input\_datagen\_source\_connector\_iterations) | Number of messages to send from each task, or -1 for unlimited | `number` | `-1` | no |
+| <a name="input_datagen_source_connector_max_interval"></a> [datagen\_source\_connector\_max\_interval](#input\_datagen\_source\_connector\_max\_interval) | Max interval between messages (ms) | `number` | `500` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | The namespace to release the Confluent Operator and Confluent Platform into. | `string` | `"confluent"` | no |
 ## Outputs
 

diff --git a/examples/complete/main.tf b/examples/complete/main.tf
@@ -46,7 +46,15 @@ module "confluent_platform" {
 
   connectors = {
     "pageviews-source" = {
-      "values" = yamldecode(file("${path.module}/values/connector.yaml"))
+      "values" = yamldecode(
+        templatefile(
+          "${path.module}/values/connector.yaml",
+          {
+            "datagen_source_connector_max_interval" : var.datagen_source_connector_max_interval,
+            "datagen_source_connector_iterations" : var.datagen_source_connector_iterations
+          }
+        )
+      )
     }
   }
 }
diff --git a/examples/complete/values/connector.yaml b/examples/complete/values/connector.yaml
@@ -8,5 +8,5 @@ spec:
     value.converter: "io.confluent.connect.avro.AvroConverter"
     value.converter.schemas.enable: "true"
     value.converter.schema.registry.url: "http://schemaregistry.confluent.svc.cluster.local:8081"
-    max.interval: "100"
-    iterations: "10000000"
+    max.interval: "${datagen_source_connector_max_interval}"
+    iterations: "${datagen_source_connector_iterations}"
diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf
@@ -9,3 +9,15 @@ variable "create_controlcenter" {
   type        = bool
   default     = true
 }
+
+variable "datagen_source_connector_max_interval" {
+  description = "Max interval between messages (ms)"
+  type        = number
+  default     = 500
+}
+
+variable "datagen_source_connector_iterations" {
+  description = "Number of messages to send from each task, or -1 for unlimited"
+  type        = number
+  default     = -1
+}
diff --git a/examples/hybrid_aws_msk/README.md b/examples/hybrid_aws_msk/README.md
@@ -0,0 +1,19 @@
+# hybrid_aws_msk
+
+This example is divided into to four parts
+
+## aws_msk
+
+First, create a new VPC and deploy an MSK and EKS cluster into it. Then try one of the confluent platform deployment.
+
+## confluent_platform
+
+Deploy the Confluent Platform components connected with an AWS MSK cluster over PLAINTEXT. The Confluent Components are also configured with PLAINTEXT.
+
+## confluent_platform_tls_only
+
+Deploy the Confluent Platform components connected with an AWS MSK cluster over TLS. The Confluent Components are also configured with TSL.
+
+## confluent_platform_iam_secure
+
+Deploy the Confluent Platform components connected with an AWS MSK cluster over TLS. Authenticate and Authorize with IAM. Please see [aws-msk-iam-auth](https://github.com/aws/aws-msk-iam-auth) for more information.
diff --git a/examples/hybrid_aws_msk/aws/README.md b/examples/hybrid_aws_msk/aws/README.md
@@ -1,13 +1,25 @@
 # aws_msk
 
-Deploy the AWS MSK cluster.
+Create a new VPC and deploy an MSK and EKS cluster into it.
+
+## Assumptions
+
+This example assumes that you have valid AWS credentials set for the default profile.
+
+## EKS Connect
+
+Add the EKS cluster context to the kube config file with the following command:
+
+```bash
+aws eks update-kubeconfig --name hybrid-aws-msk
+```
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Example
 
 ```hcl
-module "security_group" {
+module "msk_cluster_security_group" {
   source  = "terraform-aws-modules/security-group/aws"
   version = "~> 4.0"
 
@@ -17,11 +29,25 @@ module "security_group" {
 
   ingress_cidr_blocks = module.vpc.private_subnets_cidr_blocks
   ingress_rules       = ["kafka-broker-tcp", "kafka-broker-tls-tcp"]
+
+  # https://github.com/terraform-aws-modules/terraform-aws-security-group/pull/248
+  ingress_with_cidr_blocks = [
+    {
+      from_port   = 9098
+      to_port     = 9098
+      protocol    = "tcp"
+      description = "kafka-broker-sasl-iam-tcp"
+      cidr_blocks = join(",", module.vpc.private_subnets_cidr_blocks)
+    }
+  ]
 }
 
 module "msk_cluster" {
-  source  = "clowdhaus/msk-kafka-cluster/aws"
-  version = "1.2.0"
+  source = "github.com/aidanmelen/terraform-aws-msk-kafka-cluster?ref=v1.3.0"
+
+  # https://github.com/clowdhaus/terraform-aws-msk-kafka-cluster/pull/4
+  # source  = "clowdhaus/msk-kafka-cluster/aws"
+  # version = "1.3.0"
 
   name                   = var.name
   number_of_broker_nodes = 3
@@ -32,11 +58,15 @@ module "msk_cluster" {
   broker_node_client_subnets  = module.vpc.private_subnets
   broker_node_ebs_volume_size = 20
   broker_node_instance_type   = "kafka.t3.small"
-  broker_node_security_groups = [module.security_group.security_group_id]
+  broker_node_security_groups = [module.msk_cluster_security_group.security_group_id]
 
   encryption_in_transit_client_broker = "TLS_PLAINTEXT"
   encryption_in_transit_in_cluster    = true
 
+  client_unauthenticated_access_enabled = true
+  client_authentication_sasl_iam        = true
+  client_authentication_sasl_scram      = false
+
   cloudwatch_logs_enabled = true
 }
 ```
@@ -47,24 +77,30 @@ module "msk_cluster" {
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.14.8 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
-| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.0.0 |
+| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.0.0 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.12.1 |
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_confluent_operator"></a> [confluent\_operator](#module\_confluent\_operator) | ../../../modules/confluent_operator | n/a |
 | <a name="module_eks"></a> [eks](#module\_eks) | terraform-aws-modules/eks/aws | >= 18.0.0 |
-| <a name="module_msk_cluster"></a> [msk\_cluster](#module\_msk\_cluster) | clowdhaus/msk-kafka-cluster/aws | 1.2.0 |
-| <a name="module_security_group"></a> [security\_group](#module\_security\_group) | terraform-aws-modules/security-group/aws | ~> 4.0 |
+| <a name="module_iam_eks_confluent_platform_role"></a> [iam\_eks\_confluent\_platform\_role](#module\_iam\_eks\_confluent\_platform\_role) | terraform-aws-modules/iam/aws//modules/iam-eks-role | 5.3.0 |
+| <a name="module_msk_cluster"></a> [msk\_cluster](#module\_msk\_cluster) | github.com/aidanmelen/terraform-aws-msk-kafka-cluster | v1.3.0 |
+| <a name="module_msk_cluster_security_group"></a> [msk\_cluster\_security\_group](#module\_msk\_cluster\_security\_group) | terraform-aws-modules/security-group/aws | ~> 4.0 |
 | <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 |
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | The AWS region name. | `string` | `"us-west-2"` | no |
 | <a name="input_name"></a> [name](#input\_name) | The project name. | `string` | `"hybrid-aws-msk"` | no |
+| <a name="input_namespace"></a> [namespace](#input\_namespace) | The namespace to release the Confluent Operator into. | `string` | `"confluent"` | no |
 ## Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_bootstrap_brokers"></a> [bootstrap\_brokers](#output\_bootstrap\_brokers) | Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster |
+| <a name="output_bootstrap_brokers_sasl_iam"></a> [bootstrap\_brokers\_sasl\_iam](#output\_bootstrap\_brokers\_sasl\_iam) | One or more DNS names (or IP addresses) and SASL IAM port pairs |
+| <a name="output_bootstrap_brokers_tls"></a> [bootstrap\_brokers\_tls](#output\_bootstrap\_brokers\_tls) | One or more DNS names (or IP addresses) and TLS port pairs |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/hybrid_aws_msk/aws/confluent_operator.tf b/examples/hybrid_aws_msk/aws/confluent_operator.tf
@@ -0,0 +1,7 @@
+module "confluent_operator" {
+  source           = "../../../modules/confluent_operator"
+  create_namespace = true
+  namespace        = var.namespace
+  name             = "confluent-operator"
+  chart_version    = "0.517.12"
+}
diff --git a/examples/hybrid_aws_msk/aws/crds.tf → ...s/hybrid_aws_msk/aws/crds_only.tf.example b/examples/hybrid_aws_msk/aws/crds.tf → ...s/hybrid_aws_msk/aws/crds_only.tf.example
@@ -1,3 +1,15 @@
+provider "kubectl" {
+  host                   = data.aws_eks_cluster.eks.endpoint
+  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
+
+  exec {
+    api_version = "client.authentication.k8s.io/v1beta1"
+    command     = "aws"
+    # This requires the awscli to be installed locally where Terraform is executed
+    args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
+  }
+}
+
 resource "kubectl_manifest" "crds" {
   for_each = toset([
     "platform.confluent.io_clusterlinks.yaml",

diff --git a/examples/hybrid_aws_msk/aws/data.tf b/examples/hybrid_aws_msk/aws/data.tf
@@ -0,0 +1,5 @@
+data "aws_caller_identity" "current" {}
+
+data "aws_eks_cluster_auth" "eks" {
+  name = var.name
+}
diff --git a/examples/hybrid_aws_msk/aws/iam.tf b/examples/hybrid_aws_msk/aws/iam.tf
@@ -0,0 +1,62 @@
+# https://docs.aws.amazon.com/msk/latest/developerguide/security-iam-awsmanpol.html
+# https://docs.aws.amazon.com/msk/latest/developerguide/security_iam_id-based-policy-examples.html
+resource "aws_iam_policy" "aws_msk_cluster_full_access" {
+  name        = "msk-cluster-full-access"
+  path        = "/"
+  description = "MSK Cluster full access."
+
+  policy = <<-EOF
+  {
+      "Version": "2012-10-17",
+      "Statement": [
+          {
+              "Effect": "Allow",
+              "Action": [
+                  "kafka-cluster:Connect",
+                  "kafka-cluster:AlterCluster",
+                  "kafka-cluster:DescribeCluster"
+              ],
+              "Resource": [
+                  "${module.msk_cluster.arn}"
+              ]
+          },
+          {
+              "Effect": "Allow",
+              "Action": [
+                  "kafka-cluster:*Topic*",
+                  "kafka-cluster:WriteData",
+                  "kafka-cluster:ReadData"
+              ],
+              "Resource": [
+                  "arn:aws:kafka:${var.aws_region}:${data.aws_caller_identity.current.account_id}:topic/${var.name}/*"
+              ]
+          },
+          {
+              "Effect": "Allow",
+              "Action": [
+                  "kafka-cluster:AlterGroup",
+                  "kafka-cluster:DescribeGroup"
+              ],
+              "Resource": [
+                  "arn:aws:kafka:${var.aws_region}:${data.aws_caller_identity.current.account_id}:group/${var.name}/*"
+              ]
+          }
+      ]
+  }
+  EOF
+}
+
+module "iam_eks_confluent_platform_role" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-eks-role"
+  version = "5.3.0"
+
+  role_name = "confluent-platform"
+
+  cluster_service_accounts = {
+    confluent_platform = ["${module.confluent_operator.namespace}:confluent-platform"]
+  }
+
+  role_policy_arns = {
+    aws_msk_cluster_full_access = aws_iam_policy.aws_msk_cluster_full_access.arn
+  }
+}