Merge pull request #47 from cloudbees/autoscaling-review

Various issues addressed

README.md

@@ -55,6 +55,7 @@ CloudBees CI Add-on uses for its resources definition `helms release` which make
 | hostname | Route53 Hosted zone name | `string` | n/a | yes |
 | temp_license | Temporary license details | `map(string)` | n/a | yes |
 | helm_config | CloudBees CI Helm chart configuration | `any` | <pre>{<br>  "values": [<br>    ""<br>  ]<br>}</pre> | no |
+| secrets_file | Secrets file yml path containing the secrets names:values to create the Kubernetes secret cbci-secrets. It can be mounted for Casc | `string` | `"secrets-values.yml"` | no |
 
 ### Outputs
 

blueprints/01-getting-started/main.tf

@@ -21,7 +21,7 @@ locals {
   route53_zone_id  = data.aws_route53_zone.this.id
   route53_zone_arn = data.aws_route53_zone.this.arn
   #Number of AZs per region https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html
-  azs = slice(data.aws_availability_zones.available.names, 0, 1)
+  azs = slice(data.aws_availability_zones.available.names, 0, 2)
 
   tags = merge(var.tags, {
     "tf:blueprint"  = local.name

blueprints/02-at-scale/README.md

@@ -3,7 +3,7 @@
 Once you have familiarized yourself with the [Getting Started blueprint](../01-getting-started/README.md), this blueprint presents a more scalable architecture by adding the following **optional EKS Addons**:
 
 - [Cluster Autoscaler](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/cluster-autoscaler/)
-<!-- - [Node Termination Handler](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/aws-node-termination-handler/) -->
+- [Node Termination Handler](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/aws-node-termination-handler/)
 - [EFS CSI Driver](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/aws-efs-csi-driver/). It can be used by non-HA/HS (optional) and it is required by HA/HS Controllers.
 - [Metrics Server](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/aws-efs-csi-driver/). It is required by HA/HS Controllers for Horizontal Pod Autoscaling.
 

blueprints/02-at-scale/cbci-values.yml

@@ -22,17 +22,6 @@ OperationsCenter:
       scmBranch: main
       scmBundlePath: bp02
       scmPollingInterval: PT20M
-  ContainerEnv:
-    - name: SECRETS
-      value: /var/run/secrets/oc
-  ExtraVolumes:
-    - name: oc-secrets
-      secret:
-        secretName: oc-secrets
-  ExtraVolumeMounts:
-    - name: oc-secrets
-      mountPath: /var/run/secrets/oc
-      readOnly: true
 Persistence:
   StorageClass: efs
 Hibernation:

blueprints/02-at-scale/main.tf

@@ -17,8 +17,6 @@ locals {
   kubeconfig_file      = "kubeconfig_${local.name}.yaml"
   kubeconfig_file_path = abspath("${path.root}/${local.kubeconfig_file}")
 
-  cbci_namespace = "cbci"
-
   vpc_cidr = "10.0.0.0/16"
 
   #https://docs.cloudbees.com/docs/cloudbees-common/latest/supported-platforms/cloudbees-ci-cloud#_kubernetes
@@ -49,30 +47,6 @@ locals {
 
 # CloudBees CI Add-ons
 
-resource "kubernetes_namespace" "cbci" {
-
-  metadata {
-    name = local.cbci_namespace
-  }
-
-  depends_on = [
-    module.eks_blueprints_addons
-  ]
-
-}
-
-# Secrets to be passed to Casc
-# https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/docs/features/secrets.adoc#kubernetes-secrets
-resource "kubernetes_secret" "oc_secrets" {
-
-  metadata {
-    name      = "oc-secrets"
-    namespace = kubernetes_namespace.cbci.metadata[0].name
-  }
-
-  data = yamldecode(file("${path.module}/secrets-values.yml"))
-}
-
 module "eks_blueprints_addon_cbci" {
   source = "../../"
 
@@ -85,9 +59,8 @@ module "eks_blueprints_addon_cbci" {
     values           = [file("${path.module}/cbci-values.yml")]
   }
 
-  depends_on = [
-    kubernetes_secret.oc_secrets
-  ]
+  secrets_file = "${path.module}/secrets-values.yml"
+
 }
 
 # EKS Blueprints Add-ons
@@ -110,6 +83,14 @@ module "ebs_csi_driver_irsa" {
   tags = var.tags
 }
 
+data "aws_autoscaling_groups" "eks_node_groups" {
+  depends_on = [module.eks]
+  filter {
+    name   = "tag-key"
+    values = ["eks:cluster-name"]
+  }
+}
+
 module "eks_blueprints_addons" {
   source  = "aws-ia/eks-blueprints-addons/aws"
   version = "1.12.0"
@@ -145,10 +126,11 @@ module "eks_blueprints_addons" {
   external_dns_route53_zone_arns      = [local.route53_zone_arn]
   enable_aws_load_balancer_controller = true
   #02-at-scale
-  enable_aws_efs_csi_driver = true
-  enable_metrics_server     = true
-  enable_cluster_autoscaler = true
-  #enable_aws_node_termination_handler = true
+  enable_aws_efs_csi_driver             = true
+  enable_metrics_server                 = true
+  enable_cluster_autoscaler             = true
+  enable_aws_node_termination_handler   = true
+  aws_node_termination_handler_asg_arns = data.aws_autoscaling_groups.eks_node_groups.arns
 
   tags = local.tags
 }
@@ -303,7 +285,7 @@ resource "kubernetes_storage_class_v1" "gp3" {
   metadata {
     name = "gp3"
 
-    # IMPORTANT: Prometheus and Velero requires gp3 (Block Storage)
+    # IMPORTANT: Prometheus and Velero requires Block Storage
     annotations = {
       "storageclass.kubernetes.io/is-default-class" = "true"
     }
@@ -325,12 +307,10 @@ resource "kubernetes_storage_class_v1" "efs" {
 
   metadata {
     name = "efs"
-    annotations = {
-      "storageclass.kubernetes.io/is-default-class" = "false"
-    }
   }
 
   storage_provisioner = "efs.csi.aws.com"
+  reclaim_policy      = "Delete"
   parameters = {
     provisioningMode = "efs-ap" # Dynamic provisioning
     fileSystemId     = module.efs.id

main.tf

@@ -1,17 +1,66 @@
 # Copyright (c) CloudBees, Inc.
 
+locals {
+  secret_data   = fileexists(var.secrets_file) ? yamldecode(file(var.secrets_file)) : {}
+  create_secret = length(local.secret_data) > 0
+  oc_secrets_mount = [
+    <<-EOT
+      OperationsCenter:
+        ContainerEnv:
+          - name: SECRETS
+            value: /var/run/secrets/cbci
+        ExtraVolumes:
+          - name: cbci-secrets
+            secret:
+              secretName: cbci-secrets
+        ExtraVolumeMounts:
+          - name: cbci-secrets
+            mountPath: /var/run/secrets/cbci
+            readOnly: true
+      EOT
+  ]
+}
+
+resource "kubernetes_namespace" "cbci" {
+
+  metadata {
+    name = try(var.helm_config.namespace, "cbci")
+  }
+
+}
+
+# Secrets to be passed to Casc
+# https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/docs/features/secrets.adoc#kubernetes-secrets
+resource "kubernetes_secret" "oc_secrets" {
+  count = local.create_secret ? 1 : 0
+
+  metadata {
+    name      = "cbci-secrets"
+    namespace = kubernetes_namespace.cbci.metadata[0].name
+  }
+
+  data = yamldecode(file(var.secrets_file))
+}
+
 resource "helm_release" "cloudbees_ci" {
 
   name             = try(var.helm_config.name, "cloudbees-ci")
   namespace        = try(var.helm_config.namespace, "cbci")
-  create_namespace = try(var.helm_config.create_namespace, true)
+  create_namespace = false
   description      = try(var.helm_config.description, null)
   chart            = "cloudbees-core"
   #Chart versions: #https://artifacthub.io/packages/helm/cloudbees/cloudbees-core/
   #App version: https://docs.cloudbees.com/docs/release-notes/latest/cloudbees-ci/
   version    = try(var.helm_config.version, "3.15666.0+5ea03547ce92")
   repository = try(var.helm_config.repository, "https://public-charts.artifacts.cloudbees.com/repository/public/")
-  values = concat(var.helm_config.values, [templatefile("${path.module}/values.yml", {
+  values = local.create_secret ? concat(var.helm_config.values, local.oc_secrets_mount, [templatefile("${path.module}/values.yml", {
+    hostname     = var.hostname
+    cert_arn     = var.cert_arn
+    LicFirstName = var.temp_license["first_name"]
+    LicLastName  = var.temp_license["last_name"]
+    LicEmail     = var.temp_license["email"]
+    LicCompany   = var.temp_license["company"]
+    })]) : concat(var.helm_config.values, [templatefile("${path.module}/values.yml", {
     hostname     = var.hostname
     cert_arn     = var.cert_arn
     LicFirstName = var.temp_license["first_name"]
@@ -73,4 +122,8 @@ resource "helm_release" "cloudbees_ci" {
       type  = try(set_sensitive.value.type, null)
     }
   }
+
+  depends_on = [
+    kubernetes_namespace.cbci
+  ]
 }

variables.tf

@@ -30,3 +30,9 @@ variable "temp_license" {
   description = "Temporary license details"
   type        = map(string)
 }
+
+variable "secrets_file" {
+  description = "Secrets file yml path containing the secrets names:values to create the Kubernetes secret cbci-secrets. It can be mounted for Casc"
+  default     = "secrets-values.yml"
+  type        = string
+}

versions.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_version = ">= 1.0.0"
+
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.10"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = ">= 2.5.1"
+    }
+  }
+}