From 6b43a71eed5488362d94ee13e27005f059106a5a Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Lopez Date: Mon, 18 Dec 2023 18:42:40 +0100 Subject: [PATCH 1/5] First version of at-scale --- README.md | 6 +- .../.auto.tfvars.example | 0 .../README.md | 7 +- .../extdns-values.yml | 0 .../main.tf | 2 +- .../outputs.tf | 0 .../providers.tf | 0 .../variables.tf | 0 blueprints/02-at-scale/.auto.tfvars.example | 14 + blueprints/02-at-scale/README.md | 59 +++ blueprints/02-at-scale/cbci-values.yml | 25 ++ blueprints/02-at-scale/extdns-values.yml | 6 + blueprints/02-at-scale/main.tf | 422 ++++++++++++++++++ blueprints/02-at-scale/outputs.tf | 61 +++ blueprints/02-at-scale/providers.tf | 58 +++ blueprints/02-at-scale/variables.tf | 20 + blueprints/test-all.sh | 3 +- main.tf | 4 +- variables.tf | 9 +- 19 files changed, 681 insertions(+), 15 deletions(-) rename blueprints/{getting-started => 01-getting-started}/.auto.tfvars.example (100%) rename blueprints/{getting-started => 01-getting-started}/README.md (89%) rename blueprints/{getting-started => 01-getting-started}/extdns-values.yml (100%) rename blueprints/{getting-started => 01-getting-started}/main.tf (99%) rename blueprints/{getting-started => 01-getting-started}/outputs.tf (100%) rename blueprints/{getting-started => 01-getting-started}/providers.tf (100%) rename blueprints/{getting-started => 01-getting-started}/variables.tf (100%) create mode 100644 blueprints/02-at-scale/.auto.tfvars.example create mode 100644 blueprints/02-at-scale/README.md create mode 100644 blueprints/02-at-scale/cbci-values.yml create mode 100644 blueprints/02-at-scale/extdns-values.yml create mode 100644 blueprints/02-at-scale/main.tf create mode 100644 blueprints/02-at-scale/outputs.tf create mode 100644 blueprints/02-at-scale/providers.tf create mode 100644 blueprints/02-at-scale/variables.tf diff --git a/README.md b/README.md index a9cf9bee..7e726531 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ If you would like to override any defaults with the chart, you can do so by pass Easing adoption of CloudBees CI by: -- Provide just the terraform code to deploy [CloudBees CI in EKS](https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/) on top of the [AWS Terraform EKS Addons](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/) maintained by AWS. +- Providing the HCL code to deploy [CloudBees CI in EKS](https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/). - Using [AWS Terraform EKS Addons](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/) as the single point of truth for the required/recommended EKS Addons. - Provide a series of blueprints to deploy [CloudBees CI in EKS](https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/). -## License +## CloudBees License This module runs with a [Trial License for CloudBees CI](https://docs.cloudbees.com/docs/cloudbees-ci-migration/latest/trial-guide/). @@ -36,7 +36,7 @@ CloudBees CI Add-on uses for its resources definition `helms release` which make | cert_arn | Certificate ARN from AWS ACM | `string` | n/a | yes | | hostname | Route53 Hosted zone name | `string` | n/a | yes | | temp_license | Temporary license details | `map(string)` | n/a | yes | -| helm_config | CloudBees CI Helm chart configuration | `any` | `{}` | no | +| helm_config | CloudBees CI Helm chart configuration | `any` | 
{
  "values": [
    ""
  ]
}
| no | ### Outputs diff --git a/blueprints/getting-started/.auto.tfvars.example b/blueprints/01-getting-started/.auto.tfvars.example similarity index 100% rename from blueprints/getting-started/.auto.tfvars.example rename to blueprints/01-getting-started/.auto.tfvars.example diff --git a/blueprints/getting-started/README.md b/blueprints/01-getting-started/README.md similarity index 89% rename from blueprints/getting-started/README.md rename to blueprints/01-getting-started/README.md index 581da5fc..869c5643 100644 --- a/blueprints/getting-started/README.md +++ b/blueprints/01-getting-started/README.md @@ -1,6 +1,6 @@ # CloudBees CI Add-on getting started Blueprint -Get started with the CloudBees CI add-on by reviewing this example which deploys the minimum set of resources to install [CloudBees CI on EKS](https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/) following its [prerequisites](https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/installing-eks-using-helm#_prerequisites): +Get started with the CloudBees CI add-on by reviewing this example which deploys the **minimum set of required resources** to install [CloudBees CI on EKS](https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/) following its [prerequisites](https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/installing-eks-using-helm#_prerequisites): - AWS Certificate Manager - [AWS Load Balancer Controller](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/aws-load-balancer-controller/) @@ -8,11 +8,6 @@ Get started with the CloudBees CI add-on by reviewing this example which deploys Additionally, it deploys the [EBS CSI Driver](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html) to allocate EBS volumes for hosting [$JENKINS_HOME](https://docs.cloudbees.com/docs/cloudbees-ci/latest/backup-restore/jenkins-home). -The code in this directory demonstrates its compatibility with [AWS EKS Blueprint v4](https://github.com/aws-ia/terraform-aws-eks-blueprints/tree/v4.32.1) and [AWS EKS Blueprint v5](https://github.com/aws-ia/terraform-aws-eks-blueprints/tree/v5.0.0) (Additional info on [v4 to v5 migration guide](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4-to-v5/motivation/)). - -- [v4](v4/README.md) -- [v5](v5/README.md) - ## Prerequisites ### Tooling diff --git a/blueprints/getting-started/extdns-values.yml b/blueprints/01-getting-started/extdns-values.yml similarity index 100% rename from blueprints/getting-started/extdns-values.yml rename to blueprints/01-getting-started/extdns-values.yml diff --git a/blueprints/getting-started/main.tf b/blueprints/01-getting-started/main.tf similarity index 99% rename from blueprints/getting-started/main.tf rename to blueprints/01-getting-started/main.tf index ea354569..63447fc4 100644 --- a/blueprints/getting-started/main.tf +++ b/blueprints/01-getting-started/main.tf @@ -5,7 +5,7 @@ data "aws_route53_zone" "this" { data "aws_availability_zones" "available" {} locals { - name = "cbci-start-v5-i${random_integer.ramdom_id.result}" + name = "cbci-bp01-i${random_integer.ramdom_id.result}" region = "us-east-1" vpc_name = "${local.name}-vpc" diff --git a/blueprints/getting-started/outputs.tf b/blueprints/01-getting-started/outputs.tf similarity index 100% rename from blueprints/getting-started/outputs.tf rename to blueprints/01-getting-started/outputs.tf diff --git a/blueprints/getting-started/providers.tf b/blueprints/01-getting-started/providers.tf similarity index 100% rename from blueprints/getting-started/providers.tf rename to blueprints/01-getting-started/providers.tf diff --git a/blueprints/getting-started/variables.tf b/blueprints/01-getting-started/variables.tf similarity index 100% rename from blueprints/getting-started/variables.tf rename to blueprints/01-getting-started/variables.tf diff --git a/blueprints/02-at-scale/.auto.tfvars.example b/blueprints/02-at-scale/.auto.tfvars.example new file mode 100644 index 00000000..e64e6a2b --- /dev/null +++ b/blueprints/02-at-scale/.auto.tfvars.example @@ -0,0 +1,14 @@ +domain_name = "example.domain.com" # Required. Domain name used by the CloudBees CI instance. + +temp_license = { # Required. Temporary license details. + first_name = "Foo" + last_name = "Bar" + email = "foo.bar@acme.com" + company = "Acme Inc." +} + +# tags = { # Optional. Tags for the resources created. Default set to empty. Shared among all. +# "cb-owner" : "professional-services" +# "cb-user" : "crodriguezlopez" +# "cb-environment" : "demo" +# } diff --git a/blueprints/02-at-scale/README.md b/blueprints/02-at-scale/README.md new file mode 100644 index 00000000..a9d8c716 --- /dev/null +++ b/blueprints/02-at-scale/README.md @@ -0,0 +1,59 @@ +# CloudBees CI Add-on at scale Blueprint + +Once you have familiarized yourself with the [Getting Started blueprint](../01-getting-started/README.md), this blueprint presents a more scalable architecture adding the following features to the previous blueprint: + +- [Cluster Autoscaler](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/cluster-autoscaler/) + +- [EFS CSI Driver](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/aws-efs-csi-driver/). CloudBees CI HA/HS requirement. +- [Metrics Server](https://aws-ia.github.io/terraform-aws-eks-blueprints-addons/main/addons/aws-efs-csi-driver/). CloudBees CI HA/HS requirement for Horizontal Pod Autoscaling. + +Additionally, it uses [CloudBees Configuration as Code](https://docs.cloudbees.com/docs/cloudbees-ci/latest/casc-oc/casc-intro) for configuring the [Operation Center](https://docs.cloudbees.com/docs/cloudbees-ci/latest/casc-oc/) and [Controllers](https://docs.cloudbees.com/docs/cloudbees-ci/latest/casc-controller/). + +## Prerequisites + +Refer to the [Getting Started Blueprint - Prerequisites](../01-getting-started/README.md#prerequisites) section. + +## Terraform Docs + + +### Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| domain_name | Desired domain name (e.g. example.com) used as suffix for CloudBees CI subdomains (e.g. cjoc.example.com). It requires to be mapped within an existing Route 53 Hosted Zone. | `string` | n/a | yes | +| temp_license | Temporary license details | `map(string)` | n/a | yes | +| tags | Tags to apply to resources | `map(string)` | `{}` | no | + +### Outputs + +| Name | Description | +|------|-------------| +| acm_certificate_arn | ACM certificate ARN | +| cjoc_url | URL of the CloudBees CI Operations Center | +| eks_bp_addon_cbci_helm | Helm configuration for CloudBees CI Add-on. It is accesible only via state files. | +| eks_bp_addon_cbci_initial_admin_password | Operation Center Service Initial Admin Password for CloudBees CI Add-on. | +| eks_bp_addon_cbci_liveness_probe_ext | Operation Center Service External Liveness Probe for CloudBees CI Add-on. | +| eks_bp_addon_cbci_liveness_probe_int | Operation Center Service Internal Liveness Probe for CloudBees CI Add-on. | +| eks_bp_addon_cbci_namepace | Namespace for CloudBees CI Add-on. | +| eks_bp_addon_cbci_oc_ing | Operation Center Ingress for CloudBees CI Add-on. | +| eks_bp_addon_cbci_oc_pod | Operation Center Pod for CloudBees CI Add-on. | +| eks_cluster_arn | EKS cluster ARN | +| export_kubeconfig | Export KUBECONFIG environment variable to access the EKS cluster. | +| vpc_arn | VPC ID | + + +## Deploy + +Refer to the [Getting Started Blueprint - Prerequisites](../01-getting-started/README.md#deploy) section. + +## Validate + +Refer to the [Getting Started Blueprint - Prerequisites](../01-getting-started/README.md#validate) section. + +## Destroy + +Refer to the [Getting Started Blueprint - Prerequisites](../01-getting-started/README.md#destroy) section. + +## Architecture + +![Architecture]() diff --git a/blueprints/02-at-scale/cbci-values.yml b/blueprints/02-at-scale/cbci-values.yml new file mode 100644 index 00000000..2a3bcdf8 --- /dev/null +++ b/blueprints/02-at-scale/cbci-values.yml @@ -0,0 +1,25 @@ +# Copyright (c) CloudBees, Inc. + +#https://artifacthub.io/packages/helm/cloudbees/cloudbees-core/ +#https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/installing-eks-using-helm + +OperationsCenter: + Ingress: + Annotations: + alb.ingress.kubernetes.io/target-group-attributes: stickiness.enabled=true + NodeSelector: + ci_type: cb-apps + Tolerations: + - key: dedicated + operator: Equal + value: cb-apps + effect: NoSchedule +Persistence: + StorageClass: efs +Hibernation: + Enabled: true +Agents: + SeparateNamespace: + Enabled: true + Create: true + Name: cbci-agents diff --git a/blueprints/02-at-scale/extdns-values.yml b/blueprints/02-at-scale/extdns-values.yml new file mode 100644 index 00000000..27f3dabd --- /dev/null +++ b/blueprints/02-at-scale/extdns-values.yml @@ -0,0 +1,6 @@ +#https://artifacthub.io/packages/helm/external-dns/external-dns +#https://github.com/kubernetes-sigs/external-dns/tree/master/charts/external-dns/Chart.yaml +provider: "aws" +domainFilters: [ "${zoneDNS}" ] +policy: "sync" +logLevel: "debug" diff --git a/blueprints/02-at-scale/main.tf b/blueprints/02-at-scale/main.tf new file mode 100644 index 00000000..5b091dc9 --- /dev/null +++ b/blueprints/02-at-scale/main.tf @@ -0,0 +1,422 @@ +data "aws_route53_zone" "this" { + name = var.domain_name +} + +data "aws_availability_zones" "available" {} + +data "aws_caller_identity" "current" {} + +locals { + name = "cbci-bp02-i${random_integer.ramdom_id.result}" + region = "us-east-1" + + vpc_name = "${local.name}-vpc" + cluster_name = "${local.name}-eks" + efs_name = "${local.name}-efs" + kubeconfig_file = "kubeconfig_${local.name}.yaml" + kubeconfig_file_path = abspath("${path.root}/${local.kubeconfig_file}") + + vpc_cidr = "10.0.0.0/16" + + #https://docs.cloudbees.com/docs/cloudbees-common/latest/supported-platforms/cloudbees-ci-cloud#_kubernetes + k8s_version = "1.26" + + k8s_instance_types = { + # Not Scalable + "k8s-apps" = ["m5.8xlarge"] + # Scalable + "cb-apps" = ["m5d.4xlarge"] #https://aws.amazon.com/about-aws/whats-new/2018/06/introducing-amazon-ec2-m5d-instances/ + "agent" = ["m5.2xlarge"] + "agent-spot" = ["m5.2xlarge"] + } + + route53_zone_id = data.aws_route53_zone.this.id + route53_zone_arn = data.aws_route53_zone.this.arn + #Number of AZs per region https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + cjoc_url = "https://cjoc.${var.domain_name}" + + tags = merge(var.tags, { + "tf:blueprint" = local.name + "tf:repository" = "github.com/cloudbees/terraform-aws-cloudbees-ci-eks-addon" + }) + + current_account_id = data.aws_caller_identity.current.account_id + current_account_arn = data.aws_caller_identity.current.arn +} + +resource "random_integer" "ramdom_id" { + min = 1 + max = 999 +} + +################################################################################ +# EKS: Add-ons +################################################################################ + +module "eks_blueprints_addon_cbci" { + source = "../../" + + hostname = var.domain_name + cert_arn = module.acm.acm_certificate_arn + temp_license = var.temp_license + + helm_config = { + values = [file("${path.module}/cbci-values.yml")] + } + + depends_on = [ + module.eks_blueprints_addons + ] +} + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "1.12.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + oidc_provider_arn = module.eks.oidc_provider_arn + cluster_version = module.eks.cluster_version + + eks_addons = { + aws-ebs-csi-driver = { + service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + } + coredns = {} + vpc-cni = {} + kube-proxy = {} + } + #01-getting-started + enable_external_dns = true + external_dns = { + values = [templatefile("${path.module}/extdns-values.yml", { + zoneDNS = var.domain_name + })] + } + external_dns_route53_zone_arns = [local.route53_zone_arn] + enable_aws_load_balancer_controller = true + #02-at-scale + enable_aws_efs_csi_driver = true + enable_metrics_server = true + enable_cluster_autoscaler = true + #enable_aws_node_termination_handler = true + + tags = local.tags +} + +module "ebs_csi_driver_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.29.0" + + role_name_prefix = "${module.eks.cluster_name}-ebs-csi-driv" + + attach_ebs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } + + tags = var.tags +} + +################################################################################ +# EKS: Infra +################################################################################ + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "19.15.3" + + cluster_name = local.cluster_name + cluster_version = local.k8s_version + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_group_defaults = { + disk_size = 50 + iam_role_additional_policies = { + # Not required, but used in the example to access the nodes to inspect mounted volumes + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + block_device_mappings = { + # Root volume + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 24 + volume_type = "gp3" + iops = 3000 + encrypted = true + kms_key_id = module.ebs_kms_key.key_arn + delete_on_termination = true + } + } + } + } + + # Security groups based on the best practices doc https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html. + # So, by default the security groups are restrictive. Users needs to enable rules for specific ports required for App requirement or Add-ons + # See the notes below for each rule used in these examples + node_security_group_additional_rules = { + # Recommended outbound traffic for Node groups + egress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + self = true + } + # Extend node-to-node security group rules. Recommended and required for the Add-ons + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + + egress_ssh_all = { + description = "Egress all ssh to internet for github" + protocol = "tcp" + from_port = 22 + to_port = 22 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + # Allows Control Plane Nodes to talk to Worker nodes on all ports. Added this to simplify the example and further avoid issues with Add-ons communication with Control plane. + # This can be restricted further to specific port based on the requirement for each Add-on e.g., metrics-server 4443, spark-operator 8080, karpenter 8443 etc. + # Change this according to your security requirements if needed + ingress_cluster_to_node_all_traffic = { + description = "Cluster API to Nodegroup all traffic" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + source_cluster_security_group = true + } + } + + #Only Node Groups with Taints can use Autoscaling. Pods requires Selector + eks_managed_node_groups = { + #Managed by Autoscaling + mg_k8sApps = { + node_group_name = "managed-k8s-apps" + instance_types = local.k8s_instance_types["k8s-apps"] + capacity_type = "ON_DEMAND" + desired_size = 2 + }, + #Managed by Autoscaling + mg_cbApps = { + node_group_name = "managed-cb-apps" + instance_types = local.k8s_instance_types["cb-apps"] + capacity_type = "ON_DEMAND" + min_size = 1 + max_size = 6 + desired_size = 1 + taints = [{ key = "dedicated", value = "cb-apps", effect = "NO_SCHEDULE" }] + labels = { + ci_type = "cb-apps" + } + } + mg_cbAgents = { + node_group_name = "managed-agent" + instance_types = local.k8s_instance_types["agent"] + capacity_type = "ON_DEMAND" + min_size = 1 + max_size = 3 + desired_size = 1 + taints = [{ key = "dedicated", value = "build-linux", effect = "NO_SCHEDULE" }] + labels = { + ci_type = "build-linux" + } + }, + mg_cbAgents_spot = { + node_group_name = "managed-agent-spot" + instance_types = local.k8s_instance_types["agent-spot"] + capacity_type = "SPOT" + min_size = 1 + max_size = 3 + desired_size = 1 + taints = [{ key = "dedicated", value = "build-linux-spot", effect = "NO_SCHEDULE" }] + labels = { + ci_type = "build-linux-spot" + } + } + } + + tags = local.tags +} + +module "ebs_kms_key" { + source = "terraform-aws-modules/kms/aws" + version = "1.5.0" + + description = "Customer managed key to encrypt EKS managed node group volumes" + + # Policy + key_administrators = [local.current_account_arn] + key_service_roles_for_autoscaling = [ + # required for the ASG to manage encrypted volumes for nodes + "arn:aws:iam::${local.current_account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", + # required for the cluster / persistentvolume-controller to create encrypted PVCs + module.eks.cluster_iam_role_arn + ] + + # Aliases + aliases = ["eks/${local.name}/ebs"] + + tags = var.tags +} + +resource "kubernetes_annotations" "gp2" { + api_version = "storage.k8s.io/v1" + kind = "StorageClass" + # This is true because the resources was already created by the ebs-csi-driver addon + force = "true" + + metadata { + name = "gp2" + } + + annotations = { + # Modify annotations to remove gp2 as default storage class still retain the class + "storageclass.kubernetes.io/is-default-class" = "false" + } +} + +resource "kubernetes_storage_class_v1" "gp3" { + metadata { + name = "gp3" + + # IMPORTANT: Prometheus and Velero requires gp3 (Block Storage) + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + } + + storage_provisioner = "ebs.csi.aws.com" + allow_volume_expansion = true + reclaim_policy = "Delete" + volume_binding_mode = "WaitForFirstConsumer" + + parameters = { + encrypted = "true" + fsType = "ext4" + type = "gp3" + } +} + +resource "kubernetes_storage_class_v1" "efs" { + + metadata { + name = "efs" + annotations = { + "storageclass.kubernetes.io/is-default-class" = "false" + } + } + + storage_provisioner = "efs.csi.aws.com" + parameters = { + provisioningMode = "efs-ap" # Dynamic provisioning + fileSystemId = module.efs.id + directoryPerms = "700" + } + + mount_options = [ + "iam" + ] +} + +resource "null_resource" "create_kubeconfig" { + + depends_on = [module.eks] + + provisioner "local-exec" { + command = "aws eks update-kubeconfig --name ${module.eks.cluster_name} --region ${local.region} --kubeconfig ${local.kubeconfig_file}" + } +} + +################################################################################ +# Supported Resources +################################################################################ + +module "efs" { + source = "terraform-aws-modules/efs/aws" + version = "1.2.0" + + creation_token = local.efs_name + name = local.efs_name + + mount_targets = { + for k, v in zipmap(local.azs, module.vpc.private_subnets) : k => { subnet_id = v } + } + security_group_description = "${local.efs_name} EFS security group" + security_group_vpc_id = module.vpc.vpc_id + #https://docs.cloudbees.com/docs/cloudbees-ci/latest/eks-install-guide/eks-pre-install-requirements-helm#_storage_requirements + performance_mode = "generalPurpose" + throughput_mode = "elastic" + security_group_rules = { + vpc = { + # relying on the defaults provdied for EFS/NFS (2049/TCP + ingress) + description = "NFS ingress from VPC private subnets" + cidr_blocks = module.vpc.private_subnets_cidr_blocks + } + } + + tags = var.tags +} + +module "acm" { + source = "terraform-aws-modules/acm/aws" + version = "4.3.2" + + #Important: Application Services Hostname must be the same as the domain name or subject_alternative_names + domain_name = var.domain_name + subject_alternative_names = [ + "*.${var.domain_name}" # For subdomains example.${var.domain_name} + ] + + #https://docs.aws.amazon.com/acm/latest/userguide/dns-validation.html + zone_id = local.route53_zone_id + + tags = local.tags +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "5.0.0" + + name = local.vpc_name + cidr = local.vpc_cidr + + azs = local.azs + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + #https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html + #https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags + +} diff --git a/blueprints/02-at-scale/outputs.tf b/blueprints/02-at-scale/outputs.tf new file mode 100644 index 00000000..d7a6b5db --- /dev/null +++ b/blueprints/02-at-scale/outputs.tf @@ -0,0 +1,61 @@ + +output "export_kubeconfig" { + description = "Export KUBECONFIG environment variable to access the EKS cluster." + value = "export KUBECONFIG=${local.kubeconfig_file_path}" +} + +output "eks_bp_addon_cbci_helm" { + description = "Helm configuration for CloudBees CI Add-on. It is accesible only via state files." + value = module.eks_blueprints_addon_cbci.merged_helm_config + sensitive = true +} + +output "eks_bp_addon_cbci_namepace" { + description = "Namespace for CloudBees CI Add-on." + value = module.eks_blueprints_addon_cbci.cloudbees_ci_namespace +} + +output "eks_bp_addon_cbci_oc_pod" { + description = "Operation Center Pod for CloudBees CI Add-on." + value = module.eks_blueprints_addon_cbci.cloudbees_ci_oc_pod +} + +output "eks_bp_addon_cbci_oc_ing" { + description = "Operation Center Ingress for CloudBees CI Add-on." + value = module.eks_blueprints_addon_cbci.cloudbees_ci_oc_ing +} + +output "eks_bp_addon_cbci_liveness_probe_int" { + description = "Operation Center Service Internal Liveness Probe for CloudBees CI Add-on." + value = module.eks_blueprints_addon_cbci.cloudbees_ci_liveness_probe +} + +output "eks_bp_addon_cbci_liveness_probe_ext" { + description = "Operation Center Service External Liveness Probe for CloudBees CI Add-on." + value = "curl -sSf ${local.cjoc_url}/whoAmI/api/json > /dev/null" +} + +output "eks_bp_addon_cbci_initial_admin_password" { + description = "Operation Center Service Initial Admin Password for CloudBees CI Add-on." + value = module.eks_blueprints_addon_cbci.cloudbees_ci_initial_admin_password +} + +output "acm_certificate_arn" { + description = "ACM certificate ARN" + value = module.acm.acm_certificate_arn +} + +output "vpc_arn" { + description = "VPC ID" + value = module.vpc.vpc_arn +} + +output "eks_cluster_arn" { + description = "EKS cluster ARN" + value = module.eks.cluster_arn +} + +output "cjoc_url" { + description = "URL of the CloudBees CI Operations Center" + value = local.cjoc_url +} diff --git a/blueprints/02-at-scale/providers.tf b/blueprints/02-at-scale/providers.tf new file mode 100644 index 00000000..1722ebf6 --- /dev/null +++ b/blueprints/02-at-scale/providers.tf @@ -0,0 +1,58 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.5.1" + } + random = { + source = "hashicorp/random" + version = ">= 2.3.0" + } + + null = { + source = "hashicorp/null" + version = ">= 3.1.0" + } + } + +} + +provider "aws" { + region = local.region +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } + } +} diff --git a/blueprints/02-at-scale/variables.tf b/blueprints/02-at-scale/variables.tf new file mode 100644 index 00000000..1e7610c8 --- /dev/null +++ b/blueprints/02-at-scale/variables.tf @@ -0,0 +1,20 @@ + +variable "tags" { + description = "Tags to apply to resources" + default = {} + type = map(string) +} + +variable "domain_name" { + description = "Desired domain name (e.g. example.com) used as suffix for CloudBees CI subdomains (e.g. cjoc.example.com). It requires to be mapped within an existing Route 53 Hosted Zone." + type = string + validation { + condition = trim(var.domain_name, " ") != "" + error_message = "Domain name must not be en empty string." + } +} + +variable "temp_license" { + description = "Temporary license details" + type = map(string) +} diff --git a/blueprints/test-all.sh b/blueprints/test-all.sh index 34c67d5e..b349614b 100755 --- a/blueprints/test-all.sh +++ b/blueprints/test-all.sh @@ -12,7 +12,8 @@ HERE="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" export TF_LOG=DEBUG declare -a bluePrints=( - "getting-started" + #"01-getting-started" + "02-at-scale" ) test () { diff --git a/main.tf b/main.tf index 6cba71f7..d7887f6a 100644 --- a/main.tf +++ b/main.tf @@ -3,7 +3,7 @@ resource "helm_release" "cloudbees_ci" { name = try(var.helm_config.name, "cloudbees-ci") - namespace = try(var.helm_config.namespace, "cloudbees-ci") + namespace = try(var.helm_config.namespace, "cbci") create_namespace = try(var.helm_config.create_namespace, true) description = try(var.helm_config.description, null) chart = "cloudbees-core" @@ -11,7 +11,7 @@ resource "helm_release" "cloudbees_ci" { #Chart versions are linked to an app version: https://docs.cloudbees.com/docs/release-notes/latest/cloudbees-ci/ version = try(var.helm_config.version, "3.14783.0+d0af0bc462a0") repository = try(var.helm_config.repository, "https://public-charts.artifacts.cloudbees.com/repository/public/") - values = try(var.helm_config.values, [templatefile("${path.module}/values.yml", { + values = concat(var.helm_config.values, [templatefile("${path.module}/values.yml", { hostname = var.hostname cert_arn = var.cert_arn LicFirstName = var.temp_license["first_name"] diff --git a/variables.tf b/variables.tf index d38ffea1..0d269ec5 100644 --- a/variables.tf +++ b/variables.tf @@ -3,7 +3,12 @@ variable "helm_config" { description = "CloudBees CI Helm chart configuration" type = any - default = {} + default = { + values = [ + <<-EOT + EOT + ] + } } variable "hostname" { @@ -24,4 +29,4 @@ variable "cert_arn" { variable "temp_license" { description = "Temporary license details" type = map(string) -} \ No newline at end of file +} From 50a0e1d63fa4cfd13d5e017b0cf9244126a2179d Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Lopez Date: Mon, 18 Dec 2023 18:43:47 +0100 Subject: [PATCH 2/5] Simplifying the number of tools --- blueprints/.tool-versions | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/blueprints/.tool-versions b/blueprints/.tool-versions index 41979ee2..156c2936 100644 --- a/blueprints/.tool-versions +++ b/blueprints/.tool-versions @@ -1,9 +1,4 @@ -kubectl 1.28.3 -awscli 2.11.21 -terraform 1.5.1 -terraform-docs 0.16.0 -tfupdate 0.6.5 -pre-commit 3.3.2 -shfmt 3.6.0 -shellcheck 0.9.0 -terraform-ls 0.30.3 +kubectl 1.29.0 +awscli 2.15.2 +terraform 1.6.6 +k9s 0.29.1 From 602eaa7a2c79e53bb349989765c3d0ed657c4c47 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Lopez Date: Mon, 18 Dec 2023 18:49:06 +0100 Subject: [PATCH 3/5] Fix the wrong link to Makefile --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b0a87c7d..b09e4ea1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ This document provides guidelines for contributing to the module. Validate your changes inside the blueprint agent described in the [Dockerfile](blueprints/Dockerfile). which is using tooling listed in the [asdf](https://asdf-vm.com/) [tool-versions](blueprints/.tool-versions). -> **_NOTE:_** The agent and dependecies can be automated via [Makefile](../../Makefile) at the root of the project under the target `dRun`. +> **_NOTE:_** The agent and dependecies can be automated via [Makefile](Makefile) at the root of the project under the target `dRun`. ## Pre-commits: Linting, Formatting and Secrets Scanning @@ -21,4 +21,4 @@ Run `pre-commit run --all-files` ## Release Drafter -This repository uses [Release Drafter](https://github.com/release-drafter/release-drafter) thus it is recommended to use [Semantic Commit Messages](https://gist.github.com/joshbuchea/6f47e86d2510bce28f8e7f42ae84c716) to ease labelling your Pull Request accordingly. +This repository uses [Release Drafter](https://github.com/release-drafter/release-drafter) do not forget to label Pull Request accordingly. From 80321e18943d4a2878bcf240ae8562aaa749330c Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Lopez Date: Mon, 18 Dec 2023 18:59:29 +0100 Subject: [PATCH 4/5] Updating GHA for terraform DOCS --- .github/workflows/terraform-docs.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/terraform-docs.yml b/.github/workflows/terraform-docs.yml index a302064b..7615996d 100644 --- a/.github/workflows/terraform-docs.yml +++ b/.github/workflows/terraform-docs.yml @@ -24,18 +24,18 @@ jobs: output-method: inject git-push: true - - name: Render documentation for `getting-started/v4` example and push changes back to branch + - name: Render documentation for `01-getting-started` example and push changes back to branch uses: terraform-docs/gh-actions@f6d59f89a280fa0a3febf55ef68f146784b20ba0 # v1.0.0 with: config-file: "../../../.terraform-docs.yml" output-file: "README.md" git-push: true - working-dir: "./blueprints/getting-started/v4" + working-dir: "./blueprints/01-getting-started" - - name: Render documentation for `getting-started/v5` example and push changes back to branch + - name: Render documentation for `02-at-scale` example and push changes back to branch uses: terraform-docs/gh-actions@f6d59f89a280fa0a3febf55ef68f146784b20ba0 # v1.0.0 with: config-file: "../../../.terraform-docs.yml" output-file: "README.md" git-push: true - working-dir: "./blueprints/getting-started/v5" + working-dir: "./blueprints/02-at-scale" From ee454e1460d5669c764856551dcfe5a8000fc232 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Lopez Date: Mon, 18 Dec 2023 19:02:14 +0100 Subject: [PATCH 5/5] GHA, Terrafrom DOC fix path --- .github/workflows/terraform-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/terraform-docs.yml b/.github/workflows/terraform-docs.yml index 7615996d..a8e22aee 100644 --- a/.github/workflows/terraform-docs.yml +++ b/.github/workflows/terraform-docs.yml @@ -27,7 +27,7 @@ jobs: - name: Render documentation for `01-getting-started` example and push changes back to branch uses: terraform-docs/gh-actions@f6d59f89a280fa0a3febf55ef68f146784b20ba0 # v1.0.0 with: - config-file: "../../../.terraform-docs.yml" + config-file: ".terraform-docs.yml" output-file: "README.md" git-push: true working-dir: "./blueprints/01-getting-started" @@ -35,7 +35,7 @@ jobs: - name: Render documentation for `02-at-scale` example and push changes back to branch uses: terraform-docs/gh-actions@f6d59f89a280fa0a3febf55ef68f146784b20ba0 # v1.0.0 with: - config-file: "../../../.terraform-docs.yml" + config-file: ".terraform-docs.yml" output-file: "README.md" git-push: true working-dir: "./blueprints/02-at-scale"