add cluster addon support

docs/src/SUMMARY.md

@@ -12,6 +12,7 @@
     - [Subnets](./guide/network_subnets.md)
     - [Network Security Groups](./guide/network_nsgs.md)    
   - [Cluster](./guide/cluster.md)
+    - [Cluster Add-ons](./guide/cluster_addons.md)
   - [Workers](./guide/workers.md)
     - [Mode](./guide/workers_mode.md)
       - [Node Pool](./guide/workers_mode_nodepool.md)

docs/src/guide/cluster_addons.md

@@ -0,0 +1,19 @@
+# Cluster Add-ons
+
+With this module to manage both essential and optional add-ons on enhanced OKE clusters. 
+
+This module provides the option to remove [Essential addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-essential-addons) and to manage, both essential & [optional addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-optional-addons).
+
+Cluster add-on removal (using the `cluster_addons_to_remove` variable) requires the creation of the operator host.
+
+**Note**: For the cluster autoscaler you should choose **only one** of the options:
+- the stand-alone cluster-autoscaler deployment, using the [extension module](./extensions_cluster_autoscaler.md)
+- the cluster-autoscaler add-on
+
+## Example usage
+```javascript
+{{#include ../../../examples/cluster-addons/vars-cluster-addons.auto.tfvars:4:}}
+```
+
+## Reference
+* [OKE Cluster Add-ons Documentation](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons.htm)

docs/src/guide/extensions_cluster_autoscaler.md

@@ -1,5 +1,9 @@
 # Extensions: Standalone Cluster Autoscaler
 
+**Note**: For the cluster autoscaler you should choose **only one** of the options:
+- the stand-alone cluster-autoscaler deployment, using this extension
+- the [cluster-autoscaler add-on](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm#contengconfiguringclusteraddons-configurationarguments_ClusterAutoscaler), using the [addons](./cluster_addons.md).
+
 Deployed using the [cluster-autoscaler Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler) with configuration from the `worker_pools` variable.
 
 The module is using the `oke.oraclecloud.com/cluster_autoscaler` nodepool label to facilitate the understanding of how the Kubernetes cluster auto-scaler will interact with the node:

examples/cluster-addons/vars-cluster-addons.auto.tfvars

@@ -0,0 +1,21 @@
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+cluster_addons = {
+  "CertManager" = {
+    remove_addon_resources_on_delete = true
+    # The list of supported configurations for the cluster addons is here: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm#contengconfiguringclusteraddons-configurationarguments_CertificateManager
+    configurations = [
+      {
+        key          = "numOfReplicas"
+        value        = "1"
+      }
+    ]
+  }
+}
+
+cluster_addons_to_remove = {
+  Flannel = {
+    remove_k8s_resources = true
+  }
+}

module-cluster-addons.tf

@@ -0,0 +1,28 @@
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+module "cluster-addons" {
+  count  = local.cluster_enabled && lower(var.cluster_type) == "enhanced" ? 1 : 0
+  source = "./modules/cluster-addons"
+
+  operator_enabled = local.operator_enabled
+
+  cluster_addons           = var.cluster_addons
+  cluster_addons_to_remove = var.cluster_addons_to_remove
+
+  cluster_id         = coalesce(var.cluster_id, one(module.cluster[*].cluster_id))
+  kubernetes_version = var.kubernetes_version
+
+  # Bastion/operator connection
+  ssh_private_key = sensitive(local.ssh_private_key)
+  bastion_host    = local.bastion_public_ip
+  bastion_user    = var.bastion_user
+  operator_host   = local.operator_private_ip
+  operator_user   = var.operator_user
+}
+
+
+# output "supported_addons" {
+#   description = "Supported cluster addons"
+#   value       = var.output_detail ? try(one(module.cluster-addons[*].supported_addons), null) : null
+# }

modules/cluster-addons/addons.tf

@@ -0,0 +1,83 @@
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+data "oci_containerengine_addon_options" "k8s_addon_options" {
+  kubernetes_version = var.kubernetes_version
+}
+
+locals {
+  supported_addons = [for entry in data.oci_containerengine_addon_options.k8s_addon_options.addon_options : entry.name]
+  primary_addons   = ["CertManager"]
+  addons_defaults = {
+    remove_addon_resources_on_delete = true
+    configurations                   = []
+    version                          = null
+  }
+  addons_with_defaults = { for addon_name, addon_value in var.cluster_addons :
+    addon_name => merge(local.addons_defaults, addon_value)
+  }
+}
+
+resource "oci_containerengine_addon" "primary_addon" {
+  for_each = { for k, v in local.addons_with_defaults : k => v if contains(local.primary_addons, k) }
+
+  addon_name = each.key
+  cluster_id = var.cluster_id
+
+  remove_addon_resources_on_delete = lookup(each.value, "remove_addon_resources_on_delete", true)
+
+  dynamic "configurations" {
+    for_each = lookup(each.value, "configurations", [])
+    iterator = config
+
+    content {
+      key   = tostring(lookup(config.value, "key"))
+      value = tostring(lookup(config.value, "value"))
+    }
+  }
+
+  version = lookup(each.value, "version", null)
+
+  lifecycle {
+
+    precondition {
+      condition     = contains(local.supported_addons, each.key)
+      error_message = <<-EOT
+      The addon ${each.key} is not supported.
+      The list of supported addons is: ${join(", ", local.supported_addons)}.
+      EOT
+    }
+  }
+}
+
+resource "oci_containerengine_addon" "secondary_addon" {
+  for_each   = { for k, v in local.addons_with_defaults : k => v if !contains(local.primary_addons, k) }
+  depends_on = [oci_containerengine_addon.primary_addon]
+  addon_name = each.key
+  cluster_id = var.cluster_id
+
+  remove_addon_resources_on_delete = lookup(each.value, "remove_addon_resources_on_delete", true)
+
+  dynamic "configurations" {
+    for_each = lookup(each.value, "configurations", [])
+    iterator = config
+
+    content {
+      key   = tostring(lookup(config.value, "key"))
+      value = tostring(lookup(config.value, "value"))
+    }
+  }
+
+  version = lookup(each.value, "version", null)
+
+  lifecycle {
+
+    precondition {
+      condition     = contains(local.supported_addons, each.key)
+      error_message = <<-EOT
+      The addon ${each.key} is not supported.
+      The list of supported addons is: ${join(", ", local.supported_addons)}.
+      EOT
+    }
+  }
+}

modules/cluster-addons/delete_addons.tf

@@ -0,0 +1,49 @@
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+locals {
+  remove_addon_command = "oci ce cluster disable-addon --addon-name %s --cluster-id %s --is-remove-existing-add-on %t --force"
+  remove_addons_defaults = {
+    custom_commands      = []
+    remove_k8s_resources = true
+  }
+  remove_addons_with_defaults = { for addon_name, addon_value in var.cluster_addons_to_remove :
+    addon_name => merge(local.remove_addons_defaults, addon_value)
+  }
+}
+
+resource "null_resource" "remove_addons" {
+  for_each   = var.operator_enabled ? local.remove_addons_with_defaults : {}
+  depends_on = [oci_containerengine_addon.primary_addon, oci_containerengine_addon.secondary_addon]
+
+  connection {
+    bastion_host        = var.bastion_host
+    bastion_user        = var.bastion_user
+    bastion_private_key = var.ssh_private_key
+    host                = var.operator_host
+    user                = var.operator_user
+    private_key         = var.ssh_private_key
+    timeout             = "40m"
+    type                = "ssh"
+  }
+
+  provisioner "remote-exec" {
+    inline = concat(
+      [
+        "echo 'Removing ${each.key} addon'",
+        format(local.remove_addon_command, each.key, var.cluster_id, lookup(each.value, "remove_k8s_resources"))
+      ],
+      lookup(each.value, "custom_commands")
+    )
+  }
+
+  lifecycle {
+    precondition {
+      condition     = contains(local.supported_addons, each.key)
+      error_message = <<-EOT
+      The addon ${each.key} is not supported.
+      The list of supported addons is: ${join(", ", local.supported_addons)}.
+      EOT
+    }
+  }
+}

modules/cluster-addons/outputs.tf

@@ -0,0 +1,6 @@
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+output "supported_addons" {
+  value = data.oci_containerengine_addon_options.k8s_addon_options.addon_options
+}

modules/cluster-addons/variables.tf

@@ -0,0 +1,17 @@
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+# General variables
+variable "cluster_id" { type = string }
+variable "cluster_addons" { type = any }
+variable "cluster_addons_to_remove" { type = any }
+variable "kubernetes_version" { type = string }
+
+# Variables required to access the operator host
+variable "bastion_host" { type = string }
+variable "bastion_user" { type = string }
+variable "operator_enabled" { type = bool }
+variable "operator_host" { type = string }
+variable "operator_user" { type = string }
+variable "ssh_private_key" { type = string }
+

modules/cluster-addons/versions.tf

@@ -0,0 +1,13 @@
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+terraform {
+  required_version = ">= 1.2.0"
+
+  required_providers {
+    oci = {
+      source  = "oracle/oci"
+      version = ">= 4.119.0"
+    }
+  }
+}

variables-cluster-addons.tf

@@ -0,0 +1,14 @@
+# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+variable "cluster_addons" {
+  description = "Map with cluster addons not created by Terraform that should be removed. This operation is performed using oci-cli and requires the operator host to be deployed."
+  type        = any
+  default     = {}
+}
+
+variable "cluster_addons_to_remove" {
+  description = "Map with cluster addons that should be enabled.  See <a href=https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm#contengconfiguringclusteraddons-supportedarguments>ClusterAddon documentation</a> for the supported configuration of each addon."
+  type        = any
+  default     = {}
+}