Add ingress and labels configs (#157)

* Add nginx-ingress-controller

* Add labels config

config.yaml

@@ -4,6 +4,60 @@ options:
     default: "1.28/edge"
     description: |
       Snap channel to install Kubernetes worker services from
+  ingress:
+    type: boolean
+    default: true
+    description: |
+      Deploy nginx-ingress-controller to handle Ingress resources. When set to
+      true, the unit will open ports 80 and 443 to make the nginx-ingress-controller
+      endpoint accessible.
+  ingress-default-ssl-certificate:
+    type: string
+    default: ""
+    description: |
+      SSL certificate to be used by the default HTTPS server. If one of the
+      flag ingress-default-ssl-certificate or ingress-default-ssl-key is not
+      provided ingress will use a self-signed certificate. This parameter is
+      specific to nginx-ingress-controller.
+  ingress-default-ssl-key:
+    type: string
+    default: ""
+    description: |
+      Private key to be used by the default HTTPS server. If one of the flag
+      ingress-default-ssl-certificate or ingress-default-ssl-key is not
+      provided ingress will use a self-signed certificate. This parameter is
+      specific to nginx-ingress-controller.
+  ingress-ssl-chain-completion:
+    type: boolean
+    default: false
+    description: |
+      Enable chain completion for TLS certificates used by the nginx ingress
+      controller.  Set this to true if you would like the ingress controller
+      to attempt auto-retrieval of intermediate certificates.  The default
+      (false) is recommended for all production kubernetes installations, and
+      any environment which does not have outbound Internet access.
+  ingress-ssl-passthrough:
+    type: boolean
+    default: false
+    description: |
+      Enable ssl passthrough on ingress server. This allows passing the ssl
+      connection through to the workloads and not terminating it at the ingress
+      controller.
+  ingress-use-forwarded-headers:
+    type: boolean
+    default: false
+    description: |
+      If true, NGINX passes the incoming X-Forwarded-* headers to upstreams. Use this
+      option when NGINX is behind another L7 proxy / load balancer that is setting
+      these headers.
+
+      If false, NGINX ignores incoming X-Forwarded-* headers, filling them with the
+      request information it sees. Use this option if NGINX is exposed directly to
+      the internet, or it's behind a L3/packet-based load balancer that doesn't alter
+      the source IP in the packets.
+
+      Reference: https://github.com/kubernetes/ingress-nginx/blob/a9c706be12a8be418c49ab1f60a02f52f9b14e55/
+      docs/user-guide/nginx-configuration/configmap.md#use-forwarded-headers.
   kubelet-extra-args:
     type: string
     default: ""
@@ -28,6 +82,21 @@ options:
 
       For more information about KubeletConfiguration, see upstream docs:
       https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/
+  labels:
+    type: string
+    default: ""
+    description: |
+      Labels can be used to organize and to select subsets of nodes in the
+      cluster. Declare node labels in key=value format, separated by spaces.
+  nginx-image:
+    type: string
+    default: "auto"
+    description: |
+      Container image to use for the nginx ingress controller. Using "auto" will select
+      an image based on architecture.
+
+      Example:
+        quay.io/kubernetes-ingress-controller/nginx-ingress-controller-amd64:0.32.0
   proxy-extra-args:
     type: string
     default: ""

requirements.txt

@@ -4,9 +4,12 @@ charm-lib-interface-external-cloud-provider @ git+https://github.com/charmed-kub
 charm-lib-interface-kubernetes-cni @ git+https://github.com/charmed-kubernetes/charm-lib-interface-kubernetes-cni
 charm-lib-interface-tokens @ git+https://github.com/charmed-kubernetes/charm-lib-interface-tokens
 charm-lib-kubernetes-snaps @ git+https://github.com/charmed-kubernetes/charm-lib-kubernetes-snaps
+charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@main#subdirectory=ops
 charm-lib-reconciler @ git+https://github.com/charmed-kubernetes/charm-lib-reconciler
 cosl == 0.0.7
+jinja2
 ops >= 2.2.0
 ops.interface_kube_control @ git+https://github.com/juju-solutions/interface-kube-control#subdirectory=ops
 ops.interface_tls_certificates @ git+https://github.com/juju-solutions/interface-tls-certificates#subdirectory=ops
 pydantic==1.*
+tenacity

src/charm.py

@@ -5,8 +5,10 @@
 """Charmed Machine Operator for Kubernetes Worker."""
 
 import logging
+import os
 import shlex
 import subprocess
+from base64 import b64encode
 from dataclasses import dataclass
 from pathlib import Path
 from socket import gethostname
@@ -22,7 +24,10 @@
 from charms.interface_external_cloud_provider import ExternalCloudProvider
 from charms.interface_kubernetes_cni import KubernetesCniProvides
 from charms.interface_tokens import TokensRequirer
+from charms.node_base import LabelMaker
 from charms.reconciler import BlockedStatus, Reconciler
+from jinja2 import Environment, FileSystemLoader
+from kubectl import kubectl
 from ops.interface_kube_control import KubeControlRequirer
 from ops.interface_tls_certificates import CertificatesRequires
 from ops.model import MaintenanceStatus, ModelError, WaitingStatus
@@ -79,6 +84,7 @@ def __init__(self, *args):
         )
         self.external_cloud_provider = ExternalCloudProvider(self, "kube-control")
         self.kube_control = KubeControlRequirer(self)
+        self.label_maker = LabelMaker(self, kubeconfig_path="/root/.kube/config")
         self.tokens = TokensRequirer(self)
         self.reconciler = Reconciler(self, self.reconcile)
 
@@ -165,6 +171,78 @@ def _configure_kubeproxy(self, event):
             external_cloud_provider=self.external_cloud_provider,
         )
 
+    def _configure_labels(self):
+        """Configure labels."""
+        if not os.path.exists("/root/.kube/config"):
+            log.info("Waiting for kubeconfig before configuring labels")
+            return
+
+        status.add(MaintenanceStatus("Configuring node labels"))
+
+        if self.label_maker.active_labels() is not None:
+            self.label_maker.apply_node_labels()
+
+    def _configure_nginx_ingress_controller(self):
+        """Configure nginx-ingress-controller."""
+        if not os.path.exists("/root/.kube/config"):
+            log.info("Waiting for kubeconfig before configuring ingress")
+            return
+
+        status.add(MaintenanceStatus("Configuring ingress"))
+
+        manifest_dir = "/root/cdk/addons"
+        manifest_path = manifest_dir + "/ingress-daemon-set.yaml"
+
+        if self.config["ingress"]:
+            image = self.config["nginx-image"]
+            if image == "" or image == "auto":
+                registry = self.kube_control.get_registry_location() or "registry.k8s.io"
+                image = f"{registry}/ingress-nginx/controller:v1.6.4"
+
+            context = {
+                "daemonset_api_version": "apps/v1",
+                "default_ssl_certificate_option": None,
+                "enable_ssl_passthrough": self.config["ingress-ssl-passthrough"],
+                "ingress_image": image,
+                "ingress_uid": "101",
+                "juju_application": self.app.name,
+                "ssl_chain_completion": self.config["ingress-ssl-chain-completion"],
+                "use_forwarded_headers": "true"
+                if self.config["ingress-use-forwarded-headers"]
+                else "false",
+            }
+
+            ssl_cert = self.config["ingress-default-ssl-certificate"]
+            ssl_key = self.config["ingress-default-ssl-key"]
+            if ssl_cert and ssl_key:
+                context.update(
+                    {
+                        "default_ssl_certificate": b64encode(ssl_cert.encode("utf-8")).decode(
+                            "utf-8"
+                        ),
+                        "default_ssl_certificate_option": "- --default-ssl-certificate=$(POD_NAMESPACE)/default-ssl-certificate",
+                        "default_ssl_key": b64encode(ssl_key.encode("utf-8")).decode("utf-8"),
+                    }
+                )
+
+            env = Environment(loader=FileSystemLoader("templates"))
+            template = env.get_template("ingress-daemon-set.yaml")
+            output = template.render(context)
+            os.makedirs(manifest_dir, exist_ok=True)
+            with open(manifest_path, "w") as f:
+                f.write(output)
+            kubectl("apply", "-f", manifest_path)
+
+            self.unit.open_port("tcp", 80)
+            self.unit.open_port("tcp", 443)
+        else:
+            self.unit.close_port("tcp", 80)
+            self.unit.close_port("tcp", 443)
+
+            if os.path.exists(manifest_path):
+                kubectl("delete", "--ignore-not-found", "-f", manifest_path)
+                os.remove(manifest_path)
+
     def _create_kubeconfigs(self, event):
         """Generate kubeconfig files for the cluster components."""
         status.add(MaintenanceStatus("Generating Kubeconfig"))
@@ -176,7 +254,7 @@ def _create_kubeconfigs(self, event):
         if not self._check_kubecontrol_integration(event):
             return
 
-        node_user = f"system:node:{self._get_node_name()}"
+        node_user = f"system:node:{self.get_node_name()}"
         credentials = self.kube_control.get_auth_credentials(node_user)
         if not credentials:
             status.add(WaitingStatus("Waiting for kube-control credentials"))
@@ -221,11 +299,15 @@ def _create_kubeconfigs(self, event):
             token=credentials.get("proxy_token"),
         )
 
+    def get_cloud_name(self) -> str:
+        """Return cloud name."""
+        return self.external_cloud_provider.name
+
     def _get_metrics_endpoints(self) -> list:
         """Return the metrics endpoints for K8s components."""
         log.info("Building Prometheus scraping jobs.")
 
-        cos_user = f"system:cos:{self._get_node_name()}"
+        cos_user = f"system:cos:{self.get_node_name()}"
         token = self.tokens.get_token(cos_user)
 
         if not token:
@@ -286,8 +368,9 @@ def create_scrape_job(config: JobConfig):
     def _get_unit_number(self) -> int:
         return int(self.unit.name.split("/")[1])
 
-    def _get_node_name(self) -> str:
-        fqdn = self.external_cloud_provider.name == "aws"
+    def get_node_name(self) -> str:
+        """Return node name."""
+        fqdn = self.get_cloud_name() == "aws"
         return kubernetes_snaps.get_node_name(fqdn)
 
     def _install_cni_binaries(self):
@@ -326,15 +409,15 @@ def _request_kubelet_and_proxy_credentials(self):
         """Request authorization for kubelet and kube-proxy."""
         status.add(MaintenanceStatus("Requesting kubelet and kube-proxy credentials"))
 
-        node_user = f"system:node:{self._get_node_name()}"
+        node_user = f"system:node:{self.get_node_name()}"
         self.kube_control.set_auth_request(node_user)
 
     def _request_monitoring_token(self, event):
         status.add(MaintenanceStatus("Requesting COS token"))
         if not self._check_tokens_integration(event):
             return
 
-        cos_user = f"system:cos:{self._get_node_name()}"
+        cos_user = f"system:cos:{self.get_node_name()}"
         self.tokens.request_token(cos_user, OBSERVABILITY_GROUP)
 
     def reconcile(self, event):
@@ -352,6 +435,8 @@ def reconcile(self, event):
         self._configure_kernel_parameters()
         self._configure_kubelet(event)
         self._configure_kubeproxy(event)
+        self._configure_nginx_ingress_controller()
+        self._configure_labels()
 
     def _request_certificates(self):
         """Request client and server certificates."""

src/kubectl.py

@@ -0,0 +1,25 @@
+"""kubectl."""
+
+import logging
+from subprocess import CalledProcessError, check_output
+
+from tenacity import retry, stop_after_delay, wait_exponential
+
+log = logging.getLogger(__name__)
+
+
+@retry(stop=stop_after_delay(60), wait=wait_exponential())
+def kubectl(*args):
+    """Run a kubectl cli command with a config file.
+
+    Returns stdout and throws an error if the command fails.
+    """
+    command = ["kubectl", "--kubeconfig=/root/.kube/config"] + list(args)
+    log.info("Executing {}".format(command))
+    try:
+        return check_output(command).decode("utf-8")
+    except CalledProcessError as e:
+        log.error(
+            f"Command failed: {command}\nreturncode: {e.returncode}\nstdout: {e.output.decode()}"
+        )
+        raise