Add ability to use Cilium CNI, Hubble, GatewayAPI

.github/workflows/ci.yml

@@ -47,7 +47,8 @@ jobs:
             playbook: converge.yml
           - distro: debian11
             playbook: converge.yml
-
+          - distro: debian11
+            playbook: cilium.yml
           - distro: debian11
             playbook: calico.yml
 

README.md

@@ -129,7 +129,7 @@ kubernetes_pod_network:
   # cidr: '192.168.0.0/16'
 ```
 
-This role currently supports `flannel` (default), `calico` or `weave` for cluster pod networking. Choose only one for your cluster; converting between them is not done automatically and could result in broken networking; if you need to switch from one to another, it should be done outside of this role.
+This role currently supports `flannel` (default), `cilium`, `calico` or `weave` for cluster pod networking. Choose only one for your cluster; converting between them is not done automatically and could result in broken networking; if you need to switch from one to another, it should be done outside of this role.
 
 ```yaml
 kubernetes_apiserver_advertise_address: ''`
@@ -141,8 +141,7 @@ Options passed to `kubeadm init` when initializing the Kubernetes control plane.
 
 ```yaml
 kubernetes_apt_release_channel: "stable"
-kubernetes_apt_keyring_file: "/etc/apt/keyrings/kubernetes-apt-keyring.asc"
-kubernetes_apt_repository: "deb [signed-by={{ kubernetes_apt_keyring_file }}] https://pkgs.k8s.io/core:/{{ kubernetes_apt_release_channel }}:/v{{ kubernetes_version }}/deb/ /"
+kubernetes_apt_repository: "https://pkgs.k8s.io/core:/{{ kubernetes_apt_release_channel }}:/v{{ kubernetes_version }}/deb/"
 ```
 
 Apt repository options for Kubernetes installation.
@@ -168,6 +167,30 @@ kubernetes_calico_manifest_file: https://projectcalico.docs.tigera.io/manifests/
 
 Calico manifest file to apply to the Kubernetes cluster (if using Calico instead of Flannel).
 
+#### Cilium
+
+```yaml
+kubernetes_cilium_hubble_client: true
+kubernetes_cilium_values:
+  kubeProxyReplacement: true
+  gatewayAPI:
+    enabled: true
+```
+
+Cilium Helm chart values can be specified under `kubernetes_cilium_values`.
+
+##### Kube Proxy Replacement
+
+[Kube Proxy Replacement](https://docs.cilium.io/en/latest/network/kubernetes/kubeproxy-free/) is supported through this method. The is a prerequisite for Gateway API Support.
+
+##### Gateway API Support
+
+[Gateway API Support](https://docs.cilium.io/en/latest/network/servicemesh/gateway-api/gateway-api/#gs-gateway-api). This will install additional CRDs to support GatewayAPI.
+
+##### Hubble Observability
+
+In addition to setting up [Setting up Hubble Observability](https://docs.cilium.io/en/stable/gettingstarted/hubble_setup/#hubble-setup), you may choose to install the hubble client with `kubernetes_cilium_hubble_client: true`
+
 ## Dependencies
 
 None.

defaults/main.yml

@@ -11,6 +11,10 @@ kubernetes_packages:
 
 kubernetes_version: '1.25'
 kubernetes_version_rhel_package: '1.25.1'
+kubernetes_cilium_version: '1.14.5'
+kubernetes_cilium_datapath: 'native'
+kubernetes_cilium_hubble_client: false
+kubernetes_cilium_values: ""
 
 kubernetes_role: control_plane
 
@@ -52,8 +56,7 @@ kubernetes_version_kubeadm: 'stable-{{ kubernetes_version }}'
 kubernetes_ignore_preflight_errors: 'all'
 
 kubernetes_apt_release_channel: "stable"
-kubernetes_apt_keyring_file: "/etc/apt/keyrings/kubernetes-apt-keyring.asc"
-kubernetes_apt_repository: "deb [signed-by={{ kubernetes_apt_keyring_file }}] https://pkgs.k8s.io/core:/{{ kubernetes_apt_release_channel }}:/v{{ kubernetes_version }}/deb/ /"
+kubernetes_apt_repository: "https://pkgs.k8s.io/core:/{{ kubernetes_apt_release_channel }}:/v{{ kubernetes_version }}/deb/"
 
 kubernetes_yum_base_url: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version }}/rpm/"
 kubernetes_yum_gpg_key: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version }}/rpm/repodata/repomd.xml.key"

molecule/default/cilium.yml

@@ -0,0 +1,74 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+
+  vars:
+    kubernetes_cilium_hubble_client: true
+    kubernetes_cilium_values:
+      envoy:
+        enabled: true
+      kubeProxyReplacement: true
+      l7Proxy: true
+      loadBalancer:
+        l7:
+          backend: envoy
+      ingressController:
+        enabled: true
+        loadbalancerMode: dedicated
+        default: true
+      hubble:
+        relay:
+          enabled: true
+        ui:
+          enabled: true
+      gatewayAPI:
+        enabled: true
+    kubernetes_pod_network:
+      cni: 'cilium'
+      cidr: 10.244.0.0/16
+
+    # Allow swap in test environments (hard to control in some envs).
+    kubernetes_config_kubelet_configuration:
+      cgroupDriver: "systemd"
+      failSwapOn: false
+      cgroupsPerQOS: true
+      enforceNodeAllocatable: ['pods']
+    containerd_config_cgroup_driver_systemd: true
+
+  pre_tasks:
+    - name: Update apt cache.
+      apt: update_cache=true cache_valid_time=600
+      when: ansible_os_family == 'Debian'
+
+    - name: Ensure test dependencies are installed (RedHat).
+      package: name=iproute state=present
+      when: ansible_os_family == 'RedHat'
+
+    - name: Ensure test dependencies are installed (Debian).
+      package: name=iproute2 state=present
+      when: ansible_os_family == 'Debian'
+
+    - name: Gather facts.
+      action: setup
+
+  roles:
+    - role: geerlingguy.containerd
+    - role: geerlingguy.kubernetes
+
+  post_tasks:
+    - name: Get cluster info.
+      command: kubectl cluster-info
+      changed_when: false
+      register: kubernetes_info
+
+    - name: Print cluster info.
+      debug: var=kubernetes_info.stdout
+
+    - name: Get all running pods.
+      command: kubectl get pods --all-namespaces
+      changed_when: false
+      register: kubernetes_pods
+
+    - name: Print list of running pods.
+      debug: var=kubernetes_pods.stdout

tasks/cilium-client-setup.yml

@@ -0,0 +1,52 @@
+---
+- name: Check if Cilium CLI has already been Installed.
+  stat:
+    path: /usr/local/bin/cilium
+  register: cilium_init_stat
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+
+- name: Install Cilium CLI
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not cilium_init_stat.stat.exists
+  block:
+    - name: Get Cilium CLI version
+      shell: curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt
+      register: cilium_cli_version
+      changed_when: false
+
+    - name: Set CLI architecture
+      set_fact:
+        cli_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}"
+
+    - name: Download Cilium CLI
+      get_url:
+        url: "https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version.stdout }}/cilium-linux-{{ cli_arch }}.tar.gz"
+        dest: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz"
+        mode: '0644'
+
+    - name: Download Cilium CLI checksum
+      get_url:
+        url: "https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version.stdout }}/cilium-linux-{{ cli_arch }}.tar.gz.sha256sum"
+        dest: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz.sha256sum"
+        mode: '0644'
+
+    - name: Verify Cilium CLI checksum
+      shell: sha256sum --check /tmp/cilium-linux-{{ cli_arch }}.tar.gz.sha256sum
+      args:
+        chdir: /tmp
+
+    - name: Extract Cilium CLI
+      unarchive:
+        src: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz"
+        dest: /usr/local/bin
+        remote_src: true
+
+    - name: Remove downloaded files
+      file:
+        path: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz{{ item }}"
+        state: absent
+      loop:
+        - ''
+        - '.sha256sum'

tasks/control-plane-aux-setup.yml

@@ -0,0 +1,33 @@
+---
+- name: Join control node to Kubernetes control plane.
+  command: "{{ kubernetes_join_command }} --control-plane --apiserver-advertise-address {{ kubernetes_apiserver_advertise_address }} --apiserver-bind-port 6443 --certificate-key {{ kubernetes_certificate }} --ignore-preflight-errors='all'"
+  register: join_ctrl_plane
+  args:
+    creates: /etc/kubernetes/kubelet.conf
+  tags: ["skip_ansible_lint"]
+
+- name: Ensure .kube directory exists.
+  file:
+    path: ~/.kube
+    state: directory
+    mode: 0755
+
+- name: Symlink the kubectl admin.conf to ~/.kube/conf.
+  file:
+    src: /etc/kubernetes/admin.conf
+    dest: ~/.kube/config
+    state: link
+    mode: 0644
+
+- name: Allow pods on control plane (if configured).
+  command: "kubectl taint nodes {{ inventory_hostname }} node-role.kubernetes.io/control-plane-"
+  register: taint_removed
+  until: >
+    ("untainted" in taint_removed.stdout) or
+    ("not found" in taint_removed.stderr)
+  changed_when: "'untainted' in taint_removed.stdout"
+  failed_when: false
+  retries: 100
+  delay: 5
+  when:
+    - kubernetes_allow_pods_on_control_plane | bool

tasks/control-plane-setup.yml

@@ -9,22 +9,29 @@
     src: "kubeadm-kubelet-config.j2"
     dest: "{{ kubernetes_kubeadm_kubelet_config_file_path }}"
 
-- name: Initialize Kubernetes control plane with kubeadm init
+
+- name: Initialize Kubernetes control plane with kubeadm init without kube-proxy
   command: >
     kubeadm init
     --config {{ kubernetes_kubeadm_kubelet_config_file_path }}
+    --skip-phases=addon/kube-proxy
     {{ kubernetes_kubeadm_init_extra_opts }}
   register: kubeadmin_init
-  when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is not defined)
+  when:
+    - not kubernetes_init_stat.stat.exists
+    - kubernetes_ignore_preflight_errors is not defined
 
-- name: Initialize Kubernetes control plane with kubeadm init and ignore_preflight_errors
+- name: Initialize Kubernetes control plane with kubeadm init without kube-proxy and ignore_preflight_errors
   command: >
     kubeadm init
     --config {{ kubernetes_kubeadm_kubelet_config_file_path }}
     --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }}
+    --skip-phases=addon/kube-proxy
     {{ kubernetes_kubeadm_init_extra_opts }}
   register: kubeadmin_init
-  when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is defined)
+  when:
+    - not kubernetes_init_stat.stat.exists
+    - kubernetes_ignore_preflight_errors is defined
 
 - name: Print the init output to screen.
   debug:
@@ -45,45 +52,63 @@
     state: link
     mode: 0644
 
-- name: Configure Flannel networking.
-  command: "kubectl apply -f {{ kubernetes_flannel_manifest_file }}"
-  register: flannel_result
-  changed_when: "'created' in flannel_result.stdout"
-  when: kubernetes_pod_network.cni == 'flannel'
-  until: flannel_result is not failed
-  retries: 12
+- name: Allow pods on control plane (if configured).
+  command: "kubectl taint nodes {{ inventory_hostname }} node-role.kubernetes.io/control-plane-"
+  register: taint_removed
+  until: >
+    ("untainted" in taint_removed.stdout) or
+    ("not found" in taint_removed.stderr)
+  changed_when: "'untainted' in taint_removed.stdout"
+  failed_when: false
+  retries: 100
   delay: 5
+  when:
+    - kubernetes_allow_pods_on_control_plane | bool
 
-- name: Configure Calico networking.
-  command: "kubectl apply -f {{ kubernetes_calico_manifest_file }}"
-  register: calico_result
-  changed_when: "'created' in calico_result.stdout"
-  when: kubernetes_pod_network.cni == 'calico'
-  until: calico_result is not failed
-  retries: 12
-  delay: 5
+- name: Write Cilium Helm values to file
+  copy:
+    dest: "/tmp/cilium_helm.yaml"
+    content: "{{ kubernetes_cilium_values | to_yaml }}"
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not kubernetes_init_stat.stat.exists
 
-- name: Get Kubernetes version for Weave installation.
-  shell: kubectl version | base64 | tr -d '\n'
-  changed_when: false
-  register: kubectl_version
-  when: kubernetes_pod_network.cni == 'weave'
-  until: kubectl_version is not failed
+- name: Taint nodes with cilium agent-not-ready
+  command: kubectl taint nodes --all node.cilium.io/agent-not-ready=true:NoExecute
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not kubernetes_init_stat.stat.exists
+  register: cilium_taint
+  changed_when: "'tainted' in cilium_taint.stdout"
+  until: cilium_taint is not failed
   retries: 12
   delay: 5
 
-- name: Configure Weave networking.
-  command: "{{ item }}"
-  with_items:
-    - "kubectl apply -f https://cloud.weave.works/k8s/net?k8s-version={{ kubectl_version.stdout_lines[0] }}"
-  register: weave_result
-  changed_when: "'created' in weave_result.stdout"
-  when: kubernetes_pod_network.cni == 'weave'
+- name: Install Prerequisite CRDs for Cilium Gateway API support.
+  register: gateway_crds
+  changed_when: "'created' in gateway_crds.stdout"
+  command: "kubectl apply -f {{ item }}"
+  loop:
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.1.0/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.1.0/config/crd/standard/gateway.networking.k8s.io_gateways.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.1.0/config/crd/experimental/gateway.networking.k8s.io_httproutes.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.1.0/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.1.0/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.1.0/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml
 
-# TODO: Check if taint exists with something like `kubectl describe nodes`
-# instead of using kubernetes_init_stat.stat.exists check.
-- name: Allow pods on control plane (if configured).
-  command: "kubectl taint nodes --all node-role.kubernetes.io/control-plane-"
+
+- name: Configure Cilium networking
+  command: >
+    /usr/local/bin/cilium install
+    --version {{ kubernetes_cilium_version }}
+    --datapath-mode {{ kubernetes_cilium_datapath }}
+    --values /tmp/cilium_helm.yaml
   when:
-    - kubernetes_allow_pods_on_control_plane | bool
+    - kubernetes_pod_network.cni == 'cilium'
     - not kubernetes_init_stat.stat.exists
+  register: cilium_result
+  changed_when: "'Error' not in cilium_result.stdout"
+  until: cilium_result is not failed
+  retries: 20
+  delay: 5
+