blah

ansible/kubernetes/inventory/group_vars/all/k3s.yml

@@ -1,13 +1,13 @@
 ---
 # renovate: datasource=github-releases depName=k3s-io/k3s
-#k3s_release_version: "v1.27.4+k3s1"
-k3s_release_version: "v1.24.8+k3s1"
+k3s_release_version: "v1.27.4+k3s1"
+#k3s_release_version: "v1.24.8+k3s1"
 k3s_install_hard_links: true
 k3s_become: true
 k3s_debug: false
 k3s_etcd_datastore: true
-k3s_registration_address: 192.168.3.30  # dev
-k3s_use_unsupported_config: true
+k3s_registration_address: 192.168.4.11  # dev
+  #k3s_use_unsupported_config: true
 #k3s_registries:
 #  mirrors:
 #    "docker.io":

ansible/kubernetes/inventory/group_vars/master/k3s.yml

@@ -15,15 +15,8 @@ k3s_server:
   cluster-cidr: "10.42.0.0/16"
   service-cidr: "10.43.0.0/16"
   etcd-expose-metrics: true
-  kubelet-arg:
-    # https://github.com/k3s-io/k3s/issues/1264
-    - "node-status-update-frequency=4s"
   kube-controller-manager-arg:
     - "bind-address=0.0.0.0"
-    # https://github.com/k3s-io/k3s/issues/1264
-    - "node-monitor-period=4s"
-    - "node-monitor-grace-period=16s"
-    - "pod-eviction-timeout=20s"
   kube-proxy-arg:
     - "metrics-bind-address=0.0.0.0"
   kube-scheduler-arg:
@@ -33,6 +26,3 @@ k3s_server:
     - "--audit-policy-file=/var/lib/rancher/k3s/audit-policy.yaml"
     - "--audit-log-maxage=2"
     - "anonymous-auth=true"
-    # https://github.com/k3s-io/k3s/issues/1264
-    - "default-not-ready-toleration-seconds=20"
-    - "default-unreachable-toleration-seconds=20"

ansible/kubernetes/inventory/hosts_dev.yml

@@ -9,13 +9,17 @@ kubernetes:
       hosts:
         k3s-cp-00:
           ansible_host: 192.168.3.30
-    worker:
-      vars:
-        ansible_user: yin
-        ansible_ssh_port: 22
-        ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
-      hosts:
-        k3s-worker-00:
+        k3s-cp-01:
           ansible_host: 192.168.3.31
-        k3s-worker-01:
+        k3s-cp-02:
           ansible_host: 192.168.3.32
+#    worker:
+#      vars:
+#        ansible_user: yin
+#        ansible_ssh_port: 22
+#        ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+#      hosts:
+#        k3s-worker-00:
+#          ansible_host: 192.168.3.30
+#        k3s-worker-01:
+#          ansible_host: 192.168.3.32

ansible/onedr0p/.envrc

@@ -0,0 +1,8 @@
+#shellcheck disable=SC2148,SC2155
+export SOPS_AGE_KEY_FILE="$(expand_path ../../age.key)"
+export VIRTUAL_ENV="$(expand_path ../../.venv)"
+export ANSIBLE_COLLECTIONS_PATH=$(expand_path ../../.venv/galaxy)
+export ANSIBLE_ROLES_PATH=$(expand_path ../../.venv/galaxy/ansible_roles)
+export ANSIBLE_VARS_ENABLED="host_group_vars,community.sops.sops"
+export ANSIBLE_INVENTORY=$(expand_path ./inventory/hosts.yaml)
+PATH_add "$(expand_path ../../.venv/bin)"

ansible/onedr0p/inventory/group_vars/all/main.yaml

@@ -0,0 +1,22 @@
+---
+# renovate: datasource=github-releases depName=k3s-io/k3s
+k3s_release_version: "v1.27.4+k3s1"
+k3s_install_hard_links: true
+k3s_become: true
+k3s_etcd_datastore: true
+k3s_registration_address: 192.168.3.30
+k3s_use_unsupported_config: true
+# /var/lib/rancher/k3s/server/manifests
+k3s_server_manifests_urls:
+  # Kube-vip RBAC
+  - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml
+    filename: kube-vip-rbac.yaml
+  # Essential Prometheus Operator CRDs (the rest are installed with the kube-prometheus-stack helm release)
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+    filename: custom-prometheus-podmonitors.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+    filename: custom-prometheus-prometheusrules.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml
+    filename: custom-prometheus-scrapeconfigs.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+    filename: custom-prometheus-servicemonitors.yaml

ansible/onedr0p/inventory/group_vars/all/supplemental.yaml

@@ -0,0 +1,3 @@
+---
+github_username: leanxia
+timezone: America/Los_Angeles

ansible/onedr0p/inventory/group_vars/master/main.yaml

@@ -0,0 +1,28 @@
+---
+k3s_control_node: true
+k3s_server:
+  node-ip: "{{ ansible_host }}"
+  tls-san:
+    - "{{ k3s_registration_address }}"
+  https-listen-port: 6443
+  docker: false
+  flannel-backend: "none" # quote
+  disable:
+    - coredns
+    - local-storage
+    - metrics-server
+    - servicelb
+    - traefik
+  disable-network-policy: true
+  disable-cloud-controller: true
+  disable-kube-proxy: true
+  cluster-cidr: 10.32.0.0/16
+  service-cidr: 10.33.0.0/16
+  write-kubeconfig-mode: "0644"
+  etcd-expose-metrics: true
+  kube-controller-manager-arg:
+    - bind-address=0.0.0.0
+  kube-scheduler-arg:
+    - bind-address=0.0.0.0
+  kube-apiserver-arg:
+    - anonymous-auth=true

ansible/onedr0p/inventory/group_vars/worker/main.yaml

@@ -0,0 +1,4 @@
+---
+k3s_control_node: false
+k3s_agent:
+  node-ip: "{{ ansible_host }}"

ansible/onedr0p/inventory/hosts_dev.yml

@@ -0,0 +1,21 @@
+---
+kubernetes:
+  children:
+    master:
+      vars:
+        ansible_user: yin
+        ansible_ssh_port: 22
+        ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+      hosts:
+        k3s-cp-00:
+          ansible_host: 192.168.3.30
+    worker:
+      vars:
+        ansible_user: yin
+        ansible_ssh_port: 22
+        ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+      hosts:
+        k3s-worker-00:
+          ansible_host: 192.168.3.31
+        k3s-worker-01:
+          ansible_host: 192.168.3.32

ansible/onedr0p/playbooks/cluster-ceph-reset.yaml

@@ -0,0 +1,39 @@
+---
+- name: Reset Ceph Drives
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 2 seconds...
+      ansible.builtin.pause:
+        seconds: 2
+  tasks:
+    - name: Reset Ceph Drives # noqa: ignore-errors
+      ignore_errors: true
+      when: ceph_drives | default([]) | length > 0
+      block:
+        - name: Delete (/var/lib/rook)
+          ansible.builtin.file:
+            state: absent
+            path: /var/lib/rook
+        - name: Delete (/dev/mapper/ceph-*) # noqa: no-changed-when
+          ansible.builtin.shell: |
+            set -o pipefail
+            ls /dev/mapper/ceph-* | xargs -I% -- dmsetup remove_all --force % || true
+        - name: Delete (/dev/ceph-*) # noqa: no-changed-when
+          ansible.builtin.command: rm -rf /dev/ceph-*
+        - name: Delete (/dev/mapper/ceph--*) # noqa: no-changed-when
+          ansible.builtin.command: rm -rf /dev/mapper/ceph--*
+        - name: Wipe (sgdisk) # noqa: no-changed-when
+          ansible.builtin.command: "sgdisk --zap-all {{ item }}"
+          loop: "{{ ceph_drives }}"
+        - name: Wipe (dd) # noqa: no-changed-when
+          ansible.builtin.command: "dd if=/dev/zero of={{ item }} bs=1M count=100 oflag=direct,dsync"
+          loop: "{{ ceph_drives }}"
+        - name: Wipe (blkdiscard) # noqa: no-changed-when
+          ansible.builtin.command: "blkdiscard {{ item }}"
+          loop: "{{ ceph_drives }}"
+        - name: Wipe (partprobe) # noqa: no-changed-when
+          ansible.builtin.command: "partprobe {{ item }}"
+          loop: "{{ ceph_drives }}"

ansible/onedr0p/playbooks/cluster-installation.yaml

@@ -0,0 +1,69 @@
+---
+- name: Cluster Installation
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 2 seconds...
+      ansible.builtin.pause:
+        seconds: 2
+  tasks:
+    - name: Check if cluster is installed
+      check_mode: false
+      ansible.builtin.stat:
+        path: /etc/rancher/k3s/config.yaml
+      register: k3s_installed
+
+    - name: Ignore manifests templates and urls if the cluster is already installed
+      when: k3s_installed.stat.exists
+      ansible.builtin.set_fact:
+        k3s_server_manifests_templates: []
+        k3s_server_manifests_urls: []
+
+    - name: Install Kubernetes
+      ansible.builtin.include_role:
+        name: xanmanning.k3s
+        public: true
+      vars:
+        k3s_state: installed
+
+    - name: Wait for custom manifests to rollout
+      when:
+        - k3s_primary_control_node
+        - (k3s_server_manifests_templates | length > 0
+            or k3s_server_manifests_urls | length > 0)
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        kind: "{{ item.kind }}"
+        name: "{{ item.name }}"
+        namespace: "{{ item.namespace | default('') }}"
+        wait: true
+        wait_sleep: 10
+        wait_timeout: 360
+      loop:
+        - { name: cilium, kind: HelmChart, namespace: kube-system }
+        - { name: coredns, kind: HelmChart, namespace: kube-system }
+        - { name: policy, kind: CiliumL2AnnouncementPolicy }
+        - { name: pool, kind: CiliumLoadBalancerIPPool }
+        - { name: podmonitors.monitoring.coreos.com, kind: CustomResourceDefinition }
+        - { name: prometheusrules.monitoring.coreos.com, kind: CustomResourceDefinition }
+        - { name: scrapeconfigs.monitoring.coreos.com, kind: CustomResourceDefinition }
+        - { name: servicemonitors.monitoring.coreos.com, kind: CustomResourceDefinition }
+
+    - name: Coredns
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/coredns.yaml
+
+    - name: Cilium
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/cilium.yaml
+
+    - name: Cruft
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/cruft.yaml
+
+    - name: Stale Containers
+      ansible.builtin.include_tasks: tasks/stale_containers.yaml
+      vars:
+        stale_containers_state: enabled

ansible/onedr0p/playbooks/cluster-nuke.yaml

@@ -0,0 +1,61 @@
+---
+- name: Cluster Nuke
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 2 seconds...
+      ansible.builtin.pause:
+        seconds: 2
+  tasks:
+    - name: Stop Kubernetes # noqa: ignore-errors
+      ignore_errors: true
+      block:
+        - name: Stop Kubernetes
+          ansible.builtin.include_role:
+            name: xanmanning.k3s
+            public: true
+          vars:
+            k3s_state: stopped
+
+    # https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md
+    - name: Networking
+      block:
+        - name: Networking | Delete Cilium links
+          ansible.builtin.command:
+            cmd: "ip link delete {{ item }}"
+            removes: "/sys/class/net/{{ item }}"
+          loop: ["cilium_host", "cilium_net", "cilium_vxlan"]
+        - name: Networking | Flush iptables
+          ansible.builtin.iptables:
+            table: "{{ item }}"
+            flush: true
+          loop: ["filter", "nat", "mangle", "raw"]
+        - name: Networking | Flush ip6tables
+          ansible.builtin.iptables:
+            table: "{{ item }}"
+            flush: true
+            ip_version: ipv6
+          loop: ["filter", "nat", "mangle", "raw"]
+        - name: Networking | Delete CNI directory
+          ansible.builtin.file:
+            path: /etc/cni/net.d
+            state: absent
+
+    - name: Uninstall Kubernetes
+      ansible.builtin.include_role:
+        name: xanmanning.k3s
+        public: true
+      vars:
+        k3s_state: uninstalled
+
+    - name: Stale Containers
+      ansible.builtin.include_tasks: tasks/stale_containers.yaml
+      vars:
+        stale_containers_state: disabled
+
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting nodes
+        reboot_timeout: 3600