feat(ci): local custer WIP

.config/dictionaries/project.dic

@@ -165,6 +165,7 @@ toastify
 Toastify
 tojunit
 Traceback
+traefik
 TXNZD
 Typer
 unmanaged
@@ -189,4 +190,4 @@ xcodeproj
 xctest
 xctestrun
 xcworkspace
-yoroi
+yoroi

.vscode/extensions.json

@@ -18,6 +18,7 @@
         "anweiss.cddl-languageserver",
         "tintinweb.graphviz-interactive-preview",
         "terrastruct.d2",
-        "bbenoist.vagrant"
+        "bbenoist.vagrant",
+        "ms-kubernetes-tools.vscode-kubernetes-tools"
     ]
 }

.vscode/settings.recommended.json

@@ -86,5 +86,8 @@
   "rust-analyzer.rustfmt.extraArgs": [
     "+nightly"
   ],
-  "rust-analyzer.lens.enable": true
+  "rust-analyzer.lens.enable": true,
+  "vs-kubernetes": {
+    "vs-kubernetes.kubeconfig": "utilities/local-cluster/shared/k3s.yaml"
+  }
 }

docs/Earthfile

@@ -28,13 +28,23 @@ docs:
 
     DO docs-ci+BUILD
 
-# Make a locally runable container that can serve the docs.
+# local : Make a locally runable container that can serve the docs.
+#   --push will cause this image to be pushed to the local development cluster
+#   --local_registry : Set this to the hostname of your local registry or "" to inhibit.
 local:
+    # Change this to point to another local registry, OR set to "" to disable
+    # pushing to a local registry even if `--push` is passed.
+    ARG local_registry="192.168.58.10:5000"
     # Build a self contained service to show built docs locally.
     DO docs-ci+PACKAGE
 
     # Copy the static pages into the container
     COPY +docs/ /usr/share/nginx/html
 
     # This is a local only image, we do not publish it.
-    SAVE IMAGE cat-voices-docs:latest
+    SAVE IMAGE cat-voices-docs:latest
+
+    # Publish to the local development cluster 
+    IF [ "$local_registry" != "" ]
+        SAVE IMAGE --push --insecure $local_registry/cat-voices-docs:latest
+    END

utilities/local-cluster/.gitignore

@@ -1,4 +1,6 @@
 shared/*
 .vagrant
 
-!shared/.do_not_remove
+# These are shared configs, need to be checked in to git.
+!shared/k3s
+!shared/extra.hosts

utilities/local-cluster/Earthfile

@@ -4,7 +4,15 @@ VERSION 0.8
 # Needs both Vagrant and Virtualbox installed on the local machine.
 start-cluster:
     LOCALLY
+    # Check if the necessary tools are present locally
+    RUN kubectl version --client=true
+    RUN helm version --short
+    # Check if the necessary local DNS entries exist
+    RUN python scripts/check-cluster-dns.py ./shared/extra.hosts
+    # Everything checks out so far, try and start the cluster.
     RUN vagrant up
+    # Add default services
+    # Add grafana/loki
 
 # stop-cluster : stops the locally running cluster
 stop-cluster:
@@ -16,15 +24,39 @@ test-cluster:
     LOCALLY
     RUN kubectl --kubeconfig shared/k3s.yaml -o wide get nodes
 
-# test-inside-earthly : Checks if we can see the cluster from INSIDE Earthly and not locally.
-test-inside-earthly:
+# kubernetes-base : base container with tooling set up for local access
+kubernetes-base:
     FROM alpine:3.19
 
     # Install kubectl
     RUN apk update && \
         apk upgrade && \
-        apk add kubectl
+        apk add kubectl \
+                helm
 
-    COPY shared shared
+    COPY shared/k3s.yaml $HOME/.kube/config
 
-    RUN kubectl --kubeconfig shared/k3s.yaml -o wide get nodes
+    COPY --dir manifests manifests
+
+# test targets to deploy the local documentation
+deploy-docs:
+    FROM +kubernetes-base
+
+    RUN kubectl apply -f manifests/docs.yml
+
+# test target to stop the local documentation
+stop-docs:
+    FROM +kubernetes-base
+
+    RUN kubectl delete deployment cat-gateway-docs
+    RUN kubectl delete service cat-gateway-docs
+
+# show-info : list important info about the running cluster
+show-info:
+    FROM +kubernetes-base
+
+    # RUN kubectl --kubeconfig shared/k3s.yaml -o wide get nodes
+    RUN --no-cache kubectl -n kube-system get all
+    RUN --no-cache kubectl -o wide get nodes
+    RUN --no-cache kubectl -o wide get pods
+    RUN --no-cache kubectl -o wide get service

utilities/local-cluster/Readme.md

@@ -5,6 +5,8 @@ To eliminate variability and simplify local deployment, we have standardized the
 
 * [VirtualBox](https://www.virtualbox.org/)
 * [Vagrant](https://developer.hashicorp.com/vagrant/install?product_intent=vagrant)
+* [kubectl](https://kubernetes.io/docs/tasks/tools/)
+* [helm](https://helm.sh/docs/intro/install/)
 
 These tools allow us to define VMs that are consistent and provide a uniform Kubernetes environment
 for local testing.
@@ -16,14 +18,22 @@ The Cluster is based on [K3s](https://k3s.io/), which is a lightweight version o
 We set up 1 server node which hosts the control-plane and is the master node.
 Configured with:
 
-* 2 Cores
-* 2 GB of RAM
+* 4 Cores
+* 5 GB of RAM
 
 It has 3 Internal nodes, configured as:
 
 * 4 Cores
 * 4 GB of RAM
 
+## Default Services
+
+### Container Registry
+
+### Traefik
+
+### Grafana/Prometheus/Loki
+
 ## Interacting with the cluster
 
 ### Start the cluster
@@ -49,3 +59,30 @@ earthly +test-cluster
 ```sh
 earthly +test-inside-earthly
 ```
+
+## Debugging the cluster
+
+### SSH into a running VM
+
+To SSH into a VM running the cluster, use `vagrant`:
+
+```sh
+vagrant ssh server
+```
+
+```sh
+vagrant ssh agent1
+```
+
+etc.
+
+### Debug the local registry
+
+The local registry is running inside docker, on the `server` VM.
+First ssh into the VM.
+
+To display the registry service logs:
+
+```sh
+docker logs registry
+```

utilities/local-cluster/Vagrantfile

@@ -5,11 +5,10 @@ server_ip = "192.168.58.10"
 
 # Determine the maximum number of agents, and set their IP addresses
 agents = { "agent1" => "192.168.58.11",
-           "agent2" => "192.168.58.12",
-           "agent3" => "192.168.58.13" }
+           "agent2" => "192.168.58.12" }
 
-server_vcpu = "2" # Number of vCPUs in the VM
-server_memory = "2048" # 2G of Memory
+server_vcpu = "4" # Number of vCPUs in the VM
+server_memory = "51292" # 5G of Memory (An extra gig because its the server node)
 
 agent_vcpu = "4" # Number of vCPUs in the VM
 agent_memory = "4096" # 4G of Memory
@@ -20,32 +19,61 @@ is_darwin_arm64 = Vagrant::Util::Platform.architecture == "arm64" && Vagrant::Ut
 # K3s picking up the wrong interface when starting server and agent
 # https://github.com/alexellis/k3sup/issues/306
 
+static_ips = <<-SHELL
+    sudo -i
+    cat /vagrant_shared/extra.hosts >> /etc/hosts
+    SHELL
+
 server_script = <<-SHELL
     sudo -i
-    apk add curl
+    # Read preconfiguration from shared directory and install it
+    mkdir -p /etc/rancher/k3s/
+    mkdir -p /var/lib/rancher/k3s/server/manifests/
+    cp /vagrant_shared/k3s/server/manifests/* /var/lib/rancher/k3s/server/manifests/
+    cp /vagrant_shared/k3s/registries.yaml /etc/rancher/k3s/
+    # Install and start k3s server node
     export INSTALL_K3S_EXEC="--bind-address=#{server_ip} --node-external-ip=#{server_ip} --flannel-iface=eth1"
     curl -sfL https://get.k3s.io | sh -
     echo "Sleeping for 5 seconds to wait for k3s to start"
     sleep 5
-    cp /var/lib/rancher/k3s/server/token /vagrant_shared
-    cp /etc/rancher/k3s/k3s.yaml /vagrant_shared
+    # Export generated config into shared directory so we can use it outside the cluster.
+    cp /var/lib/rancher/k3s/server/token /vagrant_shared/
+    cp /etc/rancher/k3s/k3s.yaml /vagrant_shared/
     SHELL
 
 agent_script = <<-SHELL
     sudo -i
-    apk add curl
+    # Read preconfiguration from shared directory and install it
+    mkdir -p /etc/rancher/k3s/
+    cp /vagrant_shared/k3s/registries.yaml /etc/rancher/k3s/
+    # Install and start k3s agent/worker node
     export K3S_TOKEN_FILE=/vagrant_shared/token
     export K3S_URL=https://#{server_ip}:6443
     export INSTALL_K3S_EXEC="--flannel-iface=eth1"
     curl -sfL https://get.k3s.io | sh -
     SHELL
 
+registry_script = <<-SHELL
+    sudo -i
+    # Start local docker registry - running on the server VM
+    # Keeps files in the persistent shared directory
+    docker run -d -p 5000:5000 \
+      --restart=always \
+      -v /vagrant_shared/registry:/var/lib/registry \
+      -e REGISTRY_HTTP_HEADERS_Access-Control-Allow-Origin='[*]' \
+      -e REGISTRY_HTTP_HEADERS_Access-Control-Allow-Headers='[Authorization,Accept,Cache-Control]' \
+      -e REGISTRY_HTTP_HEADERS_Access-Control-Expose-Headers='[Docker-Content-Digest]' \
+      -e REGISTRY_STORAGE_DELETE_ENABLED='true' \
+      --name registry registry:2
+    SHELL
+
 Vagrant.configure("2") do |config|
-  config.vm.box = "generic/alpine319"
+  config.vm.box = "generic/debian12"
 
   config.vm.define "server", primary: true do |server|
     server.vm.network "private_network", ip: server_ip
     server.vm.hostname = "server"
+    server.vm.network "forwarded_port", guest: 5000, host: 5000
 
     if !is_darwin_arm64
       # x86 anything should work with this
@@ -62,6 +90,9 @@ Vagrant.configure("2") do |config|
         qe.smp = server_vcpu
       end
     end
+    server.vm.provision :docker
+    server.vm.provision "shell", inline: static_ips
+    server.vm.provision "shell", inline: registry_script
     server.vm.provision "shell", inline: server_script
   end
 
@@ -82,6 +113,7 @@ Vagrant.configure("2") do |config|
           qe.smp = agent_vcpu
         end
       end
+      agent.vm.provision "shell", inline: static_ips
       agent.vm.provision "shell", inline: agent_script
     end
   end

utilities/local-cluster/manifests/cat-voices-docs.yml

@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cat-gateway-docs
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: docs-nginx
+  template:
+    metadata:
+      labels:
+        app: docs-nginx
+    spec:
+      containers:
+        - name: docs-nginx
+          image: localhost/cat-voices-docs:latest
+          ports:
+            - containerPort: 80
+              name: http
+              protocol: TCP
+          resources:
+            limits:
+              cpu: "1"
+              memory: 500Mi
+            requests:
+              cpu: "0.1"
+              memory: 100Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: cat-gateway-docs
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 8080
+      targetPort: 80
+  selector:
+    app: cat-gateway-docs

utilities/local-cluster/scripts/check-cluster-dns.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import socket
+import sys
+
+
+def parse_extra_hostnames(hostnames):
+    """
+    This function takes a file of hostnames and returns a list of tuples containing the IP address and hostname for each entry.
+    The input should be a file with one hostname per line, formatted as "IP_Address Hostname".
+    The function will ignore any lines that start with "#" and any extra hostnames on the same line.
+    """
+    with open(hostnames) as f:
+        lines = f.readlines()
+        lines = [line.strip() for line in lines]
+        lines = [line for line in lines if line != "" and not line.startswith("#")]
+
+    result = []
+    for line in lines:
+        split = line.split()
+        result.append((split[0], split[1]))
+    return result
+
+
+def status(hostname, resolved_ip, ok):
+    print(f"{hostname:<25} : {resolved_ip:>15} : {ok}")
+
+
+def check_hostname(hostname, ip):
+    # Check the hostame resolves to the expected IP
+    ok = "OK"
+    try:
+        resolved_ip = socket.gethostbyname(hostname)
+        if resolved_ip != ip:
+            ok = f"FAIL. Set `{ip:<15} {hostname:<25}` in /etc/hosts."
+    except socket.error as e:
+        resolved_ip = "None"
+        ok = f"FAIL. Add `{ip:<15} {hostname:<25}` to /etc/hosts"
+
+    status(hostname, resolved_ip, ok)
+
+    return ok == "OK"
+
+
+everything_ok = True
+
+extra_hosts = parse_extra_hostnames(sys.argv[1])
+
+status("HOSTNAME", "RESOLVED IP", "STATUS")
+status("-------------------------", "---------------", "------")
+
+for host in extra_hosts:
+    everything_ok = check_hostname(host[1], host[0]) and everything_ok
+
+if not everything_ok:
+    sys.exit(1)
+sys.exit(0)