[improvement]: move vpc interface to eth0 (#358)

* move vpc interface to eth0 * fix vpcless installs and remove primary: true setting for interfaces * remove stale tag from rke2 and k3s clusters --------- Co-authored-by: Rahul Sharma <[email protected]>
linode · Jun 11, 2024 · 899fd4c · 899fd4c
1 parent e4102fc
commit 899fd4c
Show file tree

Hide file tree

Showing 16 changed files with 18 additions and 77 deletions.
diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go
@@ -31,6 +31,7 @@ import (
 	"github.com/linode/linodego"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	kutil "sigs.k8s.io/cluster-api/util"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -89,15 +90,17 @@ func (r *LinodeMachineReconciler) newCreateConfig(ctx context.Context, machineSc
 		createConfig.RootPass = uuid.NewString()
 	}
 
-	// if vpc, attach additional interface to linode (eth1)
+	// if vpc, attach additional interface as eth0 to linode
 	if machineScope.LinodeCluster.Spec.VPCRef != nil {
-		iface, err := r.getVPCInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger)
+		iface, err := r.getVPCInterfaceConfig(ctx, machineScope, logger)
 		if err != nil {
 			logger.Error(err, "Failed to get VPC interface config")
 
 			return nil, err
 		}
-		createConfig.Interfaces = append(createConfig.Interfaces, *iface)
+
+		// add VPC interface as first interface
+		createConfig.Interfaces = slices.Insert(createConfig.Interfaces, 0, *iface)
 	}
 
 	return createConfig, nil
@@ -266,7 +269,7 @@ func (r *LinodeMachineReconciler) requestsForCluster(ctx context.Context, namesp
 	return result, nil
 }
 
-func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, existingIfaces []linodego.InstanceConfigInterfaceCreateOptions, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) {
+func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) {
 	name := machineScope.LinodeCluster.Spec.VPCRef.Name
 	namespace := machineScope.LinodeCluster.Spec.VPCRef.Namespace
 	if namespace == "" {
@@ -291,14 +294,6 @@ func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, mac
 		return nil, errors.New("vpc is not available")
 	}
 
-	hasPrimary := false
-	for i := range existingIfaces {
-		if existingIfaces[i].Primary {
-			hasPrimary = true
-			break
-		}
-	}
-
 	var subnetID int
 	vpc, err := machineScope.LinodeClient.GetVPC(ctx, *linodeVPC.Spec.VPCID)
 	if err != nil {
@@ -325,8 +320,11 @@ func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, mac
 
 	return &linodego.InstanceConfigInterfaceCreateOptions{
 		Purpose:  linodego.InterfacePurposeVPC,
-		Primary:  !hasPrimary,
+		Primary:  true,
 		SubnetID: &subnetID,
+		IPv4: &linodego.VPCIPv4{
+			NAT1To1: ptr.To(("any")),
+		},
 	}, nil
 }
 

diff --git a/docs/src/topics/firewalling.md b/docs/src/topics/firewalling.md
@@ -10,9 +10,11 @@ By default, the following policies are set to audit mode(without any enforcement
     | Ports     | Use-case                 | Allowed clients       |
     |-----------|--------------------------|-----------------------|
     | 6443      | API Server Traffic       | World                 |
-    | 2379-2380 | Etcd Traffic             | World                 |
     | *         | In Cluster Communication | Intra Cluster Traffic |
 
+```admonish note
+For kubeadm clusters running outside of VPC, ports 2379 and 2380 are also allowed for etcd-traffic.
+```
 
 * [k3s](./flavors/k3s.md) cluster allow rules
 

diff --git a/docs/src/topics/vpc.md b/docs/src/topics/vpc.md
@@ -4,12 +4,12 @@ This guide covers how [VPC](https://www.linode.com/docs/products/networking/vpc/
 
 ## Default configuration
 Each linode within a cluster gets provisioned with two interfaces:
-1. eth0 (for public and nodebalancer traffic)
-2. eth1 (connected to VPC, for pod-to-pod traffic)
+1. eth0 (connected to VPC, for pod-to-pod traffic and public traffic)
+2. eth1 (for nodebalancer traffic)
 
 Key facts about VPC network configuration:
 1. VPCs are provisioned with a private subnet 10.0.0.0/8.
-2. All pod-to-pod communication happens over the VPC interface (eth1).
+2. All pod-to-pod communication happens over the VPC interface (eth0).
 3. We assign a pod CIDR of range 10.192.0.0/10 for pod-to-pod communication.
 3. By default, cilium is configured with [native routing](https://docs.cilium.io/en/stable/network/concepts/routing/#native-routing)
 4. [Kubernetes host-scope IPAM mode](https://docs.cilium.io/en/stable/network/concepts/ipam/kubernetes/) is used to assign pod CIDRs to nodes. We run [linode CCM](https://github.com/linode/linode-cloud-controller-manager) with [route-controller enabled](https://github.com/linode/linode-cloud-controller-manager?tab=readme-ov-file#routes) which automatically adds/updates routes within VPC when pod cidrs are added/updated by k8s. This enables pod-to-pod traffic to be routable within the VPC.

diff --git a/templates/addons/cilium/cilium.yaml b/templates/addons/cilium/cilium.yaml
@@ -22,7 +22,6 @@ spec:
     ipv4NativeRoutingCIDR: 10.0.0.0/8
     tunnelProtocol: ""
     enableIPv4Masquerade: true
-    egressMasqueradeInterfaces: eth0
     policyAuditMode: ${FW_AUDIT_ONLY:=true}
     hostFirewall:
       enabled: true
@@ -31,7 +30,6 @@ spec:
     k8sServiceHost: {{ .InfraCluster.spec.controlPlaneEndpoint.host }}
     k8sServicePort: {{ .InfraCluster.spec.controlPlaneEndpoint.port }}
     extraArgs:
-    - --direct-routing-device=eth1
     - --nodeport-addresses=0.0.0.0/0
     ipam:
       mode: kubernetes

diff --git a/templates/flavors/k3s/default/k3sConfigTemplate.yaml b/templates/flavors/k3s/default/k3sConfigTemplate.yaml
@@ -9,9 +9,6 @@ spec:
       agentConfig:
         nodeName: '{{ ds.meta_data.label }}'
       preK3sCommands:
-        - |
-          mkdir -p /etc/rancher/k3s/config.yaml.d/
-          echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/k3s/config.yaml.d/capi-config.yaml
         - sed -i '/swap/d' /etc/fstab
         - swapoff -a
         - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname
diff --git a/templates/flavors/k3s/default/k3sControlPlane.yaml b/templates/flavors/k3s/default/k3sControlPlane.yaml
@@ -37,7 +37,6 @@ spec:
               ipv4NativeRoutingCIDR: 10.0.0.0/8
               tunnelProtocol: ""
               enableIPv4Masquerade: true
-              egressMasqueradeInterfaces: eth0
               k8sServiceHost: 10.0.0.2
               k8sServicePort: 6443
               policyAuditMode: ${FW_AUDIT_ONLY:=true}
@@ -48,7 +47,6 @@ spec:
               k8sServiceHost: 10.0.0.2
               k8sServicePort: 6443
               extraArgs:
-                - --direct-routing-device=eth1
                 - --nodeport-addresses=0.0.0.0/0
               ipam:
                 mode: kubernetes
@@ -70,9 +68,6 @@ spec:
     agentConfig:
       nodeName: '{{ ds.meta_data.label }}'
     preK3sCommands:
-      - |
-        mkdir -p /etc/rancher/k3s/config.yaml.d/
-        echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/k3s/config.yaml.d/capi-config.yaml
       - sed -i '/swap/d' /etc/fstab
       - swapoff -a
       - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname

diff --git a/templates/flavors/k3s/vpcless/kustomization.yaml b/templates/flavors/k3s/vpcless/kustomization.yaml
@@ -60,17 +60,6 @@ patches:
           - sed -i '/swap/d' /etc/fstab
           - swapoff -a
           - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname
-  - target:
-      group: cluster.x-k8s.io
-      version: v1beta1
-      kind: Cluster
-    patch: |-
-      apiVersion: cluster.x-k8s.io/v1beta1
-      kind: Cluster
-      metadata:
-        name: ${CLUSTER_NAME}
-        labels:
-            vpcless: "true"
   - target:
       group: controlplane.cluster.x-k8s.io
       version: v1beta1

diff --git a/templates/flavors/kubeadm/default/kubeadmConfigTemplate.yaml b/templates/flavors/kubeadm/default/kubeadmConfigTemplate.yaml
@@ -21,10 +21,6 @@ spec:
             apt-get update -y
             apt-get install -y kubelet=$PATCH_VERSION* kubeadm=$PATCH_VERSION* kubectl=$PATCH_VERSION* containerd
             apt-mark hold kubelet kubeadm kubectl containerd
-            if [ -d "/sys/class/net/eth1" ]; then
-                IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)
-                sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n    node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml
-            fi
           permissions: "0500"
       preKubeadmCommands:
         - /kubeadm-pre-init.sh ${KUBERNETES_VERSION}

diff --git a/templates/flavors/kubeadm/default/kubeadmControlPlane.yaml b/templates/flavors/kubeadm/default/kubeadmControlPlane.yaml
@@ -26,10 +26,6 @@ spec:
           apt-get update -y
           apt-get install -y kubelet=$PATCH_VERSION* kubeadm=$PATCH_VERSION* kubectl=$PATCH_VERSION* containerd
           apt-mark hold kubelet kubeadm kubectl containerd
-          if [ -d "/sys/class/net/eth1" ]; then
-              IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)
-              sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n    node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml
-          fi
         permissions: "0500"
     preKubeadmCommands:
       - /kubeadm-pre-init.sh ${KUBERNETES_VERSION}

diff --git a/templates/flavors/kubeadm/default/kustomization.yaml b/templates/flavors/kubeadm/default/kustomization.yaml
@@ -4,7 +4,6 @@ resources:
   - ../../../infra
   - kubeadmConfigTemplate.yaml
   - kubeadmControlPlane.yaml
-  - allow-etcd-policy.yaml
   - ../../../addons/cilium
   - ../../../addons/cilium-network-policies
   - ../../../addons/csi-driver-linode

diff --git a/...rs/kubeadm/default/allow-etcd-policy.yaml → ...rs/kubeadm/vpcless/allow-etcd-policy.yaml b/...rs/kubeadm/default/allow-etcd-policy.yaml → ...rs/kubeadm/vpcless/allow-etcd-policy.yaml
diff --git a/templates/flavors/kubeadm/vpcless/kustomization.yaml b/templates/flavors/kubeadm/vpcless/kustomization.yaml
@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../default
+  - allow-etcd-policy.yaml
 
 patches:
   - target:
@@ -63,14 +64,3 @@ patches:
     patch: |-
       - op: remove
         path: /spec/kubeadmConfigSpec/initConfiguration/skipPhases
-  - target:
-      group: cluster.x-k8s.io
-      version: v1beta1
-      kind: Cluster
-    patch: |-
-      apiVersion: cluster.x-k8s.io/v1beta1
-      kind: Cluster
-      metadata:
-        name: ${CLUSTER_NAME}
-        labels:
-            vpcless: "true"
diff --git a/templates/flavors/rke2/default/rke2ConfigTemplate.yaml b/templates/flavors/rke2/default/rke2ConfigTemplate.yaml
@@ -16,9 +16,6 @@ spec:
             - "provider-id=linode://{{ ds.meta_data.id }}"
       # TODO: use MDS to get private IP instead
       preRKE2Commands:
-        - |
-          mkdir -p /etc/rancher/rke2/config.yaml.d/
-          echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/rke2/config.yaml.d/capi-config.yaml
         - sed -i '/swap/d' /etc/fstab
         - swapoff -a
         - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname
diff --git a/templates/flavors/rke2/default/rke2ControlPlane.yaml b/templates/flavors/rke2/default/rke2ControlPlane.yaml
@@ -24,9 +24,6 @@ spec:
     cisProfile: ${CIS_PROFILE:-"cis-1.23"}
     protectKernelDefaults: true
   preRKE2Commands:
-    - |
-      mkdir -p /etc/rancher/rke2/config.yaml.d/
-      echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/rke2/config.yaml.d/capi-config.yaml
     - sed -i '/swap/d' /etc/fstab
     - swapoff -a
     - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname

diff --git a/templates/flavors/rke2/vpcless/kustomization.yaml b/templates/flavors/rke2/vpcless/kustomization.yaml
@@ -90,14 +90,3 @@ patches:
           - sed -i '/swap/d' /etc/fstab
           - swapoff -a
           - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname
-  - target:
-      group: cluster.x-k8s.io
-      version: v1beta1
-      kind: Cluster
-    patch: |-
-      apiVersion: cluster.x-k8s.io/v1beta1
-      kind: Cluster
-      metadata:
-        name: ${CLUSTER_NAME}
-        labels:
-            vpcless: "true"
diff --git a/templates/infra/linodeMachineTemplate.yaml b/templates/infra/linodeMachineTemplate.yaml
@@ -11,7 +11,6 @@ spec:
       region: ${LINODE_REGION}
       interfaces:
         - purpose: public
-          primary: true
       authorizedKeys:
       # uncomment to include your ssh key in linode provisioning
       # - ${LINODE_SSH_PUBKEY}
@@ -28,7 +27,6 @@ spec:
       region: ${LINODE_REGION}
       interfaces:
         - purpose: public
-          primary: true
       authorizedKeys:
       # uncomment to include your ssh key in linode provisioning
       # - ${LINODE_SSH_PUBKEY}