From 899fd4c99f5ffee4f67c6c928f615518b5e50caf Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Tue, 11 Jun 2024 12:09:09 -0400 Subject: [PATCH] [improvement]: move vpc interface to eth0 (#358) * move vpc interface to eth0 * fix vpcless installs and remove primary: true setting for interfaces * remove stale tag from rke2 and k3s clusters --------- Co-authored-by: Rahul Sharma --- .../linodemachine_controller_helpers.go | 24 +++++++++---------- docs/src/topics/firewalling.md | 4 +++- docs/src/topics/vpc.md | 6 ++--- templates/addons/cilium/cilium.yaml | 2 -- .../k3s/default/k3sConfigTemplate.yaml | 3 --- .../flavors/k3s/default/k3sControlPlane.yaml | 5 ---- .../flavors/k3s/vpcless/kustomization.yaml | 11 --------- .../default/kubeadmConfigTemplate.yaml | 4 ---- .../kubeadm/default/kubeadmControlPlane.yaml | 4 ---- .../kubeadm/default/kustomization.yaml | 1 - .../allow-etcd-policy.yaml | 0 .../kubeadm/vpcless/kustomization.yaml | 12 +--------- .../rke2/default/rke2ConfigTemplate.yaml | 3 --- .../rke2/default/rke2ControlPlane.yaml | 3 --- .../flavors/rke2/vpcless/kustomization.yaml | 11 --------- templates/infra/linodeMachineTemplate.yaml | 2 -- 16 files changed, 18 insertions(+), 77 deletions(-) rename templates/flavors/kubeadm/{default => vpcless}/allow-etcd-policy.yaml (100%) diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index 1b3a8c35..e77b8400 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -31,6 +31,7 @@ import ( "github.com/linode/linodego" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" kutil "sigs.k8s.io/cluster-api/util" ctrl "sigs.k8s.io/controller-runtime" @@ -89,15 +90,17 @@ func (r *LinodeMachineReconciler) newCreateConfig(ctx context.Context, machineSc createConfig.RootPass = uuid.NewString() } - // if vpc, attach additional interface to linode (eth1) + // if vpc, attach additional interface as eth0 to linode if machineScope.LinodeCluster.Spec.VPCRef != nil { - iface, err := r.getVPCInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger) + iface, err := r.getVPCInterfaceConfig(ctx, machineScope, logger) if err != nil { logger.Error(err, "Failed to get VPC interface config") return nil, err } - createConfig.Interfaces = append(createConfig.Interfaces, *iface) + + // add VPC interface as first interface + createConfig.Interfaces = slices.Insert(createConfig.Interfaces, 0, *iface) } return createConfig, nil @@ -266,7 +269,7 @@ func (r *LinodeMachineReconciler) requestsForCluster(ctx context.Context, namesp return result, nil } -func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, existingIfaces []linodego.InstanceConfigInterfaceCreateOptions, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) { +func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) { name := machineScope.LinodeCluster.Spec.VPCRef.Name namespace := machineScope.LinodeCluster.Spec.VPCRef.Namespace if namespace == "" { @@ -291,14 +294,6 @@ func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, mac return nil, errors.New("vpc is not available") } - hasPrimary := false - for i := range existingIfaces { - if existingIfaces[i].Primary { - hasPrimary = true - break - } - } - var subnetID int vpc, err := machineScope.LinodeClient.GetVPC(ctx, *linodeVPC.Spec.VPCID) if err != nil { @@ -325,8 +320,11 @@ func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, mac return &linodego.InstanceConfigInterfaceCreateOptions{ Purpose: linodego.InterfacePurposeVPC, - Primary: !hasPrimary, + Primary: true, SubnetID: &subnetID, + IPv4: &linodego.VPCIPv4{ + NAT1To1: ptr.To(("any")), + }, }, nil } diff --git a/docs/src/topics/firewalling.md b/docs/src/topics/firewalling.md index abd92d76..4922f01e 100644 --- a/docs/src/topics/firewalling.md +++ b/docs/src/topics/firewalling.md @@ -10,9 +10,11 @@ By default, the following policies are set to audit mode(without any enforcement | Ports | Use-case | Allowed clients | |-----------|--------------------------|-----------------------| | 6443 | API Server Traffic | World | - | 2379-2380 | Etcd Traffic | World | | * | In Cluster Communication | Intra Cluster Traffic | +```admonish note +For kubeadm clusters running outside of VPC, ports 2379 and 2380 are also allowed for etcd-traffic. +``` * [k3s](./flavors/k3s.md) cluster allow rules diff --git a/docs/src/topics/vpc.md b/docs/src/topics/vpc.md index ca5f55e0..119952c2 100644 --- a/docs/src/topics/vpc.md +++ b/docs/src/topics/vpc.md @@ -4,12 +4,12 @@ This guide covers how [VPC](https://www.linode.com/docs/products/networking/vpc/ ## Default configuration Each linode within a cluster gets provisioned with two interfaces: -1. eth0 (for public and nodebalancer traffic) -2. eth1 (connected to VPC, for pod-to-pod traffic) +1. eth0 (connected to VPC, for pod-to-pod traffic and public traffic) +2. eth1 (for nodebalancer traffic) Key facts about VPC network configuration: 1. VPCs are provisioned with a private subnet 10.0.0.0/8. -2. All pod-to-pod communication happens over the VPC interface (eth1). +2. All pod-to-pod communication happens over the VPC interface (eth0). 3. We assign a pod CIDR of range 10.192.0.0/10 for pod-to-pod communication. 3. By default, cilium is configured with [native routing](https://docs.cilium.io/en/stable/network/concepts/routing/#native-routing) 4. [Kubernetes host-scope IPAM mode](https://docs.cilium.io/en/stable/network/concepts/ipam/kubernetes/) is used to assign pod CIDRs to nodes. We run [linode CCM](https://github.com/linode/linode-cloud-controller-manager) with [route-controller enabled](https://github.com/linode/linode-cloud-controller-manager?tab=readme-ov-file#routes) which automatically adds/updates routes within VPC when pod cidrs are added/updated by k8s. This enables pod-to-pod traffic to be routable within the VPC. diff --git a/templates/addons/cilium/cilium.yaml b/templates/addons/cilium/cilium.yaml index 0569c9a3..29c084cd 100644 --- a/templates/addons/cilium/cilium.yaml +++ b/templates/addons/cilium/cilium.yaml @@ -22,7 +22,6 @@ spec: ipv4NativeRoutingCIDR: 10.0.0.0/8 tunnelProtocol: "" enableIPv4Masquerade: true - egressMasqueradeInterfaces: eth0 policyAuditMode: ${FW_AUDIT_ONLY:=true} hostFirewall: enabled: true @@ -31,7 +30,6 @@ spec: k8sServiceHost: {{ .InfraCluster.spec.controlPlaneEndpoint.host }} k8sServicePort: {{ .InfraCluster.spec.controlPlaneEndpoint.port }} extraArgs: - - --direct-routing-device=eth1 - --nodeport-addresses=0.0.0.0/0 ipam: mode: kubernetes diff --git a/templates/flavors/k3s/default/k3sConfigTemplate.yaml b/templates/flavors/k3s/default/k3sConfigTemplate.yaml index bbafe3fe..d17fe247 100644 --- a/templates/flavors/k3s/default/k3sConfigTemplate.yaml +++ b/templates/flavors/k3s/default/k3sConfigTemplate.yaml @@ -9,9 +9,6 @@ spec: agentConfig: nodeName: '{{ ds.meta_data.label }}' preK3sCommands: - - | - mkdir -p /etc/rancher/k3s/config.yaml.d/ - echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/k3s/config.yaml.d/capi-config.yaml - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname diff --git a/templates/flavors/k3s/default/k3sControlPlane.yaml b/templates/flavors/k3s/default/k3sControlPlane.yaml index d4d82d17..c4aa4207 100644 --- a/templates/flavors/k3s/default/k3sControlPlane.yaml +++ b/templates/flavors/k3s/default/k3sControlPlane.yaml @@ -37,7 +37,6 @@ spec: ipv4NativeRoutingCIDR: 10.0.0.0/8 tunnelProtocol: "" enableIPv4Masquerade: true - egressMasqueradeInterfaces: eth0 k8sServiceHost: 10.0.0.2 k8sServicePort: 6443 policyAuditMode: ${FW_AUDIT_ONLY:=true} @@ -48,7 +47,6 @@ spec: k8sServiceHost: 10.0.0.2 k8sServicePort: 6443 extraArgs: - - --direct-routing-device=eth1 - --nodeport-addresses=0.0.0.0/0 ipam: mode: kubernetes @@ -70,9 +68,6 @@ spec: agentConfig: nodeName: '{{ ds.meta_data.label }}' preK3sCommands: - - | - mkdir -p /etc/rancher/k3s/config.yaml.d/ - echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/k3s/config.yaml.d/capi-config.yaml - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname diff --git a/templates/flavors/k3s/vpcless/kustomization.yaml b/templates/flavors/k3s/vpcless/kustomization.yaml index 196e5351..971e8cf5 100644 --- a/templates/flavors/k3s/vpcless/kustomization.yaml +++ b/templates/flavors/k3s/vpcless/kustomization.yaml @@ -60,17 +60,6 @@ patches: - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname - - target: - group: cluster.x-k8s.io - version: v1beta1 - kind: Cluster - patch: |- - apiVersion: cluster.x-k8s.io/v1beta1 - kind: Cluster - metadata: - name: ${CLUSTER_NAME} - labels: - vpcless: "true" - target: group: controlplane.cluster.x-k8s.io version: v1beta1 diff --git a/templates/flavors/kubeadm/default/kubeadmConfigTemplate.yaml b/templates/flavors/kubeadm/default/kubeadmConfigTemplate.yaml index 6c9c124b..33515699 100644 --- a/templates/flavors/kubeadm/default/kubeadmConfigTemplate.yaml +++ b/templates/flavors/kubeadm/default/kubeadmConfigTemplate.yaml @@ -21,10 +21,6 @@ spec: apt-get update -y apt-get install -y kubelet=$PATCH_VERSION* kubeadm=$PATCH_VERSION* kubectl=$PATCH_VERSION* containerd apt-mark hold kubelet kubeadm kubectl containerd - if [ -d "/sys/class/net/eth1" ]; then - IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1) - sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml - fi permissions: "0500" preKubeadmCommands: - /kubeadm-pre-init.sh ${KUBERNETES_VERSION} diff --git a/templates/flavors/kubeadm/default/kubeadmControlPlane.yaml b/templates/flavors/kubeadm/default/kubeadmControlPlane.yaml index e53f8593..08d4bda3 100644 --- a/templates/flavors/kubeadm/default/kubeadmControlPlane.yaml +++ b/templates/flavors/kubeadm/default/kubeadmControlPlane.yaml @@ -26,10 +26,6 @@ spec: apt-get update -y apt-get install -y kubelet=$PATCH_VERSION* kubeadm=$PATCH_VERSION* kubectl=$PATCH_VERSION* containerd apt-mark hold kubelet kubeadm kubectl containerd - if [ -d "/sys/class/net/eth1" ]; then - IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1) - sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml - fi permissions: "0500" preKubeadmCommands: - /kubeadm-pre-init.sh ${KUBERNETES_VERSION} diff --git a/templates/flavors/kubeadm/default/kustomization.yaml b/templates/flavors/kubeadm/default/kustomization.yaml index fe201802..aca59e67 100644 --- a/templates/flavors/kubeadm/default/kustomization.yaml +++ b/templates/flavors/kubeadm/default/kustomization.yaml @@ -4,7 +4,6 @@ resources: - ../../../infra - kubeadmConfigTemplate.yaml - kubeadmControlPlane.yaml - - allow-etcd-policy.yaml - ../../../addons/cilium - ../../../addons/cilium-network-policies - ../../../addons/csi-driver-linode diff --git a/templates/flavors/kubeadm/default/allow-etcd-policy.yaml b/templates/flavors/kubeadm/vpcless/allow-etcd-policy.yaml similarity index 100% rename from templates/flavors/kubeadm/default/allow-etcd-policy.yaml rename to templates/flavors/kubeadm/vpcless/allow-etcd-policy.yaml diff --git a/templates/flavors/kubeadm/vpcless/kustomization.yaml b/templates/flavors/kubeadm/vpcless/kustomization.yaml index 22907fc7..66bbda9b 100644 --- a/templates/flavors/kubeadm/vpcless/kustomization.yaml +++ b/templates/flavors/kubeadm/vpcless/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../default + - allow-etcd-policy.yaml patches: - target: @@ -63,14 +64,3 @@ patches: patch: |- - op: remove path: /spec/kubeadmConfigSpec/initConfiguration/skipPhases - - target: - group: cluster.x-k8s.io - version: v1beta1 - kind: Cluster - patch: |- - apiVersion: cluster.x-k8s.io/v1beta1 - kind: Cluster - metadata: - name: ${CLUSTER_NAME} - labels: - vpcless: "true" diff --git a/templates/flavors/rke2/default/rke2ConfigTemplate.yaml b/templates/flavors/rke2/default/rke2ConfigTemplate.yaml index fe9f3d69..cadf22b1 100644 --- a/templates/flavors/rke2/default/rke2ConfigTemplate.yaml +++ b/templates/flavors/rke2/default/rke2ConfigTemplate.yaml @@ -16,9 +16,6 @@ spec: - "provider-id=linode://{{ ds.meta_data.id }}" # TODO: use MDS to get private IP instead preRKE2Commands: - - | - mkdir -p /etc/rancher/rke2/config.yaml.d/ - echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/rke2/config.yaml.d/capi-config.yaml - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname diff --git a/templates/flavors/rke2/default/rke2ControlPlane.yaml b/templates/flavors/rke2/default/rke2ControlPlane.yaml index 7730885f..9f9ded4f 100644 --- a/templates/flavors/rke2/default/rke2ControlPlane.yaml +++ b/templates/flavors/rke2/default/rke2ControlPlane.yaml @@ -24,9 +24,6 @@ spec: cisProfile: ${CIS_PROFILE:-"cis-1.23"} protectKernelDefaults: true preRKE2Commands: - - | - mkdir -p /etc/rancher/rke2/config.yaml.d/ - echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/rke2/config.yaml.d/capi-config.yaml - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname diff --git a/templates/flavors/rke2/vpcless/kustomization.yaml b/templates/flavors/rke2/vpcless/kustomization.yaml index a228e3c6..df5a3adf 100644 --- a/templates/flavors/rke2/vpcless/kustomization.yaml +++ b/templates/flavors/rke2/vpcless/kustomization.yaml @@ -90,14 +90,3 @@ patches: - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname - - target: - group: cluster.x-k8s.io - version: v1beta1 - kind: Cluster - patch: |- - apiVersion: cluster.x-k8s.io/v1beta1 - kind: Cluster - metadata: - name: ${CLUSTER_NAME} - labels: - vpcless: "true" diff --git a/templates/infra/linodeMachineTemplate.yaml b/templates/infra/linodeMachineTemplate.yaml index 27a3c257..347cbab5 100644 --- a/templates/infra/linodeMachineTemplate.yaml +++ b/templates/infra/linodeMachineTemplate.yaml @@ -11,7 +11,6 @@ spec: region: ${LINODE_REGION} interfaces: - purpose: public - primary: true authorizedKeys: # uncomment to include your ssh key in linode provisioning # - ${LINODE_SSH_PUBKEY} @@ -28,7 +27,6 @@ spec: region: ${LINODE_REGION} interfaces: - purpose: public - primary: true authorizedKeys: # uncomment to include your ssh key in linode provisioning # - ${LINODE_SSH_PUBKEY}