fix: Wait until istio cni files appear before copying them (#1262)

* fix: Wait until istio cni files appear before copying them Signed-off-by: aerosouund <[email protected]> * fix: Use CNAO v0.94 Previous version of CNAO introduced a problem with passt binding Signed-off-by: aerosouund <[email protected]> * fix: Change CNAO custom resource to not have multus in case multus is enabled This logic is controlled by the cluster-up variable KUBEVIRT_WITH_MULTUS_V3. If this variable is passed then it will get parsed as a gocli flag, which then gets added to the node config using the nodesconfig.WithMultus(deployMultus) line in run.go In this case multus is omitted from the CNAO CR and deployed using its actual manifests Signed-off-by: aerosouund <[email protected]> * fix: Remove CNAO patch from istio operator This is the same configuration that existed prior to #1217 getting merged. Ths addition of this lead to troubles with istio passt binding, as passt wasn't able to bind to port 22 due to insufficient permissions Signed-off-by: aerosouund <[email protected]> * test: Create test cases to assert CNAO is creating the addons config with and without Multus properly when passing the multusEnabled argument Signed-off-by: aerosouund <[email protected]> --------- Signed-off-by: aerosouund <[email protected]>
kubevirt · Sep 9, 2024 · 88a4f6b · 88a4f6b
1 parent 4cc1018
commit 88a4f6b
Show file tree

Hide file tree

Showing 9 changed files with 139 additions and 65 deletions.
diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go
@@ -812,7 +812,7 @@ func provisionK8sOptions(sshClient libssh.Client, k8sClient k8s.K8sDynamicClient
 	}
 
 	if n.CNAO {
-		cnaoOpt := cnao.NewCnaoOpt(k8sClient, sshClient)
+		cnaoOpt := cnao.NewCnaoOpt(k8sClient, sshClient, n.Multus)
 		opts = append(opts, cnaoOpt)
 	}
 

diff --git a/cluster-provision/gocli/opts/cnao/cnao.go b/cluster-provision/gocli/opts/cnao/cnao.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/fs"
 	"path/filepath"
+	"regexp"
 
 	"github.com/sirupsen/logrus"
 	k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s"
@@ -16,14 +17,16 @@ import (
 var f embed.FS
 
 type cnaoOpt struct {
-	client    k8s.K8sDynamicClient
-	sshClient libssh.Client
+	client        k8s.K8sDynamicClient
+	sshClient     libssh.Client
+	multusEnabled bool
 }
 
-func NewCnaoOpt(c k8s.K8sDynamicClient, sshClient libssh.Client) *cnaoOpt {
+func NewCnaoOpt(c k8s.K8sDynamicClient, sshClient libssh.Client, multusEnabled bool) *cnaoOpt {
 	return &cnaoOpt{
-		client:    c,
-		sshClient: sshClient,
+		client:        c,
+		sshClient:     sshClient,
+		multusEnabled: multusEnabled,
 	}
 }
 
@@ -43,6 +46,12 @@ func (o *cnaoOpt) Exec() error {
 					continue
 				}
 
+				if path == "manifests/network-addons-config-example.cr.yaml" && o.multusEnabled {
+					re := regexp.MustCompile("(?m)[\r\n]+^.*multus.*$")
+					res := re.ReplaceAllString(string(yamlDoc), "")
+					yamlDoc = []byte(res)
+				}
+
 				obj, err := k8s.SerializeIntoObject(yamlDoc)
 				if err != nil {
 					logrus.Info(err.Error())

diff --git a/cluster-provision/gocli/opts/cnao/cnao_test.go b/cluster-provision/gocli/opts/cnao/cnao_test.go
@@ -6,6 +6,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"go.uber.org/mock/gomock"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s"
 	kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock"
 )
@@ -27,16 +28,41 @@ var _ = Describe("CnaoOpt", func() {
 		mockCtrl = gomock.NewController(GinkgoT())
 		client = k8s.NewTestClient()
 		sshClient = kubevirtcimocks.NewMockSSHClient(mockCtrl)
-		opt = NewCnaoOpt(client, sshClient)
 	})
 
 	AfterEach(func() {
 		mockCtrl.Finish()
 	})
 
-	It("should execute CnaoOpt successfully", func() {
+	It("should execute create CNAO with Multus", func() {
+		opt = NewCnaoOpt(client, sshClient, false)
 		sshClient.EXPECT().Command("kubectl --kubeconfig=/etc/kubernetes/admin.conf wait deployment -n cluster-network-addons cluster-network-addons-operator --for condition=Available --timeout=200s")
-		err := opt.Exec()
+		opt.Exec()
+
+		obj, err := client.Get(schema.GroupVersionKind{Group: "networkaddonsoperator.network.kubevirt.io",
+			Version: "v1",
+			Kind:    "NetworkAddonsConfig"}, "cluster", "")
+		Expect(err).NotTo(HaveOccurred())
+
+		spec, ok := obj.Object["spec"].(map[string]interface{})
+		Expect(ok).To(Equal(true))
+		Expect(spec).To(HaveKey("multus"))
+		Expect(spec).To(HaveKey("multusDynamicNetworks"))
+	})
+
+	It("should execute create CNAO without Multus", func() {
+		opt = NewCnaoOpt(client, sshClient, true)
+		sshClient.EXPECT().Command("kubectl --kubeconfig=/etc/kubernetes/admin.conf wait deployment -n cluster-network-addons cluster-network-addons-operator --for condition=Available --timeout=200s")
+		opt.Exec()
+
+		obj, err := client.Get(schema.GroupVersionKind{Group: "networkaddonsoperator.network.kubevirt.io",
+			Version: "v1",
+			Kind:    "NetworkAddonsConfig"}, "cluster", "")
 		Expect(err).NotTo(HaveOccurred())
+
+		spec, ok := obj.Object["spec"].(map[string]interface{})
+		Expect(ok).To(Equal(true))
+		Expect(spec).NotTo(HaveKey("multus"))
+		Expect(spec).NotTo(HaveKey("multusDynamicNetworks"))
 	})
 })
diff --git a/cluster-provision/gocli/opts/cnao/manifests/crd.yaml b/cluster-provision/gocli/opts/cnao/manifests/crd.yaml
@@ -59,6 +59,10 @@ spec:
                     description: NameServerIp defines the name server IP
                     type: string
                 type: object
+              kubevirtIpamController:
+                description: KubevirtIpamController plugin allows to support IPAM
+                  for secondary networks
+                type: object
               linuxBridge:
                 description: LinuxBridge plugin allows users to create a bridge and
                   add the host and the container to it
@@ -1521,20 +1525,6 @@ spec:
                           - ECDHE-RSA-CHACHA20-POLY1305
                           - DHE-RSA-AES128-GCM-SHA256
                           - DHE-RSA-AES256-GCM-SHA384
-                          - TLS_AES_128_GCM_SHA256
-                          - TLS_AES_256_GCM_SHA384
-                          - TLS_CHACHA20_POLY1305_SHA256
-                          - TLS_AES_128_GCM_SHA256
-                          - TLS_AES_256_GCM_SHA384
-                          - TLS_CHACHA20_POLY1305_SHA256
-                          - ECDHE-ECDSA-AES128-GCM-SHA256
-                          - ECDHE-RSA-AES128-GCM-SHA256
-                          - ECDHE-ECDSA-AES256-GCM-SHA384
-                          - ECDHE-RSA-AES256-GCM-SHA384
-                          - ECDHE-ECDSA-CHACHA20-POLY1305
-                          - ECDHE-RSA-CHACHA20-POLY1305
-                          - DHE-RSA-AES128-GCM-SHA256
-                          - DHE-RSA-AES256-GCM-SHA384
                           - DHE-RSA-CHACHA20-POLY1305
                           - ECDHE-ECDSA-AES128-SHA256
                           - ECDHE-RSA-AES128-SHA256
@@ -1706,6 +1696,10 @@ spec:
                     description: NameServerIp defines the name server IP
                     type: string
                 type: object
+              kubevirtIpamController:
+                description: KubevirtIpamController plugin allows to support IPAM
+                  for secondary networks
+                type: object
               linuxBridge:
                 description: LinuxBridge plugin allows users to create a bridge and
                   add the host and the container to it
@@ -3168,20 +3162,6 @@ spec:
                           - ECDHE-RSA-CHACHA20-POLY1305
                           - DHE-RSA-AES128-GCM-SHA256
                           - DHE-RSA-AES256-GCM-SHA384
-                          - TLS_AES_128_GCM_SHA256
-                          - TLS_AES_256_GCM_SHA384
-                          - TLS_CHACHA20_POLY1305_SHA256
-                          - TLS_AES_128_GCM_SHA256
-                          - TLS_AES_256_GCM_SHA384
-                          - TLS_CHACHA20_POLY1305_SHA256
-                          - ECDHE-ECDSA-AES128-GCM-SHA256
-                          - ECDHE-RSA-AES128-GCM-SHA256
-                          - ECDHE-ECDSA-AES256-GCM-SHA384
-                          - ECDHE-RSA-AES256-GCM-SHA384
-                          - ECDHE-ECDSA-CHACHA20-POLY1305
-                          - ECDHE-RSA-CHACHA20-POLY1305
-                          - DHE-RSA-AES128-GCM-SHA256
-                          - DHE-RSA-AES256-GCM-SHA384
                           - DHE-RSA-CHACHA20-POLY1305
                           - ECDHE-ECDSA-AES128-SHA256
                           - ECDHE-RSA-AES128-SHA256

diff --git a/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml b/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml
@@ -1,7 +1,8 @@
+---
 apiVersion: v1
 kind: Namespace
 metadata:
   name: cluster-network-addons
   labels:
     name: cluster-network-addons
-    openshift.io/cluster-monitoring: "true"
+    openshift.io/cluster-monitoring: "true"
diff --git a/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml b/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: networkaddonsoperator.network.kubevirt.io/v1
 kind: NetworkAddonsConfig
 metadata:
@@ -10,4 +11,3 @@ spec:
   macvtap: {}
   multus: {}
   multusDynamicNetworks: {}
-  ovs: {}
diff --git a/cluster-provision/gocli/opts/cnao/manifests/operator.yaml b/cluster-provision/gocli/opts/cnao/manifests/operator.yaml
@@ -120,6 +120,24 @@ rules:
   - services
   verbs:
   - delete
+- apiGroups:
+  - k8s.cni.cncf.io
+  resources:
+  - ipamclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+- apiGroups:
+  - k8s.cni.cncf.io
+  resources:
+  - network-attachment-definitions
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -144,12 +162,6 @@ rules:
   - create
   - patch
   - update
-- apiGroups:
-  - k8s.cni.cncf.io
-  resources:
-  - '*'
-  verbs:
-  - '*'
 - apiGroups:
   - ""
   resources:
@@ -238,6 +250,12 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - k8s.cni.cncf.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -329,6 +347,34 @@ rules:
   verbs:
   - get
   - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - patch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - cert-manager.io
+  resources:
+  - certificates
+  - issuers
+  verbs:
+  - get
+  - create
+  - update
+  - delete
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -349,7 +395,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    networkaddonsoperator.network.kubevirt.io/version: 0.87.0
+    networkaddonsoperator.network.kubevirt.io/version: 0.94.1
   labels:
     prometheus.cnao.io: "true"
   name: cluster-network-addons-operator
@@ -375,29 +421,31 @@ spec:
         - name: MULTUS_IMAGE
           value: ghcr.io/k8snetworkplumbingwg/multus-cni@sha256:3fbcc32bd4e4d15bd93c96def784a229cd84cca27942bf4858b581f31c97ee02
         - name: MULTUS_DYNAMIC_NETWORKS_CONTROLLER_IMAGE
-          value: ghcr.io/k8snetworkplumbingwg/multus-dynamic-networks-controller@sha256:dee1979d92f0a31598a6e3569ac7004be7d29e7ca9e31db23753ef263110dc04
+          value: ghcr.io/k8snetworkplumbingwg/multus-dynamic-networks-controller@sha256:83b460502671fb4f34116363a1a39b2ddfc9d14a920ee0a6413bfc3bd0580404
         - name: LINUX_BRIDGE_IMAGE
-          value: quay.io/kubevirt/cni-default-plugins@sha256:825e3f9fec1996c54a52cec806154945b38f76476b160d554c36e38dfffe5e61
+          value: quay.io/kubevirt/cni-default-plugins@sha256:0c354fa9d695b8cab97b459e8afea2f7662407a987e83f6f6f1a8af4b45726be
         - name: LINUX_BRIDGE_MARKER_IMAGE
-          value: quay.io/kubevirt/bridge-marker@sha256:5d24c6d1ecb0556896b7b81c7e5260b54173858425777b7a84df8a706c07e6d2
+          value: quay.io/kubevirt/bridge-marker@sha256:bba066e3b5ff3fb8c5e20861fe8abe51e3c9b50ad6ce3b2616af9cb5479a06d0
         - name: OVS_CNI_IMAGE
-          value: quay.io/kubevirt/ovs-cni-plugin@sha256:5f7290e2294255ab2547c3b4bf48cc2d75531ec5a43e600366e9b2719bef983f
+          value: quay.io/kubevirt/ovs-cni-plugin@sha256:e16ac74343da21abb8fb668ce71e728053d00503a992dae2164b9e94a280113e
         - name: KUBEMACPOOL_IMAGE
-          value: quay.io/kubevirt/kubemacpool@sha256:afba7d0c4a95d2d4924f6ee6ef16bbe59117877383819057f01809150829cb0c
+          value: quay.io/kubevirt/kubemacpool@sha256:20e156be33e6d3692c456081acbb91a4349f94de448f7d1f1cddd0228931b31f
         - name: MACVTAP_CNI_IMAGE
-          value: quay.io/kubevirt/macvtap-cni@sha256:434420511e09b2b5ede785a2c9062b6658ffbc26fbdd4629ce06110f9039c600
+          value: quay.io/kubevirt/macvtap-cni@sha256:850b89343ace7c7ea6b18dd8e11964613974e9d1f7377af03854d407fb15230a
         - name: KUBE_RBAC_PROXY_IMAGE
-          value: quay.io/openshift/origin-kube-rbac-proxy@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901
+          value: quay.io/openshift/origin-kube-rbac-proxy@sha256:e2def4213ec0657e72eb790ae8a115511d5b8f164a62d3568d2f1bff189917e8
         - name: KUBE_SECONDARY_DNS_IMAGE
-          value: ghcr.io/kubevirt/kubesecondarydns@sha256:77132adb5f840ceb0aadd408731a5c8b01a4b427a78084ab5e4e9b961195cb02
+          value: ghcr.io/kubevirt/kubesecondarydns@sha256:6268d84154e2483fbce8c1adacbdaf6f0839117b2d48d9fa4687cc8f76bd5130
         - name: CORE_DNS_IMAGE
           value: registry.k8s.io/coredns/coredns@sha256:a0ead06651cf580044aeb0a0feba63591858fb2e43ade8c9dea45a6a89ae7e5e
+        - name: KUBEVIRT_IPAM_CONTROLLER_IMAGE
+          value: ghcr.io/kubevirt/ipam-controller@sha256:f272eaf82c9e4fcd7fdee5d9031afba8019cff2dc842ca3e36be49de28083fee
         - name: OPERATOR_IMAGE
-          value: quay.io/kubevirt/cluster-network-addons-operator:v0.87.0
+          value: quay.io/kubevirt/cluster-network-addons-operator:v0.94.1
         - name: OPERATOR_NAME
           value: cluster-network-addons-operator
         - name: OPERATOR_VERSION
-          value: 0.87.0
+          value: 0.94.1
         - name: OPERATOR_NAMESPACE
           valueFrom:
             fieldRef:
@@ -416,8 +464,8 @@ spec:
         - name: MONITORING_SERVICE_ACCOUNT
           value: prometheus-k8s
         - name: RUNBOOK_URL_TEMPLATE
-          value: https://kubevirt.io/monitoring/runbooks/
-        image: quay.io/kubevirt/cluster-network-addons-operator:v0.87.0
+          value: https://kubevirt.io/monitoring/runbooks/%s
+        image: quay.io/kubevirt/cluster-network-addons-operator:v0.94.1
         imagePullPolicy: Always
         name: cluster-network-addons-operator
         resources:
@@ -433,7 +481,7 @@ spec:
         - --logtostderr
         - --secure-listen-address=:8443
         - --upstream=http://127.0.0.1:8080
-        image: quay.io/openshift/origin-kube-rbac-proxy@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901
+        image: quay.io/openshift/origin-kube-rbac-proxy@sha256:e2def4213ec0657e72eb790ae8a115511d5b8f164a62d3568d2f1bff189917e8
         imagePullPolicy: Always
         name: kube-rbac-proxy
         ports:
@@ -455,4 +503,4 @@ spec:
         runAsNonRoot: true
         seccompProfile:
           type: RuntimeDefault
-      serviceAccountName: cluster-network-addons-operator
+      serviceAccountName: cluster-network-addons-operator
diff --git a/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml b/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml
@@ -22,7 +22,3 @@ spec:
        - istio-system
        - kube-system
       logLevel: debug
-      cniConfFileName: "istio-cni.conf"
-    sidecarInjectorWebhook:
-      injectedAnnotations:
-        "k8s.v1.cni.cncf.io/networks": istio-cni
diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh
@@ -14,7 +14,16 @@ function deploy_kwok() {
     fi
 }
 
-
+# copy_istio_cni_conf_files copy the generated Istio CNI net conf file
+# (at '/etc/cni/multus/net.d/') to where Multus expect CNI net conf files ('/etc/cni/net.d/')
+function copy_istio_cni_conf_files() {
+    if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ] && [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then
+        for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do
+            $ssh node${nodeNum} -- "until ls /etc/cni/multus > /dev/null 2>&1; do sleep 1; done"
+            $ssh node${nodeNum} -- sudo cp -uv /etc/cni/multus/net.d/*istio*.conf /etc/cni/net.d/
+        done
+    fi
+}
 
 # configure Prometheus to select kubevirt prometheusrules
 function configure_prometheus() {
@@ -95,4 +104,9 @@ function up() {
         echo "Waiting for cluster components..."
         sleep 5
     done
+
+    # FIXME: remove 'copy_istio_cni_conf_files()' as soon as [1] and [2] are resolved
+    # [1] https://github.com/kubevirt/kubevirtci/issues/906
+    # [2] https://github.com/k8snetworkplumbingwg/multus-cni/issues/982
+    copy_istio_cni_conf_files
 }