To support running Antrea Flexible IPAM tests on KIND

Signed-off-by: KMAnju-2021 <[email protected]>

ci/jenkins/jobs/macros.yaml

@@ -254,7 +254,27 @@
           return_code=$?
           set -ex
           ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}"
-          exit $return_code 
+          exit $return_code
+                   
+- builder:
+    name: builder-kind-flexible-ipam-e2e
+    builders:
+      - shell: |-
+          #!/bin/bash
+          set -ex
+          DOCKER_REGISTRY="$(head -n1 ci/docker-registry)"
+          KIND_TIMEOUT=135
+          FULL_CLUSTER_NAME='{kind_cluster_name}'-"${{BUILD_NUMBER}}"
+          ./ci/kind/kind-setup.sh destroy --all --until ${{KIND_TIMEOUT}}
+          ./ci/kind/kind-install.sh 
+          ./ci/kind/kind-setup.sh --antrea-cni --flexible-ipam create "${{FULL_CLUSTER_NAME}}" --ipam-vlan-subnets '192.168.241.0/24,192.168.242.0/24' --ipam-vlan-ids '11,12'
+          kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config
+          set +e
+          ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind-flexible-ipam" --kind-cluster-name "${{FULL_CLUSTER_NAME}}"
+          return_code=$?
+          set -ex
+          ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}"
+          exit $return_code  
 
 - builder:
     name: builder-rancher-e2e

ci/jenkins/jobs/projects-lab.yaml

@@ -1348,3 +1348,33 @@
                 timeout: 135
                 type: absolute
             publishers: []
+      - '{name}-{test_name}-for-pull-request':
+            test_name: kind-flexible-ipam-e2e
+            node: 'antrea-kind-flexible-ipam-testbed'
+            description: 'This is the {test_name} test for {name}.'
+            branches:
+            - ${{sha1}}      
+            builders:
+              - builder-kind-flexible-ipam-e2e:
+                  kind_cluster_name: '{test_name}'
+            trigger_phrase: ^(?!Thanks for your PR).*/test-kind-flexible-ipam-e2e.*
+            white_list_target_branches: []
+            allow_whitelist_orgs_as_admins: true
+            admin_list: '{antrea_admin_list}'
+            org_list: '{antrea_org_list}'
+            white_list: '{antrea_white_list}'
+            only_trigger_phrase: true
+            trigger_permit_all: true
+            status_context: jenkins-kind-flexible-ipam-e2e
+            status_url: --none--
+            success_status: Build finished.
+            failure_status: Failed. Add comment /test-kind-flexible-ipam-e2e to re-trigger.
+            error_status: Failed. Add comment /test-kind-flexible-ipam-e2e to re-trigger.
+            triggered_status: null
+            started_status: null
+            wrappers:
+            - timeout:
+                fail: true
+                timeout: 135
+                type: absolute
+            publishers: []

ci/jenkins/test.sh

@@ -38,6 +38,7 @@ IP_MODE=""
 K8S_VERSION="1.28.2-00"
 WINDOWS_YAML_SUFFIX="windows"
 WIN_IMAGE_NODE=""
+FLEXIBLE_IPAM=false
 echo "" > WIN_DHCP
 GOLANG_RELEASE_DIR=${WORKDIR}/golang-releases
 
@@ -187,6 +188,15 @@ function clean_antrea {
     docker images | grep 'antrea' | awk '{print $3}' | xargs -r docker rmi || true
     docker images | grep '<none>' | awk '{print $3}' | xargs -r docker rmi || true
     check_and_cleanup_docker_build_cache
+
+  if [[ $TESTBED_TYPE == "kind-flexible-ipam" && $FLEXIBLE_IPAM == "true" ]]; then
+       control_plane_podcidr=$(kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' | awk '{print $1}')
+       sudo ip route del $control_plane_podcidr
+       worker_podcidr=$(kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' | awk '{print $2}')
+       sudo ip route del $worker_podcidr
+       worker2_podcidr=$(kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' | awk '{print $3}')
+       sudo ip route del $worker2_podcidr 
+  fi
 }
 
 function clean_for_windows_install_cni {
@@ -687,7 +697,6 @@ function deliver_antrea {
     if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then
         redeploy_k8s_if_ip_mode_changes
     fi
-
     echo "====== Building Antrea for the Following Commit ======"
     export GO111MODULE=on
     export GOPATH=${WORKDIR}/go
@@ -750,12 +759,16 @@ function deliver_antrea {
             scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" flow-aggregator.tar jenkins@[${IP}]:${DEFAULT_WORKDIR}/flow-aggregator.tar
             ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/antrea-ubuntu.tar; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/flow-aggregator.tar" || true
         done
-    elif [[ $TESTBED_TYPE == "kind" ]]; then
+    elif [[ $TESTBED_TYPE == "kind" || $TESTBED_TYPE == "kind-flexible-ipam" ]]; then
             kind load docker-image antrea/antrea-agent-ubuntu:latest --name ${KIND_CLUSTER}
             kind load docker-image antrea/antrea-controller-ubuntu:latest --name ${KIND_CLUSTER}
             kind load docker-image antrea/flow-aggregator:latest --name ${KIND_CLUSTER} 
             kubectl config use-context kind-${KIND_CLUSTER}
-            docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml 
+            docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml
+            if [[ $TESTBED_TYPE == "kind-flexible-ipam" ]]; then
+                  configure_kind_ipam_routes
+                  FLEXIBLE_IPAM=true
+            fi 
     elif [[ $TESTBED_TYPE == "jumper" ]]; then
         kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | while read IP; do
             scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${WORKDIR}/.ssh/id_rsa" antrea-ubuntu.tar jenkins@[${IP}]:${DEFAULT_WORKDIR}/antrea-ubuntu.tar
@@ -808,7 +821,7 @@ function run_e2e {
 
     mkdir -p "${WORKDIR}/.kube"
     mkdir -p "${WORKDIR}/.ssh"
-    if [[ $TESTBED_TYPE != "kind" ]]; then 
+    if [[ $TESTBED_TYPE != "kind" && $TESTBED_TYPE != "kind-flexible-ipam" ]]; then 
         cp -f "${WORKDIR}/kube.conf" "${WORKDIR}/.kube/config"
     fi
     generate_ssh_config
@@ -821,6 +834,8 @@ function run_e2e {
         go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus --antrea-ipam
     elif [[ $TESTBED_TYPE == "kind" ]]; then
         go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind -timeout=100m --prometheus
+    elif [[ $TESTBED_TYPE == "kind-flexible-ipam" ]]; then
+        go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind -timeout=100m --prometheus --antrea-ipam
     else
         go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus
     fi
@@ -1172,6 +1187,21 @@ EOF
     done
 }
 
+function configure_kind_ipam_routes {
+
+   control_plane_podcidr=$(kubectl get nodes $KIND_CLUSTER-control-plane -o yaml | grep podCIDR | awk 'FNR == 1 {print$2}')
+   control_plane_node_ip=$(kubectl get nodes $KIND_CLUSTER-control-plane  -o wide | awk 'FNR == 2 {print$6}')
+   sudo ip route add $control_plane_podcidr via $control_plane_node_ip
+
+   worker_podcidr=$(kubectl get nodes $KIND_CLUSTER-worker -o yaml | grep podCIDR | awk 'FNR == 1 {print$2}')
+   worker_node_ip=$(kubectl get nodes $KIND_CLUSTER-worker  -o wide | awk 'FNR == 2 {print$6}')
+   sudo ip route add $worker_podcidr via $worker_node_ip
+
+   worker2_podcidr=$(kubectl get nodes $KIND_CLUSTER-worker2 -o yaml | grep podCIDR | awk 'FNR == 1 {print$2}')
+   worker2_node_ip=$(kubectl get nodes $KIND_CLUSTER-worker2  -o wide | awk 'FNR == 2 {print$6}')
+   sudo ip route add $worker2_podcidr via $worker2_node_ip
+}
+
 export KUBECONFIG=${KUBECONFIG_PATH}
 if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then
     ./hack/generate-manifest.sh --flexible-ipam --multicast --verbose-log > build/yamls/antrea.yml
@@ -1181,6 +1211,10 @@ if [[ $TESTCASE =~ "multicast" ]]; then
     ./hack/generate-manifest.sh --encap-mode noEncap --multicast --multicast-interfaces "ens224" --verbose-log > build/yamls/antrea.yml
 fi
 
+if [[ $TESTBED_TYPE == "kind-flexible-ipam" ]]; then
+   ./hack/generate-manifest.sh --flexible-ipam --multicast --verbose-log > build/yamls/antrea.yml  
+fi
+
 source $WORKSPACE/ci/jenkins/utils.sh
 check_and_upgrade_golang
 clean_tmp

ci/kind/kind-setup.sh

@@ -32,13 +32,16 @@ SUBNETS=""
 EXTRA_NETWORKS=""
 VLAN_SUBNETS=""
 VLAN_ID=""
+IPAM_VLAN_SUBNETS=""
+IPAM_VLAN_IDS=""
 ENCAP_MODE=""
 PROXY=true
 KUBE_PROXY_MODE="iptables"
 PROMETHEUS=false
 K8S_VERSION=""
 KUBE_NODE_IPAM=true
 DEPLOY_EXTERNAL_SERVER=false
+FLEXIBLE_IPAM=false
 positional_args=()
 options=()
 
@@ -242,6 +245,44 @@ function configure_extra_networks {
   done
 }
 
+function get_bridge_id {
+  local bridge_id=$(docker network inspect kind -f {{.ID}})
+  local return_value=$?
+  if [ $return_value -eq 0 ]; then
+        echo "$bridge_id"
+    else
+        echo "Unable to get kind bridge id, Docker command failed with exit code $return_value"
+        exit 1
+    fi
+}
+
+function configure_kind_ipam_vlan_subnets {
+  if [[ -z $IPAM_VLAN_SUBNETS || -z $IPAM_VLAN_IDS ]]; then
+    return
+  fi
+  echo "Configuring Flexible IPAM kind networks"
+
+  IFS=',' read -r -a ipam_vlan_subnets <<< "$IPAM_VLAN_SUBNETS"
+  IFS=',' read -r -a ipam_vlan_ids <<< "$IPAM_VLAN_IDS"
+  i=0
+
+  for s in "${ipam_vlan_subnets[@]}"; do
+    vlan_id=${ipam_vlan_ids[$i]}
+    i=$(($i+1))
+    echo "Configuring VLAN-$vlan_id subnets"
+    bridge_id=$(get_bridge_id)
+    bridge_interface="br-${bridge_id:0:12}"
+    vlan_interface="br-${bridge_id:0:7}.$vlan_id"
+
+    docker_run_with_host_net ip link add link $bridge_interface name $vlan_interface type vlan id $vlan_id
+    docker_run_with_host_net ip link set $vlan_interface up
+    echo "Configuring extra IP $s to vlan interface $vlan_interface"
+    docker_run_with_host_net ip addr add dev $vlan_interface $s
+    docker_run_with_host_net iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT
+    docker_run_with_host_net iptables -t filter -A FORWARD -o $bridge_interface -i $vlan_interface -j ACCEPT
+  done
+}
+
 function configure_vlan_subnets {
   if [[ -z $VLAN_SUBNETS || -z $VLAN_ID ]]; then
     return
@@ -410,6 +451,7 @@ EOF
 
   configure_networks
   configure_extra_networks
+  configure_kind_ipam_vlan_subnets
   configure_vlan_subnets
   setup_external_server
   load_images
@@ -420,6 +462,9 @@ EOF
     if [[ $PROXY == false ]]; then
       cmd+=" --no-proxy"
     fi
+    if [[ $FLEXIBLE_IPAM == true ]]; then
+       cmd+=" --flexible-ipam --multicast"
+    fi
     echo "$cmd $(get_encap_mode) | kubectl apply --context kind-$CLUSTER_NAME -f -"
     eval "$cmd $(get_encap_mode) | kubectl apply --context kind-$CLUSTER_NAME -f -"
 
@@ -562,6 +607,21 @@ while [[ $# -gt 0 ]]
       VLAN_ID="$2"
       shift 2
       ;;
+    --ipam-vlan-subnets)
+      add_option "--ipam-vlan-subnets" "create"
+      IPAM_VLAN_SUBNETS="$2"
+      shift 2
+      ;;
+    --ipam-vlan-ids)
+      add_option "--ipam-vlan-ids" "create"
+      IPAM_VLAN_IDS="$2"
+      shift 2
+      ;;
+    --flexible-ipam)
+      add_option "--flaxible-ipam" "create"
+      FLEXIBLE_IPAM=true
+      shift
+      ;; 
     --images)
       add_option "--image" "create"
       IMAGES="$2"