diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 6273067b61..7c6fc74135 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -330,6 +330,17 @@ stages: k8sVersion: "" dependsOn: "containerize" + # Azure Overlay E2E Stateless CNI tests + - template: singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml + parameters: + name: "azure_overlay_stateless_e2e" + displayName: Azure Stateless CNI Overlay + os: windows + clusterType: overlay-byocni-up + clusterName: "statelesswin" + vmSize: Standard_B2ms + dependsOn: "containerize" + # AKS Swift E2E tests - template: singletenancy/aks-swift/e2e-job-template.yaml parameters: @@ -413,6 +424,7 @@ stages: dependsOn: - setup - azure_overlay_e2e + - azure_overlay_stateless_e2e - aks_swift_e2e - cilium_e2e - cilium_overlay_e2e @@ -448,6 +460,10 @@ stages: name: azure_overlay_e2e clusterName: "azovere2e" region: $(REGION_AKS_CLUSTER_TEST) + azure_overlay_stateless_e2e: + name: azure_overlay_stateless_e2e + clusterName: "statelesswin" + region: $(REGION_AKS_CLUSTER_TEST) aks_swift_e2e: name: aks_swift_e2e clusterName: "swifte2e" diff --git a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml new file mode 100644 index 0000000000..c7b536e88e --- /dev/null +++ b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml @@ -0,0 +1,111 @@ +parameters: + name: "" + displayName: "" + clusterType: "" + clusterName: "" + vmSize: "" + k8sVersion: "" + dependsOn: "" + +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_AKS_CLUSTER_TEST) + - job: windows_nodepool + displayName: Add Windows Nodepool + dependsOn: ${{ parameters.name }} + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }} + echo "Windows node are successfully added to v4 Overlay Cluster" + kubectl cluster-info + kubectl get node -owide + kubectl get po -owide -A + name: "Add_Windows_Node" + displayName: "Add windows node on v4 overlay cluster" + + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: ${{ parameters.name }}_windows + displayName: Azure Stateless CNI Overlay Test Suite | Windows - (${{ parameters.name }}) + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - template: azure-cni-overlay-stateless-e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: windows + vmSizeWin: ${{ parameters.vmSize }} + + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: windows + dependsOn: ${{ parameters.name }}_windows + datapath: true + dns: true + portforward: true + hostport: true + service: true + hybridWin: true + + - job: failedE2ELogs_windows + displayName: "Windows Failure Logs" + dependsOn: + - ${{ parameters.name }}_windows + - cni_windows + condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed') + steps: + - template: ../../templates/log-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: windows + cni: cniv2 + diff --git a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-step-template.yaml new file mode 100644 index 0000000000..c5b565a42b --- /dev/null +++ b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-step-template.yaml @@ -0,0 +1,93 @@ +parameters: + name: "" + clusterName: "" + os: "" + +steps: + - bash: | + go version + go env + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + name: "GoEnv" + displayName: "Set up the Go environment" + + - task: KubectlInstaller@0 + inputs: + kubectlVersion: latest + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + name: "kubeconfig" + displayName: "Set Kubeconfig" + + - script: | + nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'` + for node in $nodeList; do + taint=`kubectl describe node $node | grep Taints | awk '{print $2}'` + if [ $taint == "node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule" ]; then + kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule- + fi + done + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=stateless VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET_STATELESS=true VALIDATE_V4OVERLAY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true + name: "WindowsOverlayControlPlaneScaleTests" + displayName: "Windows v4Overlay ControlPlane Scale Tests" + retryCountOnTaskFailure: 2 + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + clusterName=${{ parameters.clusterName }} + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName} + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + kubectl get pods -owide -A + echo "Validating Node Restart" + CNI_TYPE=stateless RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$ + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + + - script: | + echo "Run wireserver and metadata connectivity Tests" + bash test/network/wireserver_metadata_test.sh + retryCountOnTaskFailure: 3 + name: "WireserverMetadataConnectivityTests" + displayName: "Run Wireserver and Metadata Connectivity Tests" + + - script: | + echo "IPv4 Overlay DataPath Test" + cd test/integration/datapath + sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$ + name: "WindowsV4OverlayDatapathTests" + displayName: "Windows v4Overlay Datapath Tests" + retryCountOnTaskFailure: 3 diff --git a/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml b/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml new file mode 100644 index 0000000000..1d2c921ecb --- /dev/null +++ b/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cns-win-config + namespace: kube-system +data: + cns_config.json: | + { + "TelemetrySettings": { + "TelemetryBatchSizeBytes": 16384, + "TelemetryBatchIntervalInSecs": 15, + "RefreshIntervalInSecs": 15, + "DisableAll": false, + "HeartBeatIntervalInMins": 30, + "DebugMode": false, + "SnapshotIntervalInMins": 60 + }, + "ManagedSettings": { + "PrivateEndpoint": "", + "InfrastructureNetworkID": "", + "NodeID": "", + "NodeSyncIntervalInSeconds": 30 + }, + "EnableSubnetScarcity": false, + "ChannelMode": "CRD", + "InitializeFromCNI": false, + "ManageEndpointState": true, + "ProgramSNATIPTables" : false, + "MetricsBindAddress": ":10092", + "EnableCNIConflistGeneration": false, + "CNIConflistFilepath": "C:\\k\\azurecni\\netconf\\10-azure.conflist", + "CNIConflistScenario": "v4overlay", + "EnableAsyncPodDelete": true, + "AsyncPodDeletePath": "/var/run/azure-vnet/deleteIDs" + } diff --git a/test/internal/kubernetes/utils_create.go b/test/internal/kubernetes/utils_create.go index 59918f4ff1..220d29a33a 100644 --- a/test/internal/kubernetes/utils_create.go +++ b/test/internal/kubernetes/utils_create.go @@ -24,11 +24,12 @@ import ( type CNSScenario string const ( - EnvInstallAzilium CNSScenario = "INSTALL_AZILIUM" - EnvInstallAzureVnet CNSScenario = "INSTALL_AZURE_VNET" - EnvInstallOverlay CNSScenario = "INSTALL_OVERLAY" - EnvInstallAzureCNIOverlay CNSScenario = "INSTALL_AZURE_CNI_OVERLAY" - EnvInstallDualStackOverlay CNSScenario = "INSTALL_DUALSTACK_OVERLAY" + EnvInstallAzilium CNSScenario = "INSTALL_AZILIUM" + EnvInstallAzureVnet CNSScenario = "INSTALL_AZURE_VNET" + EnvInstallAzureVnetStateless CNSScenario = "INSTALL_AZURE_VNET_STATELESS" + EnvInstallOverlay CNSScenario = "INSTALL_OVERLAY" + EnvInstallAzureCNIOverlay CNSScenario = "INSTALL_AZURE_CNI_OVERLAY" + EnvInstallDualStackOverlay CNSScenario = "INSTALL_DUALSTACK_OVERLAY" ) type cnsDetails struct { @@ -333,6 +334,7 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) cnsOverlayConfigMapPath := cnsConfigFolder + "/overlayconfigmap.yaml" cnsAzureCNIOverlayLinuxConfigMapPath := cnsConfigFolder + "/azurecnioverlaylinuxconfigmap.yaml" cnsAzureCNIOverlayWindowsConfigMapPath := cnsConfigFolder + "/azurecnioverlaywindowsconfigmap.yaml" + cnsAzureStatelessCNIOverlayWindowsConfigMapPath := cnsConfigFolder + "/azurestatelesscnioverlaywindowsconfigmap.yaml" cnsAzureCNIDualStackLinuxConfigMapPath := cnsConfigFolder + "/azurecnidualstackoverlaylinuxconfigmap.yaml" cnsAzureCNIDualStackWindowsConfigMapPath := cnsConfigFolder + "/azurecnidualstackoverlaywindowsconfigmap.yaml" cnsRolePath := cnsManifestFolder + "/role.yaml" @@ -393,6 +395,47 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) installIPMasqAgent: false, }, }, + EnvInstallAzureVnetStateless: { + corev1.Linux: { + daemonsetPath: cnsLinuxDaemonSetPath, + labelSelector: cnsLinuxLabelSelector, + rolePath: cnsRolePath, + roleBindingPath: cnsRoleBindingPath, + clusterRolePath: cnsClusterRolePath, + clusterRoleBindingPath: cnsClusterRoleBindingPath, + serviceAccountPath: cnsServiceAccountPath, + initContainerArgs: []string{ + "deploy", + "azure-vnet", "-o", "/opt/cni/bin/azure-vnet", + "azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", + }, + initContainerName: initContainerNameCNI, + volumes: volumesForAzureCNIOverlayLinux(), + initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayLinux(), + containerVolumeMounts: cnsVolumeMountsForAzureCNIOverlayLinux(), + configMapPath: cnsAzureCNIOverlayLinuxConfigMapPath, + installIPMasqAgent: true, + }, + corev1.Windows: { + daemonsetPath: cnsWindowsDaemonSetPath, + labelSelector: cnsWindowsLabelSelector, + rolePath: cnsRolePath, + roleBindingPath: cnsRoleBindingPath, + clusterRolePath: cnsClusterRolePath, + clusterRoleBindingPath: cnsClusterRoleBindingPath, + serviceAccountPath: cnsServiceAccountPath, + initContainerArgs: []string{ + "deploy", + "azure-vnet-stateless", "-o", "/k/azurecni/bin/azure-vnet.exe", + }, + initContainerName: initContainerNameCNI, + volumes: volumesForAzureCNIOverlayWindows(), + initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayWindows(), + containerVolumeMounts: cnsVolumeMountsForAzureCNIOverlayWindows(), + configMapPath: cnsAzureStatelessCNIOverlayWindowsConfigMapPath, + installIPMasqAgent: true, + }, + }, EnvInstallAzilium: { corev1.Linux: { daemonsetPath: cnsLinuxDaemonSetPath, diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index a7a43fbbef..14e72547fa 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -206,24 +206,6 @@ type AzureVnetEndpointInfo struct { PodName string } -func cnsManagedStateFileIps(result []byte) (map[string]string, error) { - var cnsResult CnsManagedState - err := json.Unmarshal(result, &cnsResult) - if err != nil { - return nil, errors.Wrapf(err, "failed to unmarshal cns endpoint list") - } - - cnsPodIps := make(map[string]string) - for _, v := range cnsResult.Endpoints { - for ifName, ip := range v.IfnameToIPMap { - if ifName == "eth0" { - cnsPodIps[ip.IPv4[0].IP.String()] = v.PodName - } - } - } - return cnsPodIps, nil -} - func cnsManagedStateFileDualStackIps(result []byte) (map[string]string, error) { var cnsResult CnsManagedState err := json.Unmarshal(result, &cnsResult) diff --git a/test/validate/validate.go b/test/validate/validate.go index 32912e8aba..f6020f3258 100644 --- a/test/validate/validate.go +++ b/test/validate/validate.go @@ -261,3 +261,21 @@ func cnsCacheStateFileIps(result []byte) (map[string]string, error) { } return cnsPodIps, nil } + +func cnsManagedStateFileIps(result []byte) (map[string]string, error) { + var cnsResult CnsManagedState + err := json.Unmarshal(result, &cnsResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal cns endpoint list") + } + + cnsPodIps := make(map[string]string) + for _, v := range cnsResult.Endpoints { + for ifName, ip := range v.IfnameToIPMap { + if ifName == "eth0" { + cnsPodIps[ip.IPv4[0].IP.String()] = v.PodName + } + } + } + return cnsPodIps, nil +} diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go index 6a69ed6f45..b721443aa2 100644 --- a/test/validate/windows_validate.go +++ b/test/validate/windows_validate.go @@ -23,6 +23,7 @@ var ( hnsNetworkCmd = []string{"powershell", "-c", "Get-HnsNetwork | ConvertTo-Json"} azureVnetCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet.json"} azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} + cnsWinManagedStateFileCmd = []string{"powershell", "-c", "cat ../../k/azurecns/azure-endpoints.json"} cnsWinCachedAssignedIPStateCmd = []string{ "powershell", "Invoke-WebRequest -Uri 127.0.0.1:10090/debug/ipaddresses", "-Method Post -ContentType application/x-www-form-urlencoded", @@ -78,6 +79,29 @@ var windowsChecksMap = map[string][]check{ cmd: cnsWinCachedAssignedIPStateCmd, }, }, + "stateless": { + { + name: "hns", + stateFileIPs: hnsStateFileIPs, + podLabelSelector: privilegedLabelSelector, + podNamespace: privilegedNamespace, + cmd: hnsEndPointCmd, + }, + { + name: "cns", + stateFileIPs: cnsManagedStateFileIps, + podLabelSelector: privilegedLabelSelector, + podNamespace: privilegedNamespace, + cmd: cnsWinManagedStateFileCmd, + }, // cns configmap "ManageEndpointState": true, | Endpoints managed in CNS State File + { + name: "cns cache", + stateFileIPs: cnsCacheStateFileIps, + podLabelSelector: cnsWinLabelSelector, + podNamespace: privilegedNamespace, + cmd: cnsWinCachedAssignedIPStateCmd, + }, + }, } type HNSEndpoint struct {