diff --git a/kubeadm/flannel/flannel-host-gw.yml b/kubeadm/flannel/flannel-host-gw.yml index 2d0acdc7..afb21e89 100644 --- a/kubeadm/flannel/flannel-host-gw.yml +++ b/kubeadm/flannel/flannel-host-gw.yml @@ -17,15 +17,34 @@ data: mkdir -force /host/k/flannel mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount - $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json + $containerRuntime = "docker" + if (Test-Path /host/etc/cni/net.d/0-containerd-nat.json) { + $containerRuntime = "containerd" + } + + Write-Host "Configuring CNI for $containerRuntime" + $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet $networkJson = wins cli net get | convertfrom-json - $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR - $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet - $cniJson.delegate.policies[2].Value.DestinationPrefix = $networkJson.AddressCIDR - Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) + if ($containerRuntime -eq "docker") { + $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json + + $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR + $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet + $cniJson.delegate.policies[2].Value.DestinationPrefix = $networkJson.AddressCIDR + + Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) + } elseif ($containerRuntime -eq "containerd") { + $cniJson = get-content /etc/kube-flannel-windows/cni-conf-containerd.json | ConvertFrom-Json + + $cniJson.delegate.AdditionalArgs[0].Value.Settings.Exceptions = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR + $cniJson.delegate.AdditionalArgs[1].Value.Settings.DestinationPrefix = $serviceSubnet + $cniJson.delegate.AdditionalArgs[2].Value.Settings.DestinationPrefix = $networkJson.AddressCIDR + + Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) + } cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel cp -force -recurse /cni/* /host/opt/cni/bin @@ -74,6 +93,53 @@ data: ] } } + cni-conf-containerd.json: | + { + "cniVersion": "0.2.0", + "name": "cbr0", + "type": "flannel", + "capabilities": { + "portMappings": true, + "dns": true + }, + "delegate": { + "type": "sdnbridge", + "optionalFlags" : { + "forceBridgeGateway" : true + }, + "AdditionalArgs": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "Settings": { + "Exceptions": [] + } + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "SDNROUTE", + "Settings": { + "DestinationPrefix": "", + "NeedEncap": true + } + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "SDNROUTE", + "Settings": { + "DestinationPrefix": "", + "NeedEncap": true + } + } + } + ] + } + } --- apiVersion: apps/v1 kind: DaemonSet diff --git a/kubeadm/flannel/flannel-overlay.yml b/kubeadm/flannel/flannel-overlay.yml index b53c7ca1..0c5eb3ab 100644 --- a/kubeadm/flannel/flannel-overlay.yml +++ b/kubeadm/flannel/flannel-overlay.yml @@ -17,20 +17,40 @@ data: mkdir -force /host/k/flannel mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount - $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json + $containerRuntime = "docker" + if (Test-Path /host/etc/cni/net.d/0-containerd-nat.json) { + $containerRuntime = "containerd" + } + + Write-Host "Configuring CNI for $containerRuntime" + $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet $networkJson = wins cli net get | convertfrom-json - $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet - $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet - Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) + if ($containerRuntime -eq "docker") { + $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json + + $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet + $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet + + Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) + } elseif ($containerRuntime -eq "containerd") { + $cniJson = get-content /etc/kube-flannel-windows/cni-conf-containerd.json | ConvertFrom-Json + + $cniJson.delegate.AdditionalArgs[0].Value.Settings.Exceptions = $serviceSubnet, $podSubnet + $cniJson.delegate.AdditionalArgs[1].Value.Settings.DestinationPrefix = $serviceSubnet + $cniJson.delegate.AdditionalArgs[2].Value.Settings.ProviderAddress = $networkJson.AddressCIDR.Split('/')[0] + + Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) + } cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel cp -force -recurse /cni/* /host/opt/cni/bin cp -force /k/flannel/* /host/k/flannel/ cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/ + wins cli process run --path /k/flannel/setup.exe --args "--mode=overlay --interface=Ethernet" wins cli route add --addresses 169.254.169.254 wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE" @@ -63,6 +83,49 @@ data: ] } } + cni-conf-containerd.json: | + { + "name": "flannel.4096", + "cniVersion": "0.2.0", + "type": "flannel", + "capabilities": { + "portMappings": true, + "dns": true + }, + "delegate": { + "type": "sdnoverlay", + "AdditionalArgs": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "Settings" : { + "Exceptions": [] + } + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "SDNROUTE", + "Settings": { + "DestinationPrefix": "", + "NeedEncap": true + } + } + }, + { + "Name":"EndpointPolicy", + "Value":{ + "Type":"ProviderAddress", + "Settings":{ + "ProviderAddress":"" + } + } + } + ] + } + } --- apiVersion: apps/v1 kind: DaemonSet diff --git a/kubeadm/kube-proxy/kube-proxy.yml b/kubeadm/kube-proxy/kube-proxy.yml index 3b22e15e..2e2a22f9 100644 --- a/kubeadm/kube-proxy/kube-proxy.yml +++ b/kubeadm/kube-proxy/kube-proxy.yml @@ -2,6 +2,29 @@ apiVersion: v1 data: run-script.ps1: |- $ErrorActionPreference = "Stop"; + + # Get newest cni conf file that is not 0-containerd-nat.json or spin until one shows up. + # With Docker the kube-proxy pod should not be scheduled to Windows nodes until host networking is configured. + # With contianerD host networking is required to schedule any pod including the CNI pods so a basic nat network is + # configured. This network should not be used by kube-proxy. + function Get-NetConfFile { + while ($true) { + if (Test-Path /host/etc/cni/net.d/) { + $files = @() + $files += Get-ChildItem -Path /host/etc/cni/net.d/ -Exclude "0-containerd-nat.json" + + if ($files.Length -gt 0) { + $file = (($files | Sort-Object LastWriteTime | Select-Object -Last 1).Name) + Write-Host "Using CNI conf file: $file" + return $file + } + } + + Write-Host "Waiting for CNI file..." + Start-Sleep 10 + } + } + mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount mkdir -force /host/k/kube-proxy @@ -9,7 +32,8 @@ data: cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? - $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name + $cniConfFile = Get-NetConfFile + $networkName = (Get-Content "/host/etc/cni/net.d/$cniConfFile" | ConvertFrom-Json).name $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName diff --git a/kubeadm/scripts/Install-Containerd.ps1 b/kubeadm/scripts/Install-Containerd.ps1 new file mode 100644 index 00000000..861c0966 --- /dev/null +++ b/kubeadm/scripts/Install-Containerd.ps1 @@ -0,0 +1,167 @@ +<# +.SYNOPSIS +Installs ContainerD on a Windows machines in preperation for joining the node to a Kubernetes cluster. + +.DESCRIPTION +This script +- Verifies that Windows Features requried for running contianers are enabled (and enables then if they are not) +- Downloads ContainerD binaries from from at the version specified. +- Downloads Windows SND CNI plugins. +- Sets up a basic nat networking config for ContainerD to use until another CNI is configured +- Registers ContainerD as a windows service. + +.PARAMETER ContainerDVersion +ContainerD version to download and use. + +.PARAMETER netAdapterName +Name of network adapter to use when configuring basic nat network. + +.EXAMPLE +PS> .\Install-Conatinerd.ps1 + +#> + +Param( + [parameter(HelpMessage = "ContainerD version to use")] + [string] $ContainerDVersion = "1.4.1", + [parameter(HelpMessage = "Name of network adapter to use when configuring basic nat network")] + [string] $netAdapterName = "Ethernet" +) + +$ErrorActionPreference = 'Stop' + +function DownloadFile($destination, $source) { + Write-Host("Downloading $source to $destination") + curl.exe --silent --fail -Lo $destination $source + + if (!$?) { + Write-Error "Download $source failed" + exit 1 + } +} + +<# +.DESCRIPTION +Computes a subnet for a gateway from the IPv4 IPAddress and PrefixLength properties +for a given network adapter. This value is used for IPAM in a nat CNI config required for +containerd. + +.NOTES +This logic is adapted from +https://github.com/containerd/containerd/blob/4a6b47d470d9f2dfc3d49f2819b968861dfa123e/script/setup/install-cni-windows + +.EXAMPLE +PS> CalculateSubNet -gateway 172.16.5.8 -prefixLength 24 +172.16.5.0/8 +#> +function CalculateSubNet { + param ( + [string]$gateway, + [int]$prefixLength + ) + $len = $prefixLength + $parts = $gateway.Split('.') + $result = @() + for ($i = 0; $i -le 3; $i++) { + if ($len -ge 8) { + $mask = 255 + + } + elseif ($len -gt 0) { + $mask = ((256 - 2 * (8 - $len))) + } + else { + $mask = 0 + } + $len -= 8 + $result += ([int]$parts[$i] -band $mask) + } + + $subnetIp = [string]::Join('.', $result) + $cidr = 32 - $prefixLength + return "${subnetIp}/$cidr" +} + +$requiredWindowsFeatures = @( + "Containers", + "Hyper-V", + "Hyper-V-PowerShell") + +function ValidateWindowsFeatures { + $allFeaturesInstalled = $true + foreach ($feature in $requiredWindowsFeatures) { + $f = Get-WindowsFeature -Name $feature + if (-not $f.Installed) { + Write-Warning "Windows feature: '$feature' is not installed." + $allFeaturesInstalled = $false + } + } + return $allFeaturesInstalled +} + +if (-not (ValidateWindowsFeatures)) { + Write-Output "Installing required windows features..." + + foreach ($feature in $requiredWindowsFeatures) { + Install-WindowsFeature -Name $feature + } + + Write-Output "Please reboot and re-run this script." + exit 0 +} + +Write-Output "Getting ContainerD binaries" +$global:ConainterDPath = "$env:ProgramFiles\containerd" +mkdir -Force $global:ConainterDPath | Out-Null +DownloadFile "$global:ConainterDPath\containerd.tar.gz" https://github.com/containerd/containerd/releases/download/v${ContainerDVersion}/containerd-${ContainerDVersion}-windows-amd64.tar.gz +tar.exe -xvf "$global:ConainterDPath\containerd.tar.gz" --strip=1 -C $global:ConainterDPath +$env:Path += ";$global:ConainterDPath" +[Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) +containerd.exe config default | Out-File "$global:ConainterDPath\config.toml" -Encoding ascii +#config file fixups +$config = Get-Content "$global:ConainterDPath\config.toml" +$config = $config -replace "bin_dir = (.)*$", "bin_dir = `"c:/opt/cni/bin`"" +$config = $config -replace "conf_dir = (.)*$", "conf_dir = `"c:/etc/cni/net.d`"" +$config | Set-Content "$global:ConainterDPath\config.toml" -Force + +mkdir -Force c:\opt\cni\bin | Out-Null +mkdir -Force c:\etc\cni\net.d | Out-Null + +Write-Output "Getting SDN CNI binaries" +DownloadFile "c:\opt\cni\cni-plugins.zip" https://github.com/microsoft/windows-container-networking/releases/download/v0.2.0/windows-container-networking-cni-amd64-v0.2.0.zip +Expand-Archive -Path "c:\opt\cni\cni-plugins.zip" -DestinationPath "c:\opt\cni\bin" -Force + +Write-Output "Creating network config for nat network" +$gateway = (Get-NetIPAddress -InterfaceAlias $netAdapterName -AddressFamily IPv4).IPAddress +$prefixLength = (Get-NetIPAddress -InterfaceAlias $netAdapterName -AddressFamily IPv4).PrefixLength + +$subnet = CalculateSubNet -gateway $gateway -prefixLength $prefixLength + +@" +{ + "cniVersion": "0.2.0", + "name": "nat", + "type": "nat", + "master": "Ethernet", + "ipam": { + "subnet": "$subnet", + "routes": [ + { + "GW": "$gateway" + } + ] + }, + "capabilities": { + "portMappings": true, + "dns": true + } +} +"@ | Set-Content "c:\etc\cni\net.d\0-containerd-nat.json" -Force + +Write-Output "Registering ContainerD as a service" +containerd.exe --register-service + +Write-Output "Starting ContainerD service" +Start-Service containerd + +Write-Output "Done - please remember to add '--cri-socket `"npipe:////./pipe/containerd-containerd`"' to your kubeadm join command" \ No newline at end of file diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1 index d4c2aa43..837fbf89 100644 --- a/kubeadm/scripts/PrepareNode.ps1 +++ b/kubeadm/scripts/PrepareNode.ps1 @@ -11,14 +11,20 @@ This script assists with joining a Windows node to a cluster. .PARAMETER KubernetesVersion Kubernetes version to download and use +.PARAMETER ContainerRuntime +Container that Kubernetes will use. (Docker or containerD) + .EXAMPLE -PS> .\PrepareNode.ps1 -KubernetesVersion v1.17.0 +PS> .\PrepareNode.ps1 -KubernetesVersion v1.19.3 -ContainerRuntime containerD #> Param( [parameter(Mandatory = $true, HelpMessage="Kubernetes version to use")] - [string] $KubernetesVersion + [string] $KubernetesVersion, + [parameter(HelpMessage="Container runtime that Kubernets will use")] + [ValidateSet("containerD", "Docker")] + [string] $ContainerRuntime = "Docker" ) $ErrorActionPreference = 'Stop' @@ -32,6 +38,18 @@ function DownloadFile($destination, $source) { } } +if ($ContainerRuntime -eq "Docker") { + if (-not(Test-Path "//./pipe/docker_engine")) { + Write-Error "Docker service was not detected - please install start Docker before calling PrepareNode.ps1 with -ContainerRuntime Docker" + exit 1 + } +} elseif ($ContainerRuntime -eq "containerD") { + if (-not(Test-Path "//./pipe/containerd-containerd")) { + Write-Error "ContainerD service was not detected - please install and start containerD before calling PrepareNode.ps1 with -ContainerRuntime containerD" + exit 1 + } +} + if (!$KubernetesVersion.StartsWith("v")) { $KubernetesVersion = "v" + $KubernetesVersion } @@ -51,9 +69,18 @@ DownloadFile $kubeletBinPath https://dl.k8s.io/$KubernetesVersion/bin/windows/am DownloadFile "$global:KubernetesPath\kubeadm.exe" https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubeadm.exe DownloadFile "$global:KubernetesPath\wins.exe" https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe -# Create host network to allow kubelet to schedule hostNetwork pods -Write-Host "Creating Docker host network" -docker network create -d nat host +if ($ContainerRuntime -eq "Docker") { + # Create host network to allow kubelet to schedule hostNetwork pods + # NOTE: For containerd the 0-containerd-nat.json network config template added by + # Install-containerd.ps1 joins pods to the host network. + Write-Host "Creating Docker host network" + docker network create -d nat host +} elseif ($ContainerRuntime -eq "containerD") { + DownloadFile "c:\k\hns.psm1" https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1 + Import-Module "c:\k\hns.psm1" + # TODO(marosset): check if network already exists before creatation + New-HnsNetwork -Type NAT -Name nat +} Write-Host "Registering wins service" wins.exe srv app run --register @@ -67,15 +94,20 @@ New-Item -path C:\var\lib\kubelet\etc\kubernetes\pki -type SymbolicLink -value C $StartKubeletFileContent = '$FileContent = Get-Content -Path "/var/lib/kubelet/kubeadm-flags.env" $global:KubeletArgs = $FileContent.TrimStart(''KUBELET_KUBEADM_ARGS='').Trim(''"'') -$netId = docker network ls -f name=host --format "{{ .ID }}" +$global:containerRuntime = {{CONTAINER_RUNTIME}} + +if ($global:containerRuntime -eq "Docker") { + $netId = docker network ls -f name=host --format "{{ .ID }}" -if ($netId.Length -lt 1) { + if ($netId.Length -lt 1) { docker network create -d nat host + } } $cmd = "C:\k\kubelet.exe $global:KubeletArgs --cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki --config=/var/lib/kubelet/config.yaml --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --hostname-override=$(hostname) --pod-infra-container-image=`"mcr.microsoft.com/oss/kubernetes/pause:1.4.1`" --enable-debugging-handlers --cgroups-per-qos=false --enforce-node-allocatable=`"`" --network-plugin=cni --resolv-conf=`"`" --log-dir=/var/log/kubelet --logtostderr=false --image-pull-progress-deadline=20m" Invoke-Expression $cmd' +$StartKubeletFileContent = $StartKubeletFileContent -replace "{{CONTAINER_RUNTIME}}", "`"$ContainerRuntime`"" Set-Content -Path $global:StartKubeletScript -Value $StartKubeletFileContent Write-Host "Installing nssm" @@ -97,6 +129,11 @@ $newPath = "$global:NssmInstallDirectory;" + Write-Host "Registering kubelet service" nssm install kubelet $global:Powershell $global:PowershellArgs $global:StartKubeletScript -nssm set kubelet DependOnService docker + +if ($ContainerRuntime -eq "Docker") { + nssm set kubelet DependOnService docker +} elseif ($ContainerRuntime -eq "containerD") { + nssm set kubelet DependOnService containerd +} New-NetFirewallRule -Name kubelet -DisplayName 'kubelet' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 10250