Skip to content

Commit

Permalink
Merge pull request #112 from marosset/kubeadm-containerd
Browse files Browse the repository at this point in the history
Adding containerd support to kubeadm related scripts/specs/etc
  • Loading branch information
k8s-ci-robot committed Nov 10, 2020
2 parents 25102ea + 64367ef commit afc5781
Show file tree
Hide file tree
Showing 5 changed files with 375 additions and 18 deletions.
76 changes: 71 additions & 5 deletions kubeadm/flannel/flannel-host-gw.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,34 @@ data:
mkdir -force /host/k/flannel
mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount
$cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
$containerRuntime = "docker"
if (Test-Path /host/etc/cni/net.d/0-containerd-nat.json) {
$containerRuntime = "containerd"
}
Write-Host "Configuring CNI for $containerRuntime"
$serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet
$podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet
$networkJson = wins cli net get | convertfrom-json
$cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR
$cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
$cniJson.delegate.policies[2].Value.DestinationPrefix = $networkJson.AddressCIDR
Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
if ($containerRuntime -eq "docker") {
$cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
$cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR
$cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
$cniJson.delegate.policies[2].Value.DestinationPrefix = $networkJson.AddressCIDR
Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
} elseif ($containerRuntime -eq "containerd") {
$cniJson = get-content /etc/kube-flannel-windows/cni-conf-containerd.json | ConvertFrom-Json
$cniJson.delegate.AdditionalArgs[0].Value.Settings.Exceptions = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR
$cniJson.delegate.AdditionalArgs[1].Value.Settings.DestinationPrefix = $serviceSubnet
$cniJson.delegate.AdditionalArgs[2].Value.Settings.DestinationPrefix = $networkJson.AddressCIDR
Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
}
cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
cp -force -recurse /cni/* /host/opt/cni/bin
Expand Down Expand Up @@ -74,6 +93,53 @@ data:
]
}
}
cni-conf-containerd.json: |
{
"cniVersion": "0.2.0",
"name": "cbr0",
"type": "flannel",
"capabilities": {
"portMappings": true,
"dns": true
},
"delegate": {
"type": "sdnbridge",
"optionalFlags" : {
"forceBridgeGateway" : true
},
"AdditionalArgs": [
{
"Name": "EndpointPolicy",
"Value": {
"Type": "OutBoundNAT",
"Settings": {
"Exceptions": []
}
}
},
{
"Name": "EndpointPolicy",
"Value": {
"Type": "SDNROUTE",
"Settings": {
"DestinationPrefix": "",
"NeedEncap": true
}
}
},
{
"Name": "EndpointPolicy",
"Value": {
"Type": "SDNROUTE",
"Settings": {
"DestinationPrefix": "",
"NeedEncap": true
}
}
}
]
}
}
---
apiVersion: apps/v1
kind: DaemonSet
Expand Down
71 changes: 67 additions & 4 deletions kubeadm/flannel/flannel-overlay.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,40 @@ data:
mkdir -force /host/k/flannel
mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount
$cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
$containerRuntime = "docker"
if (Test-Path /host/etc/cni/net.d/0-containerd-nat.json) {
$containerRuntime = "containerd"
}
Write-Host "Configuring CNI for $containerRuntime"
$serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet
$podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet
$networkJson = wins cli net get | convertfrom-json
$cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet
$cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
if ($containerRuntime -eq "docker") {
$cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
$cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet
$cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
} elseif ($containerRuntime -eq "containerd") {
$cniJson = get-content /etc/kube-flannel-windows/cni-conf-containerd.json | ConvertFrom-Json
$cniJson.delegate.AdditionalArgs[0].Value.Settings.Exceptions = $serviceSubnet, $podSubnet
$cniJson.delegate.AdditionalArgs[1].Value.Settings.DestinationPrefix = $serviceSubnet
$cniJson.delegate.AdditionalArgs[2].Value.Settings.ProviderAddress = $networkJson.AddressCIDR.Split('/')[0]
Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
}
cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
cp -force -recurse /cni/* /host/opt/cni/bin
cp -force /k/flannel/* /host/k/flannel/
cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml
cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/
wins cli process run --path /k/flannel/setup.exe --args "--mode=overlay --interface=Ethernet"
wins cli route add --addresses 169.254.169.254
wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
Expand Down Expand Up @@ -63,6 +83,49 @@ data:
]
}
}
cni-conf-containerd.json: |
{
"name": "flannel.4096",
"cniVersion": "0.2.0",
"type": "flannel",
"capabilities": {
"portMappings": true,
"dns": true
},
"delegate": {
"type": "sdnoverlay",
"AdditionalArgs": [
{
"Name": "EndpointPolicy",
"Value": {
"Type": "OutBoundNAT",
"Settings" : {
"Exceptions": []
}
}
},
{
"Name": "EndpointPolicy",
"Value": {
"Type": "SDNROUTE",
"Settings": {
"DestinationPrefix": "",
"NeedEncap": true
}
}
},
{
"Name":"EndpointPolicy",
"Value":{
"Type":"ProviderAddress",
"Settings":{
"ProviderAddress":""
}
}
}
]
}
}
---
apiVersion: apps/v1
kind: DaemonSet
Expand Down
26 changes: 25 additions & 1 deletion kubeadm/kube-proxy/kube-proxy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,38 @@ apiVersion: v1
data:
run-script.ps1: |-
$ErrorActionPreference = "Stop";
# Get newest cni conf file that is not 0-containerd-nat.json or spin until one shows up.
# With Docker the kube-proxy pod should not be scheduled to Windows nodes until host networking is configured.
# With contianerD host networking is required to schedule any pod including the CNI pods so a basic nat network is
# configured. This network should not be used by kube-proxy.
function Get-NetConfFile {
while ($true) {
if (Test-Path /host/etc/cni/net.d/) {
$files = @()
$files += Get-ChildItem -Path /host/etc/cni/net.d/ -Exclude "0-containerd-nat.json"
if ($files.Length -gt 0) {
$file = (($files | Sort-Object LastWriteTime | Select-Object -Last 1).Name)
Write-Host "Using CNI conf file: $file"
return $file
}
}
Write-Host "Waiting for CNI file..."
Start-Sleep 10
}
}
mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
mkdir -force /host/k/kube-proxy
cp -force /k/kube-proxy/* /host/k/kube-proxy
cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
$networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name
$cniConfFile = Get-NetConfFile
$networkName = (Get-Content "/host/etc/cni/net.d/$cniConfFile" | ConvertFrom-Json).name
$sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "."
yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip
yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName
Expand Down
167 changes: 167 additions & 0 deletions kubeadm/scripts/Install-Containerd.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
<#
.SYNOPSIS
Installs ContainerD on a Windows machines in preperation for joining the node to a Kubernetes cluster.
.DESCRIPTION
This script
- Verifies that Windows Features requried for running contianers are enabled (and enables then if they are not)
- Downloads ContainerD binaries from from at the version specified.
- Downloads Windows SND CNI plugins.
- Sets up a basic nat networking config for ContainerD to use until another CNI is configured
- Registers ContainerD as a windows service.
.PARAMETER ContainerDVersion
ContainerD version to download and use.
.PARAMETER netAdapterName
Name of network adapter to use when configuring basic nat network.
.EXAMPLE
PS> .\Install-Conatinerd.ps1
#>

Param(
[parameter(HelpMessage = "ContainerD version to use")]
[string] $ContainerDVersion = "1.4.1",
[parameter(HelpMessage = "Name of network adapter to use when configuring basic nat network")]
[string] $netAdapterName = "Ethernet"
)

$ErrorActionPreference = 'Stop'

function DownloadFile($destination, $source) {
Write-Host("Downloading $source to $destination")
curl.exe --silent --fail -Lo $destination $source

if (!$?) {
Write-Error "Download $source failed"
exit 1
}
}

<#
.DESCRIPTION
Computes a subnet for a gateway from the IPv4 IPAddress and PrefixLength properties
for a given network adapter. This value is used for IPAM in a nat CNI config required for
containerd.
.NOTES
This logic is adapted from
https://github.com/containerd/containerd/blob/4a6b47d470d9f2dfc3d49f2819b968861dfa123e/script/setup/install-cni-windows
.EXAMPLE
PS> CalculateSubNet -gateway 172.16.5.8 -prefixLength 24
172.16.5.0/8
#>
function CalculateSubNet {
param (
[string]$gateway,
[int]$prefixLength
)
$len = $prefixLength
$parts = $gateway.Split('.')
$result = @()
for ($i = 0; $i -le 3; $i++) {
if ($len -ge 8) {
$mask = 255

}
elseif ($len -gt 0) {
$mask = ((256 - 2 * (8 - $len)))
}
else {
$mask = 0
}
$len -= 8
$result += ([int]$parts[$i] -band $mask)
}

$subnetIp = [string]::Join('.', $result)
$cidr = 32 - $prefixLength
return "${subnetIp}/$cidr"
}

$requiredWindowsFeatures = @(
"Containers",
"Hyper-V",
"Hyper-V-PowerShell")

function ValidateWindowsFeatures {
$allFeaturesInstalled = $true
foreach ($feature in $requiredWindowsFeatures) {
$f = Get-WindowsFeature -Name $feature
if (-not $f.Installed) {
Write-Warning "Windows feature: '$feature' is not installed."
$allFeaturesInstalled = $false
}
}
return $allFeaturesInstalled
}

if (-not (ValidateWindowsFeatures)) {
Write-Output "Installing required windows features..."

foreach ($feature in $requiredWindowsFeatures) {
Install-WindowsFeature -Name $feature
}

Write-Output "Please reboot and re-run this script."
exit 0
}

Write-Output "Getting ContainerD binaries"
$global:ConainterDPath = "$env:ProgramFiles\containerd"
mkdir -Force $global:ConainterDPath | Out-Null
DownloadFile "$global:ConainterDPath\containerd.tar.gz" https://github.com/containerd/containerd/releases/download/v${ContainerDVersion}/containerd-${ContainerDVersion}-windows-amd64.tar.gz
tar.exe -xvf "$global:ConainterDPath\containerd.tar.gz" --strip=1 -C $global:ConainterDPath
$env:Path += ";$global:ConainterDPath"
[Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
containerd.exe config default | Out-File "$global:ConainterDPath\config.toml" -Encoding ascii
#config file fixups
$config = Get-Content "$global:ConainterDPath\config.toml"
$config = $config -replace "bin_dir = (.)*$", "bin_dir = `"c:/opt/cni/bin`""
$config = $config -replace "conf_dir = (.)*$", "conf_dir = `"c:/etc/cni/net.d`""
$config | Set-Content "$global:ConainterDPath\config.toml" -Force

mkdir -Force c:\opt\cni\bin | Out-Null
mkdir -Force c:\etc\cni\net.d | Out-Null

Write-Output "Getting SDN CNI binaries"
DownloadFile "c:\opt\cni\cni-plugins.zip" https://github.com/microsoft/windows-container-networking/releases/download/v0.2.0/windows-container-networking-cni-amd64-v0.2.0.zip
Expand-Archive -Path "c:\opt\cni\cni-plugins.zip" -DestinationPath "c:\opt\cni\bin" -Force

Write-Output "Creating network config for nat network"
$gateway = (Get-NetIPAddress -InterfaceAlias $netAdapterName -AddressFamily IPv4).IPAddress
$prefixLength = (Get-NetIPAddress -InterfaceAlias $netAdapterName -AddressFamily IPv4).PrefixLength

$subnet = CalculateSubNet -gateway $gateway -prefixLength $prefixLength

@"
{
"cniVersion": "0.2.0",
"name": "nat",
"type": "nat",
"master": "Ethernet",
"ipam": {
"subnet": "$subnet",
"routes": [
{
"GW": "$gateway"
}
]
},
"capabilities": {
"portMappings": true,
"dns": true
}
}
"@ | Set-Content "c:\etc\cni\net.d\0-containerd-nat.json" -Force

Write-Output "Registering ContainerD as a service"
containerd.exe --register-service

Write-Output "Starting ContainerD service"
Start-Service containerd

Write-Output "Done - please remember to add '--cri-socket `"npipe:////./pipe/containerd-containerd`"' to your kubeadm join command"
Loading

0 comments on commit afc5781

Please sign in to comment.