Automation for building flannel and kubeproxy related containers (#111)

* 1. Switch to nanoserver; 2. Update flannel to 0.13; 3. Switch to use buildx to build flannel and kubeproxy for multiple Windows versions

* rename Add-Manifest to Push-Manifest

* remove extra Write-Host

* refactoring: renames

* add more logs to buildx.psm1

* change trigger for flannel to on.push.tags.v*

* update flannel yamls

* use GitTag as suffix for flannel build

* fix suffix for flannel image tag

* simplify version comparision and add more logs

* typos fix

* remove hns.psm1 from repository because we always download actual version of it

* rallback to moby/buildkit:v0.7.2

* Add notes for `docker manifest annotate`

* add warning when manifest already exists

* add '-nanoserver' suffix to flannel and kube-proxy manifests

.github/workflows/flannel.yml

@@ -0,0 +1,33 @@
+name: flannel-images
+
+on:
+  push:
+    tags:
+      - v*
+  pull_request:
+    paths:
+      - "kubeadm/flannel/**"
+      - "kubeadm/buildx.psm1"
+      - ".github/workflows/flannel.yml"
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        shell: pwsh
+        working-directory: ./kubeadm/flannel
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build and push images
+      if: ${{ github.event_name == 'push' }}
+      run: |
+        echo ${{ secrets.DOCKER_SECRET }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+        $gittag = '-'+(($env:GITHUB_REF -split '/' | select-object -skip 2) -join '-')
+        ./build.ps1 -push -tagSuffix $gittag
+    - name: Build images (without push)
+      if: ${{ github.event_name == 'pull_request' }}
+      run: |
+        ./build.ps1

.github/workflows/kube-proxy.yml

@@ -3,15 +3,29 @@ name: kube-proxy-images
 on:
   schedule:
   - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - "kubeadm/kube-proxy/**"
+      - "kubeadm/buildx.psm1"
+      - ".github/workflows/kube-proxy.yml"
+    branches:
+      - master
 
 jobs:
   build:
-    runs-on: windows-2019
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        shell: pwsh
+        working-directory: ./kubeadm/kube-proxy
     steps:
     - uses: actions/checkout@v2
-    - uses: azure/docker-login@v1
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_SECRET }}
-    - name: Build
-      run: go run kubeadm/hack/publish-kubeproxy.go kubeadm/kube-proxy
+    - name: Build and push images
+      if: ${{ github.event_name == 'schedule' }}
+      run: |
+        echo ${{ secrets.DOCKER_SECRET }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+        ./build.ps1 -push
+    - name: Build images (without push)
+      if: ${{ github.event_name == 'pull_request' }}
+      run: |
+        ./build.ps1

kubeadm/buildx.psm1

@@ -0,0 +1,79 @@
+function Set-Builder()
+{
+    $env:DOCKER_CLI_EXPERIMENTAL = "enabled"
+    & docker buildx create --name img-builder --use --driver docker-container --driver-opt image=moby/buildkit:v0.7.2
+}
+
+function New-Build()
+{
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory = $true)]
+        [string]$name,
+        [Parameter(Mandatory = $true)]
+        [ValidateSet("docker", "registry")]
+        [string]$output,
+        [Parameter(Mandatory = $true)]
+        [string[]]$args
+    )
+
+    $command = "docker buildx build --platform windows/amd64 --output=type=$output -f Dockerfile -t $name"
+    foreach($arg in $args)
+    {
+        $command = "$command --build-arg=$arg"
+    }
+    $command = "$command ."
+    Write-Host $command
+    Invoke-Expression $command
+}
+
+function Get-ManifestName([string]$name)
+{
+    if (($name -split "/").Length -eq 1) {
+        $name = "library/$name"
+    }
+    if (($name -split "/").Length -eq 2) {
+        $name = "docker.io/$name"
+    }
+    return ($name -replace "/", "_") -replace ":", "-"
+}
+
+function Push-Manifest([string]$name, [string[]]$items, [string[]]$bases)
+{
+    $folder = Get-ManifestName -name $name
+    if (Test-Path "~/.docker/manifests/$folder")
+    {
+        Write-Warning "Manifest $name already exists and will be overridden."
+        & docker manifest rm $name | out-null
+    }
+
+    $command = "docker manifest create $name";
+    foreach($item in $items)
+    {
+        $command = "$command --amend $item"
+    }
+    Write-Host $command
+    Invoke-Expression $command
+
+    # Use `docker manifest annotate` instead of this when docker cli 20.* is ready.
+    # See details: https://github.com/docker/cli/pull/2578
+    for ($i = 0; $i -lt $items.Length; $i++) {
+        $base = $bases[$i]
+        $item = $items[$i]
+
+        $manifest = $(docker manifest inspect $base -v) | ConvertFrom-Json
+        $platform = $manifest.Descriptor.platform
+
+        $img = Get-ManifestName -name $item
+
+        $manifest = Get-Content "~/.docker/manifests/$folder/$img" | ConvertFrom-Json
+        $manifest.Descriptor.platform = $platform
+        $manifest | ConvertTo-Json -Depth 10 -Compress | Set-Content "~/.docker/manifests/$folder/$img"
+    }
+
+    & docker manifest push $name
+}
+
+Export-ModuleMember Set-Builder
+Export-ModuleMember New-Build
+Export-ModuleMember Push-Manifest

kubeadm/flannel/Dockerfile

@@ -1,30 +1,39 @@
-ARG servercoreTag="ltsc2019"
-ARG cniVersion="0.8.5"
-ARG golangTag=windowsservercore-1809
+ARG BASE="mcr.microsoft.com/powershell:nanoserver-1809"
+ARG cniVersion="0.8.7"
+ARG flannelVersion="0.13.0"
 
-FROM golang:${golangTag} as builder
-ADD setup.go build/
-RUN go build -o build/setup.exe build/setup.go
-
-FROM mcr.microsoft.com/windows/servercore:${servercoreTag}
-SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+FROM --platform=linux/amd64 golang as setup
+ENV GOOS=windows
+ENV GOARCH=amd64
+WORKDIR /build
+ADD setup.go .
+RUN go build -o setup.exe setup.go
 
+FROM --platform=linux/amd64 curlimages/curl as bins
 ARG cniVersion
+ARG flannelVersion
+
+WORKDIR /utils
+RUN curl -Lo wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
+RUN curl -Lo yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe
+
+WORKDIR /cni
+RUN curl -Lo cni.tgz https://github.com/containernetworking/plugins/releases/download/v${cniVersion}/cni-plugins-windows-amd64-v${cniVersion}.tgz
+RUN tar -xf cni.tgz
+RUN rm cni.tgz
+
+WORKDIR /flannel 
+RUN curl -Lo flanneld.exe https://github.com/coreos/flannel/releases/download/v${flannelVersion}/flanneld.exe
 
-RUN mkdir -force C:\k\flannel; \
-  pushd C:\k\flannel; \
-  curl.exe -LO https://github.com/coreos/flannel/releases/download/v0.12.0/flanneld.exe
+FROM $BASE
 
-ADD hns.psm1 /k/flannel
-COPY --from=builder /gopath/build/setup.exe /k/flannel/setup.exe
+ENV PATH="C:\Program Files\PowerShell;C:\utils;C:\Windows\system32;C:\Windows;"
 
-RUN mkdir C:\cni; \
-  pushd C:\cni; \
-  curl.exe -Lo cni.tgz https://github.com/containernetworking/plugins/releases/download/v${env:cniVersion}/cni-plugins-windows-amd64-v${env:cniVersion}.tgz; \
-  tar -xf cni.tgz; \
-  rm cni.tgz
+# wins.exe doesn't work in nanoserver with default ContainerUser.
+USER ContainerAdministrator
 
-RUN mkdir C:\utils; \
-  curl.exe -Lo C:\utils\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe; \
-  curl.exe -Lo C:\utils\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe; \
-  "[Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\utils', [EnvironmentVariableTarget]::Machine)"
+COPY --from=bins /utils /utils
+COPY --from=bins /cni /cni
+ADD https://raw.githubusercontent.com/microsoft/SDN/master/Kubernetes/windows/hns.psm1 /k/flannel/hns.psm1
+COPY --from=setup /build/setup.exe /k/flannel/setup.exe
+COPY --from=bins /flannel/flanneld.exe /k/flannel/flanneld.exe

kubeadm/flannel/build.ps1

@@ -0,0 +1,35 @@
+param(
+    [switch] $push, 
+    [string] $image = "sigwindowstools/flannel",
+    [string] $tagSuffix = ""
+)
+
+$output="docker"
+if ($push.IsPresent) {
+    $output="registry"
+}
+
+Import-Module "../buildx.psm1"
+Set-Builder
+
+$config = Get-Content .\buildconfig.json | ConvertFrom-Json
+foreach ($flannel in $config.flannel)
+{
+    Write-Host "Build images for flannel version: $flannel"
+
+    [string[]]$items = @()
+    [string[]]$bases = @()
+    foreach($tag in $config.tagsMap) 
+    {
+        $base = "$($config.baseimage):$($tag.source)"
+        $current = "$($image):v$($flannel)-$($tag.target)$($tagSuffix)"
+        $bases += $base
+        $items += $current
+        New-Build -name $current -output $output -args @("BASE=$base", "flannelVersion=$flannel")
+    }
+
+    if ($push.IsPresent)
+    {
+        Push-Manifest -name "$($image):v$flannel-nanoserver" -items $items -bases $bases
+    }
+}

kubeadm/flannel/buildconfig.json

@@ -0,0 +1,13 @@
+{
+    "flannel": [
+        "0.12.0", 
+        "0.13.0"
+    ],
+    "baseimage": "mcr.microsoft.com/powershell",
+    "tagsMap":[
+        {"source":"nanoserver-2004","target":"2004"},
+        {"source":"nanoserver-1909","target":"1909"},
+        {"source":"nanoserver-1903","target":"1903"},
+        {"source":"nanoserver-1809","target":"1809"}
+    ]
+}

kubeadm/flannel/flannel-host-gw.yml

@@ -179,9 +179,9 @@ spec:
         effect: NoSchedule
       containers:
       - name: kube-flannel
-        image: sigwindowstools/flannel:0.12.0
+        image: sigwindowstools/flannel:v0.13.0-nanoserver
         command:
-        - powershell
+        - pwsh
         args:
         - -file
         - /etc/kube-flannel-windows/run.ps1

kubeadm/flannel/flannel-overlay.yml

@@ -165,9 +165,9 @@ spec:
         effect: NoSchedule
       containers:
       - name: kube-flannel
-        image: sigwindowstools/flannel:0.12.0
+        image: sigwindowstools/flannel:v0.13.0-nanoserver
         command:
-        - powershell
+        - pwsh
         args:
         - -file
         - /etc/kube-flannel-windows/run.ps1