[K3s][Windows Port] Build script, multi-call binary, and Flannel (k3s…

…-io#7259)

* initial windows port.

Signed-off-by: Sean Yen <[email protected]>
Signed-off-by: Derek Nola <[email protected]>
Co-authored-by: Derek Nola <[email protected]>
Co-authored-by: Wei Ran <[email protected]>

cmd/k3s/main.go

@@ -9,7 +9,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
 
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/configfilearg"
@@ -46,8 +45,8 @@ func main() {
 	app := cmds.NewApp()
 	app.EnableBashCompletion = true
 	app.Commands = []cli.Command{
-		cmds.NewServerCommand(internalCLIAction(version.Program+"-server", dataDir, os.Args)),
-		cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent", dataDir, os.Args)),
+		cmds.NewServerCommand(internalCLIAction(version.Program+"-server"+programPostfix, dataDir, os.Args)),
+		cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent"+programPostfix, dataDir, os.Args)),
 		cmds.NewKubectlCommand(externalCLIAction("kubectl", dataDir)),
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
@@ -157,7 +156,7 @@ func externalCLI(cli, dataDir string, args []string) error {
 			os.Setenv("CRI_CONFIG_FILE", findCriConfig(dataDir))
 		}
 	}
-	return stageAndRun(dataDir, cli, append([]string{cli}, args...))
+	return stageAndRun(dataDir, cli, append([]string{cli}, args...), false)
 }
 
 // internalCLIAction returns a function that will call a K3s internal command, be used as the Action of a cli.Command.
@@ -173,11 +172,11 @@ func internalCLIAction(cmd, dataDir string, args []string) func(ctx *cli.Context
 
 // stageAndRunCLI calls an external binary.
 func stageAndRunCLI(cli *cli.Context, cmd string, dataDir string, args []string) error {
-	return stageAndRun(dataDir, cmd, args)
+	return stageAndRun(dataDir, cmd, args, true)
 }
 
 // stageAndRun does the actual work of setting up and calling an external binary.
-func stageAndRun(dataDir, cmd string, args []string) error {
+func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) error {
 	dir, err := extract(dataDir)
 	if err != nil {
 		return errors.Wrap(err, "extracting data")
@@ -186,9 +185,9 @@ func stageAndRun(dataDir, cmd string, args []string) error {
 
 	var pathEnv string
 	if findPreferBundledBin(args) {
-		pathEnv = filepath.Join(dir, "bin") + ":" + filepath.Join(dir, "bin/aux") + ":" + os.Getenv("PATH")
+		pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") + string(os.PathListSeparator) + os.Getenv("PATH")
 	} else {
-		pathEnv = filepath.Join(dir, "bin") + ":" + os.Getenv("PATH") + ":" + filepath.Join(dir, "bin/aux")
+		pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + os.Getenv("PATH") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux")
 	}
 	if err := os.Setenv("PATH", pathEnv); err != nil {
 		return err
@@ -204,10 +203,7 @@ func stageAndRun(dataDir, cmd string, args []string) error {
 
 	logrus.Debugf("Running %s %v", cmd, args)
 
-	if err := syscall.Exec(cmd, args, os.Environ()); err != nil {
-		return errors.Wrapf(err, "exec %s failed", cmd)
-	}
-	return nil
+	return runExec(cmd, args, calledAsInternal)
 }
 
 // getAssetAndDir returns the name of the bindata asset, along with a directory path
@@ -223,7 +219,7 @@ func getAssetAndDir(dataDir string) (string, string) {
 func extract(dataDir string) (string, error) {
 	// check if content already exists in requested data-dir
 	asset, dir := getAssetAndDir(dataDir)
-	if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil {
+	if _, err := os.Stat(filepath.Join(dir, "bin", "k3s"+programPostfix)); err == nil {
 		return dir, nil
 	}
 
@@ -232,7 +228,7 @@ func extract(dataDir string) (string, error) {
 	// dir if the assets already exist in the default path.
 	if dataDir != datadir.DefaultDataDir {
 		_, defaultDir := getAssetAndDir(datadir.DefaultDataDir)
-		if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s")); err == nil {
+		if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s"+programPostfix)); err == nil {
 			return defaultDir, nil
 		}
 	}

cmd/k3s/main_linux.go

@@ -0,0 +1,20 @@
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"os"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+const programPostfix = ""
+
+func runExec(cmd string, args []string, calledAsInternal bool) (err error) {
+	if err := syscall.Exec(cmd, args, os.Environ()); err != nil {
+		return errors.Wrapf(err, "exec %s failed", cmd)
+	}
+	return nil
+}

cmd/k3s/main_windows.go

@@ -0,0 +1,24 @@
+//go:build windows
+// +build windows
+
+package main
+
+import (
+	"os"
+	"os/exec"
+)
+
+const programPostfix = ".exe"
+
+func runExec(cmd string, args []string, calledAsInternal bool) (err error) {
+	// syscall.Exec: not supported by windows
+	if calledAsInternal {
+		args = args[1:]
+	}
+	cmdObj := exec.Command(cmd, args...)
+	cmdObj.Stdout = os.Stdout
+	cmdObj.Stderr = os.Stderr
+	cmdObj.Stdin = os.Stdin
+	cmdObj.Env = os.Environ()
+	return cmdObj.Run()
+}

go.mod

@@ -86,6 +86,7 @@ replace (
 )
 
 require (
+	github.com/Microsoft/hcsshim v0.11.1
 	github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000
@@ -184,7 +185,6 @@ require (
 	github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/Microsoft/hcsshim v0.11.1 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/Rican7/retry v0.1.0 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect

pkg/agent/flannel/setup.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"path/filepath"
+	goruntime "runtime"
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/agent/util"
@@ -23,34 +24,6 @@ import (
 )
 
 const (
-	cniConf = `{
-  "name":"cbr0",
-  "cniVersion":"1.0.0",
-  "plugins":[
-    {
-      "type":"flannel",
-      "delegate":{
-        "hairpinMode":true,
-        "forceAddress":true,
-        "isDefaultGateway":true
-      }
-    },
-    {
-      "type":"portmap",
-      "capabilities":{
-        "portMappings":true
-      }
-    },
-    {
-      "type":"bandwidth",
-      "capabilities":{
-        "bandwidth":true
-      }
-    }
-  ]
-}
-`
-
 	flannelConf = `{
 	"Network": "%CIDR%",
 	"EnableIPv6": %IPV6_ENABLED%,
@@ -60,10 +33,6 @@ const (
 }
 `
 
-	vxlanBackend = `{
-	"Type": "vxlan"
-}`
-
 	hostGWBackend = `{
 	"Type": "host-gw"
 }`
@@ -153,7 +122,20 @@ func createCNIConf(dir string, nodeConfig *config.Node) error {
 		logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.AgentConfig.FlannelCniConfFile)
 		return util.CopyFile(nodeConfig.AgentConfig.FlannelCniConfFile, p, false)
 	}
-	return util.WriteFile(p, cniConf)
+
+	cniConfJSON := cniConf
+	if goruntime.GOOS == "windows" {
+		extIface, err := LookupExtInterface(nodeConfig.FlannelIface, ipv4)
+		if err != nil {
+			return err
+		}
+
+		cniConfJSON = strings.ReplaceAll(cniConfJSON, "%IPV4_ADDRESS%", extIface.IfaceAddr.String())
+		cniConfJSON = strings.ReplaceAll(cniConfJSON, "%CLUSTER_CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
+		cniConfJSON = strings.ReplaceAll(cniConfJSON, "%SERVICE_CIDR%", nodeConfig.AgentConfig.ServiceCIDR.String())
+	}
+
+	return util.WriteFile(p, cniConfJSON)
 }
 
 func createFlannelConf(nodeConfig *config.Node) error {
@@ -205,6 +187,16 @@ func createFlannelConf(nodeConfig *config.Node) error {
 	var backendConf string
 	backendOptions := make(map[string]string)
 
+	// precheck and error out unsupported flannel backends.
+	switch nodeConfig.FlannelBackend {
+	case config.FlannelBackendHostGW:
+	case config.FlannelBackendTailscale:
+	case config.FlannelBackendWireguardNative:
+		if goruntime.GOOS == "windows" {
+			return fmt.Errorf("unsupported flannel backend '%s' for Windows", nodeConfig.FlannelBackend)
+		}
+	}
+
 	switch nodeConfig.FlannelBackend {
 	case config.FlannelBackendVXLAN:
 		backendConf = vxlanBackend

pkg/agent/flannel/setup_linux.go

@@ -0,0 +1,38 @@
+//go:build linux
+// +build linux
+
+package flannel
+
+const (
+	cniConf = `{
+  "name":"cbr0",
+  "cniVersion":"1.0.0",
+  "plugins":[
+    {
+      "type":"flannel",
+      "delegate":{
+        "hairpinMode":true,
+        "forceAddress":true,
+        "isDefaultGateway":true
+      }
+    },
+    {
+      "type":"portmap",
+      "capabilities":{
+        "portMappings":true
+      }
+    },
+    {
+      "type":"bandwidth",
+      "capabilities":{
+        "bandwidth":true
+      }
+    }
+  ]
+}
+`
+
+	vxlanBackend = `{
+	"Type": "vxlan"
+}`
+)

pkg/agent/flannel/setup_windows.go

@@ -0,0 +1,59 @@
+//go:build windows
+// +build windows
+
+package flannel
+
+const (
+	cniConf = `{
+  "name":"flannel.4096",
+  "cniVersion":"1.0.0",
+  "plugins":[
+    {
+      "type":"flannel",
+      "capabilities": {
+        "portMappings": true,
+        "dns": true
+      },
+      "delegate": {
+        "type": "win-overlay",
+        "apiVersion": 2,
+        "Policies": [{
+            "Name": "EndpointPolicy",
+            "Value": {
+                "Type": "OutBoundNAT",
+                "Settings": {
+                  "Exceptions": [
+                    "%CLUSTER_CIDR%", "%SERVICE_CIDR%"
+                  ]
+                }
+            }
+        }, {
+            "Name": "EndpointPolicy",
+            "Value": {
+                "Type": "SDNRoute",
+                "Settings": {
+                  "DestinationPrefix": "%SERVICE_CIDR%",
+                  "NeedEncap": true
+                }
+            }
+        }, {
+            "name": "EndpointPolicy",
+            "value": {
+                "Type": "ProviderAddress",
+                "Settings": {
+                    "ProviderAddress": "%IPV4_ADDRESS%"
+                }
+            }
+        }]
+      }
+    }
+  ]
+}
+`
+
+	vxlanBackend = `{
+	"Type": "vxlan",
+	"VNI": 4096,
+	"Port": 4789
+}`
+)

pkg/agent/run.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	goruntime "runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -83,6 +84,11 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	enableIPv6 := dualCluster || clusterIPv6
 	enableIPv4 := dualCluster || clusterIPv4
 
+	// dualStack or IPv6 are not supported on Windows node
+	if (goruntime.GOOS == "windows") && enableIPv6 {
+		return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node")
+	}
+
 	conntrackConfig, err := getConntrackConfig(nodeConfig)
 	if err != nil {
 		return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")

pkg/cli/crictl/crictl.go

@@ -1,11 +1,17 @@
 package crictl
 
 import (
+	"os"
+	"runtime"
+
 	"github.com/kubernetes-sigs/cri-tools/cmd/crictl"
 	"github.com/urfave/cli"
 )
 
 func Run(ctx *cli.Context) error {
+	if runtime.GOOS == "windows" {
+		os.Args = os.Args[1:]
+	}
 	crictl.Main()
 	return nil
 }