Skip to content

Commit

Permalink
Windows support
Browse files Browse the repository at this point in the history
Signed-off-by: Sean Yen <[email protected]>
  • Loading branch information
seanyen authored and manuelbuil committed Oct 16, 2023
1 parent 1f3880c commit d49c099
Show file tree
Hide file tree
Showing 22 changed files with 465 additions and 108 deletions.
24 changes: 10 additions & 14 deletions cmd/k3s/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"

"github.com/k3s-io/k3s/pkg/cli/cmds"
"github.com/k3s-io/k3s/pkg/configfilearg"
Expand Down Expand Up @@ -46,8 +45,8 @@ func main() {
app := cmds.NewApp()
app.EnableBashCompletion = true
app.Commands = []cli.Command{
cmds.NewServerCommand(internalCLIAction(version.Program+"-server", dataDir, os.Args)),
cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent", dataDir, os.Args)),
cmds.NewServerCommand(internalCLIAction(version.Program+"-server"+programPostfix, dataDir, os.Args)),
cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent"+programPostfix, dataDir, os.Args)),
cmds.NewKubectlCommand(externalCLIAction("kubectl", dataDir)),
cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
Expand Down Expand Up @@ -158,7 +157,7 @@ func externalCLI(cli, dataDir string, args []string) error {
os.Setenv("CRI_CONFIG_FILE", findCriConfig(dataDir))
}
}
return stageAndRun(dataDir, cli, append([]string{cli}, args...))
return stageAndRun(dataDir, cli, append([]string{cli}, args...), false)
}

// internalCLIAction returns a function that will call a K3s internal command, be used as the Action of a cli.Command.
Expand All @@ -174,11 +173,11 @@ func internalCLIAction(cmd, dataDir string, args []string) func(ctx *cli.Context

// stageAndRunCLI calls an external binary.
func stageAndRunCLI(cli *cli.Context, cmd string, dataDir string, args []string) error {
return stageAndRun(dataDir, cmd, args)
return stageAndRun(dataDir, cmd, args, true)
}

// stageAndRun does the actual work of setting up and calling an external binary.
func stageAndRun(dataDir, cmd string, args []string) error {
func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) error {
dir, err := extract(dataDir)
if err != nil {
return errors.Wrap(err, "extracting data")
Expand All @@ -187,9 +186,9 @@ func stageAndRun(dataDir, cmd string, args []string) error {

var pathEnv string
if findPreferBundledBin(args) {
pathEnv = filepath.Join(dir, "bin") + ":" + filepath.Join(dir, "bin/aux") + ":" + os.Getenv("PATH")
pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") + string(os.PathListSeparator) + os.Getenv("PATH")
} else {
pathEnv = filepath.Join(dir, "bin") + ":" + os.Getenv("PATH") + ":" + filepath.Join(dir, "bin/aux")
pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + os.Getenv("PATH") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux")
}
if err := os.Setenv("PATH", pathEnv); err != nil {
return err
Expand All @@ -205,10 +204,7 @@ func stageAndRun(dataDir, cmd string, args []string) error {

logrus.Debugf("Running %s %v", cmd, args)

if err := syscall.Exec(cmd, args, os.Environ()); err != nil {
return errors.Wrapf(err, "exec %s failed", cmd)
}
return nil
return runExec(cmd, args, calledAsInternal)
}

// getAssetAndDir returns the name of the bindata asset, along with a directory path
Expand All @@ -224,7 +220,7 @@ func getAssetAndDir(dataDir string) (string, string) {
func extract(dataDir string) (string, error) {
// check if content already exists in requested data-dir
asset, dir := getAssetAndDir(dataDir)
if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil {
if _, err := os.Stat(filepath.Join(dir, "bin", "k3s"+programPostfix)); err == nil {
return dir, nil
}

Expand All @@ -233,7 +229,7 @@ func extract(dataDir string) (string, error) {
// dir if the assets already exist in the default path.
if dataDir != datadir.DefaultDataDir {
_, defaultDir := getAssetAndDir(datadir.DefaultDataDir)
if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s")); err == nil {
if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s"+programPostfix)); err == nil {
return defaultDir, nil
}
}
Expand Down
20 changes: 20 additions & 0 deletions cmd/k3s/main_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
//go:build linux
// +build linux

package main

import (
"os"
"syscall"

"github.com/pkg/errors"
)

const programPostfix = ""

func runExec(cmd string, args []string, calledAsInternal bool) (err error) {
if err := syscall.Exec(cmd, args, os.Environ()); err != nil {
return errors.Wrapf(err, "exec %s failed", cmd)
}
return nil
}
24 changes: 24 additions & 0 deletions cmd/k3s/main_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
//go:build windows
// +build windows

package main

import (
"os"
"os/exec"
)

const programPostfix = ".exe"

func runExec(cmd string, args []string, calledAsInternal bool) (err error) {
// syscall.Exec: not supported by windows
if calledAsInternal {
args = args[1:]
}
cmdObj := exec.Command(cmd, args...)
cmdObj.Stdout = os.Stdout
cmdObj.Stderr = os.Stderr
cmdObj.Stdin = os.Stdin
cmdObj.Env = os.Environ()
return cmdObj.Run()
}
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ replace (
)

require (
github.com/Microsoft/hcsshim v0.11.1
github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000
github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000
github.com/containerd/aufs v1.0.0
Expand Down Expand Up @@ -181,8 +182,7 @@ require (
github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b // indirect
github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/Rican7/retry v0.1.0 // indirect
github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
Expand Down
3 changes: 2 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,9 @@ github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59
github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg=
github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Microsoft/hcsshim v0.8.25 h1:fRMwXiwk3qDwc0P05eHnh+y2v07JdtsfQ1fuAc69m9g=
github.com/Microsoft/hcsshim v0.8.25/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
Expand Down
58 changes: 25 additions & 33 deletions pkg/agent/flannel/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net"
"os/exec"
"path/filepath"
goruntime "runtime"
"strings"

"github.com/k3s-io/k3s/pkg/agent/util"
Expand All @@ -24,34 +25,6 @@ import (
)

const (
cniConf = `{
"name":"cbr0",
"cniVersion":"1.0.0",
"plugins":[
{
"type":"flannel",
"delegate":{
"hairpinMode":true,
"forceAddress":true,
"isDefaultGateway":true
}
},
{
"type":"portmap",
"capabilities":{
"portMappings":true
}
},
{
"type":"bandwidth",
"capabilities":{
"bandwidth":true
}
}
]
}
`

flannelConf = `{
"Network": "%CIDR%",
"EnableIPv6": %IPV6_ENABLED%,
Expand All @@ -61,10 +34,6 @@ const (
}
`

vxlanBackend = `{
"Type": "vxlan"
}`

hostGWBackend = `{
"Type": "host-gw"
}`
Expand Down Expand Up @@ -168,7 +137,20 @@ func createCNIConf(dir string, nodeConfig *config.Node) error {
logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.AgentConfig.FlannelCniConfFile)
return util.CopyFile(nodeConfig.AgentConfig.FlannelCniConfFile, p, false)
}
return util.WriteFile(p, cniConf)

cniConfJSON := cniConf
if goruntime.GOOS == "windows" {
extIface, err := LookupExtInterface(nodeConfig.FlannelIface, ipv4)
if err != nil {
return err
}

cniConfJSON = strings.ReplaceAll(cniConfJSON, "%IPV4_ADDRESS%", extIface.IfaceAddr.String())
cniConfJSON = strings.ReplaceAll(cniConfJSON, "%CLUSTER_CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
cniConfJSON = strings.ReplaceAll(cniConfJSON, "%SERVICE_CIDR%", nodeConfig.AgentConfig.ServiceCIDR.String())
}

return util.WriteFile(p, cniConfJSON)
}

func createFlannelConf(nodeConfig *config.Node) error {
Expand Down Expand Up @@ -234,7 +216,17 @@ func createFlannelConf(nodeConfig *config.Node) error {
}
}

// precheck and error out unsupported flannel backends.
switch backend {
case config.FlannelBackendHostGW:
case config.FlannelBackendTailscale:
case config.FlannelBackendWireguardNative:
if goruntime.GOOS == "windows" {
return fmt.Errorf("unsupported flannel backend '%s' for Windows", nodeConfig.FlannelBackend)
}
}

switch nodeConfig.FlannelBackend {
case config.FlannelBackendVXLAN:
backendConf = vxlanBackend
case config.FlannelBackendHostGW:
Expand Down
38 changes: 38 additions & 0 deletions pkg/agent/flannel/setup_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//go:build linux
// +build linux

package flannel

const (
cniConf = `{
"name":"cbr0",
"cniVersion":"1.0.0",
"plugins":[
{
"type":"flannel",
"delegate":{
"hairpinMode":true,
"forceAddress":true,
"isDefaultGateway":true
}
},
{
"type":"portmap",
"capabilities":{
"portMappings":true
}
},
{
"type":"bandwidth",
"capabilities":{
"bandwidth":true
}
}
]
}
`

vxlanBackend = `{
"Type": "vxlan"
}`
)
59 changes: 59 additions & 0 deletions pkg/agent/flannel/setup_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
//go:build windows
// +build windows

package flannel

const (
cniConf = `{
"name":"flannel.4096",
"cniVersion":"1.0.0",
"plugins":[
{
"type":"flannel",
"capabilities": {
"portMappings": true,
"dns": true
},
"delegate": {
"type": "win-overlay",
"apiVersion": 2,
"Policies": [{
"Name": "EndpointPolicy",
"Value": {
"Type": "OutBoundNAT",
"Settings": {
"Exceptions": [
"%CLUSTER_CIDR%", "%SERVICE_CIDR%"
]
}
}
}, {
"Name": "EndpointPolicy",
"Value": {
"Type": "SDNRoute",
"Settings": {
"DestinationPrefix": "%SERVICE_CIDR%",
"NeedEncap": true
}
}
}, {
"name": "EndpointPolicy",
"value": {
"Type": "ProviderAddress",
"Settings": {
"ProviderAddress": "%IPV4_ADDRESS%"
}
}
}]
}
}
]
}
`

vxlanBackend = `{
"Type": "vxlan",
"VNI": 4096,
"Port": 4789
}`
)
6 changes: 6 additions & 0 deletions pkg/agent/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net"
"os"
"path/filepath"
goruntime "runtime"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -84,6 +85,11 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
enableIPv6 := dualCluster || clusterIPv6
enableIPv4 := dualCluster || clusterIPv4

// dualStack or IPv6 are not supported on Windows node
if (goruntime.GOOS == "windows") && enableIPv6 {
return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node")
}

conntrackConfig, err := getConntrackConfig(nodeConfig)
if err != nil {
return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
Expand Down
6 changes: 6 additions & 0 deletions pkg/cli/crictl/crictl.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
package crictl

import (
"os"
"runtime"

"github.com/kubernetes-sigs/cri-tools/cmd/crictl"
"github.com/urfave/cli"
)

func Run(ctx *cli.Context) error {
if runtime.GOOS == "windows" {
os.Args = os.Args[1:]
}
crictl.Main()
return nil
}
File renamed without changes.
Loading

0 comments on commit d49c099

Please sign in to comment.