From d49c09935a93a1b56e02a502081709d1f31366b7 Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Sun, 9 Apr 2023 10:57:45 -0700 Subject: [PATCH] Windows support Signed-off-by: Sean Yen --- cmd/k3s/main.go | 24 ++-- cmd/k3s/main_linux.go | 20 +++ cmd/k3s/main_windows.go | 24 ++++ go.mod | 4 +- go.sum | 3 +- pkg/agent/flannel/setup.go | 58 ++++----- pkg/agent/flannel/setup_linux.go | 38 ++++++ pkg/agent/flannel/setup_windows.go | 59 +++++++++ pkg/agent/run.go | 6 + pkg/cli/crictl/crictl.go | 6 + ...{builtins_cri_linux.go => builtins_cri.go} | 0 pkg/containerd/builtins_windows.go | 27 ++++ pkg/daemons/agent/agent_windows.go | 1 + pkg/daemons/executor/embed.go | 4 +- pkg/daemons/executor/embed_linux.go | 16 +++ pkg/daemons/executor/embed_windows.go | 90 +++++++++++++ pkg/kubectl/main.go | 4 + scripts/build | 120 ++++++++++++------ scripts/download | 21 ++- scripts/package-cli | 35 +++-- scripts/validate | 2 +- scripts/version.sh | 11 ++ 22 files changed, 465 insertions(+), 108 deletions(-) create mode 100644 cmd/k3s/main_linux.go create mode 100644 cmd/k3s/main_windows.go create mode 100644 pkg/agent/flannel/setup_linux.go create mode 100644 pkg/agent/flannel/setup_windows.go rename pkg/containerd/{builtins_cri_linux.go => builtins_cri.go} (100%) create mode 100644 pkg/containerd/builtins_windows.go create mode 100644 pkg/daemons/executor/embed_linux.go create mode 100644 pkg/daemons/executor/embed_windows.go diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 8de363793646..934fce22749f 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -9,7 +9,6 @@ import ( "path/filepath" "strconv" "strings" - "syscall" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/configfilearg" @@ -46,8 +45,8 @@ func main() { app := cmds.NewApp() app.EnableBashCompletion = true app.Commands = []cli.Command{ - cmds.NewServerCommand(internalCLIAction(version.Program+"-server", dataDir, os.Args)), - cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent", dataDir, os.Args)), + cmds.NewServerCommand(internalCLIAction(version.Program+"-server"+programPostfix, dataDir, os.Args)), + cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent"+programPostfix, dataDir, os.Args)), cmds.NewKubectlCommand(externalCLIAction("kubectl", dataDir)), cmds.NewCRICTL(externalCLIAction("crictl", dataDir)), cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)), @@ -158,7 +157,7 @@ func externalCLI(cli, dataDir string, args []string) error { os.Setenv("CRI_CONFIG_FILE", findCriConfig(dataDir)) } } - return stageAndRun(dataDir, cli, append([]string{cli}, args...)) + return stageAndRun(dataDir, cli, append([]string{cli}, args...), false) } // internalCLIAction returns a function that will call a K3s internal command, be used as the Action of a cli.Command. @@ -174,11 +173,11 @@ func internalCLIAction(cmd, dataDir string, args []string) func(ctx *cli.Context // stageAndRunCLI calls an external binary. func stageAndRunCLI(cli *cli.Context, cmd string, dataDir string, args []string) error { - return stageAndRun(dataDir, cmd, args) + return stageAndRun(dataDir, cmd, args, true) } // stageAndRun does the actual work of setting up and calling an external binary. -func stageAndRun(dataDir, cmd string, args []string) error { +func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) error { dir, err := extract(dataDir) if err != nil { return errors.Wrap(err, "extracting data") @@ -187,9 +186,9 @@ func stageAndRun(dataDir, cmd string, args []string) error { var pathEnv string if findPreferBundledBin(args) { - pathEnv = filepath.Join(dir, "bin") + ":" + filepath.Join(dir, "bin/aux") + ":" + os.Getenv("PATH") + pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") + string(os.PathListSeparator) + os.Getenv("PATH") } else { - pathEnv = filepath.Join(dir, "bin") + ":" + os.Getenv("PATH") + ":" + filepath.Join(dir, "bin/aux") + pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + os.Getenv("PATH") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") } if err := os.Setenv("PATH", pathEnv); err != nil { return err @@ -205,10 +204,7 @@ func stageAndRun(dataDir, cmd string, args []string) error { logrus.Debugf("Running %s %v", cmd, args) - if err := syscall.Exec(cmd, args, os.Environ()); err != nil { - return errors.Wrapf(err, "exec %s failed", cmd) - } - return nil + return runExec(cmd, args, calledAsInternal) } // getAssetAndDir returns the name of the bindata asset, along with a directory path @@ -224,7 +220,7 @@ func getAssetAndDir(dataDir string) (string, string) { func extract(dataDir string) (string, error) { // check if content already exists in requested data-dir asset, dir := getAssetAndDir(dataDir) - if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil { + if _, err := os.Stat(filepath.Join(dir, "bin", "k3s"+programPostfix)); err == nil { return dir, nil } @@ -233,7 +229,7 @@ func extract(dataDir string) (string, error) { // dir if the assets already exist in the default path. if dataDir != datadir.DefaultDataDir { _, defaultDir := getAssetAndDir(datadir.DefaultDataDir) - if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s")); err == nil { + if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s"+programPostfix)); err == nil { return defaultDir, nil } } diff --git a/cmd/k3s/main_linux.go b/cmd/k3s/main_linux.go new file mode 100644 index 000000000000..76a7b641795f --- /dev/null +++ b/cmd/k3s/main_linux.go @@ -0,0 +1,20 @@ +//go:build linux +// +build linux + +package main + +import ( + "os" + "syscall" + + "github.com/pkg/errors" +) + +const programPostfix = "" + +func runExec(cmd string, args []string, calledAsInternal bool) (err error) { + if err := syscall.Exec(cmd, args, os.Environ()); err != nil { + return errors.Wrapf(err, "exec %s failed", cmd) + } + return nil +} diff --git a/cmd/k3s/main_windows.go b/cmd/k3s/main_windows.go new file mode 100644 index 000000000000..cefca3c91e67 --- /dev/null +++ b/cmd/k3s/main_windows.go @@ -0,0 +1,24 @@ +//go:build windows +// +build windows + +package main + +import ( + "os" + "os/exec" +) + +const programPostfix = ".exe" + +func runExec(cmd string, args []string, calledAsInternal bool) (err error) { + // syscall.Exec: not supported by windows + if calledAsInternal { + args = args[1:] + } + cmdObj := exec.Command(cmd, args...) + cmdObj.Stdout = os.Stdout + cmdObj.Stderr = os.Stderr + cmdObj.Stdin = os.Stdin + cmdObj.Env = os.Environ() + return cmdObj.Run() +} diff --git a/go.mod b/go.mod index 4d3b8ec58e5f..dbdd65c315b6 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ replace ( ) require ( + github.com/Microsoft/hcsshim v0.11.1 github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000 github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000 github.com/containerd/aufs v1.0.0 @@ -181,8 +182,7 @@ require ( github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b // indirect github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect - github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/Rican7/retry v0.1.0 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect diff --git a/go.sum b/go.sum index 9876d9483445..af4727281e15 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,9 @@ github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.8.25 h1:fRMwXiwk3qDwc0P05eHnh+y2v07JdtsfQ1fuAc69m9g= github.com/Microsoft/hcsshim v0.8.25/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index 82a622fd60b8..6c8013a9e10f 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -6,6 +6,7 @@ import ( "net" "os/exec" "path/filepath" + goruntime "runtime" "strings" "github.com/k3s-io/k3s/pkg/agent/util" @@ -24,34 +25,6 @@ import ( ) const ( - cniConf = `{ - "name":"cbr0", - "cniVersion":"1.0.0", - "plugins":[ - { - "type":"flannel", - "delegate":{ - "hairpinMode":true, - "forceAddress":true, - "isDefaultGateway":true - } - }, - { - "type":"portmap", - "capabilities":{ - "portMappings":true - } - }, - { - "type":"bandwidth", - "capabilities":{ - "bandwidth":true - } - } - ] -} -` - flannelConf = `{ "Network": "%CIDR%", "EnableIPv6": %IPV6_ENABLED%, @@ -61,10 +34,6 @@ const ( } ` - vxlanBackend = `{ - "Type": "vxlan" -}` - hostGWBackend = `{ "Type": "host-gw" }` @@ -168,7 +137,20 @@ func createCNIConf(dir string, nodeConfig *config.Node) error { logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.AgentConfig.FlannelCniConfFile) return util.CopyFile(nodeConfig.AgentConfig.FlannelCniConfFile, p, false) } - return util.WriteFile(p, cniConf) + + cniConfJSON := cniConf + if goruntime.GOOS == "windows" { + extIface, err := LookupExtInterface(nodeConfig.FlannelIface, ipv4) + if err != nil { + return err + } + + cniConfJSON = strings.ReplaceAll(cniConfJSON, "%IPV4_ADDRESS%", extIface.IfaceAddr.String()) + cniConfJSON = strings.ReplaceAll(cniConfJSON, "%CLUSTER_CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String()) + cniConfJSON = strings.ReplaceAll(cniConfJSON, "%SERVICE_CIDR%", nodeConfig.AgentConfig.ServiceCIDR.String()) + } + + return util.WriteFile(p, cniConfJSON) } func createFlannelConf(nodeConfig *config.Node) error { @@ -234,7 +216,17 @@ func createFlannelConf(nodeConfig *config.Node) error { } } + // precheck and error out unsupported flannel backends. switch backend { + case config.FlannelBackendHostGW: + case config.FlannelBackendTailscale: + case config.FlannelBackendWireguardNative: + if goruntime.GOOS == "windows" { + return fmt.Errorf("unsupported flannel backend '%s' for Windows", nodeConfig.FlannelBackend) + } + } + + switch nodeConfig.FlannelBackend { case config.FlannelBackendVXLAN: backendConf = vxlanBackend case config.FlannelBackendHostGW: diff --git a/pkg/agent/flannel/setup_linux.go b/pkg/agent/flannel/setup_linux.go new file mode 100644 index 000000000000..36404a01c68f --- /dev/null +++ b/pkg/agent/flannel/setup_linux.go @@ -0,0 +1,38 @@ +//go:build linux +// +build linux + +package flannel + +const ( + cniConf = `{ + "name":"cbr0", + "cniVersion":"1.0.0", + "plugins":[ + { + "type":"flannel", + "delegate":{ + "hairpinMode":true, + "forceAddress":true, + "isDefaultGateway":true + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + } + }, + { + "type":"bandwidth", + "capabilities":{ + "bandwidth":true + } + } + ] +} +` + + vxlanBackend = `{ + "Type": "vxlan" +}` +) diff --git a/pkg/agent/flannel/setup_windows.go b/pkg/agent/flannel/setup_windows.go new file mode 100644 index 000000000000..7b9c513cd364 --- /dev/null +++ b/pkg/agent/flannel/setup_windows.go @@ -0,0 +1,59 @@ +//go:build windows +// +build windows + +package flannel + +const ( + cniConf = `{ + "name":"flannel.4096", + "cniVersion":"1.0.0", + "plugins":[ + { + "type":"flannel", + "capabilities": { + "portMappings": true, + "dns": true + }, + "delegate": { + "type": "win-overlay", + "apiVersion": 2, + "Policies": [{ + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "Settings": { + "Exceptions": [ + "%CLUSTER_CIDR%", "%SERVICE_CIDR%" + ] + } + } + }, { + "Name": "EndpointPolicy", + "Value": { + "Type": "SDNRoute", + "Settings": { + "DestinationPrefix": "%SERVICE_CIDR%", + "NeedEncap": true + } + } + }, { + "name": "EndpointPolicy", + "value": { + "Type": "ProviderAddress", + "Settings": { + "ProviderAddress": "%IPV4_ADDRESS%" + } + } + }] + } + } + ] +} +` + + vxlanBackend = `{ + "Type": "vxlan", + "VNI": 4096, + "Port": 4789 +}` +) diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 7b8a9c6ade6e..31e53f1bbc7a 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -6,6 +6,7 @@ import ( "net" "os" "path/filepath" + goruntime "runtime" "strconv" "strings" "time" @@ -84,6 +85,11 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { enableIPv6 := dualCluster || clusterIPv6 enableIPv4 := dualCluster || clusterIPv4 + // dualStack or IPv6 are not supported on Windows node + if (goruntime.GOOS == "windows") && enableIPv6 { + return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node") + } + conntrackConfig, err := getConntrackConfig(nodeConfig) if err != nil { return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration") diff --git a/pkg/cli/crictl/crictl.go b/pkg/cli/crictl/crictl.go index 03b3b5b661c6..693032c3ce3c 100644 --- a/pkg/cli/crictl/crictl.go +++ b/pkg/cli/crictl/crictl.go @@ -1,11 +1,17 @@ package crictl import ( + "os" + "runtime" + "github.com/kubernetes-sigs/cri-tools/cmd/crictl" "github.com/urfave/cli" ) func Run(ctx *cli.Context) error { + if runtime.GOOS == "windows" { + os.Args = os.Args[1:] + } crictl.Main() return nil } diff --git a/pkg/containerd/builtins_cri_linux.go b/pkg/containerd/builtins_cri.go similarity index 100% rename from pkg/containerd/builtins_cri_linux.go rename to pkg/containerd/builtins_cri.go diff --git a/pkg/containerd/builtins_windows.go b/pkg/containerd/builtins_windows.go new file mode 100644 index 000000000000..aa8f01f1d7ab --- /dev/null +++ b/pkg/containerd/builtins_windows.go @@ -0,0 +1,27 @@ +//go:build ctrd +// +build ctrd + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containerd + +import ( + _ "github.com/containerd/containerd/diff/lcow" + _ "github.com/containerd/containerd/diff/windows" + _ "github.com/containerd/containerd/snapshots/lcow" + _ "github.com/containerd/containerd/snapshots/windows" +) diff --git a/pkg/daemons/agent/agent_windows.go b/pkg/daemons/agent/agent_windows.go index 36a5f505658a..3154d94e1bd3 100644 --- a/pkg/daemons/agent/agent_windows.go +++ b/pkg/daemons/agent/agent_windows.go @@ -122,5 +122,6 @@ func kubeletArgs(cfg *config.Agent) map[string]string { if cfg.ProtectKernelDefaults { argsMap["protect-kernel-defaults"] = "true" } + return argsMap } diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index 265d30ac0e64..15ad15256020 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -96,9 +96,9 @@ func (e *Embedded) Kubelet(ctx context.Context, args []string) error { return nil } -func (*Embedded) KubeProxy(ctx context.Context, args []string) error { +func (e *Embedded) KubeProxy(ctx context.Context, args []string) error { command := proxy.NewProxyCommand() - command.SetArgs(args) + command.SetArgs(daemonconfig.GetArgs(platformKubeProxyArgs(e.nodeConfig), args)) go func() { defer func() { diff --git a/pkg/daemons/executor/embed_linux.go b/pkg/daemons/executor/embed_linux.go new file mode 100644 index 000000000000..0b9f4d5e78f9 --- /dev/null +++ b/pkg/daemons/executor/embed_linux.go @@ -0,0 +1,16 @@ +//go:build linux && !no_embedded_executor +// +build linux,!no_embedded_executor + +package executor + +import ( + daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" + + // registering k3s cloud provider + _ "github.com/k3s-io/k3s/pkg/cloudprovider" +) + +func platformKubeProxyArgs(nodeConfig *daemonconfig.Node) map[string]string { + argsMap := map[string]string{} + return argsMap +} diff --git a/pkg/daemons/executor/embed_windows.go b/pkg/daemons/executor/embed_windows.go new file mode 100644 index 000000000000..e65913477c8f --- /dev/null +++ b/pkg/daemons/executor/embed_windows.go @@ -0,0 +1,90 @@ +//go:build windows && !no_embedded_executor +// +build windows,!no_embedded_executor + +package executor + +import ( + "encoding/json" + "os" + "os/exec" + "strings" + "time" + + "github.com/Microsoft/hcsshim" + "github.com/sirupsen/logrus" + + // registering k3s cloud provider + _ "github.com/k3s-io/k3s/pkg/cloudprovider" + daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" +) + +const ( + networkName = "flannel.4096" +) + +type SourceVipResponse struct { + IP4 struct { + IP string `json:"ip"` + } `json:"ip4"` +} + +func platformKubeProxyArgs(nodeConfig *daemonconfig.Node) map[string]string { + argsMap := map[string]string{} + argsMap["network-name"] = networkName + if sourceVip := waitForSourceVip(networkName, nodeConfig); sourceVip != "" { + argsMap["source-vip"] = sourceVip + } + return argsMap +} + +func waitForSourceVip(networkName string, nodeConfig *daemonconfig.Node) string { + for range time.Tick(time.Second * 5) { + network, err := hcsshim.GetHNSNetworkByName(networkName) + if err != nil { + logrus.WithError(err).Warningf("can't find HNS network, retrying %s", networkName) + continue + } + if network.ManagementIP == "" { + logrus.WithError(err).Warningf("wait for management IP, retrying %s", networkName) + continue + } + + subnet := network.Subnets[0].AddressPrefix + + configData := `{ + "cniVersion": "0.2.0", + "name": "vxlan0", + "ipam": { + "type": "host-local", + "ranges": [[{"subnet":"` + subnet + `"}]], + "dataDir": "/var/lib/cni/networks" + } + }` + + cmd := exec.Command("host-local.exe") + cmd.Env = append(os.Environ(), + "CNI_COMMAND=ADD", + "CNI_CONTAINERID=dummy", + "CNI_NETNS=dummy", + "CNI_IFNAME=dummy", + "CNI_PATH="+nodeConfig.AgentConfig.CNIBinDir, + ) + + cmd.Stdin = strings.NewReader(configData) + out, err := cmd.Output() + if err != nil { + logrus.WithError(err).Warning("Failed to execute host-local.exe") + continue + } + + var sourceVipResp SourceVipResponse + err = json.Unmarshal(out, &sourceVipResp) + if err != nil { + logrus.WithError(err).Warning("Failed to unmarshal sourceVip response") + continue + } + + return strings.TrimSpace(strings.Split(sourceVipResp.IP4.IP, "/")[0]) + } + return "" +} diff --git a/pkg/kubectl/main.go b/pkg/kubectl/main.go index fd32c21cd34f..f3d77f24a11d 100644 --- a/pkg/kubectl/main.go +++ b/pkg/kubectl/main.go @@ -4,6 +4,7 @@ import ( "fmt" "math/rand" "os" + "runtime" "strings" "time" @@ -15,6 +16,9 @@ import ( ) func Main() { + if runtime.GOOS == "windows" { + os.Args = os.Args[1:] + } kubenv := os.Getenv("KUBECONFIG") for i, arg := range os.Args { if strings.HasPrefix(arg, "--kubeconfig=") { diff --git a/scripts/build b/scripts/build index fa1af82bcb33..59ac08728ac8 100755 --- a/scripts/build +++ b/scripts/build @@ -64,6 +64,10 @@ TAGS="apparmor seccomp netcgo osusergo providerless urfave_cli_no_docs" RUNC_TAGS="apparmor seccomp" RUNC_STATIC="static" +if [ ${OS} = windows ]; then + TAGS="ctrd netcgo osusergo providerless" +fi + if [ "$SELINUX" = "true" ]; then TAGS="$TAGS selinux" RUNC_TAGS="$RUNC_TAGS selinux" @@ -90,21 +94,42 @@ if [ ${ARCH} = s390x ]; then export GOARCH="s390x" fi -rm -f \ - bin/k3s-agent \ - bin/k3s-server \ - bin/k3s-token \ - bin/k3s-etcd-snapshot \ - bin/k3s-secrets-encrypt \ - bin/k3s-certificate \ - bin/k3s-completion \ - bin/kubectl \ - bin/crictl \ - bin/ctr \ - bin/containerd \ - bin/containerd-shim \ - bin/containerd-shim-runc-v2 \ - bin/runc +k3s_binaries=( + "bin/k3s-agent" + "bin/k3s-server" + "bin/k3s-token" + "bin/k3s-etcd-snapshot" + "bin/k3s-secrets-encrypt" + "bin/k3s-certificate" + "bin/k3s-completion" + "bin/kubectl" + "bin/crictl" + "bin/ctr" +) + +containerd_binaries=( + # bin/containerd must be here in 1.25 + "bin/containerd" + "bin/containerd-shim" + "bin/containerd-shim-runc-v2" + "bin/runc" + "bin/containerd-shim-runhcs-v1" + "bin/runhcs" +) + +for i in "${k3s_binaries[@]}"; do + if [ -f "$i${BINARY_POSTFIX}" ]; then + echo "Removing $i${BINARY_POSTFIX}" + rm -f "$i${BINARY_POSTFIX}" + fi +done + +for i in "${containerd_binaries[@]}"; do + if [ -f "$i${BINARY_POSTFIX}" ]; then + echo "Removing $i${BINARY_POSTFIX}" + rm -f "$i${BINARY_POSTFIX}" + fi +done cleanup() { exit_status=$? @@ -113,7 +138,7 @@ cleanup() { } INSTALLBIN=$(pwd)/bin -if [ ! -x ${INSTALLBIN}/cni ]; then +if [ ! -x ${INSTALLBIN}/cni${BINARY_POSTFIX} ]; then ( echo Building cni TMPDIR=$(mktemp -d) @@ -121,36 +146,47 @@ if [ ! -x ${INSTALLBIN}/cni ]; then WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR cd $WORKDIR - GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni + GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni${BINARY_POSTFIX} ) fi echo Building k3s -CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s ./cmd/server -ln -s k3s ./bin/k3s-agent -ln -s k3s ./bin/k3s-server -ln -s k3s ./bin/k3s-token -ln -s k3s ./bin/k3s-etcd-snapshot -ln -s k3s ./bin/k3s-secrets-encrypt -ln -s k3s ./bin/k3s-certificate -ln -s k3s ./bin/k3s-completion -ln -s k3s ./bin/kubectl -ln -s k3s ./bin/crictl -ln -s k3s ./bin/ctr +CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s${BINARY_POSTFIX} ./cmd/server + +for i in "${k3s_binaries[@]}"; do + ln -s "k3s${BINARY_POSTFIX}" "$i${BINARY_POSTFIX}" +done export GOPATH=$(pwd)/build -echo Building containerd -pushd ./build/src/github.com/containerd/containerd -TAGS="${TAGS/netcgo/netgo}" -CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd -CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2 -popd -cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/ - -echo Building runc -pushd ./build/src/github.com/opencontainers/runc -rm -f runc -make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC -popd -cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/ +case ${OS} in + linux) + echo Building containerd + pushd ./build/src/github.com/containerd/containerd + TAGS="${TAGS/netcgo/netgo}" + CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd + CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2 + popd + cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/ + + echo Building runc + pushd ./build/src/github.com/opencontainers/runc + rm -f runc + make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC + popd + cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/ + ;; + windows) + echo Building containerd-shim-runhcs-v1 + pushd ./build/src/github.com/microsoft/hcsshim + TAGS="${TAGS/netcgo/netgo}" + CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runhcs-v1${BINARY_POSTFIX} ./cmd/containerd-shim-runhcs-v1 + CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/runhcs${BINARY_POSTFIX} ./cmd/runhcs + popd + cp -vf ./build/src/github.com/microsoft/hcsshim/bin/*${BINARY_POSTFIX} ./bin/ + ;; + *) + echo "[ERROR] unrecognized opertaing system: ${OS}" + exit 1 + ;; +esac diff --git a/scripts/download b/scripts/download index ac2c5d04be2a..0557eca715f7 100755 --- a/scripts/download +++ b/scripts/download @@ -10,6 +10,7 @@ CHARTS_URL=https://k3s.io/k3s-charts/assets CHARTS_DIR=build/static/charts RUNC_DIR=build/src/github.com/opencontainers/runc CONTAINERD_DIR=build/src/github.com/containerd/containerd +HCSSHIM_DIR=build/src/github.com/microsoft/hcsshim DATA_DIR=build/data export TZ=UTC @@ -17,12 +18,24 @@ umask 022 rm -rf ${CHARTS_DIR} rm -rf ${RUNC_DIR} rm -rf ${CONTAINERD_DIR} +rm -rf ${HCSSHIM_DIR} mkdir -p ${CHARTS_DIR} mkdir -p ${DATA_DIR} -curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf - - -git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR} +case ${OS} in + linux) + git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR} + curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf - + cp scripts/wg-add.sh bin/aux + ;; + windows) + git clone --single-branch --branch=${VERSION_HCSSHIM} --depth=1 https://github.com/microsoft/hcsshim ${HCSSHIM_DIR} + ;; + *) + echo "[ERROR] unrecognized opertaing system: ${OS}" + exit 1 + ;; +esac git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://${PKG_CONTAINERD_K3S} ${CONTAINERD_DIR} @@ -30,5 +43,3 @@ for CHART_FILE in $(grep -rlF HelmChart manifests/ | xargs yq eval --no-doc .spe CHART_NAME=$(echo $CHART_FILE | grep -oE '^(-*[a-z])+') curl -sfL ${CHARTS_URL}/${CHART_NAME}/${CHART_FILE} -o ${CHARTS_DIR}/${CHART_FILE} done - -cp scripts/wg-add.sh bin/aux diff --git a/scripts/package-cli b/scripts/package-cli index 43a28407a897..009e10db619a 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -8,13 +8,31 @@ cd $(dirname $0)/.. GO=${GO-go} for i in crictl kubectl k3s-agent k3s-server k3s-token k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s-completion; do - rm -f bin/$i - ln -s k3s bin/$i + rm -f bin/$i${BINARY_POSTFIX} + ln -s k3s${BINARY_POSTFIX} bin/$i${BINARY_POSTFIX} done -for i in bandwidth bridge firewall flannel host-local loopback portmap; do - rm -f bin/$i - ln -s cni bin/$i +cni_binaries=( + "bandwidth" + "bridge" + "firewall" + "flannel" + "host-local" + "loopback" + "portmap" +) + +if [ ${OS} = windows ]; then + cni_binaries=( + "win-overlay" + "flannel" + "host-local" + ) +fi + +for i in "${cni_binaries[@]}"; do + rm -f bin/$i${BINARY_POSTFIX} + ln -s cni${BINARY_POSTFIX} bin/$i${BINARY_POSTFIX} done cp contrib/util/check-config.sh bin/check-config @@ -22,6 +40,7 @@ cp contrib/util/check-config.sh bin/check-config rm -rf build/data mkdir -p build/data build/out mkdir -p dist/artifacts +mkdir -p ./etc ( set +x @@ -50,9 +69,9 @@ elif [ ${ARCH} = s390x ]; then BIN_SUFFIX="-s390x" fi -CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX} +CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}${BINARY_POSTFIX} -"${GO}" generate +GOOS=linux CC=gcc CXX=g++ "${GO}" generate LDFLAGS=" -X github.com/k3s-io/k3s/pkg/version.Version=$VERSION -X github.com/k3s-io/k3s/pkg/version.GitCommit=${COMMIT:0:8} @@ -60,7 +79,7 @@ LDFLAGS=" " TAGS="urfave_cli_no_docs" STATIC="-extldflags '-static'" -CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go +CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -buildvcs=false -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s stat ${CMD_NAME} diff --git a/scripts/validate b/scripts/validate index 41d09748d6f8..f01e3b11f4a9 100755 --- a/scripts/validate +++ b/scripts/validate @@ -7,7 +7,7 @@ echo Running: go mod tidy go mod tidy echo Running: go generate -go generate +GOOS=linux CC=gcc CXX=g++ go generate if [ -n "$SKIP_VALIDATE" ]; then echo Skipping validation diff --git a/scripts/version.sh b/scripts/version.sh index 42540c0bb6ed..994efb6ed837 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -2,6 +2,7 @@ GO=${GO-go} ARCH=${ARCH:-$("${GO}" env GOARCH)} +OS=${OS:-$("${GO}" env GOOS)} SUFFIX="-${ARCH}" GIT_TAG=$DRONE_TAG TREE_STATE=clean @@ -51,6 +52,11 @@ if [ -z "$VERSION_RUNC" ]; then VERSION_RUNC="v0.0.0" fi +VERSION_HCSSHIM=$(get-module-version github.com/Microsoft/hcsshim) +if [ -z "$VERSION_HCSSHIM" ]; then + VERSION_HCSSHIM="v0.0.0" +fi + VERSION_FLANNEL=$(get-module-version github.com/flannel-io/flannel) if [ -z "$VERSION_FLANNEL" ]; then VERSION_FLANNEL="v0.0.0" @@ -80,3 +86,8 @@ else VERSION="$VERSION_K8S+k3s-${COMMIT:0:8}$DIRTY" fi VERSION_TAG="$(sed -e 's/+/-/g' <<< "$VERSION")" + +BINARY_POSTFIX= +if [ ${OS} = windows ]; then + BINARY_POSTFIX=.exe +fi \ No newline at end of file