From a37ecec5e1f57f0767e4e4f1ea33449feadcf6bd Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Thu, 25 Feb 2021 15:15:54 +0100 Subject: [PATCH 01/12] Refactor InitMainController to be variadic in inttest Signed-off-by: Natanael Copa --- inttest/addons/addons_test.go | 2 +- inttest/basic/basic_test.go | 2 +- inttest/byocri/byocri_test.go | 2 +- inttest/common/footloosesuite.go | 7 ++----- inttest/dualstack/dualstack_test.go | 2 +- inttest/hacontrolplane/hacontrolplane_test.go | 2 +- inttest/kine/kine_test.go | 2 +- inttest/multicontroller/multicontroller_test.go | 2 +- inttest/singlenode/singlenode_test.go | 2 +- 9 files changed, 10 insertions(+), 13 deletions(-) diff --git a/inttest/addons/addons_test.go b/inttest/addons/addons_test.go index deb71756bdd9..7ed9f64295c7 100644 --- a/inttest/addons/addons_test.go +++ b/inttest/addons/addons_test.go @@ -45,7 +45,7 @@ func (as *AddonsSuite) TestHelmBasedAddons() { addonName := "test-addon" as.putFile("/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithAddon, addonName)) - as.Require().NoError(as.InitMainController([]string{"--config=/tmp/k0s.yaml"})) + as.Require().NoError(as.InitMainController("--config=/tmp/k0s.yaml")) as.waitForPrometheusRelease(addonName, 1) values := map[string]interface{}{ diff --git a/inttest/basic/basic_test.go b/inttest/basic/basic_test.go index c329dca6bc7d..259f89e33b15 100644 --- a/inttest/basic/basic_test.go +++ b/inttest/basic/basic_test.go @@ -36,7 +36,7 @@ type BasicSuite struct { func (s *BasicSuite) TestK0sGetsUp() { customDataDir := "/var/lib/k0s/custom-data-dir" - s.NoError(s.InitMainController([]string{fmt.Sprintf("--data-dir=%s", customDataDir)})) + s.NoError(s.InitMainController(fmt.Sprintf("--data-dir=%s", customDataDir))) s.NoError(s.RunWorkers(customDataDir, `--labels="k0sproject.io/foo=bar"`, `--kubelet-extra-args="--address=0.0.0.0 --event-burst=10"`)) kc, err := s.KubeClient("controller0", customDataDir) diff --git a/inttest/byocri/byocri_test.go b/inttest/byocri/byocri_test.go index fd3a506b2597..c16e8d47dc62 100644 --- a/inttest/byocri/byocri_test.go +++ b/inttest/byocri/byocri_test.go @@ -33,7 +33,7 @@ type BYOCRISuite struct { func (s *BYOCRISuite) TestK0sGetsUp() { - s.NoError(s.InitMainController([]string{})) + s.NoError(s.InitMainController()) s.Require().NoError(s.runDockerWorker()) kc, err := s.KubeClient("controller0", "") diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index f974ca2dc246..e7964bc62d1b 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -203,7 +203,7 @@ func getDataDir(args []string) string { } // InitMainController inits first controller assuming it's first controller in the cluster -func (s *FootlooseSuite) InitMainController(k0sArgs []string) error { +func (s *FootlooseSuite) InitMainController(k0sArgs ...string) error { controllerNode := fmt.Sprintf("controller%d", 0) ssh, err := s.SSH(controllerNode) if err != nil { @@ -211,10 +211,7 @@ func (s *FootlooseSuite) InitMainController(k0sArgs []string) error { } defer ssh.Disconnect() - opts := "" - for _, arg := range k0sArgs { - opts = fmt.Sprintf("%s %s", opts, arg) - } + opts := strings.Join(k0sArgs, " ") installCmd := fmt.Sprintf("ETCD_UNSUPPORTED_ARCH=arm64 k0s install controller --debug %s", opts) startCmd := fmt.Sprintf("ETCD_UNSUPPORTED_ARCH=arm64 nohup k0s controller --debug %s >/tmp/k0s-controller.log 2>&1 &", opts) diff --git a/inttest/dualstack/dualstack_test.go b/inttest/dualstack/dualstack_test.go index fc2d0a521d13..2f3121b24e9a 100644 --- a/inttest/dualstack/dualstack_test.go +++ b/inttest/dualstack/dualstack_test.go @@ -71,7 +71,7 @@ func (ds *DualstackSuite) getKubeConfig(node string) *restclient.Config { func (ds *DualstackSuite) SetupSuite() { ds.FootlooseSuite.SetupSuite() ds.putFile("/tmp/k0s.yaml", k0sConfigWithAddon) - ds.Require().NoError(ds.InitMainController([]string{"--config=/tmp/k0s.yaml"})) + ds.Require().NoError(ds.InitMainController("--config=/tmp/k0s.yaml")) ds.Require().NoError(ds.RunWorkers("/var/lib/k0s")) client, err := k8s.NewForConfig(ds.getKubeConfig("controller0")) ds.Require().NoError(err) diff --git a/inttest/hacontrolplane/hacontrolplane_test.go b/inttest/hacontrolplane/hacontrolplane_test.go index d74f1c019341..7529d1b844b6 100644 --- a/inttest/hacontrolplane/hacontrolplane_test.go +++ b/inttest/hacontrolplane/hacontrolplane_test.go @@ -74,7 +74,7 @@ func (s *HAControlplaneSuite) getCa(controllerIdx int) string { } func (s *HAControlplaneSuite) TestDeregistration() { - s.NoError(s.InitMainController([]string{})) + s.NoError(s.InitMainController()) token, err := s.GetJoinToken("controller", "") s.NoError(err) s.NoError(s.JoinController(1, token, "")) diff --git a/inttest/kine/kine_test.go b/inttest/kine/kine_test.go index 36af93535e15..7ea229f42a0c 100644 --- a/inttest/kine/kine_test.go +++ b/inttest/kine/kine_test.go @@ -32,7 +32,7 @@ type KineSuite struct { func (s *KineSuite) TestK0sGetsUp() { s.putFile("controller0", "/tmp/k0s.yaml", k0sConfigWithKine) - s.NoError(s.InitMainController([]string{"--config=/tmp/k0s.yaml"})) + s.NoError(s.InitMainController("--config=/tmp/k0s.yaml")) s.NoError(s.RunWorkers("")) kc, err := s.KubeClient("controller0", "") diff --git a/inttest/multicontroller/multicontroller_test.go b/inttest/multicontroller/multicontroller_test.go index f357dafd26a9..8be5bcfeb44b 100644 --- a/inttest/multicontroller/multicontroller_test.go +++ b/inttest/multicontroller/multicontroller_test.go @@ -45,7 +45,7 @@ func (s *MultiControllerSuite) TestK0sGetsUp() { s.T().Logf("ip address: %s", ipAddress) s.putFile("controller0", "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) - s.NoError(s.InitMainController([]string{"--config=/tmp/k0s.yaml"})) + s.NoError(s.InitMainController("--config=/tmp/k0s.yaml")) token, err := s.GetJoinToken("controller", "") s.NoError(err) diff --git a/inttest/singlenode/singlenode_test.go b/inttest/singlenode/singlenode_test.go index 25d6b9414ef3..79984547f08b 100644 --- a/inttest/singlenode/singlenode_test.go +++ b/inttest/singlenode/singlenode_test.go @@ -30,7 +30,7 @@ type SingleNodeSuite struct { } func (s *SingleNodeSuite) TestK0sGetsUp() { - s.NoError(s.InitMainController([]string{"--single"})) + s.NoError(s.InitMainController("--single")) kc, err := s.KubeClient("controller0", "") s.NoError(err) From 13b5b015a50bbf1ed2544e0da2aa38bd11f81f43 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Fri, 19 Mar 2021 10:39:01 +0100 Subject: [PATCH 02/12] Use alpine 3.13 for footloose inttest This provides a machine-id "service" which we can use Signed-off-by: Natanael Copa --- inttest/footloose-alpine/Dockerfile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/inttest/footloose-alpine/Dockerfile b/inttest/footloose-alpine/Dockerfile index bf2656c261b7..aa059aa7cf87 100644 --- a/inttest/footloose-alpine/Dockerfile +++ b/inttest/footloose-alpine/Dockerfile @@ -1,9 +1,10 @@ -FROM alpine:3.12 +FROM alpine:3.13 RUN apk add openrc openssh-server bash busybox-initscripts coreutils findutils iptables curl # enable syslog and sshd RUN rc-update add cgroups boot RUN rc-update add syslog boot +RUN rc-update add machine-id boot RUN rc-update add sshd default RUN rc-update add local default # remove -docker keyword so we actually mount cgroups in container @@ -12,11 +13,6 @@ RUN sed -i -e '/keyword/s/-docker//' /etc/init.d/cgroups RUN sed -i -e 's/^\(tty[0-9]\)/# \1/' /etc/inittab # enable root logins RUN sed -i -e 's/^root:!:/root::/' /etc/shadow -RUN echo "#!/bin/sh" > /etc/local.d/machine-id.start \ - && echo "if ! [ -f /etc/machine-id ]; then" >> /etc/local.d/machine-id.start \ - && echo " dd if=/dev/urandom status=none bs=16 count=1 | md5sum | cut -d' ' -f1 > /etc/machine-id" >> /etc/local.d/machine-id.start \ - && echo "fi" >> /etc/local.d/machine-id.start \ - && chmod +x /etc/local.d/machine-id.start # Put kubectl into place to ease up debugging RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl \ @@ -24,4 +20,4 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.20.0/b && mv ./kubectl /usr/local/bin/kubectl ENV KUBECONFIG=/var/lib/k0s/pki/admin.conf # This lets etcd start when running on arm in smokes -ENV ETCD_UNSUPPORTED_ARCH=arm64 \ No newline at end of file +ENV ETCD_UNSUPPORTED_ARCH=arm64 From ef4cb2e0b8a8cdaed1b072452e348b8c08a2d6ce Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Fri, 19 Mar 2021 11:24:21 +0100 Subject: [PATCH 03/12] inttest: Let RunWorkers datadir be optional Let callers pass it via the --data-dir arg Signed-off-by: Natanael Copa --- inttest/basic/basic_test.go | 5 +++-- inttest/common/footloosesuite.go | 7 ++----- inttest/dualstack/dualstack_test.go | 2 +- inttest/kine/kine_test.go | 2 +- inttest/multicontroller/multicontroller_test.go | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/inttest/basic/basic_test.go b/inttest/basic/basic_test.go index 259f89e33b15..d6c181c7c57e 100644 --- a/inttest/basic/basic_test.go +++ b/inttest/basic/basic_test.go @@ -36,8 +36,9 @@ type BasicSuite struct { func (s *BasicSuite) TestK0sGetsUp() { customDataDir := "/var/lib/k0s/custom-data-dir" - s.NoError(s.InitMainController(fmt.Sprintf("--data-dir=%s", customDataDir))) - s.NoError(s.RunWorkers(customDataDir, `--labels="k0sproject.io/foo=bar"`, `--kubelet-extra-args="--address=0.0.0.0 --event-burst=10"`)) + dataDirOpt := fmt.Sprintf("--data-dir=%s", customDataDir) + s.NoError(s.InitMainController(dataDirOpt)) + s.NoError(s.RunWorkers(dataDirOpt, `--labels="k0sproject.io/foo=bar"`, `--kubelet-extra-args="--address=0.0.0.0 --event-burst=10"`)) kc, err := s.KubeClient("controller0", customDataDir) s.NoError(err) diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index e7964bc62d1b..d17a9f782f53 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -267,22 +267,19 @@ func (s *FootlooseSuite) GetJoinToken(role string, dataDir string) (string, erro } // RunWorkers joins all the workers to the cluster -func (s *FootlooseSuite) RunWorkers(dataDir string, args ...string) error { +func (s *FootlooseSuite) RunWorkers(args ...string) error { ssh, err := s.SSH("controller0") if err != nil { return err } defer ssh.Disconnect() - token, err := s.GetJoinToken("worker", dataDir) + token, err := s.GetJoinToken("worker", getDataDir(args)) if err != nil { return err } if token == "" { return fmt.Errorf("got empty token for worker join") } - if dataDir != "" { - args = append(args, fmt.Sprintf("--data-dir=%s", dataDir)) - } workerCommand := fmt.Sprintf(`nohup k0s --debug worker %s "%s" >/tmp/k0s-worker.log 2>&1 &`, strings.Join(args, " "), token) for i := 0; i < s.WorkerCount; i++ { diff --git a/inttest/dualstack/dualstack_test.go b/inttest/dualstack/dualstack_test.go index 2f3121b24e9a..32ab33469902 100644 --- a/inttest/dualstack/dualstack_test.go +++ b/inttest/dualstack/dualstack_test.go @@ -72,7 +72,7 @@ func (ds *DualstackSuite) SetupSuite() { ds.FootlooseSuite.SetupSuite() ds.putFile("/tmp/k0s.yaml", k0sConfigWithAddon) ds.Require().NoError(ds.InitMainController("--config=/tmp/k0s.yaml")) - ds.Require().NoError(ds.RunWorkers("/var/lib/k0s")) + ds.Require().NoError(ds.RunWorkers()) client, err := k8s.NewForConfig(ds.getKubeConfig("controller0")) ds.Require().NoError(err) err = ds.WaitForNodeReady("worker0", client) diff --git a/inttest/kine/kine_test.go b/inttest/kine/kine_test.go index 7ea229f42a0c..98942c1c9f47 100644 --- a/inttest/kine/kine_test.go +++ b/inttest/kine/kine_test.go @@ -33,7 +33,7 @@ type KineSuite struct { func (s *KineSuite) TestK0sGetsUp() { s.putFile("controller0", "/tmp/k0s.yaml", k0sConfigWithKine) s.NoError(s.InitMainController("--config=/tmp/k0s.yaml")) - s.NoError(s.RunWorkers("")) + s.NoError(s.RunWorkers()) kc, err := s.KubeClient("controller0", "") s.NoError(err) diff --git a/inttest/multicontroller/multicontroller_test.go b/inttest/multicontroller/multicontroller_test.go index 8be5bcfeb44b..30a9cc0691e7 100644 --- a/inttest/multicontroller/multicontroller_test.go +++ b/inttest/multicontroller/multicontroller_test.go @@ -54,7 +54,7 @@ func (s *MultiControllerSuite) TestK0sGetsUp() { s.putFile("controller2", "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.JoinController(2, token, "")) - s.NoError(s.RunWorkers("")) + s.NoError(s.RunWorkers()) kc, err := s.KubeClient("controller0", "") s.NoError(err) From 37abbb341e9414cd2a1391f60b22b9e512143d58 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Fri, 19 Mar 2021 12:40:25 +0100 Subject: [PATCH 04/12] Merge InitMainController and JoinController to InitController Reduce duplicate code in InitMainController and JoinController by merging them. Signed-off-by: Natanael Copa --- inttest/addons/addons_test.go | 2 +- inttest/basic/basic_test.go | 2 +- inttest/byocri/byocri_test.go | 2 +- inttest/common/footloosesuite.go | 33 +++---------------- inttest/dualstack/dualstack_test.go | 2 +- inttest/hacontrolplane/hacontrolplane_test.go | 6 ++-- inttest/kine/kine_test.go | 2 +- .../multicontroller/multicontroller_test.go | 6 ++-- inttest/singlenode/singlenode_test.go | 2 +- 9 files changed, 16 insertions(+), 41 deletions(-) diff --git a/inttest/addons/addons_test.go b/inttest/addons/addons_test.go index 7ed9f64295c7..f7890c6da263 100644 --- a/inttest/addons/addons_test.go +++ b/inttest/addons/addons_test.go @@ -45,7 +45,7 @@ func (as *AddonsSuite) TestHelmBasedAddons() { addonName := "test-addon" as.putFile("/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithAddon, addonName)) - as.Require().NoError(as.InitMainController("--config=/tmp/k0s.yaml")) + as.Require().NoError(as.InitController(0, "--config=/tmp/k0s.yaml")) as.waitForPrometheusRelease(addonName, 1) values := map[string]interface{}{ diff --git a/inttest/basic/basic_test.go b/inttest/basic/basic_test.go index d6c181c7c57e..3d93e303a83e 100644 --- a/inttest/basic/basic_test.go +++ b/inttest/basic/basic_test.go @@ -37,7 +37,7 @@ type BasicSuite struct { func (s *BasicSuite) TestK0sGetsUp() { customDataDir := "/var/lib/k0s/custom-data-dir" dataDirOpt := fmt.Sprintf("--data-dir=%s", customDataDir) - s.NoError(s.InitMainController(dataDirOpt)) + s.NoError(s.InitController(0, dataDirOpt)) s.NoError(s.RunWorkers(dataDirOpt, `--labels="k0sproject.io/foo=bar"`, `--kubelet-extra-args="--address=0.0.0.0 --event-burst=10"`)) kc, err := s.KubeClient("controller0", customDataDir) diff --git a/inttest/byocri/byocri_test.go b/inttest/byocri/byocri_test.go index c16e8d47dc62..97a239e3589b 100644 --- a/inttest/byocri/byocri_test.go +++ b/inttest/byocri/byocri_test.go @@ -33,7 +33,7 @@ type BYOCRISuite struct { func (s *BYOCRISuite) TestK0sGetsUp() { - s.NoError(s.InitMainController()) + s.NoError(s.InitController(0)) s.Require().NoError(s.runDockerWorker()) kc, err := s.KubeClient("controller0", "") diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index d17a9f782f53..d13dfb27dd93 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -202,26 +202,16 @@ func getDataDir(args []string) string { return dataDir } -// InitMainController inits first controller assuming it's first controller in the cluster -func (s *FootlooseSuite) InitMainController(k0sArgs ...string) error { - controllerNode := fmt.Sprintf("controller%d", 0) +// InitController initializes a controller +func (s *FootlooseSuite) InitController(idx int, k0sArgs ...string) error { + controllerNode := fmt.Sprintf("controller%d", idx) ssh, err := s.SSH(controllerNode) if err != nil { return err } defer ssh.Disconnect() - opts := strings.Join(k0sArgs, " ") - - installCmd := fmt.Sprintf("ETCD_UNSUPPORTED_ARCH=arm64 k0s install controller --debug %s", opts) - startCmd := fmt.Sprintf("ETCD_UNSUPPORTED_ARCH=arm64 nohup k0s controller --debug %s >/tmp/k0s-controller.log 2>&1 &", opts) - - _, err = ssh.ExecWithOutput(installCmd) - if err != nil { - s.T().Logf("failed to execute '%s' on %s", installCmd, controllerNode) - return err - } - + startCmd := fmt.Sprintf("nohup k0s controller --debug %s >/tmp/k0s-controller.log 2>&1 &", strings.Join(k0sArgs, " ")) _, err = ssh.ExecWithOutput(startCmd) if err != nil { s.T().Logf("failed to execute '%s' on %s", startCmd, controllerNode) @@ -230,21 +220,6 @@ func (s *FootlooseSuite) InitMainController(k0sArgs ...string) error { return s.WaitForKubeAPI(controllerNode, getDataDir(k0sArgs)) } -// JoinController joins the cluster with a given token -func (s *FootlooseSuite) JoinController(idx int, token string, dataDir string) error { - controllerNode := fmt.Sprintf("controller%d", idx) - ssh, err := s.SSH(controllerNode) - if err != nil { - return err - } - defer ssh.Disconnect() - _, err = ssh.ExecWithOutput(fmt.Sprintf("nohup k0s controller --debug %s >/tmp/k0s-controller.log 2>&1 &", token)) - if err != nil { - return err - } - return s.WaitForKubeAPI(controllerNode, dataDir) -} - // GetJoinToken generates join token for the asked role func (s *FootlooseSuite) GetJoinToken(role string, dataDir string) (string, error) { // assume we have main on 1 node always diff --git a/inttest/dualstack/dualstack_test.go b/inttest/dualstack/dualstack_test.go index 32ab33469902..1ad24805338a 100644 --- a/inttest/dualstack/dualstack_test.go +++ b/inttest/dualstack/dualstack_test.go @@ -71,7 +71,7 @@ func (ds *DualstackSuite) getKubeConfig(node string) *restclient.Config { func (ds *DualstackSuite) SetupSuite() { ds.FootlooseSuite.SetupSuite() ds.putFile("/tmp/k0s.yaml", k0sConfigWithAddon) - ds.Require().NoError(ds.InitMainController("--config=/tmp/k0s.yaml")) + ds.Require().NoError(ds.InitController(0, "--config=/tmp/k0s.yaml")) ds.Require().NoError(ds.RunWorkers()) client, err := k8s.NewForConfig(ds.getKubeConfig("controller0")) ds.Require().NoError(err) diff --git a/inttest/hacontrolplane/hacontrolplane_test.go b/inttest/hacontrolplane/hacontrolplane_test.go index 7529d1b844b6..4cede053069e 100644 --- a/inttest/hacontrolplane/hacontrolplane_test.go +++ b/inttest/hacontrolplane/hacontrolplane_test.go @@ -74,10 +74,10 @@ func (s *HAControlplaneSuite) getCa(controllerIdx int) string { } func (s *HAControlplaneSuite) TestDeregistration() { - s.NoError(s.InitMainController()) + s.NoError(s.InitController(0)) token, err := s.GetJoinToken("controller", "") s.NoError(err) - s.NoError(s.JoinController(1, token, "")) + s.NoError(s.InitController(1, token)) ca0 := s.getCa(0) s.Contains(ca0, "-----BEGIN CERTIFICATE-----") @@ -107,7 +107,7 @@ func (s *HAControlplaneSuite) TestDeregistration() { s.Require().NoError(err) _, err = sshC1.ExecWithOutput("kill $(pidof k0s) && while pidof k0s; do sleep 0.1s; done") s.Require().NoError(err) - s.NoError(s.JoinController(1, token, "")) + s.NoError(s.InitController(1, token)) // Make one member leave the etcd cluster s.makeNodeLeave(1, membersFromJoined["controller1"]) diff --git a/inttest/kine/kine_test.go b/inttest/kine/kine_test.go index 98942c1c9f47..0520967d604e 100644 --- a/inttest/kine/kine_test.go +++ b/inttest/kine/kine_test.go @@ -32,7 +32,7 @@ type KineSuite struct { func (s *KineSuite) TestK0sGetsUp() { s.putFile("controller0", "/tmp/k0s.yaml", k0sConfigWithKine) - s.NoError(s.InitMainController("--config=/tmp/k0s.yaml")) + s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) s.NoError(s.RunWorkers()) kc, err := s.KubeClient("controller0", "") diff --git a/inttest/multicontroller/multicontroller_test.go b/inttest/multicontroller/multicontroller_test.go index 30a9cc0691e7..da9e13909daf 100644 --- a/inttest/multicontroller/multicontroller_test.go +++ b/inttest/multicontroller/multicontroller_test.go @@ -45,15 +45,15 @@ func (s *MultiControllerSuite) TestK0sGetsUp() { s.T().Logf("ip address: %s", ipAddress) s.putFile("controller0", "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) - s.NoError(s.InitMainController("--config=/tmp/k0s.yaml")) + s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) token, err := s.GetJoinToken("controller", "") s.NoError(err) s.putFile("controller1", "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) - s.NoError(s.JoinController(1, token, "")) + s.NoError(s.InitController(1, "--config=/tmp/k0s.yaml", token)) s.putFile("controller2", "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) - s.NoError(s.JoinController(2, token, "")) + s.NoError(s.InitController(2, "--config=/tmp/k0s.yaml", token)) s.NoError(s.RunWorkers()) kc, err := s.KubeClient("controller0", "") diff --git a/inttest/singlenode/singlenode_test.go b/inttest/singlenode/singlenode_test.go index 79984547f08b..dae4651e8e2d 100644 --- a/inttest/singlenode/singlenode_test.go +++ b/inttest/singlenode/singlenode_test.go @@ -30,7 +30,7 @@ type SingleNodeSuite struct { } func (s *SingleNodeSuite) TestK0sGetsUp() { - s.NoError(s.InitMainController("--single")) + s.NoError(s.InitController(0, "--single")) kc, err := s.KubeClient("controller0", "") s.NoError(err) From a736cdbc496b0ce0fe33cd73ae9dbcf203f46c7f Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Fri, 19 Mar 2021 14:17:30 +0100 Subject: [PATCH 05/12] Do not use hardcoded nodenames for inttests Introduce ControllerNode(index) and WorkerNode(index) to get the node name. Signed-off-by: Natanael Copa --- inttest/addons/addons_test.go | 9 +++---- inttest/basic/basic_test.go | 20 +++++++------- inttest/byocri/byocri_test.go | 6 ++--- inttest/common/filetools.go | 3 +-- inttest/common/footloosesuite.go | 27 ++++++++++++------- inttest/dualstack/dualstack_test.go | 9 +++---- inttest/hacontrolplane/hacontrolplane_test.go | 15 +++++------ inttest/install/singlenode_test.go | 8 +++--- inttest/kine/kine_test.go | 8 +++--- .../multicontroller/multicontroller_test.go | 12 ++++----- inttest/singlenode/singlenode_test.go | 4 +-- 11 files changed, 62 insertions(+), 59 deletions(-) diff --git a/inttest/addons/addons_test.go b/inttest/addons/addons_test.go index f7890c6da263..f35310f229b0 100644 --- a/inttest/addons/addons_test.go +++ b/inttest/addons/addons_test.go @@ -65,7 +65,7 @@ func (as *AddonsSuite) TestHelmBasedAddons() { } func (as *AddonsSuite) doPrometheusDelete(chartName string) { - cfg := as.getKubeConfig("controller0") + cfg := as.getKubeConfig(as.ControllerNode(0)) chartClient, err := clientset.New(cfg) as.Require().NoError(err) as.Require().NoError(chartClient.Charts("kube-system").Delete(context.Background(), chartName, v1.DeleteOptions{})) @@ -86,7 +86,7 @@ func (as *AddonsSuite) doPrometheusDelete(chartName string) { func (as *AddonsSuite) waitForPrometheusRelease(addonName string, rev int64) (string, string) { as.T().Logf("waiting to see prometheus release ready in kube API, generation %d", rev) - cfg := as.getKubeConfig("controller0") + cfg := as.getKubeConfig(as.ControllerNode(0)) chartClient, err := clientset.New(cfg) as.Require().NoError(err) var chartName string @@ -135,7 +135,7 @@ func (as *AddonsSuite) waitForPrometheusRelease(addonName string, rev int64) (st func (as *AddonsSuite) waitForPrometheusServerEnvs(releaseName string) error { as.T().Logf("waiting to see prometheus release to have envs set from values yaml") - kc, err := as.KubeClient("controller0", "") + kc, err := as.KubeClient(as.ControllerNode(0), "") if err != nil { return err } @@ -217,8 +217,7 @@ func TestAddonsSuite(t *testing.T) { } func (as *AddonsSuite) putFile(path string, content string) { - controllerNode := fmt.Sprintf("controller%d", 0) - ssh, err := as.SSH(controllerNode) + ssh, err := as.SSH(as.ControllerNode(0)) as.Require().NoError(err) defer ssh.Disconnect() _, err = ssh.ExecWithOutput(fmt.Sprintf("echo '%s' >%s", content, path)) diff --git a/inttest/basic/basic_test.go b/inttest/basic/basic_test.go index 3d93e303a83e..4abb9e293267 100644 --- a/inttest/basic/basic_test.go +++ b/inttest/basic/basic_test.go @@ -40,17 +40,17 @@ func (s *BasicSuite) TestK0sGetsUp() { s.NoError(s.InitController(0, dataDirOpt)) s.NoError(s.RunWorkers(dataDirOpt, `--labels="k0sproject.io/foo=bar"`, `--kubelet-extra-args="--address=0.0.0.0 --event-burst=10"`)) - kc, err := s.KubeClient("controller0", customDataDir) + kc, err := s.KubeClient(s.ControllerNode(0), customDataDir) s.NoError(err) - err = s.WaitForNodeReady("worker0", kc) + err = s.WaitForNodeReady(s.WorkerNode(0), kc) s.NoError(err) - labels, err := s.GetNodeLabels("worker0", kc) + labels, err := s.GetNodeLabels(s.WorkerNode(0), kc) s.NoError(err) s.Equal("bar", labels["k0sproject.io/foo"]) - err = s.WaitForNodeReady("worker1", kc) + err = s.WaitForNodeReady(s.WorkerNode(1), kc) s.NoError(err) pods, err := kc.CoreV1().Pods("kube-system").List(context.TODO(), v1.ListOptions{ @@ -66,14 +66,14 @@ func (s *BasicSuite) TestK0sGetsUp() { s.T().Log("waiting to see calico pods ready") s.NoError(common.WaitForCalicoReady(kc), "calico did not start") - s.Require().NoError(s.checkCertPerms("controller0")) - s.Require().NoError(s.checkCSRs("worker0", kc)) - s.Require().NoError(s.checkCSRs("worker1", kc)) + s.Require().NoError(s.checkCertPerms(s.ControllerNode(0))) + s.Require().NoError(s.checkCSRs(s.WorkerNode(0), kc)) + s.Require().NoError(s.checkCSRs(s.WorkerNode(1), kc)) - s.Require().NoError(s.verifyKubeletAddressFlag("worker0")) - s.Require().NoError(s.verifyKubeletAddressFlag("worker1")) + s.Require().NoError(s.verifyKubeletAddressFlag(s.WorkerNode(0))) + s.Require().NoError(s.verifyKubeletAddressFlag(s.WorkerNode(1))) - s.Require().NoError(common.WaitForMetricsReady(s.getKubeConfig("controller0"))) + s.Require().NoError(common.WaitForMetricsReady(s.getKubeConfig(s.ControllerNode(0)))) } func (s *BasicSuite) getKubeConfig(node string) *restclient.Config { diff --git a/inttest/byocri/byocri_test.go b/inttest/byocri/byocri_test.go index 97a239e3589b..2cf614df971b 100644 --- a/inttest/byocri/byocri_test.go +++ b/inttest/byocri/byocri_test.go @@ -36,10 +36,10 @@ func (s *BYOCRISuite) TestK0sGetsUp() { s.NoError(s.InitController(0)) s.Require().NoError(s.runDockerWorker()) - kc, err := s.KubeClient("controller0", "") + kc, err := s.KubeClient(s.ControllerNode(0), "") s.NoError(err) - err = s.WaitForNodeReady("worker0", kc) + err = s.WaitForNodeReady(s.WorkerNode(0), kc) s.NoError(err) pods, err := kc.CoreV1().Pods("kube-system").List(context.TODO(), v1.ListOptions{ @@ -64,7 +64,7 @@ func (s *BYOCRISuite) runDockerWorker() error { if token == "" { return fmt.Errorf("got empty token for worker join") } - sshWorker, err := s.SSH("worker0") + sshWorker, err := s.SSH(s.WorkerNode(0)) if err != nil { return err } diff --git a/inttest/common/filetools.go b/inttest/common/filetools.go index f2487e624a29..4f149bb315d4 100644 --- a/inttest/common/filetools.go +++ b/inttest/common/filetools.go @@ -4,8 +4,7 @@ import "fmt" // GetFile gets file from the controller with given index func (s *FootlooseSuite) GetFileFromController(controllerIdx int, path string) string { - node := fmt.Sprintf("controller%d", controllerIdx) - sshCon, err := s.SSH(node) + sshCon, err := s.SSH(s.ControllerNode(controllerIdx)) s.Require().NoError(err) content, err := sshCon.ExecWithOutput(fmt.Sprintf("cat %s", path)) s.Require().NoError(err) diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index d13dfb27dd93..e9618f573563 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -103,20 +103,30 @@ func (s *FootlooseSuite) SetupSuite() { // SSH through cluster should wait until we actually can get it through, but it doesn't for i := 0; i < 20; i++ { - err = s.Cluster.SSH("controller0", "root", "hostname") + err = s.Cluster.SSH(s.ControllerNode(0), "root", "hostname") if err == nil { break } - s.T().Logf("retrying ssh to controller0") + s.T().Logf("retrying ssh to %s", s.ControllerNode(0)) time.Sleep(300 * time.Millisecond) } if err != nil { - s.FailNowf("failed to ssh to controller0: %s", err.Error()) + s.FailNowf("failed to ssh to %s: %s", s.ControllerNode(0), err.Error()) s.T().FailNow() return } } +// ControllerNode gets the node name of given controller index +func (s *FootlooseSuite) ControllerNode(idx int) string { + return fmt.Sprintf(s.footlooseConfig.Machines[0].Spec.Name, idx) +} + +// WorkerNode gets the node name of given worker index +func (s *FootlooseSuite) WorkerNode(idx int) string { + return fmt.Sprintf(s.footlooseConfig.Machines[1].Spec.Name, idx) +} + // TearDownSuite does the cleanup work, namely destroy the footloose boxes func (s *FootlooseSuite) TearDownSuite() { // Make sure we don't fire the timer based teardown anymore @@ -204,7 +214,7 @@ func getDataDir(args []string) string { // InitController initializes a controller func (s *FootlooseSuite) InitController(idx int, k0sArgs ...string) error { - controllerNode := fmt.Sprintf("controller%d", idx) + controllerNode := s.ControllerNode(idx) ssh, err := s.SSH(controllerNode) if err != nil { return err @@ -222,8 +232,8 @@ func (s *FootlooseSuite) InitController(idx int, k0sArgs ...string) error { // GetJoinToken generates join token for the asked role func (s *FootlooseSuite) GetJoinToken(role string, dataDir string) (string, error) { - // assume we have main on 1 node always - controllerNode := fmt.Sprintf("controller%d", 0) + // assume we have main on node 0 always + controllerNode := s.ControllerNode(0) s.Contains([]string{"controller", "worker"}, role, "Bad role") ssh, err := s.SSH(controllerNode) if err != nil { @@ -243,7 +253,7 @@ func (s *FootlooseSuite) GetJoinToken(role string, dataDir string) (string, erro // RunWorkers joins all the workers to the cluster func (s *FootlooseSuite) RunWorkers(args ...string) error { - ssh, err := s.SSH("controller0") + ssh, err := s.SSH(s.ControllerNode(0)) if err != nil { return err } @@ -258,8 +268,7 @@ func (s *FootlooseSuite) RunWorkers(args ...string) error { workerCommand := fmt.Sprintf(`nohup k0s --debug worker %s "%s" >/tmp/k0s-worker.log 2>&1 &`, strings.Join(args, " "), token) for i := 0; i < s.WorkerCount; i++ { - workerNode := fmt.Sprintf("worker%d", i) - sshWorker, err := s.SSH(workerNode) + sshWorker, err := s.SSH(s.WorkerNode(i)) if err != nil { return err } diff --git a/inttest/dualstack/dualstack_test.go b/inttest/dualstack/dualstack_test.go index 1ad24805338a..649cddefb3f1 100644 --- a/inttest/dualstack/dualstack_test.go +++ b/inttest/dualstack/dualstack_test.go @@ -73,12 +73,12 @@ func (ds *DualstackSuite) SetupSuite() { ds.putFile("/tmp/k0s.yaml", k0sConfigWithAddon) ds.Require().NoError(ds.InitController(0, "--config=/tmp/k0s.yaml")) ds.Require().NoError(ds.RunWorkers()) - client, err := k8s.NewForConfig(ds.getKubeConfig("controller0")) + client, err := k8s.NewForConfig(ds.getKubeConfig(ds.ControllerNode(0))) ds.Require().NoError(err) - err = ds.WaitForNodeReady("worker0", client) + err = ds.WaitForNodeReady(ds.WorkerNode(0), client) ds.Require().NoError(err) - err = ds.WaitForNodeReady("worker1", client) + err = ds.WaitForNodeReady(ds.WorkerNode(1), client) ds.Require().NoError(err) ds.client = client @@ -100,8 +100,7 @@ func TestDualStack(t *testing.T) { } func (ds *DualstackSuite) putFile(path string, content string) { - controllerNode := fmt.Sprintf("controller%d", 0) - ssh, err := ds.SSH(controllerNode) + ssh, err := ds.SSH(ds.ControllerNode(0)) ds.Require().NoError(err) defer ssh.Disconnect() _, err = ssh.ExecWithOutput(fmt.Sprintf("echo '%s' >%s", content, path)) diff --git a/inttest/hacontrolplane/hacontrolplane_test.go b/inttest/hacontrolplane/hacontrolplane_test.go index 4cede053069e..a49c04faeeaa 100644 --- a/inttest/hacontrolplane/hacontrolplane_test.go +++ b/inttest/hacontrolplane/hacontrolplane_test.go @@ -33,8 +33,7 @@ type HAControlplaneSuite struct { func (s *HAControlplaneSuite) getMembers(fromControllerIdx int) map[string]string { // our etcd instances doesn't listen on public IP, so test is performed by calling CLI tools over ssh // which in general even makes sense, we can test tooling as well - node := fmt.Sprintf("controller%d", fromControllerIdx) - sshCon, err := s.SSH(node) + sshCon, err := s.SSH(s.ControllerNode(fromControllerIdx)) s.NoError(err) output, err := sshCon.ExecWithOutput("k0s etcd member-list") output = lastLine(output) @@ -49,8 +48,7 @@ func (s *HAControlplaneSuite) getMembers(fromControllerIdx int) map[string]strin } func (s *HAControlplaneSuite) makeNodeLeave(executeOnControllerIdx int, peerAddress string) { - node := fmt.Sprintf("controller%d", executeOnControllerIdx) - sshCon, err := s.SSH(node) + sshCon, err := s.SSH(s.ControllerNode(executeOnControllerIdx)) s.NoError(err) for i := 0; i < 20; i++ { _, err = sshCon.ExecWithOutput(fmt.Sprintf("k0s etcd leave %s", peerAddress)) @@ -64,8 +62,7 @@ func (s *HAControlplaneSuite) makeNodeLeave(executeOnControllerIdx int, peerAddr } func (s *HAControlplaneSuite) getCa(controllerIdx int) string { - node := fmt.Sprintf("controller%d", controllerIdx) - sshCon, err := s.SSH(node) + sshCon, err := s.SSH(s.ControllerNode(controllerIdx)) s.NoError(err) ca, err := sshCon.ExecWithOutput("cat /var/lib/k0s/pki/ca.crt") s.NoError(err) @@ -103,17 +100,17 @@ func (s *HAControlplaneSuite) TestDeregistration() { // Restart the second controller with a token to see it comes up // It should just ignore the token as there's CA etc already in place - sshC1, err := s.SSH("controller1") + sshC1, err := s.SSH(s.ControllerNode(1)) s.Require().NoError(err) _, err = sshC1.ExecWithOutput("kill $(pidof k0s) && while pidof k0s; do sleep 0.1s; done") s.Require().NoError(err) s.NoError(s.InitController(1, token)) // Make one member leave the etcd cluster - s.makeNodeLeave(1, membersFromJoined["controller1"]) + s.makeNodeLeave(1, membersFromJoined[s.ControllerNode(1)]) refreshedMembers := s.getMembers(0) s.Len(refreshedMembers, 1) - s.Contains(refreshedMembers, "controller0") + s.Contains(refreshedMembers, s.ControllerNode(0)) } diff --git a/inttest/install/singlenode_test.go b/inttest/install/singlenode_test.go index 7200ff5cd44b..793ecfa9dbc9 100644 --- a/inttest/install/singlenode_test.go +++ b/inttest/install/singlenode_test.go @@ -29,7 +29,7 @@ type InstallSuite struct { } func (s *InstallSuite) TestK0sGetsUp() { - ssh, err := s.SSH("controller0") + ssh, err := s.SSH(s.ControllerNode(0)) s.Require().NoError(err) defer ssh.Disconnect() @@ -39,13 +39,13 @@ func (s *InstallSuite) TestK0sGetsUp() { _, err = ssh.ExecWithOutput("rc-service k0scontroller start") s.Require().NoError(err) - err = s.WaitForKubeAPI("controller0", "") + err = s.WaitForKubeAPI(s.ControllerNode(0), "") s.Require().NoError(err) - kc, err := s.KubeClient("controller0", "") + kc, err := s.KubeClient(s.ControllerNode(0), "") s.NoError(err) - err = s.WaitForNodeReady("controller0", kc) + err = s.WaitForNodeReady(s.ControllerNode(0), kc) s.NoError(err) pods, err := kc.CoreV1().Pods("kube-system").List(context.TODO(), v1.ListOptions{ diff --git a/inttest/kine/kine_test.go b/inttest/kine/kine_test.go index 0520967d604e..db9360f3551a 100644 --- a/inttest/kine/kine_test.go +++ b/inttest/kine/kine_test.go @@ -31,17 +31,17 @@ type KineSuite struct { } func (s *KineSuite) TestK0sGetsUp() { - s.putFile("controller0", "/tmp/k0s.yaml", k0sConfigWithKine) + s.putFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithKine) s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) s.NoError(s.RunWorkers()) - kc, err := s.KubeClient("controller0", "") + kc, err := s.KubeClient(s.ControllerNode(0), "") s.NoError(err) - err = s.WaitForNodeReady("worker0", kc) + err = s.WaitForNodeReady(s.WorkerNode(0), kc) s.NoError(err) - err = s.WaitForNodeReady("worker1", kc) + err = s.WaitForNodeReady(s.WorkerNode(1), kc) s.NoError(err) pods, err := kc.CoreV1().Pods("kube-system").List(context.TODO(), v1.ListOptions{ diff --git a/inttest/multicontroller/multicontroller_test.go b/inttest/multicontroller/multicontroller_test.go index da9e13909daf..6feba0c93a8e 100644 --- a/inttest/multicontroller/multicontroller_test.go +++ b/inttest/multicontroller/multicontroller_test.go @@ -31,7 +31,7 @@ type MultiControllerSuite struct { } func (s *MultiControllerSuite) getMainIPAddress() string { - ssh, err := s.SSH("controller0") + ssh, err := s.SSH(s.ControllerNode(0)) s.Require().NoError(err) defer ssh.Disconnect() @@ -44,22 +44,22 @@ func (s *MultiControllerSuite) TestK0sGetsUp() { ipAddress := s.getMainIPAddress() s.T().Logf("ip address: %s", ipAddress) - s.putFile("controller0", "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) + s.putFile(s.ControllerNode(0), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) token, err := s.GetJoinToken("controller", "") s.NoError(err) - s.putFile("controller1", "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) + s.putFile(s.ControllerNode(1), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.InitController(1, "--config=/tmp/k0s.yaml", token)) - s.putFile("controller2", "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) + s.putFile(s.ControllerNode(2), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.InitController(2, "--config=/tmp/k0s.yaml", token)) s.NoError(s.RunWorkers()) - kc, err := s.KubeClient("controller0", "") + kc, err := s.KubeClient(s.ControllerNode(0), "") s.NoError(err) - err = s.WaitForNodeReady("worker0", kc) + err = s.WaitForNodeReady(s.WorkerNode(0), kc) s.NoError(err) pods, err := kc.CoreV1().Pods("kube-system").List(context.TODO(), v1.ListOptions{ diff --git a/inttest/singlenode/singlenode_test.go b/inttest/singlenode/singlenode_test.go index dae4651e8e2d..bbbc3d70066a 100644 --- a/inttest/singlenode/singlenode_test.go +++ b/inttest/singlenode/singlenode_test.go @@ -32,10 +32,10 @@ type SingleNodeSuite struct { func (s *SingleNodeSuite) TestK0sGetsUp() { s.NoError(s.InitController(0, "--single")) - kc, err := s.KubeClient("controller0", "") + kc, err := s.KubeClient(s.ControllerNode(0), "") s.NoError(err) - err = s.WaitForNodeReady("controller0", kc) + err = s.WaitForNodeReady(s.ControllerNode(0), kc) s.NoError(err) pods, err := kc.CoreV1().Pods("kube-system").List(context.TODO(), v1.ListOptions{ From 86419af5dd0385e7ac926d2a3f4ef8da506c8101 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Mon, 22 Mar 2021 18:07:51 +0100 Subject: [PATCH 06/12] inttest: De-duplicate PutFile Signed-off-by: Natanael Copa --- inttest/addons/addons_test.go | 13 ++----------- inttest/common/filetools.go | 11 +++++++++++ inttest/dualstack/dualstack_test.go | 12 +----------- inttest/kine/kine_test.go | 12 +----------- inttest/multicontroller/multicontroller_test.go | 15 +++------------ 5 files changed, 18 insertions(+), 45 deletions(-) diff --git a/inttest/addons/addons_test.go b/inttest/addons/addons_test.go index f35310f229b0..076e536583c9 100644 --- a/inttest/addons/addons_test.go +++ b/inttest/addons/addons_test.go @@ -43,7 +43,7 @@ type AddonsSuite struct { func (as *AddonsSuite) TestHelmBasedAddons() { addonName := "test-addon" - as.putFile("/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithAddon, addonName)) + as.PutFile(as.ControllerNode(0), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithAddon, addonName)) as.Require().NoError(as.InitController(0, "--config=/tmp/k0s.yaml")) as.waitForPrometheusRelease(addonName, 1) @@ -186,7 +186,7 @@ func (as *AddonsSuite) doPrometheusUpdate(addonName string, values map[string]in buf := bytes.NewBuffer([]byte{}) as.Require().NoError(tw.WriteToBuffer(buf)) - as.putFile(path, buf.String()) + as.PutFile(as.ControllerNode(0), path, buf.String()) } func (as *AddonsSuite) getKubeConfig(node string) *restclient.Config { @@ -216,15 +216,6 @@ func TestAddonsSuite(t *testing.T) { } -func (as *AddonsSuite) putFile(path string, content string) { - ssh, err := as.SSH(as.ControllerNode(0)) - as.Require().NoError(err) - defer ssh.Disconnect() - _, err = ssh.ExecWithOutput(fmt.Sprintf("echo '%s' >%s", content, path)) - - as.Require().NoError(err) -} - const k0sConfigWithAddon = ` spec: extensions: diff --git a/inttest/common/filetools.go b/inttest/common/filetools.go index 4f149bb315d4..03f67e9612aa 100644 --- a/inttest/common/filetools.go +++ b/inttest/common/filetools.go @@ -11,3 +11,14 @@ func (s *FootlooseSuite) GetFileFromController(controllerIdx int, path string) s return content } + +// PutFile writes content to file on given node +func (s *FootlooseSuite) PutFile(node, path, content string) { + ssh, err := s.SSH(node) + s.Require().NoError(err) + defer ssh.Disconnect() + // TODO: send data via pipe instead, so we can write data with single quotes ' + _, err = ssh.ExecWithOutput(fmt.Sprintf("echo '%s' >%s", content, path)) + + s.Require().NoError(err) +} diff --git a/inttest/dualstack/dualstack_test.go b/inttest/dualstack/dualstack_test.go index 649cddefb3f1..3282d52afbf9 100644 --- a/inttest/dualstack/dualstack_test.go +++ b/inttest/dualstack/dualstack_test.go @@ -70,7 +70,7 @@ func (ds *DualstackSuite) getKubeConfig(node string) *restclient.Config { func (ds *DualstackSuite) SetupSuite() { ds.FootlooseSuite.SetupSuite() - ds.putFile("/tmp/k0s.yaml", k0sConfigWithAddon) + ds.PutFile(ds.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithAddon) ds.Require().NoError(ds.InitController(0, "--config=/tmp/k0s.yaml")) ds.Require().NoError(ds.RunWorkers()) client, err := k8s.NewForConfig(ds.getKubeConfig(ds.ControllerNode(0))) @@ -99,16 +99,6 @@ func TestDualStack(t *testing.T) { } -func (ds *DualstackSuite) putFile(path string, content string) { - ssh, err := ds.SSH(ds.ControllerNode(0)) - ds.Require().NoError(err) - defer ssh.Disconnect() - _, err = ssh.ExecWithOutput(fmt.Sprintf("echo '%s' >%s", content, path)) - - ds.Require().NoError(err) - -} - const k0sConfigWithAddon = ` spec: network: diff --git a/inttest/kine/kine_test.go b/inttest/kine/kine_test.go index db9360f3551a..de7e1e8fb84d 100644 --- a/inttest/kine/kine_test.go +++ b/inttest/kine/kine_test.go @@ -17,7 +17,6 @@ package kine import ( "context" - "fmt" "testing" "github.com/stretchr/testify/suite" @@ -31,7 +30,7 @@ type KineSuite struct { } func (s *KineSuite) TestK0sGetsUp() { - s.putFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithKine) + s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithKine) s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) s.NoError(s.RunWorkers()) @@ -68,15 +67,6 @@ func TestKineSuite(t *testing.T) { suite.Run(t, &s) } -func (s *KineSuite) putFile(node string, path string, content string) { - ssh, err := s.SSH(node) - s.Require().NoError(err) - defer ssh.Disconnect() - _, err = ssh.ExecWithOutput(fmt.Sprintf("echo '%s' >%s", content, path)) - - s.Require().NoError(err) -} - const k0sConfigWithKine = ` spec: storage: diff --git a/inttest/multicontroller/multicontroller_test.go b/inttest/multicontroller/multicontroller_test.go index 6feba0c93a8e..7bccbfcb0cb7 100644 --- a/inttest/multicontroller/multicontroller_test.go +++ b/inttest/multicontroller/multicontroller_test.go @@ -44,15 +44,15 @@ func (s *MultiControllerSuite) TestK0sGetsUp() { ipAddress := s.getMainIPAddress() s.T().Logf("ip address: %s", ipAddress) - s.putFile(s.ControllerNode(0), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) + s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) token, err := s.GetJoinToken("controller", "") s.NoError(err) - s.putFile(s.ControllerNode(1), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) + s.PutFile(s.ControllerNode(1), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.InitController(1, "--config=/tmp/k0s.yaml", token)) - s.putFile(s.ControllerNode(2), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) + s.PutFile(s.ControllerNode(2), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.InitController(2, "--config=/tmp/k0s.yaml", token)) s.NoError(s.RunWorkers()) @@ -86,15 +86,6 @@ func TestMultiControllerSuite(t *testing.T) { suite.Run(t, &s) } -func (s *MultiControllerSuite) putFile(node string, path string, content string) { - ssh, err := s.SSH(node) - s.Require().NoError(err) - defer ssh.Disconnect() - _, err = ssh.ExecWithOutput(fmt.Sprintf("echo '%s' >%s", content, path)) - - s.Require().NoError(err) -} - const k0sConfigWithMultiController = ` spec: api: From d8fd50a970144f824053aaaebf26c22ff6bcb370 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 23 Mar 2021 11:19:21 +0100 Subject: [PATCH 07/12] Use k0s kubeconfig admin to get kubeconfig Also pass k0s args as variadic args instead of hardcoded datadir Signed-off-by: Natanael Copa --- inttest/addons/addons_test.go | 2 +- inttest/basic/basic_test.go | 2 +- inttest/byocri/byocri_test.go | 2 +- inttest/common/footloosesuite.go | 29 +++++++++---------- inttest/install/singlenode_test.go | 4 +-- inttest/kine/kine_test.go | 2 +- .../multicontroller/multicontroller_test.go | 2 +- inttest/singlenode/singlenode_test.go | 2 +- 8 files changed, 22 insertions(+), 23 deletions(-) diff --git a/inttest/addons/addons_test.go b/inttest/addons/addons_test.go index 076e536583c9..46743c82773c 100644 --- a/inttest/addons/addons_test.go +++ b/inttest/addons/addons_test.go @@ -135,7 +135,7 @@ func (as *AddonsSuite) waitForPrometheusRelease(addonName string, rev int64) (st func (as *AddonsSuite) waitForPrometheusServerEnvs(releaseName string) error { as.T().Logf("waiting to see prometheus release to have envs set from values yaml") - kc, err := as.KubeClient(as.ControllerNode(0), "") + kc, err := as.KubeClient(as.ControllerNode(0)) if err != nil { return err } diff --git a/inttest/basic/basic_test.go b/inttest/basic/basic_test.go index 4abb9e293267..a862011bda4d 100644 --- a/inttest/basic/basic_test.go +++ b/inttest/basic/basic_test.go @@ -40,7 +40,7 @@ func (s *BasicSuite) TestK0sGetsUp() { s.NoError(s.InitController(0, dataDirOpt)) s.NoError(s.RunWorkers(dataDirOpt, `--labels="k0sproject.io/foo=bar"`, `--kubelet-extra-args="--address=0.0.0.0 --event-burst=10"`)) - kc, err := s.KubeClient(s.ControllerNode(0), customDataDir) + kc, err := s.KubeClient(s.ControllerNode(0), dataDirOpt) s.NoError(err) err = s.WaitForNodeReady(s.WorkerNode(0), kc) diff --git a/inttest/byocri/byocri_test.go b/inttest/byocri/byocri_test.go index 2cf614df971b..920e940220bc 100644 --- a/inttest/byocri/byocri_test.go +++ b/inttest/byocri/byocri_test.go @@ -36,7 +36,7 @@ func (s *BYOCRISuite) TestK0sGetsUp() { s.NoError(s.InitController(0)) s.Require().NoError(s.runDockerWorker()) - kc, err := s.KubeClient(s.ControllerNode(0), "") + kc, err := s.KubeClient(s.ControllerNode(0)) s.NoError(err) err = s.WaitForNodeReady(s.WorkerNode(0), kc) diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index e9618f573563..c3380d17bbf8 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -22,7 +22,6 @@ import ( "os" "os/signal" "path" - "path/filepath" "strings" "syscall" "time" @@ -34,7 +33,6 @@ import ( "k8s.io/client-go/tools/clientcmd" "github.com/k0sproject/k0s/internal/util" - "github.com/k0sproject/k0s/pkg/constant" "github.com/weaveworks/footloose/pkg/cluster" "github.com/weaveworks/footloose/pkg/config" @@ -202,14 +200,17 @@ func (s *FootlooseSuite) keepEnvironment() bool { } } -func getDataDir(args []string) string { - dataDir := "" +func getDataDirOpt(args []string) string { for _, arg := range args { if strings.HasPrefix(arg, "--data-dir=") { - dataDir = strings.TrimPrefix(arg, "--data-dir=") + return arg } } - return dataDir + return "" +} + +func getDataDir(args []string) string { + return strings.TrimPrefix(getDataDirOpt(args), "--data-dir=") } // InitController initializes a controller @@ -227,7 +228,8 @@ func (s *FootlooseSuite) InitController(idx int, k0sArgs ...string) error { s.T().Logf("failed to execute '%s' on %s", startCmd, controllerNode) return err } - return s.WaitForKubeAPI(controllerNode, getDataDir(k0sArgs)) + + return s.WaitForKubeAPI(controllerNode, getDataDirOpt(k0sArgs)) } // GetJoinToken generates join token for the asked role @@ -323,11 +325,8 @@ func (s *FootlooseSuite) MachineForName(name string) (*cluster.Machine, error) { return nil, fmt.Errorf("no machine found with name %s", name) } -// WaitForKubeAPI return kube client by loading the admin access config from given node -func (s *FootlooseSuite) KubeClient(node string, dataDir string) (*kubernetes.Clientset, error) { - if dataDir == "" { - dataDir = constant.DataDirDefault - } +// KubeClient return kube client by loading the admin access config from given node +func (s *FootlooseSuite) KubeClient(node string, k0sKubeconfigArgs ...string) (*kubernetes.Clientset, error) { machine, err := s.MachineForName(node) if err != nil { return nil, err @@ -336,7 +335,7 @@ func (s *FootlooseSuite) KubeClient(node string, dataDir string) (*kubernetes.Cl if err != nil { return nil, err } - kubeConfigCmd := fmt.Sprintf("cat %s", filepath.Join(dataDir, "pki/admin.conf")) + kubeConfigCmd := fmt.Sprintf("k0s kubeconfig admin %s", strings.Join(k0sKubeconfigArgs, " ")) kubeConf, err := ssh.ExecWithOutput(kubeConfigCmd) if err != nil { return nil, err @@ -384,10 +383,10 @@ func (s *FootlooseSuite) GetNodeLabels(node string, kc *kubernetes.Clientset) (m // WaitForKubeAPI waits until we see kube API online on given node. // Timeouts with error return in 5 mins -func (s *FootlooseSuite) WaitForKubeAPI(node string, dataDir string) error { +func (s *FootlooseSuite) WaitForKubeAPI(node string, k0sKubeconfigArgs ...string) error { s.T().Log("starting to poll kube api") return wait.PollImmediate(100*time.Millisecond, 5*time.Minute, func() (done bool, err error) { - kc, err := s.KubeClient(node, dataDir) + kc, err := s.KubeClient(node, k0sKubeconfigArgs...) if err != nil { return false, nil } diff --git a/inttest/install/singlenode_test.go b/inttest/install/singlenode_test.go index 793ecfa9dbc9..613519f926c5 100644 --- a/inttest/install/singlenode_test.go +++ b/inttest/install/singlenode_test.go @@ -39,10 +39,10 @@ func (s *InstallSuite) TestK0sGetsUp() { _, err = ssh.ExecWithOutput("rc-service k0scontroller start") s.Require().NoError(err) - err = s.WaitForKubeAPI(s.ControllerNode(0), "") + err = s.WaitForKubeAPI(s.ControllerNode(0)) s.Require().NoError(err) - kc, err := s.KubeClient(s.ControllerNode(0), "") + kc, err := s.KubeClient(s.ControllerNode(0)) s.NoError(err) err = s.WaitForNodeReady(s.ControllerNode(0), kc) diff --git a/inttest/kine/kine_test.go b/inttest/kine/kine_test.go index de7e1e8fb84d..00a860b99c28 100644 --- a/inttest/kine/kine_test.go +++ b/inttest/kine/kine_test.go @@ -34,7 +34,7 @@ func (s *KineSuite) TestK0sGetsUp() { s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) s.NoError(s.RunWorkers()) - kc, err := s.KubeClient(s.ControllerNode(0), "") + kc, err := s.KubeClient(s.ControllerNode(0)) s.NoError(err) err = s.WaitForNodeReady(s.WorkerNode(0), kc) diff --git a/inttest/multicontroller/multicontroller_test.go b/inttest/multicontroller/multicontroller_test.go index 7bccbfcb0cb7..ee1ff32e2f2f 100644 --- a/inttest/multicontroller/multicontroller_test.go +++ b/inttest/multicontroller/multicontroller_test.go @@ -56,7 +56,7 @@ func (s *MultiControllerSuite) TestK0sGetsUp() { s.NoError(s.InitController(2, "--config=/tmp/k0s.yaml", token)) s.NoError(s.RunWorkers()) - kc, err := s.KubeClient(s.ControllerNode(0), "") + kc, err := s.KubeClient(s.ControllerNode(0)) s.NoError(err) err = s.WaitForNodeReady(s.WorkerNode(0), kc) diff --git a/inttest/singlenode/singlenode_test.go b/inttest/singlenode/singlenode_test.go index bbbc3d70066a..615e584e089b 100644 --- a/inttest/singlenode/singlenode_test.go +++ b/inttest/singlenode/singlenode_test.go @@ -32,7 +32,7 @@ type SingleNodeSuite struct { func (s *SingleNodeSuite) TestK0sGetsUp() { s.NoError(s.InitController(0, "--single")) - kc, err := s.KubeClient(s.ControllerNode(0), "") + kc, err := s.KubeClient(s.ControllerNode(0)) s.NoError(err) err = s.WaitForNodeReady(s.ControllerNode(0), kc) From f17ee2faf9a29623da1125c07fbf8fa0722534c5 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 23 Mar 2021 12:59:56 +0100 Subject: [PATCH 08/12] De-duplicate getKubeConfig in inttests Re-use the code from common/footloose.go Signed-off-by: Natanael Copa --- inttest/addons/addons_test.go | 23 ++++------------------- inttest/basic/basic_test.go | 21 +++------------------ inttest/common/footloosesuite.go | 12 +++++++++++- inttest/dualstack/dualstack_test.go | 20 +------------------- 4 files changed, 19 insertions(+), 57 deletions(-) diff --git a/inttest/addons/addons_test.go b/inttest/addons/addons_test.go index 46743c82773c..4d57e7e9e8fe 100644 --- a/inttest/addons/addons_test.go +++ b/inttest/addons/addons_test.go @@ -28,8 +28,6 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" k8s "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "github.com/k0sproject/k0s/internal/util" "github.com/k0sproject/k0s/inttest/common" @@ -65,7 +63,8 @@ func (as *AddonsSuite) TestHelmBasedAddons() { } func (as *AddonsSuite) doPrometheusDelete(chartName string) { - cfg := as.getKubeConfig(as.ControllerNode(0)) + cfg, err := as.GetKubeConfig(as.ControllerNode(0)) + as.Require().NoError(err) chartClient, err := clientset.New(cfg) as.Require().NoError(err) as.Require().NoError(chartClient.Charts("kube-system").Delete(context.Background(), chartName, v1.DeleteOptions{})) @@ -86,7 +85,8 @@ func (as *AddonsSuite) doPrometheusDelete(chartName string) { func (as *AddonsSuite) waitForPrometheusRelease(addonName string, rev int64) (string, string) { as.T().Logf("waiting to see prometheus release ready in kube API, generation %d", rev) - cfg := as.getKubeConfig(as.ControllerNode(0)) + cfg, err := as.GetKubeConfig(as.ControllerNode(0)) + as.Require().NoError(err) chartClient, err := clientset.New(cfg) as.Require().NoError(err) var chartName string @@ -189,21 +189,6 @@ func (as *AddonsSuite) doPrometheusUpdate(addonName string, values map[string]in as.PutFile(as.ControllerNode(0), path, buf.String()) } -func (as *AddonsSuite) getKubeConfig(node string) *restclient.Config { - machine, err := as.MachineForName(node) - as.Require().NoError(err) - ssh, err := as.SSH(node) - as.Require().NoError(err) - kubeConf, err := ssh.ExecWithOutput("cat /var/lib/k0s/pki/admin.conf") - as.Require().NoError(err) - cfg, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeConf)) - as.Require().NoError(err) - hostPort, err := machine.HostPort(6443) - as.Require().NoError(err) - cfg.Host = fmt.Sprintf("localhost:%d", hostPort) - return cfg -} - func TestAddonsSuite(t *testing.T) { s := AddonsSuite{ diff --git a/inttest/basic/basic_test.go b/inttest/basic/basic_test.go index a862011bda4d..585bad656f67 100644 --- a/inttest/basic/basic_test.go +++ b/inttest/basic/basic_test.go @@ -26,8 +26,6 @@ import ( capi "k8s.io/api/certificates/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" ) type BasicSuite struct { @@ -73,22 +71,9 @@ func (s *BasicSuite) TestK0sGetsUp() { s.Require().NoError(s.verifyKubeletAddressFlag(s.WorkerNode(0))) s.Require().NoError(s.verifyKubeletAddressFlag(s.WorkerNode(1))) - s.Require().NoError(common.WaitForMetricsReady(s.getKubeConfig(s.ControllerNode(0)))) -} - -func (s *BasicSuite) getKubeConfig(node string) *restclient.Config { - machine, err := s.MachineForName(node) - s.Require().NoError(err) - ssh, err := s.SSH(node) - s.Require().NoError(err) - kubeConf, err := ssh.ExecWithOutput("cat /var/lib/k0s/custom-data-dir/pki/admin.conf") - s.Require().NoError(err) - cfg, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeConf)) - s.Require().NoError(err) - hostPort, err := machine.HostPort(6443) - s.Require().NoError(err) - cfg.Host = fmt.Sprintf("localhost:%d", hostPort) - return cfg + cfg, err := s.GetKubeConfig(s.ControllerNode(0), dataDirOpt) + s.NoError(err) + s.Require().NoError(common.WaitForMetricsReady(cfg)) } func (s *BasicSuite) checkCertPerms(node string) error { diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index c3380d17bbf8..8365d5445902 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/suite" "gopkg.in/yaml.v2" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "github.com/k0sproject/k0s/internal/util" @@ -326,7 +327,7 @@ func (s *FootlooseSuite) MachineForName(name string) (*cluster.Machine, error) { } // KubeClient return kube client by loading the admin access config from given node -func (s *FootlooseSuite) KubeClient(node string, k0sKubeconfigArgs ...string) (*kubernetes.Clientset, error) { +func (s *FootlooseSuite) GetKubeConfig(node string, k0sKubeconfigArgs ...string) (*rest.Config, error) { machine, err := s.MachineForName(node) if err != nil { return nil, err @@ -349,6 +350,15 @@ func (s *FootlooseSuite) KubeClient(node string, k0sKubeconfigArgs ...string) (* return nil, errors.Wrap(err, "footloose machine has to have 6443 port mapped") } cfg.Host = fmt.Sprintf("localhost:%d", hostPort) + return cfg, nil +} + +// KubeClient return kube client by loading the admin access config from given node +func (s *FootlooseSuite) KubeClient(node string, k0sKubeconfigArgs ...string) (*kubernetes.Clientset, error) { + cfg, err := s.GetKubeConfig(node, k0sKubeconfigArgs...) + if err != nil { + return nil, err + } return kubernetes.NewForConfig(cfg) } diff --git a/inttest/dualstack/dualstack_test.go b/inttest/dualstack/dualstack_test.go index 3282d52afbf9..cd7b6a944f31 100644 --- a/inttest/dualstack/dualstack_test.go +++ b/inttest/dualstack/dualstack_test.go @@ -25,7 +25,6 @@ package dualstack import ( "context" - "fmt" "github.com/stretchr/testify/suite" v1meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,8 +33,6 @@ import ( "github.com/k0sproject/k0s/inttest/common" k8s "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" ) type DualstackSuite struct { @@ -53,27 +50,12 @@ func (ds *DualstackSuite) TestDualStackNodesHavePodCIDRs() { } -func (ds *DualstackSuite) getKubeConfig(node string) *restclient.Config { - machine, err := ds.MachineForName(node) - ds.Require().NoError(err) - ssh, err := ds.SSH(node) - ds.Require().NoError(err) - kubeConf, err := ssh.ExecWithOutput("cat /var/lib/k0s/pki/admin.conf") - ds.Require().NoError(err) - cfg, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeConf)) - ds.Require().NoError(err) - hostPort, err := machine.HostPort(6443) - ds.Require().NoError(err) - cfg.Host = fmt.Sprintf("localhost:%d", hostPort) - return cfg -} - func (ds *DualstackSuite) SetupSuite() { ds.FootlooseSuite.SetupSuite() ds.PutFile(ds.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithAddon) ds.Require().NoError(ds.InitController(0, "--config=/tmp/k0s.yaml")) ds.Require().NoError(ds.RunWorkers()) - client, err := k8s.NewForConfig(ds.getKubeConfig(ds.ControllerNode(0))) + client, err := ds.KubeClient(ds.ControllerNode(0)) ds.Require().NoError(err) err = ds.WaitForNodeReady(ds.WorkerNode(0), client) ds.Require().NoError(err) From 6c6f792fbc216568a1e540008a723fe5a8db16b8 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 23 Mar 2021 13:18:17 +0100 Subject: [PATCH 09/12] inttest: clean up disconnect of ssh Signed-off-by: Natanael Copa --- inttest/common/filetools.go | 1 + inttest/common/footloosesuite.go | 3 +++ inttest/hacontrolplane/hacontrolplane_test.go | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/inttest/common/filetools.go b/inttest/common/filetools.go index 03f67e9612aa..d4398c6cbe9f 100644 --- a/inttest/common/filetools.go +++ b/inttest/common/filetools.go @@ -6,6 +6,7 @@ import "fmt" func (s *FootlooseSuite) GetFileFromController(controllerIdx int, path string) string { sshCon, err := s.SSH(s.ControllerNode(controllerIdx)) s.Require().NoError(err) + defer sshCon.Disconnect() content, err := sshCon.ExecWithOutput(fmt.Sprintf("cat %s", path)) s.Require().NoError(err) diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index 8365d5445902..6e65a12367e8 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -156,6 +156,7 @@ func (s *FootlooseSuite) TearDownSuite() { } s.T().Logf("wrote log of node %s to %s", m.Hostname(), logPath) + ssh.Disconnect() } if s.keepEnvironment() { @@ -336,6 +337,8 @@ func (s *FootlooseSuite) GetKubeConfig(node string, k0sKubeconfigArgs ...string) if err != nil { return nil, err } + defer ssh.Disconnect() + kubeConfigCmd := fmt.Sprintf("k0s kubeconfig admin %s", strings.Join(k0sKubeconfigArgs, " ")) kubeConf, err := ssh.ExecWithOutput(kubeConfigCmd) if err != nil { diff --git a/inttest/hacontrolplane/hacontrolplane_test.go b/inttest/hacontrolplane/hacontrolplane_test.go index a49c04faeeaa..7cc76f776f01 100644 --- a/inttest/hacontrolplane/hacontrolplane_test.go +++ b/inttest/hacontrolplane/hacontrolplane_test.go @@ -35,6 +35,7 @@ func (s *HAControlplaneSuite) getMembers(fromControllerIdx int) map[string]strin // which in general even makes sense, we can test tooling as well sshCon, err := s.SSH(s.ControllerNode(fromControllerIdx)) s.NoError(err) + defer sshCon.Disconnect() output, err := sshCon.ExecWithOutput("k0s etcd member-list") output = lastLine(output) s.NoError(err) @@ -50,6 +51,7 @@ func (s *HAControlplaneSuite) getMembers(fromControllerIdx int) map[string]strin func (s *HAControlplaneSuite) makeNodeLeave(executeOnControllerIdx int, peerAddress string) { sshCon, err := s.SSH(s.ControllerNode(executeOnControllerIdx)) s.NoError(err) + defer sshCon.Disconnect() for i := 0; i < 20; i++ { _, err = sshCon.ExecWithOutput(fmt.Sprintf("k0s etcd leave %s", peerAddress)) if err == nil { @@ -64,6 +66,7 @@ func (s *HAControlplaneSuite) makeNodeLeave(executeOnControllerIdx int, peerAddr func (s *HAControlplaneSuite) getCa(controllerIdx int) string { sshCon, err := s.SSH(s.ControllerNode(controllerIdx)) s.NoError(err) + defer sshCon.Disconnect() ca, err := sshCon.ExecWithOutput("cat /var/lib/k0s/pki/ca.crt") s.NoError(err) @@ -102,6 +105,7 @@ func (s *HAControlplaneSuite) TestDeregistration() { // It should just ignore the token as there's CA etc already in place sshC1, err := s.SSH(s.ControllerNode(1)) s.Require().NoError(err) + defer sshC1.Disconnect() _, err = sshC1.ExecWithOutput("kill $(pidof k0s) && while pidof k0s; do sleep 0.1s; done") s.Require().NoError(err) s.NoError(s.InitController(1, token)) From be522d55ff2f91a65bef927c5c02fb1c2d91a649 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 23 Mar 2021 13:23:08 +0100 Subject: [PATCH 10/12] inttest: use GetFileFromController to get CA cert Signed-off-by: Natanael Copa --- inttest/hacontrolplane/hacontrolplane_test.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/inttest/hacontrolplane/hacontrolplane_test.go b/inttest/hacontrolplane/hacontrolplane_test.go index 7cc76f776f01..a26518e2189c 100644 --- a/inttest/hacontrolplane/hacontrolplane_test.go +++ b/inttest/hacontrolplane/hacontrolplane_test.go @@ -63,26 +63,16 @@ func (s *HAControlplaneSuite) makeNodeLeave(executeOnControllerIdx int, peerAddr s.NoError(err) } -func (s *HAControlplaneSuite) getCa(controllerIdx int) string { - sshCon, err := s.SSH(s.ControllerNode(controllerIdx)) - s.NoError(err) - defer sshCon.Disconnect() - ca, err := sshCon.ExecWithOutput("cat /var/lib/k0s/pki/ca.crt") - s.NoError(err) - - return ca -} - func (s *HAControlplaneSuite) TestDeregistration() { s.NoError(s.InitController(0)) token, err := s.GetJoinToken("controller", "") s.NoError(err) s.NoError(s.InitController(1, token)) - ca0 := s.getCa(0) + ca0 := s.GetFileFromController(0, "/var/lib/k0s/pki/ca.crt") s.Contains(ca0, "-----BEGIN CERTIFICATE-----") - ca1 := s.getCa(1) + ca1 := s.GetFileFromController(1, "/var/lib/k0s/pki/ca.crt") s.Contains(ca1, "-----BEGIN CERTIFICATE-----") s.Equal(ca0, ca1) From 6310751054ab6d0cf9175283c730f619863877a3 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 23 Mar 2021 13:48:38 +0100 Subject: [PATCH 11/12] inttest: use variadic args for GetJoinToken Signed-off-by: Natanael Copa --- inttest/byocri/byocri_test.go | 2 +- inttest/common/footloosesuite.go | 10 +++------- inttest/hacontrolplane/hacontrolplane_test.go | 2 +- inttest/multicontroller/multicontroller_test.go | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/inttest/byocri/byocri_test.go b/inttest/byocri/byocri_test.go index 920e940220bc..f94257b42d68 100644 --- a/inttest/byocri/byocri_test.go +++ b/inttest/byocri/byocri_test.go @@ -57,7 +57,7 @@ func (s *BYOCRISuite) TestK0sGetsUp() { } func (s *BYOCRISuite) runDockerWorker() error { - token, err := s.GetJoinToken("worker", "") + token, err := s.GetJoinToken("worker") if err != nil { return err } diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index 6e65a12367e8..b46ce8ed3e5d 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -211,10 +211,6 @@ func getDataDirOpt(args []string) string { return "" } -func getDataDir(args []string) string { - return strings.TrimPrefix(getDataDirOpt(args), "--data-dir=") -} - // InitController initializes a controller func (s *FootlooseSuite) InitController(idx int, k0sArgs ...string) error { controllerNode := s.ControllerNode(idx) @@ -235,7 +231,7 @@ func (s *FootlooseSuite) InitController(idx int, k0sArgs ...string) error { } // GetJoinToken generates join token for the asked role -func (s *FootlooseSuite) GetJoinToken(role string, dataDir string) (string, error) { +func (s *FootlooseSuite) GetJoinToken(role string, extraArgs ...string) (string, error) { // assume we have main on node 0 always controllerNode := s.ControllerNode(0) s.Contains([]string{"controller", "worker"}, role, "Bad role") @@ -244,7 +240,7 @@ func (s *FootlooseSuite) GetJoinToken(role string, dataDir string) (string, erro return "", err } defer ssh.Disconnect() - token, err := ssh.ExecWithOutput(fmt.Sprintf("k0s token create --role=%s --data-dir=%s", role, dataDir)) + token, err := ssh.ExecWithOutput(fmt.Sprintf("k0s token create --role=%s %s", role, strings.Join(extraArgs, " "))) if err != nil { return "", fmt.Errorf("can't get join token: %v", err) } @@ -262,7 +258,7 @@ func (s *FootlooseSuite) RunWorkers(args ...string) error { return err } defer ssh.Disconnect() - token, err := s.GetJoinToken("worker", getDataDir(args)) + token, err := s.GetJoinToken("worker", getDataDirOpt(args)) if err != nil { return err } diff --git a/inttest/hacontrolplane/hacontrolplane_test.go b/inttest/hacontrolplane/hacontrolplane_test.go index a26518e2189c..77311a82a2bc 100644 --- a/inttest/hacontrolplane/hacontrolplane_test.go +++ b/inttest/hacontrolplane/hacontrolplane_test.go @@ -65,7 +65,7 @@ func (s *HAControlplaneSuite) makeNodeLeave(executeOnControllerIdx int, peerAddr func (s *HAControlplaneSuite) TestDeregistration() { s.NoError(s.InitController(0)) - token, err := s.GetJoinToken("controller", "") + token, err := s.GetJoinToken("controller") s.NoError(err) s.NoError(s.InitController(1, token)) diff --git a/inttest/multicontroller/multicontroller_test.go b/inttest/multicontroller/multicontroller_test.go index ee1ff32e2f2f..592844e97eb0 100644 --- a/inttest/multicontroller/multicontroller_test.go +++ b/inttest/multicontroller/multicontroller_test.go @@ -47,7 +47,7 @@ func (s *MultiControllerSuite) TestK0sGetsUp() { s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) - token, err := s.GetJoinToken("controller", "") + token, err := s.GetJoinToken("controller") s.NoError(err) s.PutFile(s.ControllerNode(1), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithMultiController, ipAddress)) s.NoError(s.InitController(1, "--config=/tmp/k0s.yaml", token)) From ff9013283b0f156e05871473247b4b0daa5cea8f Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 23 Mar 2021 13:57:46 +0100 Subject: [PATCH 12/12] Add regression test for #790 verify that k0s returns error when it should Signed-off-by: Natanael Copa --- inttest/hacontrolplane/hacontrolplane_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/inttest/hacontrolplane/hacontrolplane_test.go b/inttest/hacontrolplane/hacontrolplane_test.go index 77311a82a2bc..c401eb012a08 100644 --- a/inttest/hacontrolplane/hacontrolplane_test.go +++ b/inttest/hacontrolplane/hacontrolplane_test.go @@ -64,6 +64,12 @@ func (s *HAControlplaneSuite) makeNodeLeave(executeOnControllerIdx int, peerAddr } func (s *HAControlplaneSuite) TestDeregistration() { + // Verify that k0s return failure (https://github.com/k0sproject/k0s/issues/790) + sshC0, err := s.SSH(s.ControllerNode(0)) + s.Require().NoError(err) + _, err = sshC0.ExecWithOutput("k0s etcd member-list") + s.Require().Error(err) + s.NoError(s.InitController(0)) token, err := s.GetJoinToken("controller") s.NoError(err)