From 9427b80c2bc7edd2afd23ddec6f1422dfd4c2fc5 Mon Sep 17 00:00:00 2001 From: Jussi Nummelin Date: Wed, 2 Dec 2020 10:35:36 +0000 Subject: [PATCH 1/4] Fix admin kubeconfig perms Signed-off-by: Jussi Nummelin --- pkg/component/server/certificates.go | 43 ++++++++++++++++++---- pkg/component/server/konnectivity.go | 2 +- pkg/constant/constant.go | 54 ++++++++++++++-------------- 3 files changed, 66 insertions(+), 33 deletions(-) diff --git a/pkg/component/server/certificates.go b/pkg/component/server/certificates.go index 200a33171891..91258c8486d1 100644 --- a/pkg/component/server/certificates.go +++ b/pkg/component/server/certificates.go @@ -126,13 +126,33 @@ func (c *Certificates) Init() error { if err != nil { return err } - if err := kubeConfig(c.K0sVars.AdminKubeconfigConfigPath, "https://localhost:6443", c.CACert, adminCert.Cert, adminCert.Key); err != nil { + if err := kubeConfig(c.K0sVars.AdminKubeconfigConfigPath, "https://localhost:6443", c.CACert, adminCert.Cert, adminCert.Key, "root"); err != nil { return err } return generateKeyPair("sa", c.K0sVars) }) + eg.Go(func() error { + // konnectivity kubeconfig + konnectivityReq := certificate.Request{ + Name: "konnectivity", + CN: "kubernetes-konnectivity", + O: "system:masters", // TODO: We need to figure out if konnectivity really needs superpowers + CACert: caCertPath, + CAKey: caCertKey, + } + konnectivityCert, err := c.CertManager.EnsureCertificate(konnectivityReq, constant.KonnectivityServerUser) + if err != nil { + return err + } + if err := kubeConfig(c.K0sVars.KonnectivityKubeconfigConfigPath, "https://localhost:6443", c.CACert, konnectivityCert.Cert, konnectivityCert.Key, constant.KonnectivityServerUser); err != nil { + return err + } + + return nil + }) + eg.Go(func() error { ccmReq := certificate.Request{ Name: "ccm", @@ -147,7 +167,7 @@ func (c *Certificates) Init() error { return err } - return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "ccm.conf"), "https://localhost:6443", c.CACert, ccmCert.Cert, ccmCert.Key) + return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "ccm.conf"), "https://localhost:6443", c.CACert, ccmCert.Cert, ccmCert.Key, constant.ControllerManagerUser) }) eg.Go(func() error { @@ -163,7 +183,7 @@ func (c *Certificates) Init() error { return err } - return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "scheduler.conf"), "https://localhost:6443", c.CACert, schedulerCert.Cert, schedulerCert.Key) + return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "scheduler.conf"), "https://localhost:6443", c.CACert, schedulerCert.Cert, schedulerCert.Key, constant.SchedulerUser) }) eg.Go(func() error { @@ -238,7 +258,7 @@ func (c *Certificates) Stop() error { return nil } -func kubeConfig(dest, url, caCert, clientCert, clientKey string) error { +func kubeConfig(dest, url, caCert, clientCert, clientKey, owner string) error { if util.FileExists(dest) { return nil } @@ -254,13 +274,24 @@ func kubeConfig(dest, url, caCert, clientCert, clientKey string) error { ClientKey: base64.StdEncoding.EncodeToString([]byte(clientKey)), } - output, err := os.Create(dest) + output, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.CertSecureMode) if err != nil { return err } defer output.Close() - return kubeconfigTemplate.Execute(output, &data) + if err = kubeconfigTemplate.Execute(output, &data); err != nil { + return err + } + + // Chown the file properly for the owner + uid, _ := util.GetUID(owner) + err = os.Chown(output.Name(), uid, -1) + if err != nil && os.Geteuid() == 0 { + return err + } + + return nil } func generateKeyPair(name string, k0sVars constant.CfgVars) error { diff --git a/pkg/component/server/konnectivity.go b/pkg/component/server/konnectivity.go index 8cb3727e718a..c975237e5f27 100644 --- a/pkg/component/server/konnectivity.go +++ b/pkg/component/server/konnectivity.go @@ -69,7 +69,7 @@ func (k *Konnectivity) Run() error { fmt.Sprintf("--uds-name=%s", filepath.Join(k.K0sVars.KonnectivitySocketDir, "konnectivity-server.sock")), fmt.Sprintf("--cluster-cert=%s", filepath.Join(k.K0sVars.CertRootDir, "server.crt")), fmt.Sprintf("--cluster-key=%s", filepath.Join(k.K0sVars.CertRootDir, "server.key")), - fmt.Sprintf("--kubeconfig=%s", k.K0sVars.AdminKubeconfigConfigPath), // FIXME: should have user rights + fmt.Sprintf("--kubeconfig=%s", k.K0sVars.KonnectivityKubeconfigConfigPath), "--mode=grpc", "--server-port=0", "--agent-port=8132", diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go index 96c6f2e1d6fc..4b9a95ff4846 100644 --- a/pkg/constant/constant.go +++ b/pkg/constant/constant.go @@ -19,19 +19,20 @@ import "fmt" // CfgVars is a struct that holds all the config variables requried for K0s type CfgVars struct { - AdminKubeconfigConfigPath string // The cluster admin kubeconfig location - BinDir string // location for all pki related binaries - CertRootDir string // CertRootDir defines the root location for all pki related artifacts - DataDir string // Data directory containing k0s state - EtcdCertDir string // EtcdCertDir contains etcd certificates - EtcdDataDir string // EtcdDataDir contains etcd state - KineSocketPath string // The unix socket path for kine - KonnectivitySocketDir string // location of konnectivity's socket path - KubeletAuthConfigPath string // KubeletAuthConfigPath defines the default kubelet auth config path - KubeletBootstrapConfigPath string // KubeletBootstrapConfigPath defines the default path for kubelet bootstrap auth config - KubeletVolumePluginDir string // location for kubelet plugins volume executables - ManifestsDir string // location for all stack manifests - RunDir string // location of supervised pid files and sockets + AdminKubeconfigConfigPath string // The cluster admin kubeconfig location + BinDir string // location for all pki related binaries + CertRootDir string // CertRootDir defines the root location for all pki related artifacts + DataDir string // Data directory containing k0s state + EtcdCertDir string // EtcdCertDir contains etcd certificates + EtcdDataDir string // EtcdDataDir contains etcd state + KineSocketPath string // The unix socket path for kine + KonnectivitySocketDir string // location of konnectivity's socket path + KubeletAuthConfigPath string // KubeletAuthConfigPath defines the default kubelet auth config path + KubeletBootstrapConfigPath string // KubeletBootstrapConfigPath defines the default path for kubelet bootstrap auth config + KubeletVolumePluginDir string // location for kubelet plugins volume executables + ManifestsDir string // location for all stack manifests + RunDir string // location of supervised pid files and sockets + KonnectivityKubeconfigConfigPath string // location for konnectivity kubeconfig // Helm config HelmHome string @@ -115,19 +116,20 @@ func GetConfig(dataDir string) CfgVars { helmHome := fmt.Sprintf("%s/helmhome", dataDir) return CfgVars{ - AdminKubeconfigConfigPath: fmt.Sprintf("%s/admin.conf", certDir), - BinDir: fmt.Sprintf("%s/bin", dataDir), - CertRootDir: certDir, - DataDir: dataDir, - EtcdCertDir: fmt.Sprintf("%s/etcd", certDir), - EtcdDataDir: fmt.Sprintf("%s/etcd", dataDir), - KineSocketPath: fmt.Sprintf("%s/kine/kine.sock:2379", runDir), - KonnectivitySocketDir: fmt.Sprintf("%s/konnectivity-server", runDir), - KubeletAuthConfigPath: fmt.Sprintf("%s/kubelet.conf", dataDir), - KubeletBootstrapConfigPath: fmt.Sprintf("%s/kubelet-bootstrap.conf", dataDir), - KubeletVolumePluginDir: "/usr/libexec/k0s/kubelet-plugins/volume/exec", - ManifestsDir: fmt.Sprintf("%s/manifests", dataDir), - RunDir: runDir, + AdminKubeconfigConfigPath: fmt.Sprintf("%s/admin.conf", certDir), + BinDir: fmt.Sprintf("%s/bin", dataDir), + CertRootDir: certDir, + DataDir: dataDir, + EtcdCertDir: fmt.Sprintf("%s/etcd", certDir), + EtcdDataDir: fmt.Sprintf("%s/etcd", dataDir), + KineSocketPath: fmt.Sprintf("%s/kine/kine.sock:2379", runDir), + KonnectivitySocketDir: fmt.Sprintf("%s/konnectivity-server", runDir), + KubeletAuthConfigPath: fmt.Sprintf("%s/kubelet.conf", dataDir), + KubeletBootstrapConfigPath: fmt.Sprintf("%s/kubelet-bootstrap.conf", dataDir), + KubeletVolumePluginDir: "/usr/libexec/k0s/kubelet-plugins/volume/exec", + ManifestsDir: fmt.Sprintf("%s/manifests", dataDir), + RunDir: runDir, + KonnectivityKubeconfigConfigPath: fmt.Sprintf("%s/konnectivity.conf", certDir), // Helm Config HelmHome: helmHome, From 5ee57c0ec4d005b73ade6885588e2f434bda5441 Mon Sep 17 00:00:00 2001 From: Jussi Nummelin Date: Wed, 2 Dec 2020 12:28:20 +0000 Subject: [PATCH 2/4] Changes controller-manager to run as api-server user as they both need access to sa.key file. Also fixes the key and kubeconfig ownership and permissions on-the-fly for upgrades. Signed-off-by: Jussi Nummelin --- pkg/component/server/certificates.go | 29 ++++++++++++++++------- pkg/component/server/controllermanager.go | 3 ++- pkg/constant/constant.go | 2 -- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/pkg/component/server/certificates.go b/pkg/component/server/certificates.go index 91258c8486d1..4f01435c15d6 100644 --- a/pkg/component/server/certificates.go +++ b/pkg/component/server/certificates.go @@ -130,7 +130,7 @@ func (c *Certificates) Init() error { return err } - return generateKeyPair("sa", c.K0sVars) + return generateKeyPair("sa", c.K0sVars, constant.ApiserverUser) }) eg.Go(func() error { @@ -161,13 +161,13 @@ func (c *Certificates) Init() error { CACert: caCertPath, CAKey: caCertKey, } - ccmCert, err := c.CertManager.EnsureCertificate(ccmReq, constant.ControllerManagerUser) + ccmCert, err := c.CertManager.EnsureCertificate(ccmReq, constant.ApiserverUser) if err != nil { return err } - return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "ccm.conf"), "https://localhost:6443", c.CACert, ccmCert.Cert, ccmCert.Key, constant.ControllerManagerUser) + return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "ccm.conf"), "https://localhost:6443", c.CACert, ccmCert.Cert, ccmCert.Key, constant.ApiserverUser) }) eg.Go(func() error { @@ -260,7 +260,7 @@ func (c *Certificates) Stop() error { func kubeConfig(dest, url, caCert, clientCert, clientKey, owner string) error { if util.FileExists(dest) { - return nil + return chownFile(dest, owner, constant.CertSecureMode) } data := struct { URL string @@ -284,9 +284,17 @@ func kubeConfig(dest, url, caCert, clientCert, clientKey, owner string) error { return err } + return chownFile(output.Name(), owner, constant.CertSecureMode) +} + +func chownFile(file, owner string, permissions os.FileMode) error { // Chown the file properly for the owner uid, _ := util.GetUID(owner) - err = os.Chown(output.Name(), uid, -1) + err := os.Chown(file, uid, -1) + if err != nil && os.Geteuid() == 0 { + return err + } + err = os.Chmod(file, permissions) if err != nil && os.Geteuid() == 0 { return err } @@ -294,12 +302,12 @@ func kubeConfig(dest, url, caCert, clientCert, clientKey, owner string) error { return nil } -func generateKeyPair(name string, k0sVars constant.CfgVars) error { +func generateKeyPair(name string, k0sVars constant.CfgVars, owner string) error { keyFile := filepath.Join(k0sVars.CertRootDir, fmt.Sprintf("%s.key", name)) pubFile := filepath.Join(k0sVars.CertRootDir, fmt.Sprintf("%s.pub", name)) if util.FileExists(keyFile) && util.FileExists(pubFile) { - return nil + return chownFile(keyFile, owner, constant.CertSecureMode) } reader := rand.Reader @@ -315,12 +323,17 @@ func generateKeyPair(name string, k0sVars constant.CfgVars) error { Bytes: x509.MarshalPKCS1PrivateKey(key), } - outFile, err := os.Create(keyFile) + outFile, err := os.OpenFile(keyFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.CertSecureMode) if err != nil { return err } defer outFile.Close() + err = chownFile(keyFile, owner, constant.CertSecureMode) + if err != nil { + return err + } + err = pem.Encode(outFile, privateKey) if err != nil { return err diff --git a/pkg/component/server/controllermanager.go b/pkg/component/server/controllermanager.go index ed0c378349ae..dfd337c554c0 100644 --- a/pkg/component/server/controllermanager.go +++ b/pkg/component/server/controllermanager.go @@ -55,7 +55,8 @@ var cmDefaultArgs = map[string]string{ // Init extracts the needed binaries func (a *ControllerManager) Init() error { var err error - a.uid, err = util.GetUID(constant.ControllerManagerUser) + // controller manager running as api-server user as they both need access to same sa.key + a.uid, err = util.GetUID(constant.ApiserverUser) if err != nil { logrus.Warning(errors.Wrap(err, "Running kube-controller-manager as root")) } diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go index 4b9a95ff4846..eb83a43eeb7a 100644 --- a/pkg/constant/constant.go +++ b/pkg/constant/constant.go @@ -75,8 +75,6 @@ const ( KineUser = "kube-apiserver" // apiserver needs to be able to read the kine unix socket // ApiserverUser defines the user to use for running k8s api-server process ApiserverUser = "kube-apiserver" - // ControllerManagerUser defines the user to use for running k8s controller manager process - ControllerManagerUser = "kube-controller-manager" // SchedulerUser defines the user to use for running k8s scheduler SchedulerUser = "kube-scheduler" // KonnectivityServerUser deinfes the user to use for konnectivity-server From 3f3835329990ceff24165c9309e86b1e27d72339 Mon Sep 17 00:00:00 2001 From: Jussi Nummelin Date: Wed, 2 Dec 2020 12:34:00 +0000 Subject: [PATCH 3/4] Add file perm checks into basic smoke test Signed-off-by: Jussi Nummelin --- inttest/basic/basic_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/inttest/basic/basic_test.go b/inttest/basic/basic_test.go index 4b6f54801dc5..f9b42ffb9bc8 100644 --- a/inttest/basic/basic_test.go +++ b/inttest/basic/basic_test.go @@ -17,6 +17,7 @@ package basic import ( "context" + "fmt" "testing" "github.com/stretchr/testify/suite" @@ -55,6 +56,27 @@ func (s *BasicSuite) TestK0sGetsUp() { s.T().Log("waiting to see calico pods ready") s.NoError(common.WaitForCalicoReady(kc), "calico did not start") + + s.Require().NoError(s.checkCertPerms("controller0")) +} + +func (s *BasicSuite) checkCertPerms(node string) error { + ssh, err := s.SSH(node) + if err != nil { + return err + } + defer ssh.Disconnect() + + output, err := ssh.ExecWithOutput(`find /var/lib/k0s/custom-data-dir/pki/ \( -name '*.key' -o -name '*.conf' \) -a \! -perm 0640`) + if err != nil { + return err + } + + if output != "" { + return fmt.Errorf("some private files having non 640 permissions: %s", output) + } + + return nil } func TestBasicSuite(t *testing.T) { From baec117321a63fcda99435aab7c1a36f0a9e2982 Mon Sep 17 00:00:00 2001 From: Jussi Nummelin Date: Wed, 2 Dec 2020 13:27:49 +0000 Subject: [PATCH 4/4] Let builds run against release branches Signed-off-by: Jussi Nummelin --- .github/workflows/go.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 6cac764b38a4..31df7e501b7b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,7 +2,9 @@ name: Go build on: push: - branches: [ main ] + branches: + - main + - release-* paths-ignore: - 'docs/**' - 'examples/**' @@ -10,7 +12,9 @@ on: - LICENSE - '**.svg' pull_request: - branches: [ main ] + branches: + - main + - release-* paths-ignore: - 'docs/**' - 'examples/**'