Merge pull request #484 from jnummelin/fix-conf-permissions

Separate kubeconfig for konnectivity and tighter permissions
k0sproject · Dec 3, 2020 · 4756ddd · 4756ddd
2 parents 9bbe4f0 + baec117
commit 4756ddd
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 44 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -2,15 +2,19 @@ name: Go build
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - main
+      - release-*
     paths-ignore:
       - 'docs/**'
       - 'examples/**'
       - '**.md'
       - LICENSE
       - '**.svg'
   pull_request:
-    branches: [ main ]
+    branches:
+      - main
+      - release-*
     paths-ignore:
       - 'docs/**'
       - 'examples/**'

diff --git a/inttest/basic/basic_test.go b/inttest/basic/basic_test.go
@@ -17,6 +17,7 @@ package basic
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
@@ -55,6 +56,27 @@ func (s *BasicSuite) TestK0sGetsUp() {
 
 	s.T().Log("waiting to see calico pods ready")
 	s.NoError(common.WaitForCalicoReady(kc), "calico did not start")
+
+	s.Require().NoError(s.checkCertPerms("controller0"))
+}
+
+func (s *BasicSuite) checkCertPerms(node string) error {
+	ssh, err := s.SSH(node)
+	if err != nil {
+		return err
+	}
+	defer ssh.Disconnect()
+
+	output, err := ssh.ExecWithOutput(`find /var/lib/k0s/custom-data-dir/pki/  \( -name '*.key' -o -name '*.conf' \) -a \! -perm 0640`)
+	if err != nil {
+		return err
+	}
+
+	if output != "" {
+		return fmt.Errorf("some private files having non 640 permissions: %s", output)
+	}
+
+	return nil
 }
 
 func TestBasicSuite(t *testing.T) {

diff --git a/pkg/component/server/certificates.go b/pkg/component/server/certificates.go
@@ -126,11 +126,31 @@ func (c *Certificates) Init() error {
 		if err != nil {
 			return err
 		}
-		if err := kubeConfig(c.K0sVars.AdminKubeconfigConfigPath, "https://localhost:6443", c.CACert, adminCert.Cert, adminCert.Key); err != nil {
+		if err := kubeConfig(c.K0sVars.AdminKubeconfigConfigPath, "https://localhost:6443", c.CACert, adminCert.Cert, adminCert.Key, "root"); err != nil {
 			return err
 		}
 
-		return generateKeyPair("sa", c.K0sVars)
+		return generateKeyPair("sa", c.K0sVars, constant.ApiserverUser)
+	})
+
+	eg.Go(func() error {
+		// konnectivity kubeconfig
+		konnectivityReq := certificate.Request{
+			Name:   "konnectivity",
+			CN:     "kubernetes-konnectivity",
+			O:      "system:masters", // TODO: We need to figure out if konnectivity really needs superpowers
+			CACert: caCertPath,
+			CAKey:  caCertKey,
+		}
+		konnectivityCert, err := c.CertManager.EnsureCertificate(konnectivityReq, constant.KonnectivityServerUser)
+		if err != nil {
+			return err
+		}
+		if err := kubeConfig(c.K0sVars.KonnectivityKubeconfigConfigPath, "https://localhost:6443", c.CACert, konnectivityCert.Cert, konnectivityCert.Key, constant.KonnectivityServerUser); err != nil {
+			return err
+		}
+
+		return nil
 	})
 
 	eg.Go(func() error {
@@ -141,13 +161,13 @@ func (c *Certificates) Init() error {
 			CACert: caCertPath,
 			CAKey:  caCertKey,
 		}
-		ccmCert, err := c.CertManager.EnsureCertificate(ccmReq, constant.ControllerManagerUser)
+		ccmCert, err := c.CertManager.EnsureCertificate(ccmReq, constant.ApiserverUser)
 
 		if err != nil {
 			return err
 		}
 
-		return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "ccm.conf"), "https://localhost:6443", c.CACert, ccmCert.Cert, ccmCert.Key)
+		return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "ccm.conf"), "https://localhost:6443", c.CACert, ccmCert.Cert, ccmCert.Key, constant.ApiserverUser)
 	})
 
 	eg.Go(func() error {
@@ -163,7 +183,7 @@ func (c *Certificates) Init() error {
 			return err
 		}
 
-		return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "scheduler.conf"), "https://localhost:6443", c.CACert, schedulerCert.Cert, schedulerCert.Key)
+		return kubeConfig(filepath.Join(c.K0sVars.CertRootDir, "scheduler.conf"), "https://localhost:6443", c.CACert, schedulerCert.Cert, schedulerCert.Key, constant.SchedulerUser)
 	})
 
 	eg.Go(func() error {
@@ -238,9 +258,9 @@ func (c *Certificates) Stop() error {
 	return nil
 }
 
-func kubeConfig(dest, url, caCert, clientCert, clientKey string) error {
+func kubeConfig(dest, url, caCert, clientCert, clientKey, owner string) error {
 	if util.FileExists(dest) {
-		return nil
+		return chownFile(dest, owner, constant.CertSecureMode)
 	}
 	data := struct {
 		URL        string
@@ -254,21 +274,40 @@ func kubeConfig(dest, url, caCert, clientCert, clientKey string) error {
 		ClientKey:  base64.StdEncoding.EncodeToString([]byte(clientKey)),
 	}
 
-	output, err := os.Create(dest)
+	output, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.CertSecureMode)
 	if err != nil {
 		return err
 	}
 	defer output.Close()
 
-	return kubeconfigTemplate.Execute(output, &data)
+	if err = kubeconfigTemplate.Execute(output, &data); err != nil {
+		return err
+	}
+
+	return chownFile(output.Name(), owner, constant.CertSecureMode)
 }
 
-func generateKeyPair(name string, k0sVars constant.CfgVars) error {
+func chownFile(file, owner string, permissions os.FileMode) error {
+	// Chown the file properly for the owner
+	uid, _ := util.GetUID(owner)
+	err := os.Chown(file, uid, -1)
+	if err != nil && os.Geteuid() == 0 {
+		return err
+	}
+	err = os.Chmod(file, permissions)
+	if err != nil && os.Geteuid() == 0 {
+		return err
+	}
+
+	return nil
+}
+
+func generateKeyPair(name string, k0sVars constant.CfgVars, owner string) error {
 	keyFile := filepath.Join(k0sVars.CertRootDir, fmt.Sprintf("%s.key", name))
 	pubFile := filepath.Join(k0sVars.CertRootDir, fmt.Sprintf("%s.pub", name))
 
 	if util.FileExists(keyFile) && util.FileExists(pubFile) {
-		return nil
+		return chownFile(keyFile, owner, constant.CertSecureMode)
 	}
 
 	reader := rand.Reader
@@ -284,12 +323,17 @@ func generateKeyPair(name string, k0sVars constant.CfgVars) error {
 		Bytes: x509.MarshalPKCS1PrivateKey(key),
 	}
 
-	outFile, err := os.Create(keyFile)
+	outFile, err := os.OpenFile(keyFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.CertSecureMode)
 	if err != nil {
 		return err
 	}
 	defer outFile.Close()
 
+	err = chownFile(keyFile, owner, constant.CertSecureMode)
+	if err != nil {
+		return err
+	}
+
 	err = pem.Encode(outFile, privateKey)
 	if err != nil {
 		return err

diff --git a/pkg/component/server/controllermanager.go b/pkg/component/server/controllermanager.go
@@ -55,7 +55,8 @@ var cmDefaultArgs = map[string]string{
 // Init extracts the needed binaries
 func (a *ControllerManager) Init() error {
 	var err error
-	a.uid, err = util.GetUID(constant.ControllerManagerUser)
+	// controller manager running as api-server user as they both need access to same sa.key
+	a.uid, err = util.GetUID(constant.ApiserverUser)
 	if err != nil {
 		logrus.Warning(errors.Wrap(err, "Running kube-controller-manager as root"))
 	}

diff --git a/pkg/component/server/konnectivity.go b/pkg/component/server/konnectivity.go
@@ -69,7 +69,7 @@ func (k *Konnectivity) Run() error {
 			fmt.Sprintf("--uds-name=%s", filepath.Join(k.K0sVars.KonnectivitySocketDir, "konnectivity-server.sock")),
 			fmt.Sprintf("--cluster-cert=%s", filepath.Join(k.K0sVars.CertRootDir, "server.crt")),
 			fmt.Sprintf("--cluster-key=%s", filepath.Join(k.K0sVars.CertRootDir, "server.key")),
-			fmt.Sprintf("--kubeconfig=%s", k.K0sVars.AdminKubeconfigConfigPath), // FIXME: should have user rights
+			fmt.Sprintf("--kubeconfig=%s", k.K0sVars.KonnectivityKubeconfigConfigPath),
 			"--mode=grpc",
 			"--server-port=0",
 			"--agent-port=8132",

diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go
@@ -19,19 +19,20 @@ import "fmt"
 
 // CfgVars is a struct that holds all the config variables requried for K0s
 type CfgVars struct {
-	AdminKubeconfigConfigPath  string // The cluster admin kubeconfig location
-	BinDir                     string // location for all pki related binaries
-	CertRootDir                string // CertRootDir defines the root location for all pki related artifacts
-	DataDir                    string // Data directory containing k0s state
-	EtcdCertDir                string // EtcdCertDir contains etcd certificates
-	EtcdDataDir                string // EtcdDataDir contains etcd state
-	KineSocketPath             string // The unix socket path for kine
-	KonnectivitySocketDir      string // location of konnectivity's socket path
-	KubeletAuthConfigPath      string // KubeletAuthConfigPath defines the default kubelet auth config path
-	KubeletBootstrapConfigPath string // KubeletBootstrapConfigPath defines the default path for kubelet bootstrap auth config
-	KubeletVolumePluginDir     string // location for kubelet plugins volume executables
-	ManifestsDir               string // location for all stack manifests
-	RunDir                     string // location of supervised pid files and sockets
+	AdminKubeconfigConfigPath        string // The cluster admin kubeconfig location
+	BinDir                           string // location for all pki related binaries
+	CertRootDir                      string // CertRootDir defines the root location for all pki related artifacts
+	DataDir                          string // Data directory containing k0s state
+	EtcdCertDir                      string // EtcdCertDir contains etcd certificates
+	EtcdDataDir                      string // EtcdDataDir contains etcd state
+	KineSocketPath                   string // The unix socket path for kine
+	KonnectivitySocketDir            string // location of konnectivity's socket path
+	KubeletAuthConfigPath            string // KubeletAuthConfigPath defines the default kubelet auth config path
+	KubeletBootstrapConfigPath       string // KubeletBootstrapConfigPath defines the default path for kubelet bootstrap auth config
+	KubeletVolumePluginDir           string // location for kubelet plugins volume executables
+	ManifestsDir                     string // location for all stack manifests
+	RunDir                           string // location of supervised pid files and sockets
+	KonnectivityKubeconfigConfigPath string // location for konnectivity kubeconfig
 
 	// Helm config
 	HelmHome             string
@@ -74,8 +75,6 @@ const (
 	KineUser = "kube-apiserver" // apiserver needs to be able to read the kine unix socket
 	// ApiserverUser defines the user to use for running k8s api-server process
 	ApiserverUser = "kube-apiserver"
-	// ControllerManagerUser defines the user to use for running k8s controller manager process
-	ControllerManagerUser = "kube-controller-manager"
 	// SchedulerUser defines the user to use for running k8s scheduler
 	SchedulerUser = "kube-scheduler"
 	// KonnectivityServerUser deinfes the user to use for konnectivity-server
@@ -115,19 +114,20 @@ func GetConfig(dataDir string) CfgVars {
 	helmHome := fmt.Sprintf("%s/helmhome", dataDir)
 
 	return CfgVars{
-		AdminKubeconfigConfigPath:  fmt.Sprintf("%s/admin.conf", certDir),
-		BinDir:                     fmt.Sprintf("%s/bin", dataDir),
-		CertRootDir:                certDir,
-		DataDir:                    dataDir,
-		EtcdCertDir:                fmt.Sprintf("%s/etcd", certDir),
-		EtcdDataDir:                fmt.Sprintf("%s/etcd", dataDir),
-		KineSocketPath:             fmt.Sprintf("%s/kine/kine.sock:2379", runDir),
-		KonnectivitySocketDir:      fmt.Sprintf("%s/konnectivity-server", runDir),
-		KubeletAuthConfigPath:      fmt.Sprintf("%s/kubelet.conf", dataDir),
-		KubeletBootstrapConfigPath: fmt.Sprintf("%s/kubelet-bootstrap.conf", dataDir),
-		KubeletVolumePluginDir:     "/usr/libexec/k0s/kubelet-plugins/volume/exec",
-		ManifestsDir:               fmt.Sprintf("%s/manifests", dataDir),
-		RunDir:                     runDir,
+		AdminKubeconfigConfigPath:        fmt.Sprintf("%s/admin.conf", certDir),
+		BinDir:                           fmt.Sprintf("%s/bin", dataDir),
+		CertRootDir:                      certDir,
+		DataDir:                          dataDir,
+		EtcdCertDir:                      fmt.Sprintf("%s/etcd", certDir),
+		EtcdDataDir:                      fmt.Sprintf("%s/etcd", dataDir),
+		KineSocketPath:                   fmt.Sprintf("%s/kine/kine.sock:2379", runDir),
+		KonnectivitySocketDir:            fmt.Sprintf("%s/konnectivity-server", runDir),
+		KubeletAuthConfigPath:            fmt.Sprintf("%s/kubelet.conf", dataDir),
+		KubeletBootstrapConfigPath:       fmt.Sprintf("%s/kubelet-bootstrap.conf", dataDir),
+		KubeletVolumePluginDir:           "/usr/libexec/k0s/kubelet-plugins/volume/exec",
+		ManifestsDir:                     fmt.Sprintf("%s/manifests", dataDir),
+		RunDir:                           runDir,
+		KonnectivityKubeconfigConfigPath: fmt.Sprintf("%s/konnectivity.conf", certDir),
 
 		// Helm Config
 		HelmHome:             helmHome,