Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[YUNIKORN-1977] E2E test for verifying user info with non kube-admin user #915

Closed
wants to merge 14 commits into from
Closed

[YUNIKORN-1977] E2E test for verifying user info with non kube-admin user #915

Changes from 4 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
62be8c1
[YUNIKORN-1977] Added E2E test for deploying pod and verifying user i…
rrajesh-cloudera Sep 14, 2024
f4e55e3
[YUNIKORN-1977] Added E2E test for deploying pod and verifying user i…
rrajesh-cloudera Sep 14, 2024
9412297
[YUNIKORN-1977] Added E2E test for deploying pod and verifying user i…
rrajesh-cloudera Sep 14, 2024
334cbb0
[YUNIKORN-1977] Added E2E test for deploying pod and verifying user i…
rrajesh-cloudera Sep 16, 2024
4daafa0
[YUNIKORN-1977] restoring old kubeconfig file
rrajesh-cloudera Sep 16, 2024
f3b153a
[YUNIKORN-1977] accomdating review comments
rrajesh-cloudera Oct 3, 2024
45f5266
[YUNIKORN-1977] accomdating review comments
rrajesh-cloudera Oct 3, 2024
e567e1c
[YUNIKORN-1977] accomdating review comments
rrajesh-cloudera Oct 3, 2024
b265689
Merge branch 'master' into YUNIKORN-1977
rrajesh-cloudera Oct 3, 2024
b2e717a
[YUNIKORN-1977] Accomdating PR Suggestions
rrajesh-cloudera Oct 17, 2024
30954d5
Resolved merge conflicts
rrajesh-cloudera Oct 17, 2024
735ceaf
[Yunikorn-1977] Accomdating PR Review Comments
rrajesh-cloudera Oct 17, 2024
a232066
[YUNIKORN-1977] accomdating PR Review comments
rrajesh-cloudera Oct 19, 2024
2e25c0e
[YUNIKORN-1977] accomdating PR Review comments
rrajesh-cloudera Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
265 changes: 264 additions & 1 deletion test/e2e/user_group_limit/user_group_limit_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,31 +19,44 @@
package user_group_limit_test

import (
"context"
"encoding/json"
"fmt"
"net/url"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"

"github.com/apache/yunikorn-core/pkg/common/configs"
"github.com/apache/yunikorn-core/pkg/common/resources"
"github.com/apache/yunikorn-core/pkg/webservice/dao"
tests "github.com/apache/yunikorn-k8shim/test/e2e"

amCommon "github.com/apache/yunikorn-k8shim/pkg/admission/common"
amconf "github.com/apache/yunikorn-k8shim/pkg/admission/conf"

"github.com/apache/yunikorn-k8shim/pkg/common/constants"
tests "github.com/apache/yunikorn-k8shim/test/e2e"

"github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager"
"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common"
"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s"
"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn"

siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common"
"github.com/apache/yunikorn-scheduler-interface/lib/go/si"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"

v1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)

type TestType int
Expand Down Expand Up @@ -75,6 +88,7 @@ var (
"log.core.scheduler.ugm.level": "debug",
amconf.AMAccessControlBypassAuth: constants.True,
}
oldKubeconfigContent []byte
)

var _ = ginkgo.BeforeSuite(func() {
Expand Down Expand Up @@ -911,6 +925,200 @@ var _ = ginkgo.Describe("UserGroupLimit", func() {
checkUsageWildcardGroups(groupTestType, group2, sandboxQueue1, []*v1.Pod{group2Sandbox1Pod1, group2Sandbox1Pod2, group2Sandbox1Pod3})
})

ginkgo.It("Verify User info for the non kube admin user", func() {
var clientset *kubernetes.Clientset
var namespace = "default"
var serviceAccountName = "test-user-sa"
var podName = "test-pod"
var secretName = "test-user-sa-token" // #nosec G101
clientset = kClient.GetClient()
ginkgo.By("Update config")
// The wait wrapper still can't fully guarantee that the config in AdmissionController has been updated.
admissionCustomConfig = map[string]string{
"log.core.scheduler.ugm.level": "debug",
amconf.AMAccessControlBypassAuth: constants.False,
}
yunikorn.WaitForAdmissionControllerRefreshConfAfterAction(func() {
yunikorn.UpdateCustomConfigMapWrapperWithMap(oldConfigMap, "", admissionCustomConfig, func(sc *configs.SchedulerConfig) error {
// remove placement rules so we can control queue
sc.Partitions[0].PlacementRules = nil

err := common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{
Name: "default",
Limits: []configs.Limit{
{
Limit: "user entry",
Users: []string{user1},
MaxApplications: 1,
MaxResources: map[string]string{
siCommon.Memory: fmt.Sprintf("%dM", mediumMem),
},
},
{
Limit: "user2 entry",
Users: []string{user2},
MaxApplications: 2,
MaxResources: map[string]string{
siCommon.Memory: fmt.Sprintf("%dM", largeMem),
},
},
}})
if err != nil {
return err
}
return common.AddQueue(sc, constants.DefaultPartition, constants.RootQueue, configs.QueueConfig{Name: "sandbox2"})
})
})
// Backup the existing kubeconfig
oldKubeconfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
if _, err := os.Stat(oldKubeconfigPath); !os.IsNotExist(err) {
oldKubeconfigContent, err = os.ReadFile(oldKubeconfigPath)
gomega.Ω(err).NotTo(HaveOccurred())
}
rrajesh-cloudera marked this conversation as resolved.
Show resolved Hide resolved
// Create Service Account
ginkgo.By("Creating Service Account...")
_, err := clientset.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
},
}, metav1.CreateOptions{})
gomega.Ω(err).NotTo(HaveOccurred())
// Create a ClusterRole with necessary permissions
ginkgo.By("Creating ClusterRole...")
clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: "pod-creator-role",
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"pods", "serviceaccounts"},
Verbs: []string{"create", "get", "list", "watch", "delete"},
},
},
}
_, err = kClient.CreateClusterRole(clusterRole)
gomega.Ω(err).NotTo(HaveOccurred())
// Create a ClusterRoleBinding to bind the ClusterRole to the service account
ginkgo.By("Creating ClusterRoleBinding...")
clusterRoleBinding := &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "pod-creator-role-binding",
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "pod-creator-role",
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: "test-user-sa",
Namespace: "default",
},
},
}
_, err = kClient.CreateClusterRoleBinding(clusterRoleBinding.ObjectMeta.Name, clusterRoleBinding.RoleRef.Name, clusterRoleBinding.Subjects[0].Namespace, clusterRoleBinding.Subjects[0].Name)
gomega.Ω(err).NotTo(HaveOccurred())
// Create a Secret for the Service Account
ginkgo.By("Creating Secret for the Service Account...")
_, err = clientset.CoreV1().Secrets(namespace).Create(context.TODO(), &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Annotations: map[string]string{
"kubernetes.io/service-account.name": serviceAccountName,
},
},
Type: v1.SecretTypeServiceAccountToken,
}, metav1.CreateOptions{})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use KubeCtl.CreateSecret() if possible. If the annotation is necessary, create a new method with the name CreateSecretWithAnnotation(secret *v1.Secret, namespace string, annotations map[string]string) to avoid calling this directly.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated the code with requested changes.

gomega.Ω(err).NotTo(HaveOccurred())
// Get the token value from the Secret
ginkgo.By("Getting the token value from the Secret...")
userTokenValue, err := GetKubeConfigValue("kubectl get secret/test-user-sa-token -o=go-template='{{.data.token}}' | base64 --decode")
gomega.Ω(err).NotTo(HaveOccurred())
currentContext, err := GetKubeConfigValue("kubectl config current-context")
gomega.Ω(err).NotTo(HaveOccurred())
currentCluster, err := GetKubeConfigValue("kubectl config current-context")
gomega.Ω(err).NotTo(HaveOccurred())
clusterCA, err := GetKubeConfigValue("kubectl config view --raw -o go-template='{{range .clusters}}{{if eq .name \"kind-yk8s\"}}{{index .cluster \"certificate-authority-data\"}}{{end}}{{end}}'")
gomega.Ω(err).NotTo(HaveOccurred())
clusterServer, err := GetKubeConfigValue(fmt.Sprintf("kubectl config view --raw -o=go-template='{{range .clusters}}{{if eq .name \"%s\"}}{{ .cluster.server }}{{end}}{{ end }}'", currentCluster))
gomega.Ω(err).NotTo(HaveOccurred())
kubeconfigPath := filepath.Join(os.TempDir(), "test-user-config")
ginkgo.By("Creating kubeconfig file...")
err = createKubeconfig(kubeconfigPath, currentContext, clusterCA, clusterServer, userTokenValue)
gomega.Ω(err).NotTo(gomega.HaveOccurred())
config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
gomega.Ω(err).NotTo(HaveOccurred())
clientset, err = kubernetes.NewForConfig(config)
gomega.Ω(err).NotTo(HaveOccurred())
// Create Pod
ginkgo.By("Creating Pod...")
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Annotations: map[string]string{
"created-by": fmt.Sprintf("system:serviceaccount:%s:%s", namespace, serviceAccountName),
},
Labels: map[string]string{"applicationId": "test-app"},
},
Spec: v1.PodSpec{
ServiceAccountName: serviceAccountName,
Containers: []v1.Container{
{
Name: "nginx",
Image: "nginx",
Ports: []v1.ContainerPort{{ContainerPort: 80}},
},
},
},
}
_, err = clientset.CoreV1().Pods(namespace).Create(context.TODO(), pod, metav1.CreateOptions{})
gomega.Ω(err).NotTo(HaveOccurred())
createdPod, err := clientset.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
gomega.Ω(err).NotTo(HaveOccurred())
fmt.Println(createdPod.Annotations)
// Verify User Info
ginkgo.By("Verifying User Info...")
userInfo, err := GetUserInfoFromPodAnnotation(createdPod)
gomega.Ω(err).NotTo(gomega.HaveOccurred())
rrajesh-cloudera marked this conversation as resolved.
Show resolved Hide resolved
// user info should contain the substring "system:serviceaccount:default:test-user-sa"
gomega.Ω(strings.Contains(fmt.Sprintf("%v", userInfo), "system:serviceaccount:default:test-user-sa")).To(gomega.BeTrue())
if oldKubeconfigContent != nil {
err := os.WriteFile(oldKubeconfigPath, oldKubeconfigContent, 0600)
gomega.Ω(err).NotTo(HaveOccurred())
}
// Remove the new kubeconfig file
err = os.Remove(kubeconfigPath)
gomega.Ω(err).NotTo(gomega.HaveOccurred())
rrajesh-cloudera marked this conversation as resolved.
Show resolved Hide resolved
os.Unsetenv("KUBECONFIG")
rrajesh-cloudera marked this conversation as resolved.
Show resolved Hide resolved
err = kClient.SetClient()
gomega.Ω(err).NotTo(gomega.HaveOccurred())
rrajesh-cloudera marked this conversation as resolved.
Show resolved Hide resolved
// Cleanup
ginkgo.By("Cleaning up resources...")
err = clientset.CoreV1().Pods(namespace).Delete(context.TODO(), podName, metav1.DeleteOptions{})
gomega.Ω(err).NotTo(HaveOccurred())
err = clientset.CoreV1().ServiceAccounts(namespace).Delete(context.TODO(), serviceAccountName, metav1.DeleteOptions{})
gomega.Ω(err).NotTo(HaveOccurred())
err = kClient.DeleteClusterRole("pod-creator-role")
gomega.Ω(err).NotTo(HaveOccurred())
err = kClient.DeleteClusterRoleBindings("pod-creator-role-binding")
gomega.Ω(err).NotTo(HaveOccurred())
rrajesh-cloudera marked this conversation as resolved.
Show resolved Hide resolved
queueName2 := "root_22"
yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "", func(sc *configs.SchedulerConfig) error {
// remove placement rules so we can control queue
sc.Partitions[0].PlacementRules = nil
var err error
if err = common.AddQueue(sc, "default", "root", configs.QueueConfig{
Name: queueName2,
Resources: configs.Resources{Guaranteed: map[string]string{"memory": fmt.Sprintf("%dM", 200)}},
Properties: map[string]string{"preemption.delay": "1s"},
}); err != nil {
return err
}
return nil
})
})
ginkgo.AfterEach(func() {
tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns.Name})

Expand All @@ -927,6 +1135,19 @@ var _ = ginkgo.Describe("UserGroupLimit", func() {
})
})

func GetUserInfoFromPodAnnotation(pod *v1.Pod) (interface{}, error) {
userInfo, ok := pod.Annotations[amCommon.UserInfoAnnotation]
if !ok {
return nil, fmt.Errorf("user info not found in pod annotation")
}
var userInfoObj interface{}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We know the type of this object. It's si.UserGroupInformation, let's use that.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated the code with requested changes.

err := json.Unmarshal([]byte(userInfo), &userInfoObj)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal user info from pod annotation")
}
return userInfoObj, nil
}

func deploySleepPod(usergroup *si.UserGroupInformation, queuePath string, expectedRunning bool, reason string) *v1.Pod {
usergroupJsonBytes, err := json.Marshal(usergroup)
Ω(err).NotTo(gomega.HaveOccurred())
Expand Down Expand Up @@ -1022,3 +1243,45 @@ func checkUsageWildcardGroups(testType TestType, name string, queuePath string,
Ω(resourceUsageDAO.ResourceUsage.Resources["pods"]).To(gomega.Equal(resources.Quantity(len(expectedRunningPods))))
Ω(resourceUsageDAO.RunningApplications).To(gomega.ConsistOf(appIDs...))
}

func createKubeconfig(path, currentContext, clusterCA, clusterServer, userTokenValue string) error {
kubeconfigTemplate := `
apiVersion: v1
kind: Config
current-context: ${CURRENT_CONTEXT}
contexts:
- name: ${CURRENT_CONTEXT}
context:
cluster: ${CURRENT_CONTEXT}
user: test-user
clusters:
- name: ${CURRENT_CONTEXT}
cluster:
certificate-authority-data: ${CLUSTER_CA}
server: ${CLUSTER_SERVER}
users:
- name: test-user
user:
token: ${USER_TOKEN_VALUE}
`
// Replace placeholders in the template
kubeconfigContent := strings.ReplaceAll(kubeconfigTemplate, "${CURRENT_CONTEXT}", currentContext)
kubeconfigContent = strings.ReplaceAll(kubeconfigContent, "${CLUSTER_CA}", clusterCA)
kubeconfigContent = strings.ReplaceAll(kubeconfigContent, "${CLUSTER_SERVER}", clusterServer)
kubeconfigContent = strings.ReplaceAll(kubeconfigContent, "${USER_TOKEN_VALUE}", userTokenValue)

// Write the kubeconfig YAML to the file
err := os.WriteFile(path, []byte(kubeconfigContent), 0600)
gomega.Ω(err).NotTo(gomega.HaveOccurred())
return nil
}
Copy link
Contributor

@pbacsko pbacsko Sep 16, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All of this should NOT be necessary. Way too complicated to mess around with separate kubectl calls.

You can do this:

config, _ := kClient.GetKubeConfig() // handle error in real code

newConf := config.DeepCopy()  // copy existing config
newConf.TLSClientConfig.CertFile = ""  // remove cert file
newConf.TLSClientConfig.KeyFile = ""  // remove key file
newConf.BearerToken = "<base64Token>"  // set token that is retrieved in the test
_ = kClient.SetClientFromConfig(newConf)

After this point, kClient will use the token for authentication there's no need to delete/restore anything.

New method in KubeCtl:

func (k *KubeCtl) SetClientFromConfig(conf *rest.Config) error {
    k.kubeConfig = conf.DeepCopy()
    k.clientSet, err = kubernetes.NewForConfig(k.kubeConfig)   // creates new clientset 
    return err
}

Also, try to retrieve the secret token using KubeCtl. We might need to create a new method for it, but again, it shouldn't involve running the kubectl command:

func (k *KubeCtl) GetSecret(namespace string) (*v1.Secret, error) {
    return k.clientSet.CoreV1().Secrets(namespace).Get(context.TODO(), namespace, metav1.GetOptions{})
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure @pbacsko will accommodate the requested changes.


// GetKubeConfigValue is a helper function to execute a kubectl command and get the output
func GetKubeConfigValue(command string) (string, error) {
cmd := exec.Command("bash", "-c", command)
output, err := cmd.Output()
if err != nil {
return "", err
}
return strings.TrimSpace(string(output)), nil
}