From c7e450405bdfa756d3c7c8a18cf24217832bcd4f Mon Sep 17 00:00:00 2001 From: Surax98 Date: Tue, 18 Jul 2023 07:34:57 +0000 Subject: [PATCH 1/5] removed kubectl dependencies from interlink and sidecars --- go.mod | 2 +- pkg/sidecars/slurm/aux.go | 68 +++++++++------------------------- pkg/sidecars/slurm/handlers.go | 10 +++++ pkg/virtualkubelet/execute.go | 19 +++------- 4 files changed, 35 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index bb30a1f1..2d6220fa 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 + k8s.io/client-go v11.0.0+incompatible k8s.io/kubernetes v1.15.2 ) @@ -59,7 +60,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.0.0 // indirect - k8s.io/client-go v11.0.0+incompatible // indirect k8s.io/klog v0.3.3 // indirect k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22 // indirect k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a // indirect diff --git a/pkg/sidecars/slurm/aux.go b/pkg/sidecars/slurm/aux.go index 61d6e561..08dbdeaa 100644 --- a/pkg/sidecars/slurm/aux.go +++ b/pkg/sidecars/slurm/aux.go @@ -1,6 +1,7 @@ package slurm import ( + "context" "encoding/base64" "fmt" "log" @@ -15,6 +16,8 @@ import ( commonIL "github.com/intertwin-eu/interlink/pkg/common" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) type JidStruct struct { @@ -23,6 +26,9 @@ type JidStruct struct { } var prefix string +var ctx context.Context +var kubecfg *rest.Config +var clientset *kubernetes.Clientset func prepare_envs(container v1.Container) []string { env := make([]string, 1) @@ -263,39 +269,19 @@ func mountConfigMaps(container v1.Container, pod *v1.Pod) ([]string, []string) { mode := os.FileMode(*podVolumeSpec.ConfigMap.DefaultMode) podConfigMapDir := filepath.Join(commonIL.InterLinkConfigInst.DataRootFolder, pod.Namespace+"-"+string(pod.UID)+"/", "configMaps/", vol.Name) - cmd := []string{"get configmap " + cmvs.Name + " -o template --template='{{.data}}' -n " + pod.Namespace} - shell := exec2.ExecTask{ - Command: "kubectl", - Args: cmd, - Shell: true, - } - - execReturn, _ := shell.Execute() - fmt.Println(execReturn) - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "map[", "") - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "]", "") - returnedConfigMapsArray := make([]string, 0) - - if strings.Compare(execReturn.Stdout, "") != 0 { - returnedConfigMapsArray = strings.Split(execReturn.Stdout, " ") - } - - if returnedConfigMapsArray != nil { - for _, element := range returnedConfigMapsArray { - parts := strings.Split(element, ":") - key := parts[0] - value := parts[1] - configMaps[key] = value + configMap, err := clientset.CoreV1().ConfigMaps(pod.Namespace).Get(cmvs.Name, metav1.GetOptions{}) + if configMap.Data != nil { + for key := range configMap.Data { + configMaps[key] = configMap.Data[key] path := filepath.Join(podConfigMapDir, key) path += (":" + mountSpec.MountPath + "/" + key + ",") configMapNamePaths = append(configMapNamePaths, path) if strings.Compare(os.Getenv("SHARED_FS"), "true") != 0 { env := string(container.Name) + "_CFG_" + key - os.Setenv(env, value) + os.Setenv(env, configMap.Data[key]) envs = append(envs, env) } - } } @@ -311,7 +297,7 @@ func mountConfigMaps(container v1.Container, pod *v1.Pod) ([]string, []string) { Shell: true, } - execReturn, _ = shell.Execute() + execReturn, _ := shell.Execute() if strings.Compare(execReturn.Stdout, "") != 0 { log.Panicln(err) } @@ -366,36 +352,18 @@ func mountSecrets(container v1.Container, pod *v1.Pod) ([]string, []string) { // fmt.Println(mode) podSecretDir := filepath.Join(commonIL.InterLinkConfigInst.DataRootFolder, pod.Namespace+"-"+string(pod.UID)+"/", "secrets/", vol.Name) - cmd := []string{"get secret " + svs.SecretName + " -o jsonpath='{.data}' -n " + pod.Namespace} - shell := exec2.ExecTask{ - Command: "kubectl", - Args: cmd, - Shell: true, - } - - execReturn, _ := shell.Execute() - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\"", "") - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "{", "") - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "}", "") - returnedSecretsArray := make([]string, 0) - - if strings.Compare(execReturn.Stdout, "") != 0 { - returnedSecretsArray = strings.Split(execReturn.Stdout, " ") - } - - if returnedSecretsArray != nil { - for _, element := range returnedSecretsArray { - parts := strings.Split(element, ":") - key := parts[0] - value, _ := base64.StdEncoding.DecodeString(parts[1]) - secrets[key] = value + secret, err := clientset.CoreV1().Secrets(pod.Namespace).Get(svs.SecretName, metav1.GetOptions{}) + if secret.Data != nil { + for key := range secret.Data { + secrets[key] = secret.Data[key] + decodedSecret, _ := base64.StdEncoding.DecodeString(string(secret.Data[key])) path := filepath.Join(podSecretDir, key) path += (":" + mountSpec.MountPath + "/" + key + ",") secretNamePaths = append(secretNamePaths, path) if strings.Compare(os.Getenv("SHARED_FS"), "true") != 0 { env := string(container.Name) + "_SECRET_" + key - os.Setenv(env, string(value)) + os.Setenv(env, string(decodedSecret)) envs = append(envs, env) } } diff --git a/pkg/sidecars/slurm/handlers.go b/pkg/sidecars/slurm/handlers.go index 1c2e2554..b7ae1b40 100644 --- a/pkg/sidecars/slurm/handlers.go +++ b/pkg/sidecars/slurm/handlers.go @@ -1,6 +1,7 @@ package slurm import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -13,6 +14,8 @@ import ( commonIL "github.com/intertwin-eu/interlink/pkg/common" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) var JID []JidStruct @@ -189,5 +192,12 @@ func SetKubeCFGHandler(w http.ResponseWriter, r *http.Request) { os.Setenv("KUBECONFIG", path+"config") fmt.Println(os.Getenv("KUBECONFIG")) + ctx = context.Background() + kubecfg, err = clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + if err != nil { + log.Println("Unable to retrieve config file") + } + clientset = kubernetes.NewForConfigOrDie(kubecfg) + w.Write([]byte("200")) } diff --git a/pkg/virtualkubelet/execute.go b/pkg/virtualkubelet/execute.go index ef4fba92..518c47fb 100644 --- a/pkg/virtualkubelet/execute.go +++ b/pkg/virtualkubelet/execute.go @@ -8,10 +8,8 @@ import ( "io/ioutil" "net/http" "os" - "strings" common "github.com/CARV-ICS-FORTH/knoc/common" - exec "github.com/alexellis/go-execute/pkg/v1" commonIL "github.com/intertwin-eu/interlink/pkg/common" "github.com/containerd/containerd/log" @@ -137,20 +135,15 @@ func checkPodsStatus(p *VirtualKubeletProvider, ctx context.Context, token strin for podIndex, podStatus := range ret.PodStatus { if podStatus.PodStatus == 1 { NoReq++ - cmd := []string{"delete", "pod", ret.PodName[podIndex].Name, "-n", "vk"} - shell := exec.ExecTask{ - Command: "kubectl", - Args: cmd, - Shell: true, - } - execReturn, _ := shell.Execute() - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") + pod, _ := p.GetPod(ctx, commonIL.InterLinkConfigInst.Namespace, ret.PodName[podIndex].Name) + err := p.DeletePod(ctx, pod) + fmt.Println(err) - if execReturn.Stderr != "" { - log.L.Println("Could not delete pod. " + execReturn.Stderr) + if err != nil { + log.L.Println("Could not delete pod. " + pod.Name) } else { - log.L.Println("Pod " + ret.PodName[podIndex].Name + " successfully deleted") + log.L.Println("Pod " + pod.Name + " successfully deleted") } } } From ed1225b0f949163035198a15bf5fa7aae5d9987e Mon Sep 17 00:00:00 2001 From: Surax98 Date: Tue, 18 Jul 2023 15:22:33 +0000 Subject: [PATCH 2/5] temp files now in /tmp instead of .tmp. --- kustomizations/InterLinkConfig.yaml | 6 +++--- main.go | 4 ++++ pkg/common/func.go | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/kustomizations/InterLinkConfig.yaml b/kustomizations/InterLinkConfig.yaml index 69dac763..ff619eaf 100644 --- a/kustomizations/InterLinkConfig.yaml +++ b/kustomizations/InterLinkConfig.yaml @@ -1,7 +1,7 @@ VKTokenFile: "$HOME/interLink/token" -InterlinkURL: "http://localhost" -SidecarURL: "http://localhost" -InterlinkPort: "30002" +InterlinkURL: "https://intertwin.vega.izum.si" +SidecarURL: "http://intertwin.vega.izum.si" +InterlinkPort: "30443" SidecarService: "slurm" #docker, slurm SbatchPath: "/usr/bin/sbatch" ScancelPath: "/usr/bin/scancel" diff --git a/main.go b/main.go index 81f38c30..0c3ccfce 100644 --- a/main.go +++ b/main.go @@ -17,8 +17,11 @@ package main import ( "context" + "crypto/tls" "strings" + "net/http" + "github.com/intertwin-eu/interlink/pkg/virtualkubelet" "github.com/sirupsen/logrus" cli "github.com/virtual-kubelet/node-cli" @@ -39,6 +42,7 @@ var ( ) func main() { + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} ctx, cancel := context.WithCancel(context.Background()) defer cancel() ctx = cli.ContextWithCancelOnSignal(ctx) diff --git a/pkg/common/func.go b/pkg/common/func.go index c4de9387..c0dc911c 100644 --- a/pkg/common/func.go +++ b/pkg/common/func.go @@ -128,7 +128,7 @@ func NewServiceAccount() { var sa string var script string - path := ".tmp/" + path := "/tmp/" err := os.MkdirAll(path, os.ModePerm) if err != nil { From d4b5f6e425c110ff157dc1302a08fecdce4b5b26 Mon Sep 17 00:00:00 2001 From: Surax98 <33400876+Surax98@users.noreply.github.com> Date: Wed, 19 Jul 2023 09:13:40 +0200 Subject: [PATCH 3/5] Update InterLinkConfig.yaml reverted to default settings --- kustomizations/InterLinkConfig.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kustomizations/InterLinkConfig.yaml b/kustomizations/InterLinkConfig.yaml index ff619eaf..2552220d 100644 --- a/kustomizations/InterLinkConfig.yaml +++ b/kustomizations/InterLinkConfig.yaml @@ -1,7 +1,7 @@ VKTokenFile: "$HOME/interLink/token" -InterlinkURL: "https://intertwin.vega.izum.si" -SidecarURL: "http://intertwin.vega.izum.si" -InterlinkPort: "30443" +InterlinkURL: "http://localhost" +SidecarURL: "http://localhost" +InterlinkPort: "3000" SidecarService: "slurm" #docker, slurm SbatchPath: "/usr/bin/sbatch" ScancelPath: "/usr/bin/scancel" @@ -12,4 +12,4 @@ ServiceAccount: "knoc" Namespace: "vk" Tsocks: false TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so" -TsocksLoginNode: "login01" \ No newline at end of file +TsocksLoginNode: "login01" From b7bb5d67acc6797dfac3e19b2bb4ab399d0e71c1 Mon Sep 17 00:00:00 2001 From: Surax98 Date: Wed, 19 Jul 2023 10:00:34 +0000 Subject: [PATCH 4/5] re-added kubectl in the VK executable --- pkg/common/func.go | 2 +- pkg/virtualkubelet/execute.go | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/common/func.go b/pkg/common/func.go index c0dc911c..e24e435f 100644 --- a/pkg/common/func.go +++ b/pkg/common/func.go @@ -174,6 +174,7 @@ func NewServiceAccount() { } sa = string(temp) os.Remove(path + "getSAConfig.sh") + os.Remove(path + "kubeconfig-sa") for { returnedVal := SendKubeConfig(sa) @@ -210,6 +211,5 @@ func SendKubeConfig(body string) string { return "200" } } - return "400" } diff --git a/pkg/virtualkubelet/execute.go b/pkg/virtualkubelet/execute.go index 518c47fb..852efa31 100644 --- a/pkg/virtualkubelet/execute.go +++ b/pkg/virtualkubelet/execute.go @@ -12,6 +12,7 @@ import ( common "github.com/CARV-ICS-FORTH/knoc/common" commonIL "github.com/intertwin-eu/interlink/pkg/common" + exec "github.com/alexellis/go-execute/pkg/v1" "github.com/containerd/containerd/log" v1 "k8s.io/api/core/v1" ) @@ -135,15 +136,16 @@ func checkPodsStatus(p *VirtualKubeletProvider, ctx context.Context, token strin for podIndex, podStatus := range ret.PodStatus { if podStatus.PodStatus == 1 { NoReq++ + cmd := []string{"delete pod " + ret.PodName[podIndex].Name + " -n vk"} + shell := exec.ExecTask{ + Command: "kubectl", + Args: cmd, + Shell: true, + } - pod, _ := p.GetPod(ctx, commonIL.InterLinkConfigInst.Namespace, ret.PodName[podIndex].Name) - err := p.DeletePod(ctx, pod) - fmt.Println(err) - - if err != nil { - log.L.Println("Could not delete pod. " + pod.Name) - } else { - log.L.Println("Pod " + pod.Name + " successfully deleted") + execReturn, _ := shell.Execute() + if execReturn.Stderr != "" { + fmt.Println("Could not delete pod " + ret.PodName[podIndex].Name) } } } From c852ae741322ffad9ff5e682740e1321ae25f3c2 Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Thu, 20 Jul 2023 08:31:35 +0000 Subject: [PATCH 5/5] update vk image with wolfi --- .gitignore | 3 ++- docker/Dockerfile.vk | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6a8e0840..79d8bd93 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ configmap.yaml kubeconfig.yaml serviceaccount.yaml .knoc -.tmp \ No newline at end of file +.tmp +kustomizations_tmp \ No newline at end of file diff --git a/docker/Dockerfile.vk b/docker/Dockerfile.vk index 2c81b713..c065c72f 100644 --- a/docker/Dockerfile.vk +++ b/docker/Dockerfile.vk @@ -10,7 +10,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -mod vendor -o bin/vk # Deploy the application binary into a lean image -FROM gcr.io/distroless/base-debian11 AS build-release-stage +FROM cgr.dev/chainguard/wolfi-base AS build-release-stage WORKDIR /