From cf37e614cc70fc5ee8f6b65697633a101d301095 Mon Sep 17 00:00:00 2001 From: Robert Jandow <38583713+robertjndw@users.noreply.github.com> Date: Mon, 25 Sep 2023 08:50:03 +0200 Subject: [PATCH 1/2] Update build.yml Fix typo in scheduler package name --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 059be2e..eda6b65 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: - dockerfile: ./HadesAPI/Dockerfile image: ghcr.io/mtze/hades/hades-api - dockerfile: ./HadesScheduler/Dockerfile - image: ghcr.io/mtze/hades/hades-schduler-k8s + image: ghcr.io/mtze/hades/hades-scheduler-k8s steps: - name: Compute Tag uses: actions/github-script@v6 From 756ffe1dc53812f374d95dafe8b3ea8ee8b142ca Mon Sep 17 00:00:00 2001 From: Matthias Linhuber Date: Mon, 25 Sep 2023 11:21:58 +0200 Subject: [PATCH 2/2] Implement Kubernetes job runner (#13) * Adapt k8s job config to use a shared volume * Add credentials to clone command * Kubernetes build now working --- HadesScheduler/kube/kube.go | 82 ++++++++++++++++++++++++++++++-- HadesScheduler/kube/kube_test.go | 64 ++++++++++++------------- shared/utils/git.go | 5 +- 3 files changed, 112 insertions(+), 39 deletions(-) diff --git a/HadesScheduler/kube/kube.go b/HadesScheduler/kube/kube.go index 232c2c5..4e8d596 100644 --- a/HadesScheduler/kube/kube.go +++ b/HadesScheduler/kube/kube.go @@ -4,7 +4,6 @@ import ( "context" "os" "path/filepath" - "strings" "time" log "github.com/sirupsen/logrus" @@ -20,7 +19,9 @@ import ( ) const ( - waitForNamespace = 5 * time.Second + waitForNamespace = 5 * time.Second + cloneContainerImage = "alpine/git:latest" + sharedVolumeName = "shared" ) type JobScheduler interface { @@ -64,6 +65,12 @@ func (k Scheduler) ScheduleJob(buildJob payload.BuildJob) error { log.Infof("Scheduling job %s", buildJob.BuildConfig.ExecutionContainer) + _, err := createExecutionScriptConfigMap(clientset, namespace.Name, buildJob) + if err != nil { + log.WithError(err).Error("error creating configmap") + return err + } + job, err := createJob(clientset, namespace.Name, buildJob) if err != nil { @@ -182,11 +189,16 @@ func deleteNamespace(clientset *kubernetes.Clientset, namespace string) { func createJob(clientset *kubernetes.Clientset, namespace string, buildJob payload.BuildJob) (*batchv1.Job, error) { log.Infof("Creating job %v in namespace %s", buildJob, namespace) - buildCommand := "sleep 10" + //TODO: Use function to generate build command + sharedVolumeName := "shared-volume-" + buildJob.Name jobs := clientset.BatchV1().Jobs(namespace) var backOffLimit int32 = 0 + //TODO: Use function to generate clone command + cloneCommand := utils.BuildCloneCommands(buildJob.Credentials, buildJob.BuildConfig.Repositories...) + log.Debugf("Clone command: %s", cloneCommand) + jobSpec := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: buildJob.Name, @@ -195,11 +207,53 @@ func createJob(clientset *kubernetes.Clientset, namespace string, buildJob paylo Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "clone", + Image: cloneContainerImage, + Command: []string{"/bin/sh", "-c"}, + Args: []string{cloneCommand}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: sharedVolumeName, + MountPath: "/shared", + }, + }, + }, + }, Containers: []corev1.Container{ { Name: buildJob.Name, Image: buildJob.BuildConfig.ExecutionContainer, - Command: strings.Split(buildCommand, " "), + Command: []string{"/bin/sh", "/tmp/build-script/build.sh"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: sharedVolumeName, + MountPath: "/shared", + }, + { + Name: "build-script", + MountPath: "/tmp/build-script", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: sharedVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, // An emptyDir volume is shared among containers in the same Pod + }, + }, + { + Name: "build-script", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: buildJob.Name, + }, + }, + }, }, }, RestartPolicy: corev1.RestartPolicyNever, @@ -220,3 +274,23 @@ func createJob(clientset *kubernetes.Clientset, namespace string, buildJob paylo log.Debugf("Job details: %v", job) return job, nil } + +func createExecutionScriptConfigMap(clientset *kubernetes.Clientset, namespace string, buildJob payload.BuildJob) (*corev1.ConfigMap, error) { + log.Infof("Creating configmap for execution script %v in namespace %s", buildJob, namespace) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: buildJob.Name, + }, + Data: map[string]string{ + "build.sh": "cd /shared && " + buildJob.BuildConfig.BuildScript, + }, + } + cm, err := clientset.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) + if err != nil { + log.WithError(err).Error("error creating configmap") + return nil, err + } + + return cm, nil + +} diff --git a/HadesScheduler/kube/kube_test.go b/HadesScheduler/kube/kube_test.go index 5440f69..eb86b2c 100644 --- a/HadesScheduler/kube/kube_test.go +++ b/HadesScheduler/kube/kube_test.go @@ -2,8 +2,6 @@ package kube import ( "testing" - - "github.com/Mtze/HadesCI/shared/payload" ) func TestKubeconfigInitialization(t *testing.T) { @@ -30,37 +28,37 @@ func TestDeleteNamespace(t *testing.T) { deleteNamespace(client, "test") } -func TestCreateJob(t *testing.T) { - client := initializeKubeconfig() +// func TestCreateJob(t *testing.T) { +// client := initializeKubeconfig() - testBuildJob := payload.BuildJob{ - Name: "Test Build", - Credentials: struct { - Username string `json:"username" binding:"required"` - Password string `json:"password" binding:"required"` - }{ - Username: "testuser", - Password: "testpassword", - }, - BuildConfig: struct { - Repositories []payload.Repository `json:"repositories" binding:"required,dive"` - ExecutionContainer string `json:"executionContainer" binding:"required"` - }{ - Repositories: []payload.Repository{ - { - Path: "/tmp/testrepo1", - URL: "https://github.com/testuser/testrepo1.git", - }, - { - Path: "/tmp/testrepo2", - URL: "https://github.com/testuser/testrepo2.git", - }, - }, - ExecutionContainer: "docker", - }, - } +// testBuildJob := payload.BuildJob{ +// Name: "Test Build", +// Credentials: struct { +// Username string `json:"username" binding:"required"` +// Password string `json:"password" binding:"required"` +// }{ +// Username: "testuser", +// Password: "testpassword", +// }, +// BuildConfig: struct { +// Repositories []payload.Repository `json:"repositories" binding:"required,dive"` +// ExecutionContainer string `json:"executionContainer" binding:"required"` +// }{ +// Repositories: []payload.Repository{ +// { +// Path: "/tmp/testrepo1", +// URL: "https://github.com/testuser/testrepo1.git", +// }, +// { +// Path: "/tmp/testrepo2", +// URL: "https://github.com/testuser/testrepo2.git", +// }, +// }, +// ExecutionContainer: "docker", +// }, +// } - namespace := "default" +// namespace := "default" - createJob(client, namespace, testBuildJob) -} +// createJob(client, namespace, testBuildJob) +// } diff --git a/shared/utils/git.go b/shared/utils/git.go index 6660dc7..a6015b4 100644 --- a/shared/utils/git.go +++ b/shared/utils/git.go @@ -2,16 +2,17 @@ package utils import ( "fmt" - "github.com/Mtze/HadesCI/shared/payload" "net/url" "strings" + + "github.com/Mtze/HadesCI/shared/payload" ) func BuildCloneCommand(username, password string, repo payload.Repository) string { username = url.PathEscape(username) password = url.PathEscape(password) cloneURL := strings.Replace(repo.URL, "https://", fmt.Sprintf("https://%s:%s@", username, password), 1) - return fmt.Sprintf("git clone %s %s", cloneURL, repo.Path) + return fmt.Sprintf("git clone %s %s", cloneURL, "/shared"+repo.Path) } func BuildCloneCommands(credentials payload.Credentials, repos ...payload.Repository) string {