From a0b80cf7ce923c0abe917b94848513982abe0ee0 Mon Sep 17 00:00:00 2001 From: Robert Jandow <38583713+robertjndw@users.noreply.github.com> Date: Fri, 1 Dec 2023 11:50:12 +0100 Subject: [PATCH] Add Cloning Container (#28) * Add clone container for cloning via env variables * Update workspace for CloneContainer * Adapt Dockerfile to build it from CI/CD * New API prototype * Update HadesCloneContainer/Dockerfile --------- Co-authored-by: Matthias Linhuber --- .github/workflows/build.yml | 2 + HadesAPI/main.go | 8 +- HadesAPI/router.go | 5 +- HadesCloneContainer/Dockerfile | 26 ++ .../docker-compose.example.yml | 14 + HadesCloneContainer/go.mod | 32 ++ HadesCloneContainer/go.sum | 147 +++++++++ HadesCloneContainer/main.go | 102 ++++++ HadesScheduler/config/images.go | 7 - HadesScheduler/docker/container.go | 14 - HadesScheduler/docker/docker.go | 220 ++++--------- HadesScheduler/docker/image.go | 48 +++ HadesScheduler/go.mod | 36 -- HadesScheduler/go.sum | 112 ------- HadesScheduler/kube/kube.go | 309 ------------------ HadesScheduler/kube/kube_test.go | 30 -- HadesScheduler/kube/utils.go | 6 - HadesScheduler/main.go | 13 +- go.work | 1 + go.work.sum | 25 +- shared/payload/example.json | 28 ++ shared/payload/payload.go | 31 +- shared/queue/queue.go | 9 +- shared/utils/git.go | 27 -- 24 files changed, 515 insertions(+), 737 deletions(-) create mode 100644 HadesCloneContainer/Dockerfile create mode 100644 HadesCloneContainer/docker-compose.example.yml create mode 100644 HadesCloneContainer/go.mod create mode 100644 HadesCloneContainer/go.sum create mode 100644 HadesCloneContainer/main.go delete mode 100644 HadesScheduler/config/images.go create mode 100644 HadesScheduler/docker/image.go delete mode 100644 HadesScheduler/kube/kube.go delete mode 100644 HadesScheduler/kube/kube_test.go delete mode 100644 HadesScheduler/kube/utils.go create mode 100644 shared/payload/example.json delete mode 100644 shared/utils/git.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 349cccc..0e9bc51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,8 @@ jobs: image: ghcr.io/mtze/hades/hades-api - dockerfile: ./HadesScheduler/Dockerfile image: ghcr.io/mtze/hades/hades-scheduler + - dockerfile: ./HadesCloneContainer/Dockerfile + image: ghcr.io/mtze/hades/hades-clone-container steps: - name: Compute Tag diff --git a/HadesAPI/main.go b/HadesAPI/main.go index 9fa1b2c..9251028 100644 --- a/HadesAPI/main.go +++ b/HadesAPI/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "github.com/Mtze/HadesCI/shared/queue" "github.com/Mtze/HadesCI/shared/utils" @@ -17,6 +18,10 @@ type HadesAPIConfig struct { } func main() { + if is_debug := os.Getenv("DEBUG"); is_debug == "true" { + log.SetLevel(log.DebugLevel) + log.Warn("DEBUG MODE ENABLED") + } var cfg HadesAPIConfig utils.LoadConfig(&cfg) @@ -26,7 +31,8 @@ func main() { log.Debug("Connecting to RabbitMQ: ", rabbitmqURL) BuildQueue, err = queue.Init("builds", rabbitmqURL) if err != nil { - log.Panic(err) + log.WithError(err).Fatal("Failed to connect to RabbitMQ") + return } log.Infof("Starting HadesAPI on port %d", cfg.APIPort) diff --git a/HadesAPI/router.go b/HadesAPI/router.go index 0cb832d..9486fff 100644 --- a/HadesAPI/router.go +++ b/HadesAPI/router.go @@ -15,7 +15,8 @@ func ping(c *gin.Context) { } func AddBuildToQueue(c *gin.Context) { - var payload payload.BuildJob + var payload payload.RESTPayload + payload.Priority = 3 // Default priority if err := c.ShouldBind(&payload); err != nil { log.WithError(err).Error("Failed to bind JSON") c.String(http.StatusBadRequest, "Failed to bind JSON") @@ -23,5 +24,5 @@ func AddBuildToQueue(c *gin.Context) { } log.Debug("Received build request ", payload) - BuildQueue.Enqueue(c.Request.Context(), payload) + BuildQueue.Enqueue(c.Request.Context(), payload.QueuePayload, uint8(payload.Priority)) } diff --git a/HadesCloneContainer/Dockerfile b/HadesCloneContainer/Dockerfile new file mode 100644 index 0000000..b03ff52 --- /dev/null +++ b/HadesCloneContainer/Dockerfile @@ -0,0 +1,26 @@ +# Use an official Go runtime as a parent image +FROM golang:1.21-alpine AS builder + +# Set the working directory inside the container +WORKDIR /app + +# Copy the Go application source code into the container +COPY . . + +# Build the Go application +RUN go build -o hades-clone ./HadesCloneContainer + +# Start a new stage for the minimal runtime container +FROM alpine:latest + +# Set the working directory inside the minimal runtime container +WORKDIR /app + +# Copy the built binary from the builder container into the minimal runtime container +COPY --from=builder /app/hades-clone . + +# Ensure the binary is executable +RUN chmod +x /app/hades-clone + +# Run your Go application and then sleep indefinitely +ENTRYPOINT ["/app/hades-clone"] \ No newline at end of file diff --git a/HadesCloneContainer/docker-compose.example.yml b/HadesCloneContainer/docker-compose.example.yml new file mode 100644 index 0000000..6f21c86 --- /dev/null +++ b/HadesCloneContainer/docker-compose.example.yml @@ -0,0 +1,14 @@ +version: '3.7' +services: + hadesclonecontainer: + build: + context: . + dockerfile: ./Dockerfile + environment: + - HADES_HELLO_URL=http://hades_hello_1:8080/example.git + - HADES_HELLO_USERNAME=user + - HADES_HELLO_PASSWORD=secret + - HADES_HELLO_PATH=./hello + - REPOSITORY_DIR=/opt/repositories + volumes: + - ./repo:/opt/repositories diff --git a/HadesCloneContainer/go.mod b/HadesCloneContainer/go.mod new file mode 100644 index 0000000..ce31fba --- /dev/null +++ b/HadesCloneContainer/go.mod @@ -0,0 +1,32 @@ +module github.com/Mtze/HadesCI/hadesCloneContainer + +go 1.21.0 + +require ( + github.com/go-git/go-git/v5 v5.10.1 + github.com/sirupsen/logrus v1.9.3 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect + github.com/cloudflare/circl v1.3.3 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect + github.com/skeema/knownhosts v1.2.1 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/tools v0.13.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect +) diff --git a/HadesCloneContainer/go.sum b/HadesCloneContainer/go.sum new file mode 100644 index 0000000..168c330 --- /dev/null +++ b/HadesCloneContainer/go.sum @@ -0,0 +1,147 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.10.1 h1:tu8/D8i+TWxgKpzQ3Vc43e+kkhXqtsZCKI/egajKnxk= +github.com/go-git/go-git/v5 v5.10.1/go.mod h1:uEuHjxkHap8kAl//V5F/nNWwqIYtP/402ddd05mp0wg= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= +github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/HadesCloneContainer/main.go b/HadesCloneContainer/main.go new file mode 100644 index 0000000..36a4969 --- /dev/null +++ b/HadesCloneContainer/main.go @@ -0,0 +1,102 @@ +package main + +import ( + "os" + "path" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + log "github.com/sirupsen/logrus" +) + +var dir = "/opt/repositories/" + +func isValidPath(path string) bool { + _, err := os.Stat(path) + return err == nil || os.IsExist(err) +} + +func main() { + if env_dir := os.Getenv("REPOSITORY_DIR"); env_dir != "" && isValidPath(env_dir) { + dir = env_dir + } + log.Info("Starting HadesCloneContainer") + log.Info("Cloning repositories to ", dir) + + repos := getReposFromEnv() + for _, repo := range repos { + repodir := dir + // URL is mandatory + if repo["URL"] == "" { + log.Warn("Skipping repository without URL") + continue + } + clone_options := &git.CloneOptions{ + URL: repo["URL"], + } + // Check if username and password are set and use them for authentication + if repo["USERNAME"] != "" && repo["PASSWORD"] != "" { + clone_options.Auth = &http.BasicAuth{ + Username: repo["USERNAME"], + Password: repo["PASSWORD"], + } + } + // Check if a branch is specified + if repo["BRANCH"] != "" { + clone_options.ReferenceName = plumbing.ReferenceName("refs/heads/" + repo["BRANCH"]) + } + + if repo["PATH"] != "" { + repodir = path.Join(repodir, repo["PATH"]) + } else { + parts := strings.Split(repo["URL"], "/") + if len(parts) > 0 { + repoName := strings.TrimSuffix(parts[len(parts)-1], ".git") + repodir = path.Join(repodir, repoName) + } + } + + // Execute the clone operation + _, err := git.PlainClone(repodir, false, clone_options) + if err != nil { + log.WithError(err).Error("Failed to clone repository") + continue + } + log.Infof("Cloned repository %s to %s", repo["URL"], repodir) + } +} + +// Returns a map of all repositories and their respective metadata +// The metadata is a map of key-value pairs +func getReposFromEnv() map[string]map[string]string { + // Map to hold the environment variables + repoVars := make(map[string]map[string]string) + + // Iterate over all environment variables + for _, e := range os.Environ() { + pair := strings.SplitN(e, "=", 2) + key := pair[0] + value := pair[1] + + // Check if the environment variable starts with "HADES_" + if strings.HasPrefix(key, "HADES_") { + // Should be in the form HADES__ + parts := strings.SplitN(key, "_", 3) + if len(parts) < 3 { + continue + } + group := parts[1] + varType := parts[2] + + // Group the variables + if _, ok := repoVars[group]; !ok { + repoVars[group] = make(map[string]string) + } + repoVars[group][varType] = value + } + } + + return repoVars +} diff --git a/HadesScheduler/config/images.go b/HadesScheduler/config/images.go deleted file mode 100644 index f9dbe83..0000000 --- a/HadesScheduler/config/images.go +++ /dev/null @@ -1,7 +0,0 @@ -package config - -const ( - CloneContainerImage = "alpine/git:latest" - ResultContainerImage = "alpine:latest" - SharedVolumeName = "shared" -) diff --git a/HadesScheduler/docker/container.go b/HadesScheduler/docker/container.go index 82caecc..8907800 100644 --- a/HadesScheduler/docker/container.go +++ b/HadesScheduler/docker/container.go @@ -7,26 +7,12 @@ import ( "io" "os" - "github.com/Mtze/HadesCI/hadesScheduler/config" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" log "github.com/sirupsen/logrus" ) -var defaultHostConfig = container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: mount.TypeVolume, - Source: config.SharedVolumeName, - Target: "/shared", - }, - }, - AutoRemove: true, -} - func writeContainerLogsToFile(ctx context.Context, client *client.Client, containerID string, logFilePath string) error { out, err := os.Create(logFilePath) if err != nil { diff --git a/HadesScheduler/docker/docker.go b/HadesScheduler/docker/docker.go index be9d241..c211a39 100644 --- a/HadesScheduler/docker/docker.go +++ b/HadesScheduler/docker/docker.go @@ -3,13 +3,8 @@ package docker import ( "context" "fmt" - "io" - "sync" - "time" - - "github.com/Mtze/HadesCI/hadesScheduler/config" - "os" + "time" "github.com/Mtze/HadesCI/shared/payload" "github.com/Mtze/HadesCI/shared/utils" @@ -22,6 +17,7 @@ import ( ) var cli *client.Client +var global_envs []string = []string{} type Scheduler struct{} @@ -41,188 +37,100 @@ func init() { } } -func (d Scheduler) ScheduleJob(job payload.BuildJob) error { +func (d Scheduler) ScheduleJob(job payload.QueuePayload) error { ctx := context.Background() - startOfPull := time.Now() - // Pull the images - err := pullImages(ctx, cli, job.BuildConfig.ExecutionContainer, config.CloneContainerImage, config.ResultContainerImage) - if err != nil { - log.WithError(err).Error("Failed to pull images") - return err - } - log.Debugf("Pulled images in %s", time.Since(startOfPull)) - - startOfVolume := time.Now() + // Create a unique volume name for this job + volumeName := fmt.Sprintf("shared-%d", time.Now().UnixNano()) // Create the shared volume - err = createSharedVolume(ctx, cli, config.SharedVolumeName) - if err != nil { + if err := createSharedVolume(ctx, cli, volumeName); err != nil { log.WithError(err).Error("Failed to create shared volume") return err } - log.Debugf("Create Shared Volume in %s", time.Since(startOfVolume)) + + // Read the global env variables from the job metadata + for k, v := range job.Metadata { + global_envs = append(global_envs, fmt.Sprintf("%s=%s", k, v)) + } + + for _, step := range job.Steps { + executeStep(ctx, cli, step, volumeName) + } + // Delete the shared volume after the job is done defer func() { time.Sleep(1 * time.Second) - startOfDelete := time.Now() - err = deleteSharedVolume(ctx, cli, config.SharedVolumeName) - if err != nil { + if err := deleteSharedVolume(ctx, cli, volumeName); err != nil { log.WithError(err).Error("Failed to delete shared volume") } - log.Debugf("Delete Shared Volume in %s", time.Since(startOfDelete)) - }() - startOfClone := time.Now() - // Clone the repository - err = cloneRepository(ctx, cli, job.Credentials, job.BuildConfig.Repositories...) - if err != nil { - log.WithError(err).Error("Failed to clone repository") - return err - } - log.Debugf("Clone repo in %s", time.Since(startOfClone)) - - startOfExecute := time.Now() - err = executeRepository(ctx, cli, job.BuildConfig) - if err != nil { - log.WithError(err).Error("Failed to execute repository") - return err - } - log.Debugf("Execute repo in %s", time.Since(startOfExecute)) - log.Debugf("Total time: %s", time.Since(startOfPull)) - - // TODO enable deletion of shared volume - time.Sleep(1 * time.Second) - startOfDelete := time.Now() - err = deleteSharedVolume(ctx, cli, config.SharedVolumeName) - if err != nil { - log.WithError(err).Error("Failed to delete shared volume") - return err - } - log.Debugf("Delete Shared Volume in %s", time.Since(startOfDelete)) + global_envs = []string{} + }() return nil } -func pullImages(ctx context.Context, client *client.Client, images ...string) error { - var wg sync.WaitGroup - errorsCh := make(chan error, len(images)) - - for _, image := range images { - wg.Add(1) - - go func(img string) { - defer wg.Done() - - response, err := client.ImagePull(ctx, img, types.ImagePullOptions{}) - if err != nil { - errorsCh <- fmt.Errorf("failed to pull image %s: %v", img, err) - return - } - defer response.Close() - io.Copy(io.Discard, response) // consume the response to prevent potential leaks - }(image) +func executeStep(ctx context.Context, client *client.Client, step payload.Step, volumeName string) error { + // Pull the images + err := pullImages(ctx, cli, step.Image) + if err != nil { + log.WithError(err).Errorf("Failed to pull image %s", step.Image) + return err } - // wait for all goroutines to complete - wg.Wait() - close(errorsCh) - - // Collect errors - var errors []error - for err := range errorsCh { - errors = append(errors, err) + // Copy the global envs and add the step specific ones + var envs []string + copy(envs, global_envs) + for k, v := range step.Metadata { + envs = append(envs, fmt.Sprintf("%s=%s", k, v)) } - if len(errors) > 0 { - return fmt.Errorf("encountered %d errors while pulling images: %+v", len(errors), errors) + container_config := container.Config{ + Image: step.Image, + Env: envs, + WorkingDir: "/shared", // Set the working directory to the shared volume } - return nil -} - -func cloneRepository(ctx context.Context, client *client.Client, credentials payload.Credentials, repositories ...payload.Repository) error { - // Use the index to modify the slice in place - for i := range repositories { - repositories[i].Path = "/shared" + repositories[i].Path - } - commandStr := utils.BuildCloneCommands(credentials, repositories...) - log.Debug(commandStr) - - // Create the container - resp, err := client.ContainerCreate(ctx, &container.Config{ - Image: config.CloneContainerImage, - Entrypoint: []string{"/bin/sh", "-c"}, - Cmd: []string{commandStr}, - Volumes: map[string]struct{}{ - "/shared": {}, + host_config := container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: mount.TypeVolume, + Source: volumeName, + Target: "/shared", + }, }, - }, &defaultHostConfig, nil, nil, "") - if err != nil { - return err + AutoRemove: true, } - // Start the container - err = client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) - if err != nil { - return err - } - - statusCh, errCh := client.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) - select { - case err := <-errCh: + // Create the bash script if there is one + if step.Script != "" { + scriptPath, err := writeBashScriptToFile(step.Script) if err != nil { - log.WithError(err).Errorf("Error waiting for container with ID %s", resp.ID) + log.WithError(err).Error("Failed to write bash script to a temporary file") return err } - case status := <-statusCh: - if status.StatusCode != 0 { - log.Errorf("Container with ID %s exited with status %d", resp.ID, status.StatusCode) - return fmt.Errorf("container exited with status %d", status.StatusCode) - } - } + defer os.Remove(scriptPath) - log.Infof("Container %s started with ID: %s\n", config.CloneContainerImage, resp.ID) - return nil -} + host_config.Mounts = append(host_config.Mounts, mount.Mount{ + Type: mount.TypeBind, + Source: scriptPath, + Target: "/tmp/script.sh", + }) -func executeRepository(ctx context.Context, client *client.Client, buildConfig payload.BuildConfig) error { - // First, write the Bash script to a temporary file - scriptPath, err := writeBashScriptToFile("cd /shared", buildConfig.BuildScript) - if err != nil { - log.WithError(err).Error("Failed to write bash script to a temporary file") - return err - } - defer os.Remove(scriptPath) - - hostConfigWithScript := defaultHostConfig - hostConfigWithScript.Mounts = append(defaultHostConfig.Mounts, mount.Mount{ - Type: mount.TypeBind, - Source: scriptPath, - Target: "/tmp/script.sh", - }) - // Create the container - resp, err := client.ContainerCreate(ctx, &container.Config{ - Image: buildConfig.ExecutionContainer, - Entrypoint: []string{"/bin/sh", "/tmp/script.sh"}, - Volumes: map[string]struct{}{ - "/shared": {}, - "/tmp/script.sh": {}, // this volume will hold our script - }, - }, &hostConfigWithScript, nil, nil, "") - if err != nil { - return err + // Overwrite the default entrypoint + container_config.Entrypoint = []string{"/bin/bash", "/tmp/script.sh"} } - // Copy the script to the container - err = copyFileToContainer(ctx, client, resp.ID, scriptPath, "/tmp") + resp, err := client.ContainerCreate(ctx, &container_config, &host_config, nil, nil, "") if err != nil { - log.WithError(err).Error("Failed to copy script to container") + log.WithError(err).Errorf("Failed to create container %s", step.Image) return err } // Start the container err = client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) if err != nil { + log.WithError(err).Errorf("Failed to start container %s with ID %s", step.Image, resp.ID) return err } @@ -231,24 +139,16 @@ func executeRepository(ctx context.Context, client *client.Client, buildConfig p select { case err := <-errCh: if err != nil { - log.WithError(err).Errorf("Error waiting for container %s with ID %s", buildConfig.ExecutionContainer, resp.ID) + log.WithError(err).Errorf("Error waiting for container %s with ID %s", step.Image, resp.ID) return err } case status := <-statusCh: if status.StatusCode != 0 { - log.Errorf("Container %s with ID %s exited with status %d", buildConfig.ExecutionContainer, resp.ID, status.StatusCode) + log.Errorf("Container %s with ID %s exited with status %d", step.Image, resp.ID, status.StatusCode) return fmt.Errorf("container exited with status %d", status.StatusCode) } } - // Fetch logs and write to a file - logFilePath := "./logfile.log" // TODO make this configurable - err = writeContainerLogsToFile(ctx, client, resp.ID, logFilePath) - if err != nil { - log.WithError(err).Errorf("Failed to write logs of container %s with ID %s", buildConfig.ExecutionContainer, resp.ID) - return err - } - - log.Infof("Container %s with ID: %s completed", buildConfig.ExecutionContainer, resp.ID) + log.Debugf("Container %s with ID: %s completed", step.Image, resp.ID) return nil } diff --git a/HadesScheduler/docker/image.go b/HadesScheduler/docker/image.go new file mode 100644 index 0000000..d8e5c17 --- /dev/null +++ b/HadesScheduler/docker/image.go @@ -0,0 +1,48 @@ +package docker + +import ( + "context" + "fmt" + "io" + "sync" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +func pullImages(ctx context.Context, client *client.Client, images ...string) error { + var wg sync.WaitGroup + errorsCh := make(chan error, len(images)) + + for _, image := range images { + wg.Add(1) + + go func(img string) { + defer wg.Done() + + response, err := client.ImagePull(ctx, img, types.ImagePullOptions{}) + if err != nil { + errorsCh <- fmt.Errorf("failed to pull image %s: %v", img, err) + return + } + defer response.Close() + io.Copy(io.Discard, response) // consume the response to prevent potential leaks + }(image) + } + + // wait for all goroutines to complete + wg.Wait() + close(errorsCh) + + // Collect errors + var errors []error + for err := range errorsCh { + errors = append(errors, err) + } + + if len(errors) > 0 { + return fmt.Errorf("encountered %d errors while pulling images: %+v", len(errors), errors) + } + + return nil +} diff --git a/HadesScheduler/go.mod b/HadesScheduler/go.mod index 4dac373..a0ebd6e 100644 --- a/HadesScheduler/go.mod +++ b/HadesScheduler/go.mod @@ -8,62 +8,26 @@ require ( github.com/Mtze/HadesCI/shared v0.0.0-00010101000000-000000000000 github.com/docker/docker v24.0.7+incompatible github.com/sirupsen/logrus v1.9.3 - k8s.io/api v0.28.2 - k8s.io/apimachinery v0.28.2 - k8s.io/client-go v0.28.2 ) require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/caarlos0/env/v9 v9.0.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/imdario/mergo v0.3.16 // indirect github.com/joho/godotenv v1.5.1 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/term v0.5.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rabbitmq/amqp091-go v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.8.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.30.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230918164632-68afd615200d // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/HadesScheduler/go.sum b/HadesScheduler/go.sum index 7f99aaa..81dbd37 100644 --- a/HadesScheduler/go.sum +++ b/HadesScheduler/go.sum @@ -4,7 +4,6 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -16,73 +15,21 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= -github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= @@ -93,120 +40,61 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA= github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230918164632-68afd615200d h1:/CFeJBjBrZvHX09rObS2+2iEEDevMWYc1v3aIYAjIYI= -k8s.io/kube-openapi v0.0.0-20230918164632-68afd615200d/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/HadesScheduler/kube/kube.go b/HadesScheduler/kube/kube.go deleted file mode 100644 index 112359a..0000000 --- a/HadesScheduler/kube/kube.go +++ /dev/null @@ -1,309 +0,0 @@ -package kube - -import ( - "context" - "os" - "path/filepath" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/Mtze/HadesCI/shared/payload" - "github.com/Mtze/HadesCI/shared/utils" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" -) - -const ( - waitForNamespace = 5 * time.Second - cloneContainerImage = "alpine/git:latest" - sharedVolumeName = "shared" -) - -type JobScheduler interface { - ScheduleJob(job payload.BuildJob) error -} - -type Scheduler struct{} - -var clientset *kubernetes.Clientset -var namespace *corev1.Namespace -var k8sCfg K8sConfig - -func Init() { - log.Debug("Kube Init called") - utils.LoadConfig(&k8sCfg) - - var err error - - hadesCInamespace := k8sCfg.HadesCInamespace - - clientset = initializeKubeconfig() - - // Ensure that the namespace exists - - log.Debugf("Ensure that %s namespace exists", hadesCInamespace) - namespace, err = getNamespace(clientset, hadesCInamespace) - if err != nil { - log.Infof("Namespace '%s' does not exist - Trying creating a new one", hadesCInamespace) - - namespace, err = createNamespace(clientset, hadesCInamespace) - if err != nil { - log.WithError(err).Panic("error getting existing namespace - no more options to try - exiting") - } - log.Infof("Namespace '%s' created", namespace.Name) - } - log.Debugf("Using namespace '%s'", namespace.Name) - -} - -func (k Scheduler) ScheduleJob(buildJob payload.BuildJob) error { - - log.Infof("Scheduling job %s", buildJob.BuildConfig.ExecutionContainer) - - _, err := createExecutionScriptConfigMap(clientset, namespace.Name, buildJob) - if err != nil { - log.WithError(err).Error("error creating configmap") - return err - } - - job, err := createJob(clientset, namespace.Name, buildJob) - - if err != nil { - log.WithError(err).Error("error creating job") - return err - } - - _ = job - - log.Infof("Job %v scheduled to the Cluster", buildJob) - - return nil -} - -// This function inizializes the kubeconfig clientset using the kubeconfig file in the useres home directory -func initializeKubeconfig() *kubernetes.Clientset { - - var kubeConfig *rest.Config - - // Check if kubeconfig is explicitly set - if k8sCfg.Kubeconfig != "" { - log.Infof("Using kubeconfig: %s", k8sCfg.Kubeconfig) - var err error - kubeConfig, err = clientcmd.BuildConfigFromFlags("", k8sCfg.Kubeconfig) - if err != nil { - log.WithError(err).Panic("error getting Kubernetes clientset") - } - } else { - log.Info("Kubeconfig not set - using default location") - // Load kubeconfig from default location - userHomeDir, err := os.UserHomeDir() - if err != nil { - log.WithError(err).Panic("error getting user home dir") - } - kubeConfigPath := filepath.Join(userHomeDir, ".kube", "config") - log.Infof("Using kubeconfig: %s", kubeConfigPath) - - // Create kubeconfig object - kubeConfig, err = clientcmd.BuildConfigFromFlags("", kubeConfigPath) - if err != nil { - log.WithError(err).Panic("error getting Kubernetes clientset") - } - } - - clientset, err := kubernetes.NewForConfig(kubeConfig) - if err != nil { - log.WithError(err).Panic("error getting Kubernetes clientset") - } - - return clientset - -} - -func getPods(clientset *kubernetes.Clientset, namespace string) *corev1.PodList { - - pods, err := clientset.CoreV1().Pods("kube-system").List(context.Background(), v1.ListOptions{}) - - if err != nil { - log.WithError(err).Error("error getting pods") - } - for _, pod := range pods.Items { - log.Infof("Pod name: %s", pod.Name) - - } - - return pods -} - -func createNamespace(clientset *kubernetes.Clientset, namespace string) (*corev1.Namespace, error) { - log.Infof("Creating namespace %s", namespace) - - ns, err := clientset.CoreV1().Namespaces().Create( - context.Background(), - &corev1.Namespace{ - ObjectMeta: v1.ObjectMeta{ - Name: namespace, - }, - }, v1.CreateOptions{}) - - if err != nil { - log.WithError(err).Error("error creating namespace") - return nil, err - } - - // sleep for 5 seconds to give the namespace time to be created - time.Sleep(waitForNamespace) - - return ns, nil -} - -func getNamespaces(clientset *kubernetes.Clientset) *corev1.NamespaceList { - log.Debugf("Getting namespaces") - - namespaces, err := clientset.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) - - if err != nil { - log.WithError(err).Error("error getting namespaces") - } - - for _, namespace := range namespaces.Items { - log.Debugf("Namespace name: %s", namespace.Name) - } - return namespaces -} - -func getNamespace(clientset *kubernetes.Clientset, namespace string) (*corev1.Namespace, error) { - log.Debugf("Getting namespace %s", namespace) - - ns, err := clientset.CoreV1().Namespaces().Get(context.Background(), namespace, v1.GetOptions{}) - - if err != nil { - log.WithError(err).Error("error getting namespace") - return nil, err - } - - return ns, nil -} - -func deleteNamespace(clientset *kubernetes.Clientset, namespace string) { - log.Infof("Deleting namespace %s", namespace) - - err := clientset.CoreV1().Namespaces().Delete(context.Background(), namespace, v1.DeleteOptions{}) - - if err != nil { - log.WithError(err).Error("error deleting namespace") - } -} - -func createJob(clientset *kubernetes.Clientset, namespace string, buildJob payload.BuildJob) (*batchv1.Job, error) { - log.Infof("Creating job %v in namespace %s", buildJob, namespace) - - sharedVolumeName := "shared-volume-" + buildJob.Name - - jobs := clientset.BatchV1().Jobs(namespace) - var backOffLimit int32 = 0 - - cloneCommand := utils.BuildCloneCommands(buildJob.Credentials, buildJob.BuildConfig.Repositories...) - log.Debugf("Clone command: %s", cloneCommand) - - jobSpec := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: buildJob.Name, - Namespace: namespace, - }, - Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - InitContainers: []corev1.Container{ - { - Name: "clone", - Image: cloneContainerImage, - Command: []string{"/bin/sh", "-c"}, - Args: []string{cloneCommand}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: sharedVolumeName, - MountPath: "/shared", - }, - }, - }, - }, - Containers: []corev1.Container{ - { - Name: buildJob.Name, - Image: buildJob.BuildConfig.ExecutionContainer, - Command: []string{"/bin/sh", "/tmp/build-script/build.sh"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: sharedVolumeName, - MountPath: "/shared", - }, - { - Name: "build-script", - MountPath: "/tmp/build-script", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: sharedVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, // An emptyDir volume is shared among containers in the same Pod - }, - }, - { - Name: "build-script", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: buildJob.Name, - }, - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - BackoffLimit: &backOffLimit, - }, - } - - job, err := jobs.Create(context.TODO(), jobSpec, metav1.CreateOptions{}) - if err != nil { - log.WithError(err).Error("error creating job") - return nil, err - } - - //print job details - log.Infof("Created K8s job %s successfully", buildJob.Name) - log.Debugf("Job details: %v", job) - return job, nil -} - -func createExecutionScriptConfigMap(clientset *kubernetes.Clientset, namespace string, buildJob payload.BuildJob) (*corev1.ConfigMap, error) { - log.Infof("Creating configmap for execution script %v in namespace %s", buildJob.Name, namespace) - - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: buildJob.Name, - }, - Data: map[string]string{ - "build.sh": "cd /shared && " + buildJob.BuildConfig.BuildScript, - }, - } - cm, err := clientset.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) - if err != nil { - log.WithError(err).Error("error creating configmap") - return nil, err - } - - return cm, nil - -} diff --git a/HadesScheduler/kube/kube_test.go b/HadesScheduler/kube/kube_test.go deleted file mode 100644 index 1c8ebe0..0000000 --- a/HadesScheduler/kube/kube_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package kube - -import ( - "testing" -) - -func TestKubeconfigInitialization(t *testing.T) { - client := initializeKubeconfig() - - if client == nil { - t.Error("client is nil") - } -} - -func TestGetPods(t *testing.T) { - client := initializeKubeconfig() - namespace := "default" - getPods(client, namespace) -} - -func TestCreateNamespace(t *testing.T) { - client := initializeKubeconfig() - createNamespace(client, "test") -} - -func TestDeleteNamespace(t *testing.T) { - client := initializeKubeconfig() - deleteNamespace(client, "test") -} - diff --git a/HadesScheduler/kube/utils.go b/HadesScheduler/kube/utils.go deleted file mode 100644 index 27dc693..0000000 --- a/HadesScheduler/kube/utils.go +++ /dev/null @@ -1,6 +0,0 @@ -package kube - -type K8sConfig struct { - HadesCInamespace string `env:"HADES_CI_NAMESPACE" envDefault:"hades-ci"` - Kubeconfig string `env:"KUBECONFIG" envDefault:""` -} diff --git a/HadesScheduler/main.go b/HadesScheduler/main.go index dff5815..dab144a 100644 --- a/HadesScheduler/main.go +++ b/HadesScheduler/main.go @@ -5,7 +5,6 @@ import ( "os" "github.com/Mtze/HadesCI/hadesScheduler/docker" - "github.com/Mtze/HadesCI/hadesScheduler/kube" "github.com/Mtze/HadesCI/shared/payload" "github.com/Mtze/HadesCI/shared/queue" "github.com/Mtze/HadesCI/shared/utils" @@ -16,7 +15,7 @@ import ( var BuildQueue *queue.Queue type JobScheduler interface { - ScheduleJob(job payload.BuildJob) error + ScheduleJob(job payload.QueuePayload) error } func main() { @@ -45,15 +44,15 @@ func main() { var scheduler JobScheduler switch executorCfg.Executor { - case "k8s": - log.Info("Started HadesScheduler in Kubernetes mode") - kube.Init() - scheduler = kube.Scheduler{} + // case "k8s": + // log.Info("Started HadesScheduler in Kubernetes mode") + // kube.Init() + // scheduler = kube.Scheduler{} case "docker": log.Info("Started HadesScheduler in Docker mode") scheduler = docker.Scheduler{} default: - log.Panic("Invalid executor specified") + log.Fatal("Invalid executor specified") } BuildQueue.Dequeue(scheduler.ScheduleJob) diff --git a/go.work b/go.work index 615e0da..c7e9fe0 100644 --- a/go.work +++ b/go.work @@ -2,6 +2,7 @@ go 1.21.0 use ( ./HadesAPI + ./HadesCloneContainer ./HadesScheduler ./shared ) diff --git a/go.work.sum b/go.work.sum index c240cb8..cedca7d 100644 --- a/go.work.sum +++ b/go.work.sum @@ -2,16 +2,29 @@ cloud.google.com/go/compute/metadata v0.2.0 h1:nBbNSZyDpkNlo3DepaaLKVuO7ClyifSAm cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= +k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= diff --git a/shared/payload/example.json b/shared/payload/example.json new file mode 100644 index 0000000..46e39c8 --- /dev/null +++ b/shared/payload/example.json @@ -0,0 +1,28 @@ +{ + "name": "", + "metadata": {}, + "timestamp": "2021-01-01T00:00:00.000Z", + "priority": 1, // optional, default 3 + "jobs" : [ + { + "id" : 1, // mandatory to declare the order of execution + "name" : "Clone", + "image" : "git:alpine", // mandatory + "script" : "echo hello", // optional + "metadata" : { + "GIT_USERNAME[_*]" : "user", + "GIT_PASSWORD[_*]" : "pass", + "GIT_URL_*": "fake.fake", + "GIT_BRANCH_*": "main", + "GIT_COMMIT_*": "hadfuisahsdfui" + } + }, + { + "id" : 2, + "name" : "Execute", + "image" : "java:alpine", + "script" : "maven run", + "metadata" : {} + } + ] +} \ No newline at end of file diff --git a/shared/payload/payload.go b/shared/payload/payload.go index 49dbeb5..0c504f1 100644 --- a/shared/payload/payload.go +++ b/shared/payload/payload.go @@ -1,24 +1,23 @@ package payload -type BuildJob struct { - Name string `json:"name" binding:"required"` +import "time" - Credentials Credentials `json:"credentials" binding:"required"` - BuildConfig BuildConfig `json:"buildConfig" binding:"required"` +type RESTPayload struct { + Priority int `json:"priority"` + QueuePayload } -type Credentials struct { - Username string `json:"username" binding:"required"` - Password string `json:"password" binding:"required"` +type QueuePayload struct { + Name string `json:"name"` + Timestamp time.Time `json:"timestamp"` + Metadata map[string]string `json:"metadata"` + Steps []Step `json:"steps"` } -type BuildConfig struct { - Repositories []Repository `json:"repositories" binding:"required,dive"` - ExecutionContainer string `json:"executionContainer" binding:"required"` - BuildScript string `json:"buildScript" binding:"required"` -} - -type Repository struct { - URL string `json:"url" binding:"required,url"` - Path string `json:"path" binding:"required,dirpath"` +type Step struct { + ID int `json:"id"` + Name string `json:"name"` + Image string `json:"image"` + Script string `json:"script"` + Metadata map[string]string `json:"metadata"` } diff --git a/shared/queue/queue.go b/shared/queue/queue.go index a3e3c48..c762dd6 100644 --- a/shared/queue/queue.go +++ b/shared/queue/queue.go @@ -50,11 +50,11 @@ func Init(queueName, url string) (*Queue, error) { log.WithError(err).Error("error declaring RabbitMQ queue") return nil, err } - log.Info("Queue initialized", q) + log.Debug("Queue initialized", q) return &q, nil } -func (q *Queue) Enqueue(ctx context.Context, msg payload.BuildJob) error { +func (q *Queue) Enqueue(ctx context.Context, msg payload.QueuePayload, prio uint8) error { log.Debugf("Enqueue function called with ctx %+v message: %v", ctx, msg) body, err := json.Marshal(msg) @@ -71,6 +71,7 @@ func (q *Queue) Enqueue(ctx context.Context, msg payload.BuildJob) error { amqp.Publishing{ ContentType: "text/plain", Body: []byte(body), + Priority: prio, }) if err != nil { log.WithError(err).Error("error publishing message") @@ -79,7 +80,7 @@ func (q *Queue) Enqueue(ctx context.Context, msg payload.BuildJob) error { return nil } -func (q *Queue) Dequeue(callback func(job payload.BuildJob) error) error { +func (q *Queue) Dequeue(callback func(job payload.QueuePayload) error) error { msgs, err := q.channel.Consume( q.queue.Name, // queue "", // consumer @@ -95,7 +96,7 @@ func (q *Queue) Dequeue(callback func(job payload.BuildJob) error) error { } for d := range msgs { - var buildJob payload.BuildJob + var buildJob payload.QueuePayload err := json.Unmarshal(d.Body, &buildJob) if err != nil { log.WithError(err).Warn("error unmarshalling message %s", d.Body) diff --git a/shared/utils/git.go b/shared/utils/git.go deleted file mode 100644 index a6015b4..0000000 --- a/shared/utils/git.go +++ /dev/null @@ -1,27 +0,0 @@ -package utils - -import ( - "fmt" - "net/url" - "strings" - - "github.com/Mtze/HadesCI/shared/payload" -) - -func BuildCloneCommand(username, password string, repo payload.Repository) string { - username = url.PathEscape(username) - password = url.PathEscape(password) - cloneURL := strings.Replace(repo.URL, "https://", fmt.Sprintf("https://%s:%s@", username, password), 1) - return fmt.Sprintf("git clone %s %s", cloneURL, "/shared"+repo.Path) -} - -func BuildCloneCommands(credentials payload.Credentials, repos ...payload.Repository) string { - var builder strings.Builder - for i, repo := range repos { - if i > 0 { - builder.WriteString(" && ") - } - builder.WriteString(BuildCloneCommand(credentials.Username, credentials.Password, repo)) - } - return builder.String() -}