Skip to content

Commit

Permalink
Merge pull request #105 from Surax98/quickfix
Browse files Browse the repository at this point in the history 
Better status management
  • Loading branch information
@dciangot
dciangot authored Oct 12, 2023
2 parents f191e78 + 3b1a633 commit 74005b4
Show file tree
Hide file tree
Showing 15 changed files with 329 additions and 264 deletions.
2 changes: 2 additions & 0 deletions kustomizations/InterLinkConfig.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ InterlinkPort: "3000"
SidecarPort: "4000"
SbatchPath: "/usr/bin/sbatch"
ScancelPath: "/usr/bin/scancel"
SqueuePath: "/usr/bin/squeue"
CommandPrefix: ""
ExportPodData: true
DataRootFolder: ".knoc/"
Expand All @@ -13,3 +14,4 @@ Namespace: "knoc"
Tsocks: false
TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so"
TsocksLoginNode: "login01"
BashPath: /bin/bash
3 changes: 1 addition & 2 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"context"
"crypto/tls"
"fmt"
"io/ioutil"
"net"
"os"
"path"
Expand Down Expand Up @@ -113,7 +112,7 @@ func main() {
}

var kubecfg *rest.Config
kubecfgFile, err := ioutil.ReadFile(os.Getenv("KUBECONFIG"))
kubecfgFile, err := os.ReadFile(os.Getenv("KUBECONFIG"))
if err != nil {
log.G(ctx).Error(err)
log.G(ctx).Info("Trying InCluster configuration")
Expand Down
5 changes: 4 additions & 1 deletion pkg/common/func.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"os"
"time"

"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"

"github.com/containerd/containerd/log"
Expand All @@ -20,6 +21,7 @@ import (
)

var InterLinkConfigInst InterLinkConfig
var Clientset *kubernetes.Clientset

func NewInterLinkConfig() {
if InterLinkConfigInst.set == false {
Expand Down Expand Up @@ -126,7 +128,8 @@ func NewServiceAccount() error {

defer f.Close()

script = "SERVICE_ACCOUNT_NAME=" + InterLinkConfigInst.ServiceAccount + "\n" +
script = "#!" + InterLinkConfigInst.BashPath + "\n" +
"SERVICE_ACCOUNT_NAME=" + InterLinkConfigInst.ServiceAccount + "\n" +
"CONTEXT=$(kubectl config current-context)\n" +
"NAMESPACE=" + InterLinkConfigInst.Namespace + "\n" +
"NEW_CONTEXT=" + InterLinkConfigInst.Namespace + "\n" +
Expand Down
18 changes: 4 additions & 14 deletions pkg/common/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,10 @@ import (
v1 "k8s.io/api/core/v1"
)

const (
RUNNING = 0
STOP = 1
UNKNOWN = 2
)

type PodStatus struct {
PodName string `json:"name"`
PodNamespace string `json:"namespace"`
PodStatus uint `json:"status"`
PodName string `json:"name"`
PodNamespace string `json:"namespace"`
Containers []v1.ContainerStatus `json:"containers"`
}

type RetrievedContainer struct {
Expand Down Expand Up @@ -45,6 +39,7 @@ type InterLinkConfig struct {
Sidecarurl string `yaml:"SidecarURL"`
Sbatchpath string `yaml:"SbatchPath"`
Scancelpath string `yaml:"ScancelPath"`
Squeuepath string `yaml:"SqueuePath"`
Interlinkport string `yaml:"InterlinkPort"`
Sidecarport string `yaml:"SidecarPort"`
Commandprefix string `yaml:"CommandPrefix"`
Expand Down Expand Up @@ -85,8 +80,3 @@ type LogStruct struct {
ContainerName string `json:"ContainerName"`
Opts ContainerLogOpts `json:"Opts"`
}

type JidStruct struct {
PodName string `json:"PodName"`
JIDs []string `json:"JIDs"`
}
38 changes: 11 additions & 27 deletions pkg/interlink/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"io"
"net/http"
"time"

"github.com/containerd/containerd/log"
commonIL "github.com/intertwin-eu/interlink/pkg/common"
Expand All @@ -29,37 +28,23 @@ func CreateHandler(w http.ResponseWriter, r *http.Request) {
var retrieved_data []commonIL.RetrievedPodData
for _, pod := range req2 {

data := []commonIL.RetrievedPodData{}
data := commonIL.RetrievedPodData{}
if commonIL.InterLinkConfigInst.ExportPodData {
data, err = getData(pod)
if err != nil {
statusCode = http.StatusInternalServerError
w.WriteHeader(statusCode)
return
}
log.G(Ctx).Debug(data)
//log.G(Ctx).Debug(data)

}
data = []commonIL.RetrievedPodData{}
if commonIL.InterLinkConfigInst.ExportPodData {
data, err = getData(pod)
if err != nil {
statusCode = http.StatusInternalServerError
w.WriteHeader(statusCode)
return
}
log.G(Ctx).Debug(data)
}

if data == nil {
data = append(data, commonIL.RetrievedPodData{Pod: *pod})
}

retrieved_data = append(retrieved_data, data...)
retrieved_data = append(retrieved_data, data)

if retrieved_data != nil {
bodyBytes, err = json.Marshal(retrieved_data)
log.G(Ctx).Debug(string(bodyBytes))
//log.G(Ctx).Debug(string(bodyBytes))
reader := bytes.NewReader(bodyBytes)

req, err = http.NewRequest(http.MethodPost, commonIL.InterLinkConfigInst.Sidecarurl+":"+commonIL.InterLinkConfigInst.Sidecarport+"/create", reader)
Expand All @@ -72,14 +57,13 @@ func CreateHandler(w http.ResponseWriter, r *http.Request) {

log.G(Ctx).Info("InterLink: forwarding Create call to sidecar")
var resp *http.Response
for {
resp, err = http.DefaultClient.Do(req)
if err != nil {
log.G(Ctx).Error(err)
time.Sleep(time.Second * 5)
} else {
break
}

resp, err = http.DefaultClient.Do(req)
if err != nil {
statusCode = http.StatusInternalServerError
w.WriteHeader(statusCode)
log.G(Ctx).Error(err)
return
}

statusCode = resp.StatusCode
Expand Down
2 changes: 1 addition & 1 deletion pkg/interlink/delete.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) {
}
log.G(Ctx).Debug("InterLink: " + string(returnValue))
var returnJson []commonIL.PodStatus
returnJson = append(returnJson, commonIL.PodStatus{PodName: pod.Name, PodNamespace: pod.Namespace, PodStatus: commonIL.STOP})
returnJson = append(returnJson, commonIL.PodStatus{PodName: pod.Name, PodNamespace: pod.Namespace})

bodyBytes, err = json.Marshal(returnJson)
if err != nil {
Expand Down
41 changes: 17 additions & 24 deletions pkg/interlink/func.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,28 +9,25 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func getData(pod *v1.Pod) ([]commonIL.RetrievedPodData, error) {
var retrieved_data []commonIL.RetrievedPodData
func getData(pod *v1.Pod) (commonIL.RetrievedPodData, error) {
var retrieved_data commonIL.RetrievedPodData
retrieved_data.Pod = *pod
for _, container := range pod.Spec.Containers {
log.G(Ctx).Info("- Retrieving Secrets and ConfigMaps for the Docker Sidecar. Container: " + container.Name)

data, err := retrieve_data(container, pod)
if err != nil {
log.G(Ctx).Error(err)
return nil, err
}

if data.Containers != nil {
data.Pod = *pod
retrieved_data = append(retrieved_data, data)
return commonIL.RetrievedPodData{}, err
}
retrieved_data.Containers = append(retrieved_data.Containers, data)
}

return retrieved_data, nil
}

func retrieve_data(container v1.Container, pod *v1.Pod) (commonIL.RetrievedPodData, error) {
retrieved_data := commonIL.RetrievedPodData{}
func retrieve_data(container v1.Container, pod *v1.Pod) (commonIL.RetrievedContainer, error) {
retrieved_data := commonIL.RetrievedContainer{}
for _, mount_var := range container.VolumeMounts {
log.G(Ctx).Debug("-- Retrieving data for mountpoint " + mount_var.Name)

Expand All @@ -50,16 +47,15 @@ func retrieve_data(container v1.Container, pod *v1.Pod) (commonIL.RetrievedPodDa

if err != nil {
log.G(Ctx).Error(err)
return commonIL.RetrievedPodData{}, err
return commonIL.RetrievedContainer{}, err
} else {
log.G(Ctx).Debug("---- Retrieved ConfigMap " + podVolumeSpec.ConfigMap.Name)
}

if configMap != nil {
if retrieved_data.Containers == nil {
retrieved_data.Containers = append(retrieved_data.Containers, commonIL.RetrievedContainer{Name: container.Name})
}
retrieved_data.Containers[len(retrieved_data.Containers)-1].ConfigMaps = append(retrieved_data.Containers[len(retrieved_data.Containers)-1].ConfigMaps, *configMap)

retrieved_data.Name = container.Name
retrieved_data.ConfigMaps = append(retrieved_data.ConfigMaps, *configMap)
}

} else if podVolumeSpec != nil && podVolumeSpec.Secret != nil {
Expand All @@ -70,24 +66,21 @@ func retrieve_data(container v1.Container, pod *v1.Pod) (commonIL.RetrievedPodDa

if err != nil {
log.G(Ctx).Error(err)
return commonIL.RetrievedPodData{}, err
return commonIL.RetrievedContainer{}, err
} else {
log.G(Ctx).Debug("---- Retrieved Secret " + svs.SecretName)
}

if secret.Data != nil {
if retrieved_data.Containers == nil {
retrieved_data.Containers = append(retrieved_data.Containers, commonIL.RetrievedContainer{Name: container.Name})
}
retrieved_data.Containers[len(retrieved_data.Containers)-1].Secrets = append(retrieved_data.Containers[len(retrieved_data.Containers)-1].Secrets, *secret)
retrieved_data.Name = container.Name
retrieved_data.Secrets = append(retrieved_data.Secrets, *secret)
}

} else if podVolumeSpec != nil && podVolumeSpec.EmptyDir != nil {
edPath := filepath.Join(commonIL.InterLinkConfigInst.DataRootFolder, pod.Namespace+"-"+string(pod.UID)+"/"+"emptyDirs/"+vol.Name)
if retrieved_data.Containers == nil {
retrieved_data.Containers = append(retrieved_data.Containers, commonIL.RetrievedContainer{Name: container.Name})
}
retrieved_data.Containers[len(retrieved_data.Containers)-1].EmptyDirs = append(retrieved_data.Containers[len(retrieved_data.Containers)-1].EmptyDirs, edPath)

retrieved_data.Name = container.Name
retrieved_data.EmptyDirs = append(retrieved_data.EmptyDirs, edPath)
}
}
}
Expand Down
34 changes: 18 additions & 16 deletions pkg/sidecars/docker/Delete.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,23 +61,25 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) {
continue
}

cmd = []string{"rm", execReturn.Stdout}
shell = exec.ExecTask{
Command: "docker",
Args: cmd,
Shell: true,
}
execReturn, _ = shell.Execute()
execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "")
if execReturn.Stdout != "" {
cmd = []string{"rm", execReturn.Stdout}
shell = exec.ExecTask{
Command: "docker",
Args: cmd,
Shell: true,
}
execReturn, _ = shell.Execute()
execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "")

if execReturn.Stderr != "" {
log.G(Ctx).Error("-- Error deleting container " + container.Name)
statusCode = http.StatusInternalServerError
w.WriteHeader(statusCode)
w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs"))
return
} else {
log.G(Ctx).Info("- Deleted container " + container.Name)
if execReturn.Stderr != "" {
log.G(Ctx).Error("-- Error deleting container " + container.Name)
statusCode = http.StatusInternalServerError
w.WriteHeader(statusCode)
w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs"))
return
} else {
log.G(Ctx).Info("- Deleted container " + container.Name)
}
}

os.RemoveAll(commonIL.InterLinkConfigInst.DataRootFolder + pod.Namespace + "-" + string(pod.UID))
Expand Down
25 changes: 18 additions & 7 deletions pkg/sidecars/docker/Status.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,11 @@ func StatusHandler(w http.ResponseWriter, r *http.Request) {
return
}

for _, pod := range req {
for i, pod := range req {
resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodNamespace: pod.Namespace})
for _, container := range pod.Spec.Containers {
log.G(Ctx).Debug("- Getting status for container " + container.Name)
cmd := []string{"ps -aqf name=" + container.Name}
cmd := []string{"ps -af name=^" + container.Name + "$ --format \"{{.Status}}\""}

shell := exec.ExecTask{
Command: "docker",
Expand All @@ -55,12 +56,22 @@ func StatusHandler(w http.ResponseWriter, r *http.Request) {
break
}

if execReturn.Stdout == "" {
log.G(Ctx).Info("-- Container " + container.Name + " is not running")
resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodNamespace: pod.Namespace, PodStatus: commonIL.STOP})
containerstatus := strings.Split(execReturn.Stdout, " ")

if execReturn.Stdout != "" {
if containerstatus[0] == "Created" {
log.G(Ctx).Info("-- Container " + container.Name + " is going ready...")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false})
} else if containerstatus[0] == "Up" {
log.G(Ctx).Info("-- Container " + container.Name + " is running")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true})
} else if containerstatus[0] == "Exited" {
log.G(Ctx).Info("-- Container " + container.Name + " has been stopped")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false})
}
} else {
log.G(Ctx).Info("-- Container " + container.Name + " is running")
resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodNamespace: pod.Namespace, PodStatus: commonIL.RUNNING})
log.G(Ctx).Info("-- Container " + container.Name + " doesn't exist")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false})
}
}
}
Expand Down
Loading

0 comments on commit 74005b4

Please sign in to comment.