initial commit

Gopkg.toml

@@ -0,0 +1,30 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[prune]
+  go-tests = true
+  unused-packages = true

Readme.md

@@ -0,0 +1,16 @@
+# spot-termination-handler
+
+The tool is supposed to run on instances of a spot fleet backing an ECS cluster. Runs in a 5s loop polling the instance metadata for a spot termination notification (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html#spot-instance-termination-notices). If there is a notification the instance is marked as "DRAINING" to evacuate the workload to other instances before termination.
+
+## requirements
+
+```
+brew install go
+brew install dep
+```
+
+## build
+
+```
+go build
+```

main.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+  "github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecs"
+  "io/ioutil"
+  "encoding/json"
+  "net/http"
+  "time"
+  "log"
+)
+
+type ECSMetadata struct {
+    Cluster              string `json:"Cluster"`
+    ContainerInstanceArn string `json:"ContainerInstanceArn"`
+}
+
+func main() {
+  client := &http.Client{Timeout: time.Second * 10}
+
+  for {
+    time.Sleep(5 * time.Second)
+    response, err := client.Get("http://169.254.169.254/latest/meta-data/spot/termination-time")
+    if err != nil {
+      panic(err)
+    }
+
+    if response.StatusCode == 200 {
+      log.Print("Spot instance termination notice detected")
+      break
+    }
+  }
+
+  response, err := client.Get("http://localhost:51678/v1/metadata")
+  if err != nil {
+    panic(err);
+  }
+  buf, err := ioutil.ReadAll(response.Body)
+  if err != nil {
+    panic(err);
+  }
+  var ecsMetadata ECSMetadata
+  err = json.Unmarshal(buf, &ecsMetadata)
+  if err != nil {
+    panic(err);
+  }
+	svc := ecs.New(session.New(&aws.Config{}))
+  status := "DRAINING"
+  state := &ecs.UpdateContainerInstancesStateInput{
+    Cluster: &ecsMetadata.Cluster,
+    ContainerInstances: []*string{&ecsMetadata.ContainerInstanceArn},
+    Status: &status,
+  }
+  log.Print("Putting instance in state DRAINING")
+  _, err = svc.UpdateContainerInstancesState(state)
+  if err != nil {
+    panic(err);
+  }
+
+  log.Print("Sleeping for 120s until termination")
+  time.Sleep(120 * time.Second)
+}

spot-termination-handler.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Start spot instance termination handler monitoring script
+
+[Service]
+Restart=always
+ExecStart=/usr/local/bin/spot-instance-termination-handler
+
+[Install]
+WantedBy=multi-user.target