From a7cb630ab3199ee72a000ec3be7dea7ca54cc68a Mon Sep 17 00:00:00 2001
From: Magnus Kulke <magnus.kulke@moovel.com>
Date: Wed, 3 Jul 2019 14:49:07 +0200
Subject: [PATCH] initial commit

---
 Gopkg.lock                       | 61 +++++++++++++++++++++++++++++++
 Gopkg.toml                       | 30 +++++++++++++++
 Readme.md                        | 16 ++++++++
 main.go                          | 63 ++++++++++++++++++++++++++++++++
 spot-termination-handler.service |  9 +++++
 5 files changed, 179 insertions(+)
 create mode 100644 Gopkg.lock
 create mode 100644 Gopkg.toml
 create mode 100644 Readme.md
 create mode 100644 main.go
 create mode 100644 spot-termination-handler.service

diff --git a/Gopkg.lock b/Gopkg.lock
new file mode 100644
index 0000000..455a7ea
--- /dev/null
+++ b/Gopkg.lock
@@ -0,0 +1,61 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:b3e74203542b3be655caf6319afe23b37f7f11cec5e266032dc610f0dfdd5347"
+  name = "github.com/aws/aws-sdk-go"
+  packages = [
+    "aws",
+    "aws/awserr",
+    "aws/awsutil",
+    "aws/client",
+    "aws/client/metadata",
+    "aws/corehandlers",
+    "aws/credentials",
+    "aws/credentials/ec2rolecreds",
+    "aws/credentials/endpointcreds",
+    "aws/credentials/processcreds",
+    "aws/credentials/stscreds",
+    "aws/csm",
+    "aws/defaults",
+    "aws/ec2metadata",
+    "aws/endpoints",
+    "aws/request",
+    "aws/session",
+    "aws/signer/v4",
+    "internal/ini",
+    "internal/sdkio",
+    "internal/sdkrand",
+    "internal/sdkuri",
+    "internal/shareddefaults",
+    "private/protocol",
+    "private/protocol/json/jsonutil",
+    "private/protocol/jsonrpc",
+    "private/protocol/query",
+    "private/protocol/query/queryutil",
+    "private/protocol/rest",
+    "private/protocol/xml/xmlutil",
+    "service/ecs",
+    "service/sts",
+  ]
+  pruneopts = "UT"
+  revision = "06512ecac9db8f44189277d87f426e6dc53cc86b"
+  version = "v1.20.14"
+
+[[projects]]
+  digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca"
+  name = "github.com/jmespath/go-jmespath"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "c2b33e84"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/aws/aws-sdk-go/aws",
+    "github.com/aws/aws-sdk-go/aws/session",
+    "github.com/aws/aws-sdk-go/service/ecs",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1
diff --git a/Gopkg.toml b/Gopkg.toml
new file mode 100644
index 0000000..d7072c2
--- /dev/null
+++ b/Gopkg.toml
@@ -0,0 +1,30 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[prune]
+  go-tests = true
+  unused-packages = true
diff --git a/Readme.md b/Readme.md
new file mode 100644
index 0000000..fc8e475
--- /dev/null
+++ b/Readme.md
@@ -0,0 +1,16 @@
+# spot-termination-handler
+
+The tool is supposed to run on instances of a spot fleet backing an ECS cluster. Runs in a 5s loop polling the instance metadata for a spot termination notification (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html#spot-instance-termination-notices). If there is a notification the instance is marked as "DRAINING" to evacuate the workload to other instances before termination.
+
+## requirements
+
+```
+brew install go
+brew install dep
+```
+
+## build
+
+```
+go build
+```
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..15694dc
--- /dev/null
+++ b/main.go
@@ -0,0 +1,63 @@
+package main
+
+import (
+  "github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecs"
+  "io/ioutil"
+  "encoding/json"
+  "net/http"
+  "time"
+  "log"
+)
+
+type ECSMetadata struct {
+    Cluster              string `json:"Cluster"`
+    ContainerInstanceArn string `json:"ContainerInstanceArn"`
+}
+
+func main() {
+  client := &http.Client{Timeout: time.Second * 10}
+
+  for {
+    time.Sleep(5 * time.Second)
+    response, err := client.Get("http://169.254.169.254/latest/meta-data/spot/termination-time")
+    if err != nil {
+      panic(err)
+    }
+    
+    if response.StatusCode == 200 {
+      log.Print("Spot instance termination notice detected")
+      break
+    }
+  }
+
+  response, err := client.Get("http://localhost:51678/v1/metadata")
+  if err != nil {
+    panic(err);
+  }
+  buf, err := ioutil.ReadAll(response.Body)
+  if err != nil {
+    panic(err);
+  }
+  var ecsMetadata ECSMetadata
+  err = json.Unmarshal(buf, &ecsMetadata)
+  if err != nil {
+    panic(err);
+  }
+	svc := ecs.New(session.New(&aws.Config{}))
+  status := "DRAINING"
+  state := &ecs.UpdateContainerInstancesStateInput{
+    Cluster: &ecsMetadata.Cluster,
+    ContainerInstances: []*string{&ecsMetadata.ContainerInstanceArn},
+    Status: &status,
+  }
+  log.Print("Putting instance in state DRAINING")
+  _, err = svc.UpdateContainerInstancesState(state)
+  if err != nil {
+    panic(err);
+  }
+
+  log.Print("Sleeping for 120s until termination")
+  time.Sleep(120 * time.Second)
+}
diff --git a/spot-termination-handler.service b/spot-termination-handler.service
new file mode 100644
index 0000000..6d74989
--- /dev/null
+++ b/spot-termination-handler.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Start spot instance termination handler monitoring script
+
+[Service]
+Restart=always
+ExecStart=/usr/local/bin/spot-instance-termination-handler
+
+[Install]
+WantedBy=multi-user.target