Merge pull request #2 from AndrewChubatiuk/fix-scaler

Updated scaling

.circleci/config.yml

@@ -1,5 +1,8 @@
 version: 2.1
 
+orbs:
+  docker: circleci/[email protected]
+
 executors:
   go-linux:
     resource_class: medium
@@ -57,20 +60,33 @@ workflows:
     jobs:
       - check-deps-go:
           filters:
-              tags:
-                only: /^v\d+\.\d+\.\d+$/
+            tags:
+              only: /^v\d+\.\d+\.\d+$/
       - lint-go:
           filters:
-              tags:
-                only: /^v\d+\.\d+\.\d+$/
+            tags:
+              only: /^v\d+\.\d+\.\d+$/
       - linux-build-go:
           filters:
-              tags:
-                only: /^v\d+\.\d+\.\d+$/
+            tags:
+              only: /^v\d+\.\d+\.\d+$/
       - linux-test-go:
           filters:
-              tags:
-                only: /^v\d+\.\d+\.\d+$/
+            tags:
+              only: /^v\d+\.\d+\.\d+$/
+      - docker/publish:
+          attach-at: ./bin/
+          deploy: true
+          image: achubatiuk/nomad-autoscaler
+          tag: $CIRCLE_BRANCH
+          requires:
+            - check-deps-go
+            - lint-go
+            - linux-build-go
+            - linux-test-go
+          filters:
+            tags:
+              only: /^v\d+\.\d+\.\d+$/
       - release:
           requires:
             - check-deps-go

.gitignore

@@ -1,2 +1,9 @@
 bin/*
-
+.terraform.lock.hcl
+*.tfstate
+*.tfstate.backup
+*.tfstate.*.backup
+*.tfplan
+**/.terraform/*
+creds.json
+.vscode

Dockerfile

@@ -0,0 +1,2 @@
+FROM hashicorp/nomad-autoscaler:0.3.6
+ADD bin/hcloud-server /plugins/

Makefile

@@ -32,7 +32,7 @@ build:
 	@CGO_ENABLED=0 GO111MODULE=on \
 	go build \
 	-ldflags $(GO_LDFLAGS) \
-	-o ./bin/nomad-hcloud-autoscaler
+	-o ./bin/hcloud-server
 	@echo "==> Done"
 
 .PHONY: lint
@@ -66,5 +66,5 @@ test: ## Test the source code
 .PHONY: clean
 clean:
 	@echo "==> Cleaning build artifacts..."
-	@rm -f ./bin/nomad-hcloud-autoscaler
+	@rm -f ./bin/*
 	@echo "==> Done"

README.md

@@ -1,6 +1,15 @@
 # nomad-hcloud-autoscaler
 
-Example configuration
+## Demo
+Run `terraform apply` in [demo](demo/setup) folder to create: 
+ - nomad server which runs services for:
+    - nomad-autoscaler
+    - prometheus
+    - redis
+Autoscaler scales hcloud nodes for redis. After successful run both Nomad and Consul are wide-world open and credentials for both you can find in terraform output
+
+
+## Configuration
 
 `config.hcl`
 ```
@@ -75,7 +84,7 @@ template {
             hcloud_user_data = ""
             hcloud_ssh_keys = "XXX"
             hcloud_server_type = "cx11"
-            hcloud_name_prefix = "XXX"
+            hcloud_group_id = "XXX"
             hcloud_labels = "XXX_node=true"
             hcloud_networks = "XXX"
         }

demo/modules/hcloud-server/output.tf

@@ -0,0 +1,7 @@
+output "ipv4_addresses" {
+  value = hcloud_server.server.*.ipv4_address
+}
+
+output "ids" {
+  value = hcloud_server.server.*.id
+}

demo/modules/hcloud-server/servers.tf

@@ -0,0 +1,18 @@
+data "hcloud_datacenters" "dc" {}
+
+resource "random_shuffle" "dc" {
+  count        = var.server_count
+  input        = local.locations
+  result_count = 1
+}
+
+resource "hcloud_server" "server" {
+  count       = var.server_count
+  name        = "${var.prefix}-${count.index}"
+  image       = var.image
+  datacenter  = random_shuffle.dc[count.index].result[0]
+  ssh_keys    = var.ssh_keys
+  server_type = var.server_type
+  user_data   = var.user_data
+  labels      = merge(var.labels, { "Name" = var.prefix })
+}

demo/modules/hcloud-server/variables.tf

@@ -0,0 +1,29 @@
+variable "location" {
+  type    = string
+  default = null
+}
+
+variable "labels" {
+  type    = map(string)
+  default = {}
+}
+
+variable "prefix" {}
+
+variable "server_count" {}
+
+variable "ssh_keys" {
+  type = list(string)
+}
+
+variable "server_type" {}
+
+variable "user_data" {}
+
+variable "image" {
+  default = "ubuntu-20.04"
+}
+
+locals {
+  locations = [for dc in data.hcloud_datacenters.dc.names : dc if var.location != null && can(regex("^${var.location}-dc\\d+$", dc))]
+}

demo/modules/hcloud-server/versions.tf

@@ -0,0 +1,11 @@
+terraform {
+  required_providers {
+    hcloud = {
+      source = "hetznercloud/hcloud"
+    }
+    random = {
+      source = "hashicorp/random"
+    }
+  }
+  required_version = ">= 0.13"
+}

demo/modules/nomad-service/service.tf

@@ -0,0 +1,3 @@
+resource "nomad_job" "service" {
+  jobspec = file(var.path)
+}

demo/modules/nomad-service/variables.tf

@@ -0,0 +1 @@
+variable "path" {}

demo/modules/user-data/data/bootstrap.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+systemctl restart consul
+
+export CONSUL_HTTP_ADDR=http://$(ip -o -4 addr list ${interface} | head -n1 | awk '{print $4}' | cut -d/ -f1):8500
+until [[ $(curl -s -f $CONSUL_HTTP_ADDR/v1/status/leader) =~ \"[0-9.]+:8300\" ]] ; do
+  echo "Waiting for consul to become available..."
+  sleep 5
+done
+%{ if length(servers) == 0 }
+export CONSUL_HTTP_TOKEN=$(consul acl bootstrap -format=json | jq -r '.SecretID')
+%{ else }
+export CONSUL_HTTP_TOKEN=${consul_token}
+%{ endif }
+echo "CONSUL_HTTP_TOKEN=$CONSUL_HTTP_TOKEN" >> /etc/nomad.d/nomad.env
+echo "CONSUL_HTTP_ADDR=$CONSUL_HTTP_ADDR" >> /etc/nomad.d/nomad.env
+echo "CONSUL_HTTP_TOKEN=$CONSUL_HTTP_TOKEN" >> /etc/consul.d/consul.env
+
+systemctl restart nomad
+
+export NOMAD_ADDR=http://$(ip -o -4 addr list ${interface} | head -n1 | awk '{print $4}' | cut -d/ -f1):4646
+until [[ $(curl -s -f $NOMAD_ADDR/v1/status/leader) =~ \"[0-9.]+:4647\" ]] ; do
+  echo "Waiting for nomad to become available..."
+  sleep 5
+done
+%{ if length(servers) == 0 }
+export NOMAD_TOKEN=$(nomad acl bootstrap -json | jq -r '.SecretID')
+%{ else }
+export NOMAD_TOKEN=${ nomad_token }
+%{ endif }
+echo "NOMAD_TOKEN=$NOMAD_TOKEN" >> /etc/nomad.d/nomad.env
+echo "NOMAD_TOKEN=$NOMAD_TOKEN" >> /etc/consul.d/consul.env
+
+systemctl enable consul nomad
+
+cat > /tmp/creds.json <<EOL
+{
+    "nomad": "$${NOMAD_TOKEN}",
+    "consul": "$${CONSUL_HTTP_TOKEN}"
+}
+EOL

demo/modules/user-data/data/consul.hcl.tmpl

@@ -0,0 +1,14 @@
+server           = %{ if length(servers) == 0 }true
+bootstrap_expect = 1%{else}false%{ endif }
+data_dir         = "/opt/consul"
+bind_addr        = "{{ GetInterfaceIP \"${interface}\" }}"
+advertise_addr   = "{{ GetInterfaceIP \"${interface}\" }}"
+%{ if length(servers) > 0 }
+start_join       = ["${ join("\",\"", servers) }"]
+%{ endif }
+client_addr      = "{{ GetInterfaceIP \"${interface}\" }}"
+acl = {
+  enabled        = true
+  default_policy = "deny"
+  down_policy    = "extend-cache"
+}

demo/modules/user-data/data/nomad.hcl.tmpl

@@ -0,0 +1,30 @@
+region    = "global"
+log_level = "INFO"
+data_dir  =  "/opt/nomad"
+bind_addr = "0.0.0.0"
+server = {
+  enabled          = %{ if length(servers) == 0 }true%{else}false%{ endif }
+  bootstrap_expect = 1
+}
+advertise = {
+  http = "{{ GetInterfaceIP \"${interface}\" }}"
+  rpc  = "{{ GetInterfaceIP \"${interface}\" }}"
+  serf = "{{ GetInterfaceIP \"${interface}\" }}"
+}
+client = {
+  node_class = "${node_class}"
+  enabled    = true%{ if length(servers) > 0 }
+  servers    = ["${ join("\",\"", servers) }"]
+%{ endif }
+}
+telemetry = {
+  prometheus_metrics         = true
+  publish_allocation_metrics = true
+  publish_node_metrics       = true
+}
+acl = {
+  enabled           = true
+  token_ttl         = "30s"
+  policy_ttl        = "60s"
+  replication_token = ""
+}

demo/modules/user-data/output.tf

@@ -0,0 +1,3 @@
+output "data" {
+  value = "#cloud-config\n${yamlencode(local.user_data)}"
+}

demo/modules/user-data/variables.tf

@@ -0,0 +1,44 @@
+locals {
+  user_data = {
+    apt = {
+      sources = {
+        "hashicorp-releases.list" = {
+          keyid  = "9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
+          source = "deb [arch=amd64] https://apt.releases.hashicorp.com $RELEASE main"
+        }
+        "docker.list" = {
+          keyid  = "E8A032E094D8EB4EA189D270DA418C88A3219F7B"
+          source = "deb [arch=amd64] https://download.docker.com/linux/ubuntu $RELEASE stable"
+        }
+      }
+    }
+    packages = [
+      "nomad",
+      "consul",
+      "docker-ce",
+      "docker-ce-cli",
+      "containerd.io",
+      "jq",
+    ]
+    runcmd = [
+      ["/usr/bin/bootstrap.sh"],
+    ]
+    write_files = [
+      {
+        path        = "/etc/nomad.d/nomad.hcl"
+        permissions = "0644"
+        content     = templatefile("${path.module}/data/nomad.hcl.tmpl", var.input)
+        }, {
+        path        = "/etc/consul.d/consul.hcl"
+        permissions = "0644"
+        content     = templatefile("${path.module}/data/consul.hcl.tmpl", var.input)
+        }, {
+        path        = "/usr/bin/bootstrap.sh"
+        permissions = "0755"
+        content     = templatefile("${path.module}/data/bootstrap.sh", var.input)
+      }
+    ]
+  }
+}
+
+variable "input" {}