[WIP] Adding Spinnaker/Kustomize Bootstrap Template

.gitignore

@@ -29,3 +29,4 @@ secrets.yaml
 
 # vscode
 .vscode/
+.truss/

bootstrap-templates/spin-kz/.terraform-version

@@ -0,0 +1 @@
+0.12.29

bootstrap-templates/spin-kz/.truss-manifest.yaml

@@ -0,0 +1,22 @@
+name: spin-kz
+description: Deploy your application using Spinnaker and Kustomize
+version: 0.1.0
+params:
+  - name: name
+    type: string
+    prompt: What's the name of your app?
+  - name: role
+    type: string
+    prompt: What's the role of your service? (i.e. api, web)
+  - name: httpPort
+    type: string
+    prompt: What is your app's HTTP port?
+  - name: githubRepo
+    type: string
+    prompt: Full https URL of your GitHub Repo
+  - name: image
+    type: string
+    prompt: What is the name of your Docker image? Don't include any tags
+  - name: smoketestImage
+    type: string
+    prompt: What is the name of your Smoketest Docker image? Don't include any tags

bootstrap-templates/spin-kz/README.md

@@ -0,0 +1,23 @@
+# Deploying {{ .Params.name }}
+
+This is how you deploy {{ .Params.name }} on Truss! Spinnaker pipelins are available at https://prod.spinnaker.bridgeops.sh/#/applications/{{ .Params.name }}/executions
+
+## Directory Structure
+
+- `{{ .TrussDir }}/` - Holds your tenant configuration and Spinnaker Application
+  - `/modules/{{ .Params.name }}-{{ .Params.role }}` - Holds the deployment configuration for a single instance of {{ .Params.name }}
+    - `/kustomize` - Kubernetes manifests for {{ .Params.name }}'s common infrastructure components
+      - `/deployment` -  Base config for deployment
+      - `/postdeploy` - Base config for post-deploy job
+      - `/predeploy` - Base config for pre-deploy job
+  - `/{edge|staging|prod}/{cmh|dub|syd}/` - Holds deployment configuration for a given environment/region of {{ .Params.name }}
+    - `/kustomize` - Environment-specific infra overrides
+      - `/deployment` - Environment-speicifc deployment overrides
+      - `/postdeploy` - Environment-specific post-deploy overrides
+      - `/predeploy` - Environment-specific pre-deploy overrides
+
+## Runbook
+
+- Provision your tenant: `cd {{ .TrussDir }} && terraform init && terraform apply`
+- Provision a given environment (i.e. edge-cmh): `cd {{ .TrussDir }}/edge/cmh && terraform init && terraform apply`
+- Retrieve realtime logs (i.e. edge-cmh): `truss wrap -e cmh-edge -- kubectl -n {{ .Params.name }}-edge logs -c {{ .Params.name }}-{{ .Params.role }} deployment/{{ .Params.name }}-{{ .Params.role }}`

bootstrap-templates/spin-kz/common.tf

@@ -0,0 +1,32 @@
+module "truss-tenant" {
+  source   = "git::ssh://[email protected]/instructure/truss//modules/truss-tenant"
+  name     = "{{ .Params.name }}"
+  istio    = true
+  apps = [{
+    name = "{{ .Params.role }}"
+    vault = [{
+      path         = "secret/data/bridge/{env}/{region}/shared/*"
+      capabilities = ["read", "list"]
+      }, {
+      path         = "secret/data/bridge/{env}/{region}/{{ .Params.name }}/*"
+      capabilities = ["read", "list"]
+    }]
+  }]
+  # iamStatements = [{
+  #   Effect   = "Allow"
+  #   Resource = ["arn:aws:s3:::*"]
+  #   Action   = ["s3:*"]
+  # }]
+}
+
+resource "spinnaker_application" "application" {
+  name          = "{{ .Params.name }}"
+  email         = "[email protected]"
+  instance_port = {{ .Params.httpPort }}
+
+  permissions {
+    read    = ["bridge-engineering-all"]
+    write   = ["bridge-engineering-all"]
+    execute = ["bridge-engineering-all"]
+  }
+}

bootstrap-templates/spin-kz/config.tf

@@ -0,0 +1,25 @@
+terraform {
+  backend "s3" {
+    bucket   = "bridge-shared-terraform-us-east-2"
+    key      = "{{ .Params.name }}/{{ .TrussDir }}/terraform.tfstate"
+    region   = "us-east-2"
+    acl      = "bucket-owner-full-control"
+    encrypt  = true
+    role_arn = "arn:aws:iam::127178877223:role/xacct/ops-admin"
+  }
+}
+
+provider "aws" {
+  region              = "us-east-2"
+  allowed_account_ids = ["127178877223"]
+
+  assume_role {
+    role_arn = "arn:aws:iam::127178877223:role/xacct/ops-admin"
+  }
+}
+
+provider "spinnaker" {
+  address   = "https://api-prod.spinnaker.bridgeops.sh"
+  cert_path = "~/.spin/shared-prod/spinnaker-client.crt"
+  key_path  = "~/.spin/shared-prod/spinnaker-client.key"
+}

bootstrap-templates/spin-kz/edge/cmh/deployment/kustomization.yaml

@@ -0,0 +1,17 @@
+namespace: "{{ .Params.name }}-edge"
+bases:
+  - ../../../modules/{{ .Params.name }}-{{ .Params.role }}/kustomize/deployment
+
+patchesStrategicMerge:
+  - |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: {{ .Params.name }}-{{ .Params.role }}
+    spec:
+      template:
+        metadata:
+          annotations:
+            vault.security.banzaicloud.io/vault-role: {{ .Params.name }}-edge-{{ .Params.role }}
+        spec:
+          serviceAccountName: {{ .Params.name }}-edge-{{ .Params.role }}

bootstrap-templates/spin-kz/edge/cmh/kustomization.yaml

@@ -0,0 +1,24 @@
+namespace: {{ .Params.name }}-edge
+bases:
+  - ../../modules/{{ .Params.name }}-{{ .Params.role }}/kustomize
+
+commonLabels:
+  env: edge
+  region: cmh
+
+configMapGenerator:
+  - name: {{ .Params.name }}-{{ .Params.role }}
+    behavior: merge
+    literals:
+      - AWS_REGION=us-east-2
+      - INST_DOG_TAGS='{"project":"{{ .Params.name }}","environment":"edge","region":"us-east-2"}'
+
+patchesStrategicMerge:
+  - |-
+    apiVersion: networking.istio.io/v1alpha3
+    kind: VirtualService
+    metadata:
+      name: {{ .Params.name }}-{{ .Params.role }}
+    spec:
+      hosts:
+        - {{ .Params.name }}-edge.nonprod-cmh.truss.bridgeops.sh

bootstrap-templates/spin-kz/edge/cmh/main.tf

@@ -0,0 +1,27 @@
+terraform {
+  backend "s3" {
+    bucket   = "bridge-shared-terraform-us-east-2"
+    key      = "{{ .Params.name }}/{{ .TrussDir }}/edge/cmh/terraform.tfstate"
+    region   = "us-east-2"
+    acl      = "bucket-owner-full-control"
+    encrypt  = true
+    role_arn = "arn:aws:iam::127178877223:role/xacct/ops-admin"
+  }
+}
+
+provider "aws" {
+  region              = "us-east-2"
+  allowed_account_ids = ["127178877223"]
+
+  assume_role {
+    role_arn = "arn:aws:iam::127178877223:role/xacct/ops-admin"
+  }
+}
+
+module "app" {
+  source = "../../modules/{{ .Params.name }}-{{ .Params.role }}"
+
+  truss_env   = "nonprod"
+  app_env     = "edge"
+  region_code = "cmh"
+}

bootstrap-templates/spin-kz/edge/cmh/postdeploy/kustomization.yaml

@@ -0,0 +1,17 @@
+namespace: "{{ .Params.name }}-edge"
+bases:
+  - ../../../modules/{{ .Params.name }}-{{ .Params.role }}/kustomize/postdeploy
+
+patchesStrategicMerge:
+  - |-
+    apiVersion: batch/v1
+    kind: Job
+    metadata:
+      name: "{{ .Params.name }}-{{ .Params.role }}-postdeploy-${trigger['parameters']['sha']}"
+    spec:
+      template:
+        metadata:
+          annotations:
+            vault.security.banzaicloud.io/vault-role: {{ .Params.name }}-edge-{{ .Params.role }}
+        spec:
+          serviceAccountName: {{ .Params.name }}-edge-{{ .Params.role }}

bootstrap-templates/spin-kz/edge/cmh/predeploy/kustomization.yaml

@@ -0,0 +1,17 @@
+namespace: "{{ .Params.name }}-edge"
+bases:
+  - ../../../modules/{{ .Params.name }}-{{ .Params.role }}/kustomize/predeploy
+
+patchesStrategicMerge:
+  - |-
+    apiVersion: batch/v1
+    kind: Job
+    metadata:
+      name: "{{ .Params.name }}-{{ .Params.role }}-predeploy-${trigger['parameters']['sha']}"
+    spec:
+      template:
+        metadata:
+          annotations:
+            vault.security.banzaicloud.io/vault-role: {{ .Params.name }}-edge-{{ .Params.role }}
+        spec:
+          serviceAccountName: {{ .Params.name }}-edge-{{ .Params.role }}

bootstrap-templates/spin-kz/modules/{{ .Params.name }}-{{ .Params.role }}/kustomize/deployment/deployment.yaml

@@ -0,0 +1,65 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Params.name }}-{{ .Params.role }}"
+  annotations:
+    traffic.spinnaker.io/load-balancers: '["service {{ .Params.name }}-{{ .Params.role }}"]'
+  labels:
+    version: "${trigger['parameters']['sha']}"
+spec:
+  replicas: 2
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      service-role: "{{ .Params.role }}"
+  template:
+    metadata:
+      labels:
+        version: "${trigger['parameters']['sha']}"
+      annotations:
+        sidecar.istio.io/rewriteAppHTTPProbers: "true"
+        traffic.sidecar.istio.io/excludeOutboundPorts: "8200"
+        vault.security.banzaicloud.io/vault-addr: "https://vault.vault.svc.cluster.local:8200"
+        vault.security.banzaicloud.io/vault-path: kubernetes
+        vault.security.banzaicloud.io/vault-skip-verify: "true"
+        # ad.datadoghq.com/{{ .Params.name }}-{{ .Params.role }}.check_names: '["openmetrics"]'
+        # ad.datadoghq.com/{{ .Params.name }}-{{ .Params.role }}.init_configs: "[{}]"
+        # ad.datadoghq.com/{{ .Params.name }}-{{ .Params.role }}.instances: "[{}]"
+    spec:
+      containers:
+        - envFrom:
+            - configMapRef:
+                name: "{{ .Params.name }}-{{ .Params.role }}"
+          name: "{{ .Params.name }}-{{ .Params.role }}"
+          image: "{{ .Params.image }}:${trigger['parameters']['sha']}"
+          imagePullPolicy: Always
+          ports:
+            - containerPort: {{ .Params.httpPort }}
+              name: http
+              protocol: TCP
+
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              httpHeaders: []
+              path: /health-check
+              port: http
+            initialDelaySeconds: 60
+            periodSeconds: 15
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              httpHeaders: []
+              path: /health-check
+              port: http
+            initialDelaySeconds: 60
+            periodSeconds: 15
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+            requests:
+              cpu: 100m
+              memory: 100Mi
+      imagePullSecrets:
+        - name: starlord

bootstrap-templates/spin-kz/modules/{{ .Params.name }}-{{ .Params.role }}/kustomize/deployment/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - deployment.yaml

bootstrap-templates/spin-kz/modules/{{ .Params.name }}-{{ .Params.role }}/kustomize/horizontalPodAutoscaler.yaml

@@ -0,0 +1,44 @@
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ .Params.name }}-{{ .Params.role }}
+
+  # Configure your scaling parameters
+  minReplicas: 2
+  maxReplicas: 10
+  metrics:
+  # Uncomment the following lines to scale by CPU
+  # - type: Resource
+  #   resource:
+  #     name: cpu
+  #     target:
+  #       type: Utilization
+  #       averageUtilization: 80
+
+  # Uncomment the following lines to scale by Memory
+  # - type: Resource
+  #   resource:
+  #     name: memory
+  #     target:
+  #       type: Utilization
+  #       averageUtilization: 80
+
+  # Uncomment the following lines to scale by Datadog metrics
+  # - type: External
+  #   external:
+  #     metric:
+  #       name: sidekiq.queue_latency
+  #       selector:
+  #         matchLabels:
+  #           # These matchLabels refer to Datadog tags, adjust as necessary
+  #           region: cmh
+  #           env: prod
+  #           queuename: myqueue
+  #     target:
+  #       type: Value
+  #       Value: "60"

bootstrap-templates/spin-kz/modules/{{ .Params.name }}-{{ .Params.role }}/kustomize/kustomization.yaml

@@ -0,0 +1,33 @@
+---
+# Use this to add labels to all of your deployment's resources
+commonLabels:
+  app: {{ .Params.name }}
+  role: {{ .Params.role }}
+
+# Use this to add annotations to all of your deployment's resources
+commonAnnotations: {}
+
+# Resources to include in the deployment
+resources:
+  - service.yaml
+  - podDisruptionBudget.yaml
+  - virtualService.yaml
+  # Uncomment this line to enable autoscaling. You'll need to add `app_resources`
+  # in ../spinnaker.tf for this to work. Lastly, customize your scaling in
+  # `./horizontalPodAutoscaler.yaml`. See https://truss.bridgeops.sh/#/howto/autoscaling
+  # - horizontalPodAutoscaler.yaml
+
+# Generate config maps for your deployment. This should be overridden in an
+# environment-specific kustomization.yaml. Defaults go here.
+configMapGenerator:
+  - name: {{ .Params.name }}-{{ .Params.role }}
+    literals:
+      - RAILS_ENV=production
+      - INST_DOG_TAGS='{}'
+      - INST_STATSD_HOST="datadog"
+      - INST_STATSD_APPEND_HOSTNAME="false"
+
+# Since we're using Spinnaker to create the Deployment, we need to disable this
+# fancy Kustomize feature.
+generatorOptions:
+  disableNameSuffixHash: true

bootstrap-templates/spin-kz/modules/{{ .Params.name }}-{{ .Params.role }}/kustomize/podDisruptionBudget.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: pdb
+spec:
+  # How many pods can Kubernetes make unavailable during cluster upgrades?
+  maxUnavailable: 1
+  selector:
+    matchLabels: {}