From 8e1b18526d2805ebd84d0ea02bd22b01a1b0ea6c Mon Sep 17 00:00:00 2001 From: jk464 Date: Mon, 5 Feb 2024 17:05:56 +0000 Subject: [PATCH] Enabled internal TLS between k8s pods by default --- .github/workflows/e2e.yaml | 11 +++ CHANGELOG.md | 1 + Chart.yaml | 2 +- templates/_helpers.tpl | 53 ++++++++++- templates/ca.yaml | 19 ++++ templates/certificate.yaml | 29 ++++++ templates/configmaps_st2-conf.yaml | 41 +++++++++ templates/configmaps_st2-urls.yaml | 12 +++ templates/deployments.yaml | 138 +++++++++++++++++++++++++++-- templates/jobs.yaml | 20 +++++ templates/services.yaml | 16 +++- templates/tests/st2tests-pod.yaml | 14 +++ tests/integration/st2tests.sh | 6 +- values.yaml | 55 +++++++++++- 14 files changed, 402 insertions(+), 15 deletions(-) create mode 100644 templates/ca.yaml create mode 100644 templates/certificate.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a67759f0..88c4e3ce 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -39,6 +39,17 @@ jobs: with: k3s-channel: ${{ matrix.k3s-channel }} + - name: Setup cert-manager + run: | + helm repo add jetstack https://charts.jetstack.io --force-update + helm repo update + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.crds.yaml + helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.14.5 + - name: Update stackstorm-ha chart dependencies run: | set -x diff --git a/CHANGELOG.md b/CHANGELOG.md index c73a35e5..9351a86a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Development * Updated our tests/unit to support newer versions of `unittests` - for now bumping to `v0.4.4` as `v0.5.0` has a bug that impacts us (see helm-unittest/helm-unittest#329), but testing around the bug shows `v0.5.x` should also "just work" (#414) (by @jk464) +* Enable the use of TLS between all internal components (aside from Redis), and enable it by default (#401) (by @jk464) ## v1.1.0 * Fix syntax with ensure-packs-volumes-are-writable job (#403, #411) (by @skiedude) diff --git a/Chart.yaml b/Chart.yaml index a418bdd5..6d40e86b 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -33,7 +33,7 @@ dependencies: repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami condition: rabbitmq.enabled - name: mongodb - version: 10.0.1 + version: 13.18.5 repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami condition: mongodb.enabled - name: external-dns diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 9cadeeab..2f7a09f3 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -199,7 +199,7 @@ Reduce duplication of the st2.*.conf volume details {{- define "stackstorm-ha.init-containers-wait-for-db" -}} {{- if index .Values "mongodb" "enabled" }} -{{- $mongodb_port := (int (index .Values "mongodb" "service" "port")) }} +{{- $mongodb_port := (int (index .Values "mongodb" "service" "ports" "mongodb")) }} - name: wait-for-db image: {{ template "stackstorm-ha.utilityImage" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} @@ -422,3 +422,54 @@ Create the custom env list for each deployment value: {{ $value | quote }} {{- end }} {{- end -}} + +{{/* +Generate CA +*/}} +{{- define "stackstorm-ha.internal_tls.ca" }} +data: +{{- if (default false ((($.Values.secret).ca))) }} + tls.crt: "{{ .Values.secret.ca.crt }}" + tls.key: "{{ .Values.secret.ca.key }}" +{{- else }} +{{- $ca := genCA "StackStorm CA" 365 }} + tls.crt: "{{ $ca.Cert | b64enc}}" + tls.key: "{{ $ca.Key | b64enc}}" +{{- end -}} +{{- end -}} + +{{/* +Set up values for Internal TLS +*/}} +{{- define "stackstorm-ha.internal_tls.cert_volume.mount" -}} +{{- if or .Values.st2.tls.enabled .Values.mongodb.tls.enabled .Values.rabbitmq.tls.enabled }} +- name: {{ .Values.st2.tls.secretName }} + mountPath: {{ .Values.st2.tls.mountPath }}/ + readOnly: true +{{- end }} +{{- end -}} +{{- define "stackstorm-ha.internal_tls.cert_volume.volume" -}} +{{- if or .Values.st2.tls.enabled .Values.mongodb.tls.enabled .Values.rabbitmq.tls.enabled }} +- name: {{ .Values.st2.tls.secretName }} + secret: + secretName: {{ .Values.st2.tls.secretName }} +{{- end }} +{{- end -}} + +{{/* +Define st2web ports +*/}} +{{- define "stackstorm-ha.st2web.http_port" -}} +{{- if ne (default 0 ((($.Values.st2web.securityContext).runAsUser) | int)) 0 -}} +8080 +{{- else -}} +80 +{{- end -}} +{{- end -}} +{{- define "stackstorm-ha.st2web.https_port" -}} +{{- if ne (default 0 ((($.Values.st2web.securityContext).runAsUser) | int)) 0 -}} +8443 +{{- else -}} +443 +{{- end -}} +{{- end -}} diff --git a/templates/ca.yaml b/templates/ca.yaml new file mode 100644 index 00000000..f3d109e1 --- /dev/null +++ b/templates/ca.yaml @@ -0,0 +1,19 @@ +{{- if not ( .Values.st2.tls.certificate_issuer.existing ) -}} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ .Values.st2.tls.certificate_issuer.name }} + namespace: "{{ $.Release.Namespace }}" +spec: + ca: + secretName: {{ .Values.st2.tls.certificate_issuer.name }}-tls +--- +apiVersion: v1 +{{- include "stackstorm-ha.internal_tls.ca" . }} +kind: Secret +metadata: + name: {{ .Values.st2.tls.certificate_issuer.name }}-tls + namespace: "{{ $.Release.Namespace }}" +type: kubernetes.io/tls +{{- end -}} diff --git a/templates/certificate.yaml b/templates/certificate.yaml new file mode 100644 index 00000000..4104a886 --- /dev/null +++ b/templates/certificate.yaml @@ -0,0 +1,29 @@ +{{- if or .Values.st2.tls.enabled .Values.mongodb.tls.enabled .Values.rabbitmq.tls.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.st2.tls.secretName }} + namespace: "{{ $.Release.Namespace }}" + labels: + app: stackstorm + heritage: {{.Release.Service | quote}} + release: {{.Release.Name | quote}} + chart: {{ replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name }} +spec: + secretName: {{ .Values.st2.tls.secretName }} + dnsNames: + - "*.{{ $.Release.Namespace }}.svc.{{ $.Values.clusterDomain }}" +{{ include "stackstorm-ha.mongodb-nodes" $ | splitList "," | toYaml | indent 4 }} + ipAddresses: + - "127.0.0.1" + renewBefore: 360h # 15d + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 3072 + issuerRef: + name: {{ .Values.st2.tls.certificate_issuer.name }} + kind: Issuer + group: cert-manager.io +{{- end -}} diff --git a/templates/configmaps_st2-conf.yaml b/templates/configmaps_st2-conf.yaml index c2473b3a..6193167b 100644 --- a/templates/configmaps_st2-conf.yaml +++ b/templates/configmaps_st2-conf.yaml @@ -11,19 +11,49 @@ data: # The order of merging: st2.conf < st2.docker.conf < st2.user.conf st2.docker.conf: | [auth] + {{- if .Values.rabbitmq.tls.enabled }} + api_url = https://{{ .Release.Name }}-st2api:9111/ + {{- else }} api_url = http://{{ .Release.Name }}-st2api:9101/ + {{- end -}} + {{- if and .Values.st2.tls.enabled .Values.st2auth.tls.enabled }} + use_ssl = True + key = {{ .Values.st2.tls.mountPath }}/tls.key + cert = {{ .Values.st2.tls.mountPath }}/tls.crt + debug = False + enable = True + {{- else }} + use_ssl = False + {{- end }} + [system_user] user = {{ .Values.st2.system_user.user }} ssh_key_file = {{ tpl .Values.st2.system_user.ssh_key_file . }} {{- if index .Values "redis" "enabled" }} + [coordination] url = redis://{{ template "stackstorm-ha.redis-password" $ }}{{ template "stackstorm-ha.redis-nodes" $ }} {{- end }} {{- if index .Values "rabbitmq" "enabled" }} + [messaging] + {{- if .Values.rabbitmq.tls.enabled }} + url = amqp://{{ required "rabbitmq.auth.username is required!" (index .Values "rabbitmq" "auth" "username") }}:{{ required "rabbitmq.auth.password is required!" (index .Values "rabbitmq" "auth" "password") }}@{{ .Release.Name }}-rabbitmq:5671{{ required "rabbitmq.ingress.path is required!" (index .Values "rabbitmq" "ingress" "path") }} + {{- else }} url = amqp://{{ required "rabbitmq.auth.username is required!" (index .Values "rabbitmq" "auth" "username") }}:{{ required "rabbitmq.auth.password is required!" (index .Values "rabbitmq" "auth" "password") }}@{{ .Release.Name }}-rabbitmq:5672{{ required "rabbitmq.ingress.path is required!" (index .Values "rabbitmq" "ingress" "path") }} + {{- end -}} {{- end }} {{- if index .Values "mongodb" "enabled" }} + {{- if .Values.rabbitmq.tls.enabled }} + ssl = True + ssl_ca_certs = {{ .Values.st2.tls.mountPath }}/ca.crt + ssl_cert_reqs = optional + ssl_certfile = {{ .Values.st2.tls.mountPath }}/tls.crt + ssl_keyfile = {{ .Values.st2.tls.mountPath }}/tls.key + {{- else }} + ssl = False + {{- end }} + [database] {{- if index .Values "mongodb" "auth" "enabled" }} host = mongodb://{{ template "stackstorm-ha.mongodb-nodes" $ }}/{{ required "mongodb.auth.database is required!" (index .Values "mongodb" "auth" "database") }}?authSource={{ required "mongodb.auth.database is required!" (index .Values "mongodb" "auth" "database") }}&replicaSet={{ index .Values "mongodb" "replicaSetName" }} @@ -36,10 +66,21 @@ data: port = {{ index .Values "mongodb" "service" "port" }} {{- end }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} + {{- if .Values.mongodb.tls.enabled }} + ssl = True + ssl_ca_certs = {{ .Values.st2.tls.mountPath }}/ca.crt + ssl_cert_reqs = optional + ssl_certfile = {{ .Values.st2.tls.mountPath }}/tls.crt + ssl_keyfile = {{ .Values.st2.tls.mountPath }}/tls.key + {{- else }} + ssl = False + {{- end }} + [keyvalue] encryption_key_path = /etc/st2/keys/datastore_key.json {{- end }} {{- if .Values.st2.rbac.enabled }} + [rbac] enable = True backend = default diff --git a/templates/configmaps_st2-urls.yaml b/templates/configmaps_st2-urls.yaml index e02dccc1..53158c18 100644 --- a/templates/configmaps_st2-urls.yaml +++ b/templates/configmaps_st2-urls.yaml @@ -7,6 +7,18 @@ metadata: description: StackStorm service URLs, used across entire st2 cluster labels: {{- include "stackstorm-ha.labels" (list $ "st2") | nindent 4 }} data: + {{- if and .Values.st2.tls.enabled .Values.st2auth.tls.enabled }} + ST2_AUTH_URL: https://{{ .Release.Name }}-st2auth:9100/ + {{- else }} ST2_AUTH_URL: http://{{ .Release.Name }}-st2auth:9100/ + {{- end }} + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + ST2_API_URL: https://{{ .Release.Name }}-st2api:9111/ + {{- else }} ST2_API_URL: http://{{ .Release.Name }}-st2api:9101/ + {{- end }} + {{- if and .Values.st2.tls.enabled .Values.st2stream.tls.enabled }} + ST2_STREAM_URL: https://{{ .Release.Name }}-st2stream:9112/ + {{- else }} ST2_STREAM_URL: http://{{ .Release.Name }}-st2stream:9102/ + {{- end }} diff --git a/templates/deployments.yaml b/templates/deployments.yaml index b5ea91c3..64a7996c 100644 --- a/templates/deployments.yaml +++ b/templates/deployments.yaml @@ -86,6 +86,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} - name: htpasswd-vol mountPath: /etc/st2/htpasswd @@ -115,6 +116,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} - name: htpasswd-vol emptyDir: @@ -198,8 +200,11 @@ spec: {{- with .Values.securityContext }} securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{/* When using internal TLS don't expose API Port outside pod */}} + {{- if not (and .Values.st2.tls.enabled .Values.st2api.tls.enabled) }} ports: - containerPort: 9101 + {{- end }} # TODO: Add liveness/readiness probes (#3) #livenessProbe: #readinessProbe: @@ -214,6 +219,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -244,10 +250,45 @@ spec: {{- end }} resources: {{- toYaml .Values.st2api.resources | nindent 10 }} + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + # StackStorm doesn't provide TLS for the API endpoint. + # Therefore when using Internal TLS, we need use ghostunnel (https://github.com/ghostunnel/ghostunnel) to provide a TLS proxy layer + - name: tls-proxy + image: {{ .Values.ghostunnel.image.repository }}/{{ .Values.ghostunnel.image.name }}:{{ .Values.ghostunnel.image.tag }} + imagePullPolicy: {{ .Values.ghostunnel.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: {{- toYaml . | nindent 10 }} + {{- end }} + args: + - server + - --listen=:9111 + - --target=127.0.0.1:9101 + - --key={{ .Values.st2.tls.mountPath }}/tls.key + - --cert={{ .Values.st2.tls.mountPath }}/tls.crt + - --disable-authentication + resources: + requests: + memory: 25Mi + cpu: 100m + limits: + memory: 50Mi + cpu: 200m + ports: + - containerPort: 9111 + name: https + protocol: TCP + volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 10 }} + {{- range .Values.st2api.extra_volumes }} + - name: {{ required "Each volume must have a 'name' in st2api.extra_volumes" .name }} + {{- tpl (required "Each volume must have a 'mount' definition in st2api.extra_volumes" .mount | toYaml) $ | nindent 12 }} + {{- end }} + {{- end }} {{- if .Values.st2api.serviceAccount.attach }} serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: @@ -331,8 +372,11 @@ spec: {{- with .Values.securityContext }} securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{/* When using internal TLS don't expose Stream Port outside pod */}} + {{- if not (and .Values.st2.tls.enabled .Values.st2stream.tls.enabled) }} ports: - containerPort: 9102 + {{- end }} # TODO: Add liveness/readiness probes (#3) #livenessProbe: #readinessProbe: @@ -347,6 +391,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- range .Values.st2stream.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2stream.extra_volumes" .name }} @@ -368,10 +413,45 @@ spec: {{- end }} resources: {{- toYaml .Values.st2stream.resources | nindent 10 }} + {{- if and .Values.st2.tls.enabled .Values.st2stream.tls.enabled }} + # StackStorm doesn't provide TLS for the Stream endpoint. + # Therefore when using Internal TLS, we need use ghostunnel (https://github.com/ghostunnel/ghostunnel) to provide a TLS proxy layer + - name: tls-proxy + image: {{ .Values.ghostunnel.image.repository }}/{{ .Values.ghostunnel.image.name }}:{{ .Values.ghostunnel.image.tag }} + imagePullPolicy: {{ .Values.ghostunnel.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: {{- toYaml . | nindent 10 }} + {{- end }} + args: + - server + - --listen=:9112 + - --target=127.0.0.1:9102 + - --key={{ .Values.st2.tls.mountPath }}/tls.key + - --cert={{ .Values.st2.tls.mountPath }}/tls.crt + - --disable-authentication + resources: + requests: + memory: 25Mi + cpu: 100m + limits: + memory: 50Mi + cpu: 200m + ports: + - containerPort: 9112 + name: https + protocol: TCP + volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} + {{- range .Values.st2stream.extra_volumes }} + - name: {{ required "Each volume must have a 'name' in st2stream.extra_volumes" .name }} + {{- tpl (required "Each volume must have a 'mount' definition in st2stream.extra_volumes" .mount | toYaml) $ | nindent 12 }} + {{- end }} + {{- end }} {{- if .Values.st2stream.serviceAccount.attach }} serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- range .Values.st2stream.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2stream.extra_volumes" .name }} @@ -439,13 +519,15 @@ spec: securityContext: {{- toYaml . | nindent 10 }} {{- end }} ports: - - containerPort: {{ eq (get .Values.st2web.env "ST2WEB_HTTPS" | toString) "1" | ternary 443 80 }} + {{- $https_port := include "stackstorm-ha.st2web.https_port" . }} + {{- $http_port := include "stackstorm-ha.st2web.http_port" . }} + - containerPort: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}{{ $https_port }}{{- else -}}{{ $http_port }}{{- end }} # Probe to check if app is running. Failure will lead to a pod restart. livenessProbe: httpGet: - scheme: {{ eq (get .Values.st2web.env "ST2WEB_HTTPS" | toString) "1" | ternary "HTTPS" "HTTP" }} + scheme: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}HTTPS{{- else -}}HTTP{{- end }} path: / - port: {{ eq (get .Values.st2web.env "ST2WEB_HTTPS" | toString) "1" | ternary 443 80 }} + port: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}{{ $https_port }}{{- else -}}{{ $http_port }}{{- end }} initialDelaySeconds: 1 # Probe to check if app is ready to serve traffic. Failure will lead to temp stop serving traffic. # TODO: Failing to add readinessProbe, since st2 requires authorization (401) and we don't have `/healthz` endpoints yet (https://github.com/StackStorm/st2/issues/4020) @@ -458,17 +540,32 @@ spec: # path: /api/ # port: 443 # initialDelaySeconds: 3 + env: {{- if .Values.st2web.env }} - env: {{- include "stackstorm-ha.customEnv" .Values.st2web | nindent 8 }} + {{- include "stackstorm-ha.customEnv" .Values.st2web | nindent 8 }} {{- end }} + {{- if and .Values.st2.tls.enabled .Values.st2web.tls.enabled }} + - name: "ST2WEB_HTTPS" + value: "1" + {{- end }} + - name: "ST2_PORT_HTTP" + value: "{{ $http_port }}" + - name: "ST2_PORT_HTTPS" + value: "{{ $https_port }}" envFrom: - configMapRef: name: {{ .Release.Name }}-st2-urls - {{- if or .Values.st2web.config .Values.st2web.extra_volumes .Values.st2web.postStartScript }} + {{- if or .Values.st2web.config .Values.st2web.extra_volumes .Values.st2web.postStartScript .Values.st2web.tls.enabled }} volumeMounts: {{- else }} volumeMounts: [] {{- end }} + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 10 }} + {{- if and .Values.st2.tls.enabled .Values.st2web.tls.enabled }} + - name: {{ .Values.st2.tls.secretName }}-st2web + mountPath: "/etc/ssl/st2/" + readOnly: true + {{- end }} {{- range .Values.st2web.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2web.extra_volumes" .name }} {{- tpl (required "Each volume must have a 'mount' definition in st2web.extra_volumes" .mount | toYaml) $ | nindent 12 }} @@ -497,11 +594,22 @@ spec: {{- if .Values.st2web.serviceAccount.attach }} serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} - {{- if or .Values.st2web.config .Values.st2web.extra_volumes .Values.st2web.postStartScript }} + {{- if or .Values.st2web.config .Values.st2web.extra_volumes .Values.st2web.postStartScript .Values.st2web.tls.enabled }} volumes: {{- else }} volumes: [] {{- end }} + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} + {{- if and .Values.st2.tls.enabled .Values.st2web.tls.enabled }} + - name: {{ .Values.st2.tls.secretName }}-st2web + secret: + secretName: {{ .Values.st2.tls.secretName }} + items: + - key: tls.crt + path: st2.crt + - key: tls.key + path: st2.key + {{- end }} {{- range .Values.st2web.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2web.extra_volumes" .name }} {{- tpl (required "Each volume must have a 'volume' definition in st2web.extra_volumes" .volume | toYaml) $ | nindent 10 }} @@ -593,6 +701,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -618,6 +727,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -711,6 +821,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- range .Values.st2timersengine.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2timersengine.extra_volumes" .name }} @@ -731,6 +842,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- range .Values.st2timersengine.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2timersengine.extra_volumes" .name }} @@ -819,6 +931,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -844,6 +957,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -938,6 +1052,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -963,6 +1078,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: @@ -1055,6 +1171,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- range .Values.st2notifier.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2notifier.extra_volumes" .name }} @@ -1075,6 +1192,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- range .Values.st2notifier.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2notifier.extra_volumes" .name }} @@ -1242,6 +1360,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" $ | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" $ | nindent 8 }} {{- include "stackstorm-ha.packs-volume-mounts" $ | nindent 8 }} {{- if $some_sensors_per_pod }} @@ -1274,6 +1393,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" $ }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" $ | nindent 8 }} {{- if ne "disable" (default "" $.Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: @@ -1398,6 +1518,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} - name: st2-ssh-key-vol mountPath: {{ tpl .Values.st2.system_user.ssh_key_file . | dir | dir }}/.ssh-key-vol/ @@ -1427,6 +1548,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: @@ -1529,6 +1651,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- range .Values.st2garbagecollector.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2garbagecollector.extra_volumes" .name }} @@ -1549,6 +1672,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- range .Values.st2garbagecollector.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2garbagecollector.extra_volumes" .name }} @@ -1672,6 +1796,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if .Values.st2.rbac.enabled }} - name: st2-rbac-roles-vol @@ -1712,6 +1837,7 @@ spec: resources: {{- toYaml .Values.st2client.resources | nindent 10 }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: diff --git a/templates/jobs.yaml b/templates/jobs.yaml index dfbca877..19774b89 100644 --- a/templates/jobs.yaml +++ b/templates/jobs.yaml @@ -51,6 +51,7 @@ spec: {{- end }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} - name: st2-rbac-roles-vol mountPath: /opt/stackstorm/rbac/roles/ @@ -65,6 +66,7 @@ spec: # TODO: Find out default resource limits for this specific service (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} - name: st2-rbac-roles-vol configMap: @@ -143,7 +145,11 @@ spec: - 'sh' - '-c' - > + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + until nc -z -w 2 {{ .Release.Name }}-st2api 9111 && echo st2api ready; + {{- else }} until nc -z -w 2 {{ .Release.Name }}-st2api 9101 && echo st2api ready; + {{- end }} do sleep 2; done # Sidecar container for generating st2client config with st2 username & password pair and sharing produced file with the main container @@ -163,6 +169,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ # `st2 login` doesn't exit on failure correctly, use old methods instead. See bug: https://github.com/StackStorm/st2/issues/4338 @@ -196,6 +203,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ - name: st2-apikeys-vol @@ -208,6 +216,7 @@ spec: # TODO: Find out default resource limits for this specific service (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} - name: st2client-config-vol emptyDir: medium: Memory @@ -289,6 +298,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ # `st2 login` doesn't exit on failure correctly, use old methods instead. See bug: https://github.com/StackStorm/st2/issues/4338 @@ -323,6 +333,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ @@ -336,6 +347,7 @@ spec: # TODO: Find out default resource limits for this specific service (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} - name: st2client-config-vol emptyDir: @@ -418,6 +430,7 @@ spec: {{- end }} command: {{- toYaml $.Values.jobs.preRegisterContentCommand | nindent 8 }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.overrides-config-mounts" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- include "stackstorm-ha.pack-configs-volume-mount" . | nindent 8 }} @@ -447,6 +460,7 @@ spec: {{- end }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.overrides-config-mounts" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- include "stackstorm-ha.packs-volume-mounts-for-register-job" . | nindent 8 }} @@ -458,6 +472,7 @@ spec: # TODO: Find out default resource limits for this specific service (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.overrides-configs" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- include "stackstorm-ha.packs-volumes" . | nindent 8 }} @@ -567,6 +582,7 @@ spec: echo DONE volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.packs-volume-mounts" $ | nindent 8 }} {{/* do not include the pack-configs-volume-mount helper here */}} - name: st2-pack-configs-vol @@ -575,6 +591,7 @@ spec: # TODO: Find out default resource limits for this specific job (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.packs-volumes" $ | nindent 8 }} {{- if $.Values.st2.packs.volumes.configs }} {{/* do not include the pack-configs-volume helper here */}} @@ -659,6 +676,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ # `st2 login` doesn't exit on failure correctly, use old methods instead. See bug: https://github.com/StackStorm/st2/issues/4338 @@ -693,6 +711,7 @@ spec: volumeMounts: - name: st2client-config-vol mountPath: /root/.st2/ + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.overrides-config-mounts" $ | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" $ | nindent 8 }} {{- include "stackstorm-ha.packs-volume-mounts-for-register-job" $ | nindent 8 }} @@ -708,6 +727,7 @@ spec: - name: st2client-config-vol emptyDir: medium: Memory + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.overrides-configs" $ | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" $ | nindent 8 }} {{- include "stackstorm-ha.packs-volumes" $ | nindent 8 }} diff --git a/templates/services.yaml b/templates/services.yaml index 6b0e385d..57ca35aa 100644 --- a/templates/services.yaml +++ b/templates/services.yaml @@ -48,7 +48,11 @@ spec: {{- end }} ports: - protocol: TCP + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + port: 9111 + {{- else }} port: 9101 + {{- end }} --- kind: Service @@ -74,7 +78,11 @@ spec: {{- end }} ports: - protocol: TCP + {{- if and .Values.st2.tls.enabled .Values.st2stream.tls.enabled }} + port: 9112 + {{- else }} port: 9102 + {{- end }} --- kind: Service @@ -99,8 +107,12 @@ spec: {{- end }} {{- end }} ports: - - protocol: TCP - port: {{ eq (get .Values.st2web.env "ST2WEB_HTTPS" | toString) "1" | ternary 443 80 }} + {{- $https_port := include "stackstorm-ha.st2web.https_port" . }} + {{- $http_port := include "stackstorm-ha.st2web.http_port" . }} + - name: st2web + protocol: TCP + port: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}{{ $https_port }}{{- else -}}{{ $http_port }}{{- end }} + targetPort: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}{{ $https_port }}{{- else -}}{{ $http_port }}{{- end }} {{ if .Values.st2chatops.enabled -}} --- diff --git a/templates/tests/st2tests-pod.yaml b/templates/tests/st2tests-pod.yaml index 5c7ed8c9..e742af92 100644 --- a/templates/tests/st2tests-pod.yaml +++ b/templates/tests/st2tests-pod.yaml @@ -51,7 +51,20 @@ spec: value: "{{ .Chart.AppVersion }}" - name: ST2_RBAC_ENABLED value: "{{ .Values.st2.rbac.enabled }}" + {{- if and .Values.st2.tls.enabled .Values.st2auth.tls.enabled }} + - name: ST2_AUTH_TLS + value: "--cacert {{ .Values.st2.tls.mountPath }}/ca.crt" + {{- end }} + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + - name: ST2_API_TLS + value: "--cacert {{ .Values.st2.tls.mountPath }}/ca.crt" + {{- end }} + {{- if and .Values.st2.tls.enabled .Values.st2stream.tls.enabled }} + - name: ST2_STEAM_TLS + value: "--cacert {{ .Values.st2.tls.mountPath }}/ca.crt" + {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 4 }} - name: tools mountPath: /tools - name: tests @@ -60,6 +73,7 @@ spec: - /tools/bats/bin/bats - /tests/st2tests.sh volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 4 }} - name: tools emptyDir: {} - name: tests diff --git a/tests/integration/st2tests.sh b/tests/integration/st2tests.sh index 44470a62..a0228643 100755 --- a/tests/integration/st2tests.sh +++ b/tests/integration/st2tests.sh @@ -13,19 +13,19 @@ load "${BATS_HELPERS_DIR}/bats-file/load.bash" } @test 'ST2_AUTH_URL service endpoint is accessible and working' { - run curl -v ${ST2_API_URL} + run curl -v ${ST2_API_URL} ${ST2_AUTH_TLS} assert_line --partial 'Content-Type: application/json' assert_line --partial 'St2-Api-Key' } @test 'ST2_API_URL service endpoint is accessible and working' { - run curl -v ${ST2_API_URL} + run curl -v ${ST2_API_URL} ${ST2_API_TLS} assert_line --partial 'Content-Type: application/json' assert_line --partial 'St2-Api-Key' } @test 'ST2_STREAM_URL service endpoint is accessible and working' { - run curl -v ${ST2_API_URL} + run curl -v ${ST2_API_URL} ${ST2_STREAM_TLS} assert_line --partial 'Content-Type: application/json' assert_line --partial 'St2-Api-Key' } diff --git a/values.yaml b/values.yaml index 8f4d90fc..5795e837 100644 --- a/values.yaml +++ b/values.yaml @@ -96,11 +96,11 @@ st2: [api] allow_origin = '*' # fixes no replicaset found bug; + [database] # Connection and server selection timeout (in ms). connection_timeout = 5000 - #Override Definitions can be added here. #https://docs.stackstorm.com/latest/packs.html#overriding-pack-defaults overrides: {} @@ -296,6 +296,15 @@ st2: # roles: # - "admin" + # Controls configuring TLS between internal inter-pod communications + tls: + enabled: true + secretName: "internal-tls" + mountPath: "/etc/ssl/internal" + certificate_issuer: + existing: false + name: stackstorm-issuer + ## ## Default SecurityContext for pods and containers. ## Overrides available for st2web, st2actionrunner, st2sensorcontainer, st2client pods, and custom packs images. @@ -392,6 +401,9 @@ st2web: # see: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ postStartScript: "" preStopSleep: "10" + # Use TLS on HTTP connections to st2web (i.e. between pod and ingress) + tls: + enabled: true # https://docs.stackstorm.com/reference/ha.html#st2auth # Multiple st2auth processes can be behind a load balancer in an active-active configuration. @@ -432,6 +444,9 @@ st2auth: ## Note that Helm templating is supported in 'mount' and 'volume' extra_volumes: [] # see examples under st2actionrunner.extra_volumes + # Use TLS on HTTP connections to st2auth (i.e. between pod and ingress, and inter-pod) + tls: + enabled: true # https://docs.stackstorm.com/reference/ha.html#st2api # Multiple st2api process can be behind a load balancer in an active-active configuration. @@ -472,6 +487,9 @@ st2api: ## Note that Helm templating is supported in 'mount' and 'volume' extra_volumes: [] # see examples under st2actionrunner.extra_volumes + # Use TLS on HTTP connections to st2api (i.e. between pod and ingress, and inter-pod) + tls: + enabled: true # https://docs.stackstorm.com/reference/ha.html#st2stream # Multiple st2stream process can be behind a load balancer in an active-active configuration. @@ -512,6 +530,9 @@ st2stream: ## Note that Helm templating is supported in 'mount' and 'volume' extra_volumes: [] # see examples under st2actionrunner.extra_volumes + # Use TLS on HTTP connections to st2stream (i.e. between pod and ingress, and inter-pod) + tls: + enabled: true # https://docs.stackstorm.com/reference/ha.html#st2rulesengine # Multiple st2rulesengine processes can run in active-active with only connections to MongoDB and RabbitMQ. All these will share the TriggerInstance load and naturally pick up more work if one or more of the processes becomes unavailable. @@ -1050,6 +1071,13 @@ mongodb: arbiter: enabled: false resources: {} + tls: + enabled: true + replicaset: + existingSecrets: + - internal-tls + - internal-tls + - internal-tls ## ## RabbitMQ configuration (3rd party chart dependency) @@ -1067,6 +1095,10 @@ rabbitmq: forceBoot: true # Authentication Details auth: + tls: + enabled: true + existingSecret: "internal-tls" + # Fail over to username admin if LDAP is down: username: admin # TODO: Use default random 10 character password, but need to fetch this string for use by downstream services password: 9jS+w1u07NbHtZke1m+jW4Cj @@ -1097,7 +1129,9 @@ rabbitmq: # As RabbitMQ enabled prometheus operator monitoring by default, disable it for non-prometheus users metrics: enabled: false - + tls: + enabled: true + existingSecret: "internal-tls" ## ## Redis HA configuration (3rd party chart dependency) ## @@ -1133,6 +1167,13 @@ redis: usePassword: false metrics: enabled: false + # tls: + # enabled: true + # authClients: false + # certificatesSecret: internal-tls + # certFilename: tls.crt + # certKeyFilename: tls.key + # certCAFilename: ca.crt ## ## Settings to be applied to all stackstorm-ha pods @@ -1168,3 +1209,13 @@ external-dns: aws: zoneType: "public" domainFilters: [] + +## +## Image details for ghostunnel +## +ghostunnel: + image: + name: ghostunnel + repository: ghostunnel + tag: v1.6.0 + pullPolicy: IfNotPresent