From 22ba75f75d7c2e739e51e978df7b3ced51710838 Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Fri, 20 Mar 2020 23:51:17 -0700 Subject: [PATCH 1/5] Add support for using NATS Account Server as the resolver Signed-off-by: Waldemar Quevedo --- helm/charts/nats-account-server/Chart.yaml | 20 ++++ .../templates/_helpers.tpl | 6 ++ .../templates/configmap.yaml | 33 +++++++ .../templates/service.yaml | 15 +++ .../templates/statefulset.yaml | 92 +++++++++++++++++++ helm/charts/nats-account-server/values.yaml | 47 ++++++++++ helm/charts/nats/templates/configmap.yaml | 7 ++ helm/charts/nats/values.yaml | 13 ++- 8 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 helm/charts/nats-account-server/Chart.yaml create mode 100644 helm/charts/nats-account-server/templates/_helpers.tpl create mode 100644 helm/charts/nats-account-server/templates/configmap.yaml create mode 100644 helm/charts/nats-account-server/templates/service.yaml create mode 100644 helm/charts/nats-account-server/templates/statefulset.yaml create mode 100644 helm/charts/nats-account-server/values.yaml diff --git a/helm/charts/nats-account-server/Chart.yaml b/helm/charts/nats-account-server/Chart.yaml new file mode 100644 index 00000000..2d824633 --- /dev/null +++ b/helm/charts/nats-account-server/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v2 +appVersion: "0.8.6" +description: A Helm chart for the NATS.io JWT Account Server +name: nats-account-server +keywords: + - nats + - messaging + - cncf + - jwt + - auth +version: 0.3.0 +home: http://github.com/nats-io/k8s +maintainers: + - name: Waldemar Quevedo + github: https://github.com/wallyqs + email: wally@nats.io + - name: Colin Sullivan + github: https://github.com/ColinSullivan1 + email: colin@nats.io +icon: https://nats.io/img/logo.png diff --git a/helm/charts/nats-account-server/templates/_helpers.tpl b/helm/charts/nats-account-server/templates/_helpers.tpl new file mode 100644 index 00000000..36023b63 --- /dev/null +++ b/helm/charts/nats-account-server/templates/_helpers.tpl @@ -0,0 +1,6 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Release.Name -}} +{{- end -}} diff --git a/helm/charts/nats-account-server/templates/configmap.yaml b/helm/charts/nats-account-server/templates/configmap.yaml new file mode 100644 index 00000000..4b771425 --- /dev/null +++ b/helm/charts/nats-account-server/templates/configmap.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "name" . }}-config + labels: + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +data: + accountserver.conf: | + # Host/Port for the NATS Account Server + http { + host: "0.0.0.0" + port: 9090 + } + + # Operator JWT used to validate the accounts. + operatorjwtpath: "/etc/nats-config/operator/{{ .Values.operator.operatorjwt.configMap.key }}" + + # System Account JWT + systemaccountjwtpath: "/etc/nats-config/sys/{{ .Values.operator.systemaccountjwt.configMap.key }}" + + # NATS Server connection + nats { + servers: [{{ .Values.nats.url }}] + usercredentials: "/etc/nats-config/syscreds/{{ .Values.nats.credentials.secret.key }}" + } + + {{- if eq .Values.store.type "file"}} + store { + dir: "/store" + } + {{- end }} \ No newline at end of file diff --git a/helm/charts/nats-account-server/templates/service.yaml b/helm/charts/nats-account-server/templates/service.yaml new file mode 100644 index 00000000..3a194fdd --- /dev/null +++ b/helm/charts/nats-account-server/templates/service.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "name" . }} + labels: + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +spec: + selector: + app: {{ template "name" . }} + clusterIP: None + ports: + - name: server + port: 9090 diff --git a/helm/charts/nats-account-server/templates/statefulset.yaml b/helm/charts/nats-account-server/templates/statefulset.yaml new file mode 100644 index 00000000..be4b1fd1 --- /dev/null +++ b/helm/charts/nats-account-server/templates/statefulset.yaml @@ -0,0 +1,92 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "name" . }} + labels: + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +spec: + selector: + matchLabels: + app: {{ template "name" . }} + replicas: 1 + serviceName: {{ template "name" . }} + {{- if eq .Values.store.type "file"}} + volumeClaimTemplates: + - metadata: + name: {{ template "name" . }}-pvc + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.store.file.storageSize }} + {{- end }} + template: + metadata: + labels: + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + spec: + volumes: + - name: config-volume + configMap: + name: {{ template "name" . }}-config + - name: nats-sys-creds + secret: + secretName: {{ .Values.nats.credentials.secret.name }} + - name: system-account-jwt-volume + configMap: + name: {{ .Values.operator.systemaccountjwt.configMap.name }} + - name: operator-jwt-volume + configMap: + name: {{ .Values.operator.operatorjwt.configMap.name }} + + ######################## + # # + # NATS Account Server # + # # + ######################## + terminationGracePeriodSeconds: 60 + containers: + - name: nats-account-server + image: {{ .Values.accountserver.image }} + imagePullPolicy: {{ .Values.accountserver.pullPolicy }} + ports: + - containerPort: 9090 + # hostPort: 9090 + name: server + command: + - "nats-account-server" + - "-c" + - "/etc/nats-config/conf/accountserver.conf" + + volumeMounts: + - name: config-volume + mountPath: /etc/nats-config/conf + - name: system-account-jwt-volume + mountPath: /etc/nats-config/sys + - name: operator-jwt-volume + mountPath: /etc/nats-config/operator + - name: nats-sys-creds + mountPath: /etc/nats-config/syscreds + {{- if eq .Values.store.type "file"}} + - name: {{ template "name" . }}-pvc + mountPath: /store + {{- end }} + + # Liveness/Readiness probes against the monitoring. + # + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + timeoutSeconds: 5 diff --git a/helm/charts/nats-account-server/values.yaml b/helm/charts/nats-account-server/values.yaml new file mode 100644 index 00000000..3efbca41 --- /dev/null +++ b/helm/charts/nats-account-server/values.yaml @@ -0,0 +1,47 @@ +# Copyright 2020 The NATS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +accountserver: + image: "synadia/nats-account-server:0.8.4" + pullPolicy: IfNotPresent + +store: + type: file + file: + storageSize: 1Gi + +# NATS Server connection settings. +nats: + # NATS Service to which we can connect. + url: "nats://nats:4222" + + # Credentials to connect to the NATS Server. + credentials: + secret: + name: nats-sys-creds + key: sys.creds + +# Trusted Operator mode settings. +operator: + # Reference to the system account jwt. + systemaccountjwt: + configMap: + name: nats-sys-jwt + key: SYS.jwt + + # Reference to the operator jwt. + operatorjwt: + configMap: + name: operator-jwt + key: KO.jwt diff --git a/helm/charts/nats/templates/configmap.yaml b/helm/charts/nats/templates/configmap.yaml index abff5f3e..02df7c51 100644 --- a/helm/charts/nats/templates/configmap.yaml +++ b/helm/charts/nats/templates/configmap.yaml @@ -118,4 +118,11 @@ data: resolver: MEMORY include "accounts/{{ .Values.auth.resolver.configMap.key }}" {{- end }} + + {{- if eq .Values.auth.resolver.type "URL" }} + resolver: URL({{ .Values.auth.resolver.url }}) + operator: {{ .Values.auth.operator }} + system_account: {{ .Values.auth.systemAccount }} + {{- end }} + {{- end }} diff --git a/helm/charts/nats/values.yaml b/helm/charts/nats/values.yaml index e1d7a22a..004c3d31 100644 --- a/helm/charts/nats/values.yaml +++ b/helm/charts/nats/values.yaml @@ -91,8 +91,19 @@ exporter: # Authentication setup auth: enabled: false + + # Operator JWT + operator: + + # Public key of the System Account + systemAccount: + resolver: - type: memory + type: URL + + # URL resolver settings + url: "http://nats-account-server:9090/jwt/v1/accounts/" + # # Use a configmap reference which will be mounted # into the container. From 5da76aa7d1acf7ce3e77f6740ad2d00534d703dd Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Mon, 23 Mar 2020 10:54:20 -0700 Subject: [PATCH 2/5] Add NATS Account Server Helm chart Signed-off-by: Waldemar Quevedo --- .../templates/configmap.yaml | 2 + helm/charts/nats-account-server/values.yaml | 42 +++++++-------- helm/charts/nats/templates/configmap.yaml | 2 +- helm/charts/nats/templates/nats-box.yaml | 16 ++++++ helm/charts/nats/templates/statefulset.yaml | 11 ++++ helm/charts/nats/values.yaml | 51 ++++++++++++------- 6 files changed, 85 insertions(+), 39 deletions(-) diff --git a/helm/charts/nats-account-server/templates/configmap.yaml b/helm/charts/nats-account-server/templates/configmap.yaml index 4b771425..c125c28f 100644 --- a/helm/charts/nats-account-server/templates/configmap.yaml +++ b/helm/charts/nats-account-server/templates/configmap.yaml @@ -17,6 +17,7 @@ data: # Operator JWT used to validate the accounts. operatorjwtpath: "/etc/nats-config/operator/{{ .Values.operator.operatorjwt.configMap.key }}" + {{- if .Values.nats.url }} # System Account JWT systemaccountjwtpath: "/etc/nats-config/sys/{{ .Values.operator.systemaccountjwt.configMap.key }}" @@ -25,6 +26,7 @@ data: servers: [{{ .Values.nats.url }}] usercredentials: "/etc/nats-config/syscreds/{{ .Values.nats.credentials.secret.key }}" } + {{- end }} {{- if eq .Values.store.type "file"}} store { diff --git a/helm/charts/nats-account-server/values.yaml b/helm/charts/nats-account-server/values.yaml index 3efbca41..eed8008a 100644 --- a/helm/charts/nats-account-server/values.yaml +++ b/helm/charts/nats-account-server/values.yaml @@ -22,26 +22,26 @@ store: storageSize: 1Gi # NATS Server connection settings. -nats: - # NATS Service to which we can connect. - url: "nats://nats:4222" - - # Credentials to connect to the NATS Server. - credentials: - secret: - name: nats-sys-creds - key: sys.creds +# nats: +# # # NATS Service to which we can connect. +# # url: "nats://nats:4222" +# # +# # # Credentials to connect to the NATS Server. +# # credentials: +# # secret: +# # name: nats-sys-creds +# # key: sys.creds # Trusted Operator mode settings. -operator: - # Reference to the system account jwt. - systemaccountjwt: - configMap: - name: nats-sys-jwt - key: SYS.jwt - - # Reference to the operator jwt. - operatorjwt: - configMap: - name: operator-jwt - key: KO.jwt +# operator: +# # # Reference to the system account jwt. +# # systemaccountjwt: +# # configMap: +# # name: nats-sys-jwt +# # key: SYS.jwt +# # +# # # Reference to the Operator JWT. +# # operatorjwt: +# # configMap: +# # name: operator-jwt +# # key: KO.jwt diff --git a/helm/charts/nats/templates/configmap.yaml b/helm/charts/nats/templates/configmap.yaml index 02df7c51..3e3cefb0 100644 --- a/helm/charts/nats/templates/configmap.yaml +++ b/helm/charts/nats/templates/configmap.yaml @@ -121,7 +121,7 @@ data: {{- if eq .Values.auth.resolver.type "URL" }} resolver: URL({{ .Values.auth.resolver.url }}) - operator: {{ .Values.auth.operator }} + operator: /etc/nats-config/operator/{{ .Values.auth.operatorjwt.configMap.key }} system_account: {{ .Values.auth.systemAccount }} {{- end }} diff --git a/helm/charts/nats/templates/nats-box.yaml b/helm/charts/nats/templates/nats-box.yaml index 7a89e229..6489d166 100644 --- a/helm/charts/nats/templates/nats-box.yaml +++ b/helm/charts/nats/templates/nats-box.yaml @@ -8,6 +8,13 @@ metadata: app: {{ .Release.Name }}-box chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} spec: + volumes: + {{- if .Values.natsbox.credentials }} + - name: nats-sys-creds + secret: + secretName: {{ .Values.natsbox.credentials.secret.name }} + {{- end }} + containers: - name: nats-box image: {{ .Values.natsbox.image }} @@ -15,8 +22,17 @@ spec: env: - name: NATS_URL value: {{ .Release.Name }} + {{- if .Values.natsbox.credentials }} + - name: USER_CREDS + value: /etc/nats-config/creds/{{ .Values.natsbox.credentials.secret.key }} + {{- end }} command: - "tail" - "-f" - "/dev/null" + {{- if .Values.natsbox.credentials }} + volumeMounts: + - name: nats-sys-creds + mountPath: /etc/nats-config/creds + {{- end }} {{- end }} diff --git a/helm/charts/nats/templates/statefulset.yaml b/helm/charts/nats/templates/statefulset.yaml index 66856951..ad9fd80b 100644 --- a/helm/charts/nats/templates/statefulset.yaml +++ b/helm/charts/nats/templates/statefulset.yaml @@ -43,6 +43,12 @@ spec: configMap: name: {{ .Values.auth.resolver.configMap.name }} {{- end }} + + {{- if eq .Values.auth.resolver.type "URL" }} + - name: operator-jwt-volume + configMap: + name: {{ .Values.auth.operatorjwt.configMap.name }} + {{- end }} {{- end }} {{ if and .Values.nats.externalAccess .Values.nats.advertise }} @@ -154,6 +160,11 @@ spec: - name: resolver-volume mountPath: /etc/nats-config/accounts {{- end }} + + {{- if eq .Values.auth.resolver.type "URL" }} + - name: operator-jwt-volume + mountPath: /etc/nats-config/operator + {{- end }} {{- end }} # Liveness/Readiness probes against the monitoring. diff --git a/helm/charts/nats/values.yaml b/helm/charts/nats/values.yaml index 004c3d31..5bceba27 100644 --- a/helm/charts/nats/values.yaml +++ b/helm/charts/nats/values.yaml @@ -76,6 +76,11 @@ natsbox: image: synadia/nats-box:0.3.0 pullPolicy: IfNotPresent + # credentials: + # secret: + # name: nats-sys-creds + # key: sys.creds + # The NATS config reloader image to use. reloader: enabled: true @@ -92,22 +97,34 @@ exporter: auth: enabled: false - # Operator JWT - operator: + # Reference to the Operator JWT. + # operatorjwt: + # configMap: + # name: operator-jwt + # key: KO.jwt # Public key of the System Account - systemAccount: - - resolver: - type: URL - - # URL resolver settings - url: "http://nats-account-server:9090/jwt/v1/accounts/" - - # - # Use a configmap reference which will be mounted - # into the container. - # - configMap: - name: nats-accounts - key: resolver.conf + # systemAccount: + + # resolver: + # ############################ + # # # + # # Memory resolver settings # + # # # + # ############################## + # # type: memory + # # + # # Use a configmap reference which will be mounted + # # into the container. + # # + # # configMap: + # # name: nats-accounts + # # key: resolver.conf + # + # ########################## + # # # + # # URL resolver settings # + # # # + # ########################## + # # type: URL + # # url: "http://nats-account-server:9090/jwt/v1/accounts/" From 143840b2b62baad03628832a3091a1b864922be4 Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Tue, 24 Mar 2020 16:36:43 -0700 Subject: [PATCH 3/5] Add support for gateways and leafnode remotes Signed-off-by: Waldemar Quevedo --- helm/charts/nats/templates/configmap.yaml | 54 +++++++++++++++++++++ helm/charts/nats/templates/nats-box.yaml | 2 + helm/charts/nats/templates/statefulset.yaml | 25 ++++++++++ helm/charts/nats/values.yaml | 14 ++++++ 4 files changed, 95 insertions(+) diff --git a/helm/charts/nats/templates/configmap.yaml b/helm/charts/nats/templates/configmap.yaml index 3e3cefb0..6471dd39 100644 --- a/helm/charts/nats/templates/configmap.yaml +++ b/helm/charts/nats/templates/configmap.yaml @@ -41,12 +41,66 @@ data: include "advertise/client_advertise.conf" {{- end }} + ################# + # # + # NATS Leafnode # + # # + ################# {{- if .Values.leafnodes.enabled }} leafnodes { listen: "0.0.0.0:7422" {{ if and .Values.nats.advertise .Values.nats.externalAccess }} include "advertise/gateway_advertise.conf" {{ end }} + + remotes: [ + {{- range .Values.leafnodes.remotes }} + { + {{- with .url }} + url: {{ . }} + {{- end }} + + {{- with .credentials }} + credentials: "/etc/nats-creds/{{ .secret.name }}/{{ .secret.key }}" + {{- end }} + } + {{- end }} + ] + } + {{ end }} + + ################# + # # + # NATS Gateways # + # # + ################# + {{- if .Values.gateway.enabled }} + gateway { + name: {{ .Values.gateway.name }} + port: 7522 + + {{ if and .Values.nats.advertise .Values.nats.externalAccess }} + include "advertise/gateway_advertise.conf" + {{ end }} + + # Gateways array here + gateways: [ + {{- range .Values.gateway.gateways }} + { + {{- with .name }} + name: {{ . }} + {{- end }} + + {{- with .url }} + url: {{ . | quote }} + {{- end }} + + {{- with .urls }} + urls: {{ . | quote }} + {{- end }} + }, + {{- end }} + ] } {{ end }} diff --git a/helm/charts/nats/templates/nats-box.yaml b/helm/charts/nats/templates/nats-box.yaml index 6489d166..09ac9306 100644 --- a/helm/charts/nats/templates/nats-box.yaml +++ b/helm/charts/nats/templates/nats-box.yaml @@ -25,6 +25,8 @@ spec: {{- if .Values.natsbox.credentials }} - name: USER_CREDS value: /etc/nats-config/creds/{{ .Values.natsbox.credentials.secret.key }} + - name: USER2_CREDS + value: /etc/nats-config/creds/{{ .Values.natsbox.credentials.secret.key }} {{- end }} command: - "tail" diff --git a/helm/charts/nats/templates/statefulset.yaml b/helm/charts/nats/templates/statefulset.yaml index ad9fd80b..ea88de88 100644 --- a/helm/charts/nats/templates/statefulset.yaml +++ b/helm/charts/nats/templates/statefulset.yaml @@ -57,6 +57,19 @@ spec: emptyDir: {} {{ end }} + {{- if .Values.leafnodes.enabled }} + # + # Leafnode credential volumes + # + {{- range .Values.leafnodes.remotes }} + {{- with .credentials }} + - name: {{ .secret.name }}-volume + secret: + secretName: {{ .secret.name }} + {{- end }} + {{- end }} + {{- end }} + {{ if and .Values.nats.externalAccess .Values.nats.advertise }} # Assume that we only use the service account in case we want to # figure out what is the current external public IP from the server @@ -165,6 +178,18 @@ spec: - name: operator-jwt-volume mountPath: /etc/nats-config/operator {{- end }} + + {{- if .Values.leafnodes.enabled }} + # + # Leafnode credential volumes + # + {{- range .Values.leafnodes.remotes }} + {{- with .credentials }} + - name: {{ .secret.name }}-volume + mountPath: /etc/nats-creds/{{ .secret.name }} + {{- end }} + {{- end }} + {{- end }} {{- end }} # Liveness/Readiness probes against the monitoring. diff --git a/helm/charts/nats/values.yaml b/helm/charts/nats/values.yaml index 5bceba27..851a81b7 100644 --- a/helm/charts/nats/values.yaml +++ b/helm/charts/nats/values.yaml @@ -59,7 +59,21 @@ cluster: # leafnodes: enabled: false + # remotes: + # - url: "tls://connect.ngs.global:7422" +# Gateway connections to create a super cluster +# +# https://docs.nats.io/nats-server/configuration/gateways +# +gateway: + enabled: false + name: 'default' + # List of remote gateways + # gateways: + # - name: other + # url: tls://my-gateway-url:7522 + # In case of both external access and advertisements being # enabled, an initializer container will be used to gather # the public ips. From f60c9a9275d4942bfa21535b9ea0bc1890c25795 Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Wed, 25 Mar 2020 14:16:09 -0700 Subject: [PATCH 4/5] Update values.yaml --- helm/charts/nats/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/charts/nats/values.yaml b/helm/charts/nats/values.yaml index 851a81b7..2ed1c41f 100644 --- a/helm/charts/nats/values.yaml +++ b/helm/charts/nats/values.yaml @@ -72,7 +72,7 @@ gateway: # List of remote gateways # gateways: # - name: other - # url: tls://my-gateway-url:7522 + # url: nats://my-gateway-url:7522 # In case of both external access and advertisements being # enabled, an initializer container will be used to gather From 79456413f552d08152c9e98bed9ec29551d155a4 Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Thu, 2 Apr 2020 17:20:03 -0700 Subject: [PATCH 5/5] Make resolver optional Signed-off-by: Waldemar Quevedo --- helm/charts/nats/templates/configmap.yaml | 4 +++- helm/charts/nats/values.yaml | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/helm/charts/nats/templates/configmap.yaml b/helm/charts/nats/templates/configmap.yaml index 6471dd39..2d93ac18 100644 --- a/helm/charts/nats/templates/configmap.yaml +++ b/helm/charts/nats/templates/configmap.yaml @@ -174,7 +174,9 @@ data: {{- end }} {{- if eq .Values.auth.resolver.type "URL" }} - resolver: URL({{ .Values.auth.resolver.url }}) + {{- with .Values.auth.resolver.url }} + resolver: URL({{ . }}) + {{- end }} operator: /etc/nats-config/operator/{{ .Values.auth.operatorjwt.configMap.key }} system_account: {{ .Values.auth.systemAccount }} {{- end }} diff --git a/helm/charts/nats/values.yaml b/helm/charts/nats/values.yaml index 2ed1c41f..dc130828 100644 --- a/helm/charts/nats/values.yaml +++ b/helm/charts/nats/values.yaml @@ -1,6 +1,10 @@ -# NATS Server Configuration +############################### +# # +# NATS Server Configuration # +# # +############################### nats: - image: nats:2.1.4-alpine3.11 + image: nats:2.1.6-alpine3.11 pullPolicy: IfNotPresent # Toggle whether to enable external access.