From e10a21247f67a674998329e8ce3da369ddf79cb1 Mon Sep 17 00:00:00 2001 From: Kaan Yagci <9104546+kaanyagci@users.noreply.github.com> Date: Thu, 29 Feb 2024 12:03:55 +0100 Subject: [PATCH] feat: optional minio + add s3 support with localstack (#784) Signed-off-by: Kaan Yagci --- charts/substra-backend/CHANGELOG.md | 6 +++ charts/substra-backend/Chart.lock | 7 ++- charts/substra-backend/Chart.yaml | 6 ++- charts/substra-backend/templates/_helpers.tpl | 54 ++++++++++++++++++- .../templates/configmap-settings.yaml | 2 +- .../templates/secret-objectstore.yaml | 4 +- .../templates/statefulset-builder.yaml | 6 +-- .../templates/statefulset-worker.yaml | 2 +- charts/substra-backend/values.yaml | 33 +++++++++++- 9 files changed, 107 insertions(+), 13 deletions(-) diff --git a/charts/substra-backend/CHANGELOG.md b/charts/substra-backend/CHANGELOG.md index 49bad9f55..740ae7eab 100644 --- a/charts/substra-backend/CHANGELOG.md +++ b/charts/substra-backend/CHANGELOG.md @@ -2,6 +2,12 @@ ## [] - Unreleased +## [24.1.0] - 2024-02-29 + +### Added + +- Minio is optional. S3 bucket is now available through LocalStack + ## [24.0.3] - 2024-02-26 ### Changed diff --git a/charts/substra-backend/Chart.lock b/charts/substra-backend/Chart.lock index 466135272..c127d292c 100644 --- a/charts/substra-backend/Chart.lock +++ b/charts/substra-backend/Chart.lock @@ -14,5 +14,8 @@ dependencies: - name: minio repository: https://charts.bitnami.com/bitnami version: 12.8.12 -digest: sha256:f7f76249b4efc0de4514cdc62dbd91e8bc48769b996c9b277568fb85d7ce5683 -generated: "2023-10-10T12:10:32.20442+02:00" +- name: localstack + repository: https://localstack.github.io/helm-charts + version: 0.6.9 +digest: sha256:5f78a36fc3eed5d0f7fe40c86d18993d16494a87ced2c78e4dfb79f63e79a3f6 +generated: "2024-02-16T17:47:16.781406+01:00" diff --git a/charts/substra-backend/Chart.yaml b/charts/substra-backend/Chart.yaml index 158efdba8..b4cb8d820 100644 --- a/charts/substra-backend/Chart.yaml +++ b/charts/substra-backend/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: substra-backend home: https://github.com/Substra -version: 24.0.3 +version: 24.1.0 appVersion: 0.43.0 kubeVersion: ">= 1.19.0-0" description: Main package for Substra @@ -33,3 +33,7 @@ dependencies: repository: https://charts.bitnami.com/bitnami version: 12.8.12 condition: minio.enabled + - name: localstack + repository: https://localstack.github.io/helm-charts + version: 0.6.9 + condition: localstack.enabled \ No newline at end of file diff --git a/charts/substra-backend/templates/_helpers.tpl b/charts/substra-backend/templates/_helpers.tpl index b5859626e..283c504a3 100644 --- a/charts/substra-backend/templates/_helpers.tpl +++ b/charts/substra-backend/templates/_helpers.tpl @@ -201,9 +201,11 @@ The hostname we should connect to (external is defined, otherwise integrated) `wait-minio` container initialisation used inside of `initContainers` */}} {{- define "common.waitMinIOContainer" -}} +{{- if or .Values.minio.enabled .Values.localstack.enabled }} - name: wait-minio image: jwilder/dockerize:0.6.1 - command: ['dockerize', '-wait', 'tcp://{{ .Release.Name }}-minio:9000'] + command: ['dockerize', '-wait', 'tcp://{{ template "substra-backend.objectStore.url" .}}'] +{{- end }} {{- end -}} @@ -272,4 +274,54 @@ The hostname we should connect to (external is defined, otherwise integrated) env: - name: DJANGO_SETTINGS_MODULE value: backend.settings.{{ .Values.settings }} +{{- end -}} + + +{{/* +Define service URL based on MinIO or LocalStack enablement +*/}} +{{- define "substra-backend.objectStore.url" -}} + {{- if .Values.minio.enabled -}} + {{- printf "%s-minio:9000" .Release.Name -}} + {{- else if .Values.localstack.enabled -}} + {{- printf "%s-localstack:4566" .Release.Name -}} + {{- end -}} +{{- end -}} + + +{{/* +Define objectstore access key based on MinIO or LocalStack enablement +*/}} +{{- define "substra-backend.objectStore.accessKey" -}} + {{- if .Values.minio.enabled -}} + {{- .Values.minio.auth.rootUser }} + {{- else if .Values.localstack.enabled -}} + {{- include "substra-backend.localstack.envValue" (dict "name" "AWS_ACCESS_KEY_ID" "context" .) -}} + {{- end -}} +{{- end -}} + +{{/* +Define objectstore secret key bassed on MinIO and Localstack enablemement +*/}} +{{- define "substra-backend.objectStore.secretKey" -}} + {{- if .Values.minio.enabled -}} + {{- .Values.minio.auth.rootPassword }} + {{- else if .Values.localstack.enabled -}} + {{- include "substra-backend.localstack.envValue" (dict "name" "AWS_SECRET_ACCESS_KEY" "context" .) -}} + {{- end -}} +{{- end -}} + +{{/* +Retrieve AWS environment variable value +*/}} +{{- define "substra-backend.localstack.envValue" -}} +{{- $envName := .name -}} +{{- $context := .context -}} +{{- $value := "" -}} +{{- range $context.Values.localstack.environment -}} + {{- if eq .name $envName -}} + {{- $value = .value -}} + {{- end -}} +{{- end -}} +{{- $value -}} {{- end -}} \ No newline at end of file diff --git a/charts/substra-backend/templates/configmap-settings.yaml b/charts/substra-backend/templates/configmap-settings.yaml index 8b979d052..1c4bd5090 100644 --- a/charts/substra-backend/templates/configmap-settings.yaml +++ b/charts/substra-backend/templates/configmap-settings.yaml @@ -19,7 +19,7 @@ data: KANIKO_IMAGE: {{ include "common.images.name" .Values.kaniko.image }} KANIKO_MIRROR: {{ .Values.kaniko.mirror | quote }} - OBJECTSTORE_URL: {{ .Release.Name }}-minio:9000 + OBJECTSTORE_URL: {{ include "substra-backend.objectStore.url" . | quote }} WORKER_REPLICA_SET_NAME: {{ template "substra.fullname" . }}-worker ENABLE_DATASAMPLE_STORAGE_IN_SERVERMEDIAS: {{ .Values.DataSampleStorageInServerMedia | quote }} diff --git a/charts/substra-backend/templates/secret-objectstore.yaml b/charts/substra-backend/templates/secret-objectstore.yaml index 49f4d2941..b2e8b1417 100644 --- a/charts/substra-backend/templates/secret-objectstore.yaml +++ b/charts/substra-backend/templates/secret-objectstore.yaml @@ -7,5 +7,5 @@ metadata: app.kubernetes.io/name: {{ include "substra.name" . }} type: Opaque data: - OBJECTSTORE_ACCESSKEY: {{ .Values.minio.auth.rootUser | b64enc }} - OBJECTSTORE_SECRETKEY: {{ .Values.minio.auth.rootPassword | b64enc }} + OBJECTSTORE_ACCESSKEY: {{ include "substra-backend.objectStore.accessKey" . | b64enc }} + OBJECTSTORE_SECRETKEY: {{ include "substra-backend.objectStore.secretKey" . | b64enc }} diff --git a/charts/substra-backend/templates/statefulset-builder.yaml b/charts/substra-backend/templates/statefulset-builder.yaml index 7c077b23c..81b84c54f 100644 --- a/charts/substra-backend/templates/statefulset-builder.yaml +++ b/charts/substra-backend/templates/statefulset-builder.yaml @@ -74,9 +74,7 @@ spec: - mountPath: /tmp/certs/ name: ssl-certs {{- end }} - - name: wait-minio - image: jwilder/dockerize:0.6.1 - command: ['dockerize', '-wait', 'tcp://{{ .Release.Name }}-minio:9000'] + {{- include "common.waitMinIOContainer" . | nindent 6 }} {{- if .Values.kaniko.cache.warmer.cachedImages }} - name: kaniko-cache-warmer image: {{ include "common.images.name" .Values.kaniko.cache.warmer.image }} @@ -159,7 +157,7 @@ spec: - name: KANIKO_DOCKER_CONFIG_SECRET_NAME value: {{ .Values.kaniko.dockerConfigSecretName | quote }} - name: OBJECTSTORE_URL - value: {{ .Release.Name }}-minio:9000 + value: {{ include "substra-backend.objectStore.url" . | quote }} ports: - name: http containerPort: 8000 diff --git a/charts/substra-backend/templates/statefulset-worker.yaml b/charts/substra-backend/templates/statefulset-worker.yaml index 1a8623307..4dca75754 100644 --- a/charts/substra-backend/templates/statefulset-worker.yaml +++ b/charts/substra-backend/templates/statefulset-worker.yaml @@ -134,7 +134,7 @@ spec: - name: COMPUTE_POD_MAX_STARTUP_WAIT_SECONDS value: {{ .Values.worker.computePod.maxStartupWaitSeconds | quote }} - name: OBJECTSTORE_URL - value: {{ .Release.Name }}-minio:9000 + value: {{ include "substra-backend.objectStore.url" . | quote }} - name: ENABLE_DATASAMPLE_STORAGE_IN_SERVERMEDIAS value: {{ .Values.DataSampleStorageInServerMedia | quote }} {{- with .Values.extraEnv }} diff --git a/charts/substra-backend/values.yaml b/charts/substra-backend/values.yaml index ccc4c4d0a..3af6e85f3 100644 --- a/charts/substra-backend/values.yaml +++ b/charts/substra-backend/values.yaml @@ -854,7 +854,7 @@ docker-registry: ## @skip minio ## minio: - enabled: true + enabled: false auth: rootUser: minio rootPassword: minio1234 @@ -863,6 +863,37 @@ minio: ## required to take into account new access and secret keys forceNewKeys: true +## @skip localstack +## +localstack: + enabled: true + service: + edgeService: + nodePort: "" + environment: + - name: SERVICES + value: s3 + - name: DEBUG + value: "1" + - name: DATA_DIR + value: "/tmp/localstack/data" + - name: PORT_WEB_UI + value: "8080" + - name: LAMBDA_EXECUTOR + value: "local" + - name: KINESIS_ERROR_PROBABILITY + value: "0.0" + - name: DOCKER_HOST + value: "unix:///var/run/docker.sock" + - name: AWS_ACCESS_KEY_ID + value: "helloAws" + - name: AWS_SECRET_ACCESS_KEY + value: "mySuperSecureAWSAccessKey1234" + + persistence: + enabled: true + accessMode: ReadWriteOnce + size: 5Gi ## @section Helm hooks ## hooks: