feat: optional minio + add s3 support with localstack (#784)

Signed-off-by: Kaan Yagci <[email protected]>
Substra · Feb 29, 2024 · e10a212 · e10a212
1 parent 8d35146
commit e10a212
Show file tree

Hide file tree

Showing 9 changed files with 107 additions and 13 deletions.
diff --git a/charts/substra-backend/CHANGELOG.md b/charts/substra-backend/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 ## [] - Unreleased
 
+## [24.1.0] - 2024-02-29
+
+### Added
+
+- Minio is optional. S3 bucket is now available through LocalStack
+
 ## [24.0.3] - 2024-02-26
 
 ### Changed

diff --git a/charts/substra-backend/Chart.lock b/charts/substra-backend/Chart.lock
@@ -14,5 +14,8 @@ dependencies:
 - name: minio
   repository: https://charts.bitnami.com/bitnami
   version: 12.8.12
-digest: sha256:f7f76249b4efc0de4514cdc62dbd91e8bc48769b996c9b277568fb85d7ce5683
-generated: "2023-10-10T12:10:32.20442+02:00"
+- name: localstack
+  repository: https://localstack.github.io/helm-charts
+  version: 0.6.9
+digest: sha256:5f78a36fc3eed5d0f7fe40c86d18993d16494a87ced2c78e4dfb79f63e79a3f6
+generated: "2024-02-16T17:47:16.781406+01:00"
diff --git a/charts/substra-backend/Chart.yaml b/charts/substra-backend/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: substra-backend
 home: https://github.com/Substra
-version: 24.0.3
+version: 24.1.0
 appVersion: 0.43.0
 kubeVersion: ">= 1.19.0-0"
 description: Main package for Substra
@@ -33,3 +33,7 @@ dependencies:
     repository: https://charts.bitnami.com/bitnami
     version: 12.8.12
     condition: minio.enabled
+  - name: localstack
+    repository: https://localstack.github.io/helm-charts
+    version: 0.6.9
+    condition: localstack.enabled
diff --git a/charts/substra-backend/templates/_helpers.tpl b/charts/substra-backend/templates/_helpers.tpl
@@ -201,9 +201,11 @@ The hostname we should connect to (external is defined, otherwise integrated)
 `wait-minio` container initialisation used inside of `initContainers`
 */}}
 {{- define "common.waitMinIOContainer" -}}
+{{- if or .Values.minio.enabled .Values.localstack.enabled }}
 - name: wait-minio
   image: jwilder/dockerize:0.6.1
-  command: ['dockerize', '-wait', 'tcp://{{ .Release.Name }}-minio:9000']
+  command: ['dockerize', '-wait', 'tcp://{{ template "substra-backend.objectStore.url" .}}']
+{{- end }}
 {{- end -}}
 
 
@@ -272,4 +274,54 @@ The hostname we should connect to (external is defined, otherwise integrated)
   env:
   - name: DJANGO_SETTINGS_MODULE
     value: backend.settings.{{ .Values.settings }}
+{{- end -}}
+
+
+{{/*
+Define service URL based on MinIO or LocalStack enablement
+*/}}
+{{- define "substra-backend.objectStore.url" -}}
+    {{- if .Values.minio.enabled -}}
+        {{- printf "%s-minio:9000" .Release.Name -}}
+    {{- else if .Values.localstack.enabled -}}
+        {{- printf "%s-localstack:4566" .Release.Name -}}
+    {{- end -}}
+{{- end -}}
+
+
+{{/*
+Define objectstore access key based on MinIO or LocalStack enablement
+*/}}
+{{- define "substra-backend.objectStore.accessKey" -}}
+    {{- if .Values.minio.enabled -}}
+        {{- .Values.minio.auth.rootUser }}
+    {{- else if .Values.localstack.enabled -}}
+        {{- include "substra-backend.localstack.envValue" (dict "name" "AWS_ACCESS_KEY_ID" "context" .) -}}
+    {{- end -}}
+{{- end -}}
+
+{{/*
+Define objectstore secret key bassed on MinIO and Localstack enablemement
+*/}}
+{{- define "substra-backend.objectStore.secretKey" -}}
+  {{- if .Values.minio.enabled -}}
+        {{- .Values.minio.auth.rootPassword }}
+    {{- else if .Values.localstack.enabled -}}
+        {{- include "substra-backend.localstack.envValue" (dict "name" "AWS_SECRET_ACCESS_KEY" "context" .) -}}
+    {{- end -}}
+{{- end -}}
+
+{{/*
+Retrieve AWS environment variable value
+*/}}
+{{- define "substra-backend.localstack.envValue" -}}
+{{- $envName := .name -}}
+{{- $context := .context -}}
+{{- $value := "" -}}
+{{- range $context.Values.localstack.environment -}}
+  {{- if eq .name $envName -}}
+    {{- $value = .value -}}
+  {{- end -}}
+{{- end -}}
+{{- $value -}}
 {{- end -}}
diff --git a/charts/substra-backend/templates/configmap-settings.yaml b/charts/substra-backend/templates/configmap-settings.yaml
@@ -19,7 +19,7 @@ data:
   KANIKO_IMAGE: {{ include "common.images.name" .Values.kaniko.image }}
   KANIKO_MIRROR: {{ .Values.kaniko.mirror | quote }}
 
-  OBJECTSTORE_URL: {{ .Release.Name }}-minio:9000
+  OBJECTSTORE_URL: {{ include "substra-backend.objectStore.url" . | quote }}
 
   WORKER_REPLICA_SET_NAME: {{ template "substra.fullname" . }}-worker
   ENABLE_DATASAMPLE_STORAGE_IN_SERVERMEDIAS: {{ .Values.DataSampleStorageInServerMedia | quote }}

diff --git a/charts/substra-backend/templates/secret-objectstore.yaml b/charts/substra-backend/templates/secret-objectstore.yaml
@@ -7,5 +7,5 @@ metadata:
     app.kubernetes.io/name: {{ include "substra.name" . }}
 type: Opaque
 data:
-  OBJECTSTORE_ACCESSKEY: {{ .Values.minio.auth.rootUser | b64enc }}
-  OBJECTSTORE_SECRETKEY: {{ .Values.minio.auth.rootPassword | b64enc }}
+  OBJECTSTORE_ACCESSKEY: {{ include "substra-backend.objectStore.accessKey" . | b64enc }}
+  OBJECTSTORE_SECRETKEY: {{ include "substra-backend.objectStore.secretKey" . | b64enc }}
diff --git a/charts/substra-backend/templates/statefulset-builder.yaml b/charts/substra-backend/templates/statefulset-builder.yaml
@@ -74,9 +74,7 @@ spec:
           - mountPath: /tmp/certs/
             name: ssl-certs
       {{- end }}
-      - name: wait-minio
-        image: jwilder/dockerize:0.6.1
-        command: ['dockerize', '-wait', 'tcp://{{ .Release.Name }}-minio:9000']
+      {{- include "common.waitMinIOContainer" . | nindent 6 }}
       {{- if .Values.kaniko.cache.warmer.cachedImages }}
       - name: kaniko-cache-warmer
         image: {{ include "common.images.name" .Values.kaniko.cache.warmer.image }}
@@ -159,7 +157,7 @@ spec:
           - name: KANIKO_DOCKER_CONFIG_SECRET_NAME
             value: {{ .Values.kaniko.dockerConfigSecretName | quote }}
           - name: OBJECTSTORE_URL
-            value: {{ .Release.Name }}-minio:9000
+            value: {{ include "substra-backend.objectStore.url" . | quote }}
         ports:
           - name: http
             containerPort: 8000

diff --git a/charts/substra-backend/templates/statefulset-worker.yaml b/charts/substra-backend/templates/statefulset-worker.yaml
@@ -134,7 +134,7 @@ spec:
             - name: COMPUTE_POD_MAX_STARTUP_WAIT_SECONDS
               value: {{ .Values.worker.computePod.maxStartupWaitSeconds | quote }}
             - name: OBJECTSTORE_URL
-              value: {{ .Release.Name }}-minio:9000
+              value: {{ include "substra-backend.objectStore.url" . | quote }}
             - name: ENABLE_DATASAMPLE_STORAGE_IN_SERVERMEDIAS
               value: {{ .Values.DataSampleStorageInServerMedia | quote }}
           {{- with .Values.extraEnv }}

diff --git a/charts/substra-backend/values.yaml b/charts/substra-backend/values.yaml
@@ -854,7 +854,7 @@ docker-registry:
 ## @skip minio
 ##
 minio:
-  enabled: true
+  enabled: false
   auth:
     rootUser: minio
     rootPassword: minio1234
@@ -863,6 +863,37 @@ minio:
     ## required to take into account new access and secret keys
     forceNewKeys: true
 
+## @skip localstack
+##
+localstack:
+  enabled: true
+  service:
+    edgeService:
+      nodePort: ""
+  environment:
+    - name: SERVICES
+      value: s3
+    - name: DEBUG
+      value: "1"
+    - name: DATA_DIR
+      value: "/tmp/localstack/data"
+    - name: PORT_WEB_UI
+      value: "8080"
+    - name: LAMBDA_EXECUTOR
+      value: "local"
+    - name: KINESIS_ERROR_PROBABILITY
+      value: "0.0"
+    - name: DOCKER_HOST
+      value: "unix:///var/run/docker.sock"
+    - name: AWS_ACCESS_KEY_ID
+      value: "helloAws"
+    - name: AWS_SECRET_ACCESS_KEY
+      value: "mySuperSecureAWSAccessKey1234"
+
+  persistence:
+    enabled: true
+    accessMode: ReadWriteOnce
+    size: 5Gi
 ## @section Helm hooks
 ##
 hooks: