From cba53a2d9a38e1631a2aa0aaf7956453166da3e2 Mon Sep 17 00:00:00 2001 From: Guilhem Barthes Date: Thu, 24 Aug 2023 16:38:11 +0200 Subject: [PATCH] fix: rebase errors Signed-off-by: Guilhem Barthes --- .../substrapp/compute_tasks/image_builder.py | 1 + backend/substrapp/tasks/tasks_compute_task.py | 10 - .../templates/deployment-builder.yaml | 240 ------------------ charts/substra-backend/templates/rbac.yaml | 55 +--- 4 files changed, 2 insertions(+), 304 deletions(-) delete mode 100644 charts/substra-backend/templates/deployment-builder.yaml diff --git a/backend/substrapp/compute_tasks/image_builder.py b/backend/substrapp/compute_tasks/image_builder.py index 84a540942..49ce491e7 100644 --- a/backend/substrapp/compute_tasks/image_builder.py +++ b/backend/substrapp/compute_tasks/image_builder.py @@ -4,6 +4,7 @@ from tempfile import TemporaryDirectory import structlog +from django.conf import settings import orchestrator import substrapp.clients.organization as organization_client diff --git a/backend/substrapp/tasks/tasks_compute_task.py b/backend/substrapp/tasks/tasks_compute_task.py index db608fd0f..436f7f581 100644 --- a/backend/substrapp/tasks/tasks_compute_task.py +++ b/backend/substrapp/tasks/tasks_compute_task.py @@ -26,9 +26,6 @@ import orchestrator from backend.celery import app -from builder.tasks.tasks_build_image import build_image -from substrapp import models -from substrapp import utils from substrapp.clients import organization as organization_client from substrapp.compute_tasks import compute_task as task_utils from substrapp.compute_tasks import errors as compute_task_errors @@ -149,13 +146,6 @@ def queue_compute_task(channel_name: str, task: orchestrator.ComputeTask) -> Non ) return - # add image build to the Celery queue - with get_orchestrator_client(channel_name) as client: - function = client.query_function(task.function_key) - builder_queue = get_builder_queue() - # TODO switch to function.model_dump_json() as soon as pydantic is updated to > 2.0 - build_image.apply_async((function.json(),), queue=builder_queue, task_id=function.key) - with get_orchestrator_client(channel_name) as client: if not task_utils.is_task_runnable(task.key, client): return # avoid creating a Celery task diff --git a/charts/substra-backend/templates/deployment-builder.yaml b/charts/substra-backend/templates/deployment-builder.yaml deleted file mode 100644 index 9b102fbff..000000000 --- a/charts/substra-backend/templates/deployment-builder.yaml +++ /dev/null @@ -1,240 +0,0 @@ -{{- if .Values.builder.enabled }} -## Headless service doesn't get its own file -apiVersion: v1 -kind: Service -metadata: - name: {{ template "substra.fullname" . }}-builder - labels: - {{- include "substra.labels" . | nindent 4 }} - app.kubernetes.io/name: {{ template "substra.name" . }}-builder -spec: - clusterIP: None - selector: - app.kubernetes.io/component: substra-builder - app.kubernetes.io/instance: {{ .Release.Name }} ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ template "substra.fullname" . }}-builder - labels: - {{ include "substra.labels" . | nindent 4 }} - app.kubernetes.io/name: {{ template "substra.name" . }}-builder -spec: - replicas: {{ .Values.builder.replicaCount }} - serviceName: {{ template "substra.fullname" . }}-builder - selector: - matchLabels: - app.kubernetes.io/name: {{ template "substra.name" . }}-builder - {{ include "substra.selectorLabels" . | nindent 8}} - template: - metadata: - labels: - app.kubernetes.io/name: {{ template "substra.name" . }}-builder - {{ include "substra.labels" . | nindent 8 }} - app.kubernetes.io/component: substra-builder - annotations: - # This will cause the pod to restart if the content of the ConfigMap is updated through Helm - checksum-cm-orchestrator: {{ include (print $.Template.BasePath "/configmap-orchestrator.yaml") . | sha256sum }} - checksum-cm-settings: {{ include (print $.Template.BasePath "/configmap-settings.yaml") . | sha256sum }} - checksum-secret-objectstore : {{ include (print $.Template.BasePath "/secret-objectstore.yaml") . | sha256sum }} - checksum-secret-redis: {{ include (print $.Template.BasePath "/secret-redis.yaml") . | sha256sum }} - spec: - {{- if .Values.builder.podSecurityContext.enabled }} - securityContext: - fsGroup: {{ .Values.builder.podSecurityContext.fsGroup }} - runAsUser: {{ .Values.builder.podSecurityContext.runAsUser }} - runAsGroup: {{ .Values.builder.podSecurityContext.runAsGroup }} - {{- end }} - {{- with .Values.builder.image.pullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ template "substra.fullname" . }}-worker - containers: - - name: builder - image: {{ include "substra-backend.images.name" (dict "img" .Values.builder.image "defaultTag" $.Chart.AppVersion) }} - imagePullPolicy: "{{ .Values.builder.image.pullPolicy }}" - command: ["/bin/bash"] - {{- if eq .Values.settings "prod" }} - args: ["-c", "celery -A backend worker -E -l info -Q {{ .Values.organizationName }}.builder,{{ .Values.organizationName }}.builder-${HOSTNAME##*-},{{ .Values.organizationName }}.broadcast,celery --hostname {{ .Values.organizationName }}.builder-${HOSTNAME##*-}"] - {{ else }} - args: ["-c", "watchmedo auto-restart --directory=./ --pattern=*.py --recursive -- celery -A backend worker -E -l info -Q {{ .Values.organizationName }}.builder,{{ .Values.organizationName }}.builder-${HOSTNAME##*-},{{ .Values.organizationName }}.broadcast,celery --hostname {{ .Values.organizationName }}.builder-${HOSTNAME##*-}"] - {{ end }} - envFrom: - # TODO: Remove dependency for LDEGER_MSP_ID - - configMapRef: - name: {{ include "substra.fullname" . }}-orchestrator - - configMapRef: - name: {{ include "substra.fullname" . }}-orchestrator - - configMapRef: - name: {{ include "substra.fullname" . }}-settings - - configMapRef: - name: {{ include "substra.fullname" . }}-redis - - configMapRef: - name: {{ include "substra.fullname" . }}-registry - # TODO: Remove once moved ImageResitryEntrypoint logic - - configMapRef: - name: {{ include "substra.fullname" . }}-database - - secretRef: - name: {{ include "substra.fullname" . }}-objectstore - - secretRef: - name: {{ include "substra.fullname" . }}-redis - # TODO: Remove once moved ImageResitryEntrypoint logic - - secretRef: - name: {{ include "substra-backend.database.secret-name" . }} - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: DJANGO_SETTINGS_MODULE - value: backend.settings.celery.{{ .Values.settings }} - - name: DEFAULT_DOMAIN - value: "{{ .Values.server.defaultDomain }}" - - name: "CELERY_WORKER_CONCURRENCY" - value: {{ .Values.builder.concurrency | quote }} - - name: WORKER_PVC_DOCKER_CACHE - value: docker-cache - - name: WORKER_PVC_SUBTUPLE - value: subtuple - {{- if .Values.privateCa.enabled }} - - name: REQUESTS_CA_BUNDLE - value: /etc/ssl/certs/ca-certificates.crt - {{- end }} - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: KANIKO_DOCKER_CONFIG_SECRET_NAME - value: {{ .Values.kaniko.dockerConfigSecretName | quote }} - - name: OBJECTSTORE_URL - value: {{ .Release.Name }}-minio:9000 - ports: - - name: http - containerPort: 8000 - protocol: TCP - volumeMounts: - - name: subtuple - mountPath: /var/substra/medias/subtuple - {{- if .Values.privateCa.enabled }} - - mountPath: /etc/ssl/certs - name: ssl-certs - {{- end }} - {{ if .Values.orchestrator.tls.enabled }} - - name: orchestrator-tls-cacert - mountPath: /var/substra/orchestrator/tls/server - {{ if .Values.orchestrator.tls.mtls.enabled }} - - name: orchestrator-tls-client-pair - mountPath: /var/substra/orchestrator/tls/client - {{ end }} - {{ end }} - resources: - {{- toYaml .Values.builder.resources | nindent 12 }} - initContainers: - {{- if .Values.privateCa.enabled }} - - name: add-cert - image: {{ include "common.images.name" .Values.privateCa.image }} - imagePullPolicy: {{ .Values.privateCa.image.pullPolicy }} - securityContext: - runAsUser: 0 - command: ['sh', '-c'] - args: - - | - {{- if .Values.privateCa.image.apkAdd }} - apt update - apt install -y ca-certificates openssl - {{- end }} - update-ca-certificates && cp /etc/ssl/certs/* /tmp/certs/ - volumeMounts: - - mountPath: /usr/local/share/ca-certificates/{{ .Values.privateCa.configMap.fileName }} - name: private-ca - subPath: {{ .Values.privateCa.configMap.fileName }} - - mountPath: /tmp/certs/ - name: ssl-certs - {{- end }} - - name: wait-minio - image: jwilder/dockerize:0.6.1 - command: ['dockerize', '-wait', 'tcp://{{ .Release.Name }}-minio:9000'] - {{- if .Values.kaniko.cache.warmer.cachedImages }} - - name: kaniko-cache-warmer - image: {{ include "common.images.name" .Values.kaniko.cache.warmer.image }} - args: - - "--cache-dir=/cache" - {{- range .Values.kaniko.cache.warmer.cachedImages }} - - "--image={{ . }}" - {{- end }} - - "--verbosity=debug" - volumeMounts: - - name: docker-cache - mountPath: /cache - readOnly: False - {{- if .Values.kaniko.dockerConfigSecretName }} - - name: docker-config - mountPath: /kaniko/.docker - {{- end }} - {{- end}} - volumes: - {{- if .Values.privateCa.enabled }} - - name: ssl-certs - emptyDir: {} - - name: private-ca - configMap: - name: {{ .Values.privateCa.configMap.name }} - {{- end }} - {{ if .Values.orchestrator.tls.enabled }} - - name: orchestrator-tls-cacert - configMap: - name: {{ .Values.orchestrator.tls.cacert }} - {{ if .Values.orchestrator.tls.mtls.enabled }} - - name: orchestrator-tls-client-pair - secret: - secretName: {{ .Values.orchestrator.tls.mtls.clientCertificate }} - {{ end }} - {{ end }} - {{- if .Values.kaniko.dockerConfigSecretName }} - - name: docker-config - secret: - secretName: {{ .Values.kaniko.dockerConfigSecretName }} - items: - - key: .dockerconfigjson - path: config.json - {{- end }} - {{- with .Values.builder.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.builder.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.builder.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumeClaimTemplates: - - metadata: - name: subtuple - spec: - accessModes: [ "ReadWriteOnce" ] - {{ include "common.storage.class" .Values.builder.persistence }} - resources: - requests: - storage: {{ .Values.builder.persistence.size }} - - metadata: - name: docker-cache - spec: - accessModes: [ "ReadWriteOnce" ] - {{ include "common.storage.class" .Values.kaniko.cache.persistence }} - resources: - requests: - storage: {{ .Values.kaniko.cache.persistence.size }} -{{- end }} diff --git a/charts/substra-backend/templates/rbac.yaml b/charts/substra-backend/templates/rbac.yaml index 5be1c9aa8..ff98028b9 100644 --- a/charts/substra-backend/templates/rbac.yaml +++ b/charts/substra-backend/templates/rbac.yaml @@ -182,57 +182,4 @@ roleRef: kind: Role name: {{ template "substra.fullname" . }}-builder apiGroup: rbac.authorization.k8s.io -{{- end -}} -{{- if .Values.builder.rbac.create }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ template "substra.fullname" . }}-builder - labels: - {{ include "substra.labels" . | nindent 4 }} - app.kubernetes.io/name: {{ template "substra.name" . }} ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "substra.fullname" . }}-builder - labels: - {{ include "substra.labels" . | nindent 4 }} - app.kubernetes.io/name: {{ template "substra.name" . }} -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: ["pods/log", "pods/status"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["pods", "pods/exec"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: [""] - resources: ["services"] - verbs: ["get"] - {{- if .Values.psp.create }} - - apiGroups: [""] - resources: ["podsecuritypolicies"] - verbs: ["use"] - resourceNames: - - {{ template "substra.fullname" . }}-psp - {{- end }} ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "substra.fullname" . }}-builder - labels: - {{ include "substra.labels" . | nindent 4 }} - app.kubernetes.io/name: {{ template "substra.name" . }} -subjects: - - kind: ServiceAccount - name: {{ template "substra.fullname" . }}-builder -roleRef: - kind: Role - name: {{ template "substra.fullname" . }}-builder - apiGroup: rbac.authorization.k8s.io -{{- end }} +{{- end -}} \ No newline at end of file