diff --git a/backend/substrapp/kubernetes_utils.py b/backend/substrapp/kubernetes_utils.py index 110474428..61c8d0404 100644 --- a/backend/substrapp/kubernetes_utils.py +++ b/backend/substrapp/kubernetes_utils.py @@ -55,9 +55,11 @@ def get_resources_requirements_from_yaml( """Return a kubernetes.client.V1ResourceRequirements object from a yaml string.""" resources_dict = yaml.safe_load(yaml_resources) - return kubernetes.client.V1ResourceRequirements( - requests=resources_dict["requests"], limits=resources_dict["limits"] - ) + requests = resources_dict.get("requests", None) + + limits = resources_dict.get("limits", None) + + return kubernetes.client.V1ResourceRequirements(requests=requests, limits=limits) def pod_exists_by_label_selector(k8s_client: kubernetes.client.CoreV1Api, label_selector: str) -> bool: diff --git a/charts/substra-backend/CHANGELOG.md b/charts/substra-backend/CHANGELOG.md index 8123866b9..5973765be 100644 --- a/charts/substra-backend/CHANGELOG.md +++ b/charts/substra-backend/CHANGELOG.md @@ -2,6 +2,12 @@ +## [26.5.1] - 2024-05-21 + +### Changed + +- Add `resources-dev.yaml` file to overwrite resources requirement to a minimal value in a dev setup (#907) + ## [26.5.0] - 2024-04-29 ### Added diff --git a/charts/substra-backend/Chart.yaml b/charts/substra-backend/Chart.yaml index 1616d429f..cdd6ee622 100644 --- a/charts/substra-backend/Chart.yaml +++ b/charts/substra-backend/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: substra-backend home: https://github.com/Substra -version: 26.5.0 +version: 26.5.1 appVersion: 0.45.0 kubeVersion: ">= 1.19.0-0" description: Main package for Substra diff --git a/charts/substra-backend/README.md b/charts/substra-backend/README.md index 82229aaf9..199db82ac 100644 --- a/charts/substra-backend/README.md +++ b/charts/substra-backend/README.md @@ -72,8 +72,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `server.ingress.extraHosts` | The list of additional hostnames to be covered with this ingress record | `[]` | | `server.ingress.extraTls` | The tls configuration for hostnames to be coverred by the ingress | `[]` | | `server.ingress.ingressClassName` | _IngressClass_ that will be used to implement the Ingress | `nil` | -| `server.resources.requests.cpu` | Server container cpu request | `200m` | -| `server.resources.requests.memory` | Server container memory request | `512Mi` | +| `server.resources.requests.cpu` | Server container cpu request | `1000m` | +| `server.resources.requests.memory` | Server container memory request | `6Gi` | | `server.resources.limits.cpu` | Server container cpu limit | `2000m` | | `server.resources.limits.memory` | Server container memory limit | `12Gi` | | `server.persistence.storageClass` | Specify the _StorageClass_ used to provision the volume. Or the default _StorageClass_ will be used. Set it to `-` to disable dynamic provisioning | `""` | @@ -122,8 +122,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `worker.podSecurityContext.runAsUser` | User ID for the pod | `1001` | | `worker.podSecurityContext.runAsGroup` | Group ID for the pod | `1001` | | `worker.podSecurityContext.fsGroup` | FileSystem group ID for the pod | `1001` | -| `worker.resources.requests.cpu` | Worker container cpu request | `200m` | -| `worker.resources.requests.memory` | Worker container memory request | `512Mi` | +| `worker.resources.requests.cpu` | Worker container cpu request | `1000m` | +| `worker.resources.requests.memory` | Worker container memory request | `4Gi` | | `worker.resources.limits.cpu` | Worker container cpu limit | `2000m` | | `worker.resources.limits.memory` | Worker container memory limit | `8Gi` | | `worker.nodeSelector` | Node labels for pod assignment | `{}` | @@ -138,8 +138,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `worker.computePod.securityContext.fsGroup` | Set the filesystem group for the Compute pod | `1001` | | `worker.computePod.securityContext.runAsUser` | Set the user for the Compute pod | `1001` | | `worker.computePod.securityContext.runAsGroup` | Set the group for the Compute pod | `1001` | -| `worker.computePod.resources.requests.cpu` | Worker compute pod container cpu request | `500m` | -| `worker.computePod.resources.requests.memory` | Worker compute pod container memory request | `512Mi` | +| `worker.computePod.resources.requests.cpu` | Worker compute pod container cpu request | `1000m` | +| `worker.computePod.resources.requests.memory` | Worker compute pod container memory request | `1Gi` | | `worker.computePod.resources.limits.memory` | Worker compute pod container memory limit | `64Gi` | | `worker.events.enabled` | Enable event service | `true` | | `worker.events.image.registry` | Substra event app image registry | `ghcr.io` | @@ -147,8 +147,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `worker.events.image.tag` | Substra event app image tag (defaults to AppVersion) | `nil` | | `worker.events.image.pullPolicy` | Substra event app image pull policy | `IfNotPresent` | | `worker.events.image.pullSecrets` | Specify image pull secrets | `[]` | -| `worker.events.resources.requests.cpu` | Worker events container cpu request | `100m` | -| `worker.events.resources.requests.memory` | Worker events container memory request | `50Mi` | +| `worker.events.resources.requests.cpu` | Worker events container cpu request | `500m` | +| `worker.events.resources.requests.memory` | Worker events container memory request | `200Mi` | | `worker.events.resources.limits.cpu` | Worker events container cpu limit | `500m` | | `worker.events.resources.limits.memory` | Worker events container memory limit | `400Mi` | | `worker.events.podSecurityContext.enabled` | Enable security context | `true` | @@ -176,8 +176,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `schedulerWorker.nodeSelector` | Node labels for pod assignment | `{}` | | `schedulerWorker.tolerations` | Toleration labels for pod assignment | `[]` | | `schedulerWorker.affinity` | Affinity settings for pod assignment | `{}` | -| `schedulerWorker.resources.requests.cpu` | Scheduler container cpu request | `50m` | -| `schedulerWorker.resources.requests.memory` | Scheduler container memory request | `50Mi` | +| `schedulerWorker.resources.requests.cpu` | Scheduler container cpu request | `250m` | +| `schedulerWorker.resources.requests.memory` | Scheduler container memory request | `200Mi` | | `schedulerWorker.resources.limits.cpu` | Scheduler container cpu limit | `250m` | | `schedulerWorker.resources.limits.memory` | Scheduler container memory limit | `400Mi` | | `schedulerWorker.podSecurityContext.enabled` | Enable security context | `true` | @@ -196,8 +196,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `scheduler.image.tag` | Substra backend tasks scheduler image tag (defaults to AppVersion) | `nil` | | `scheduler.image.pullPolicy` | Substra backend task scheduler image pull policy | `IfNotPresent` | | `scheduler.image.pullSecrets` | Specify image pull secrets | `[]` | -| `scheduler.resources.requests.cpu` | Scheduler container cpu request | `50m` | -| `scheduler.resources.requests.memory` | Scheduler container memory request | `50Mi` | +| `scheduler.resources.requests.cpu` | Scheduler container cpu request | `250m` | +| `scheduler.resources.requests.memory` | Scheduler container memory request | `200Mi` | | `scheduler.resources.limits.cpu` | Scheduler container cpu limit | `250m` | | `scheduler.resources.limits.memory` | Scheduler container memory limit | `400Mi` | | `scheduler.nodeSelector` | Node labels for pod assignment | `{}` | @@ -225,8 +225,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `builder.podSecurityContext.runAsUser` | User ID for the pod | `1001` | | `builder.podSecurityContext.runAsGroup` | Group ID for the pod | `1001` | | `builder.podSecurityContext.fsGroup` | FileSystem group ID for the pod | `1001` | -| `builder.resources.requests.cpu` | Builder container cpu request | `200m` | -| `builder.resources.requests.memory` | Builder container memory request | `512Mi` | +| `builder.resources.requests.cpu` | Builder container cpu request | `2000m` | +| `builder.resources.requests.memory` | Builder container memory request | `4Gi` | | `builder.resources.limits.cpu` | Builder container cpu limit | `2000m` | | `builder.resources.limits.memory` | Builder container memory limit | `8Gi` | | `builder.nodeSelector` | Node labels for pod assignment | `{}` | @@ -260,8 +260,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `api.events.image.tag` | Substra event app image tag (defaults to AppVersion) | `nil` | | `api.events.image.pullPolicy` | Substra event app image pull policy | `IfNotPresent` | | `api.events.image.pullSecrets` | Specify image pull secrets | `[]` | -| `api.events.resources.requests.cpu` | Api events container cpu request | `100m` | -| `api.events.resources.requests.memory` | Api events container memory request | `50Mi` | +| `api.events.resources.requests.cpu` | Api events container cpu request | `500m` | +| `api.events.resources.requests.memory` | Api events container memory request | `200Mi` | | `api.events.resources.limits.cpu` | Api events container cpu limit | `500m` | | `api.events.resources.limits.memory` | Api events container memory limit | `400Mi` | | `api.events.podSecurityContext.enabled` | Enable security context | `true` | @@ -296,8 +296,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | `kaniko.image.registry` | Kaniko image registry | `gcr.io` | | `kaniko.image.repository` | Kaniko image repository | `kaniko-project/executor` | | `kaniko.image.tag` | Kaniko image tag | `v1.8.1` | -| `kaniko.resources.requests.cpu` | Kaniko container cpu request | `500m` | -| `kaniko.resources.requests.memory` | Kaniko container memory request | `256Mi` | +| `kaniko.resources.requests.cpu` | Kaniko container cpu request | `1000m` | +| `kaniko.resources.requests.memory` | Kaniko container memory request | `4Gi` | | `kaniko.resources.limits.memory` | Kaniko container memory limit | `32Gi` | | `kaniko.mirror` | If set to `true` pull base images from the local registry. | `false` | | `kaniko.dockerConfigSecretName` | A Docker config to use for pulling base images | `nil` | @@ -320,8 +320,8 @@ See [UPGRADE.md](https://github.com/Substra/substra-backend/blob/main/charts/sub | Name | Description | Value | | ------------------------------------------------------------ | -------------------------------------- | ------- | -| `registryPrepopulate.waitRegistry.resources.requests.cpu` | Wait registry container cpu request | `100m` | -| `registryPrepopulate.waitRegistry.resources.requests.memory` | Wait registry container memory request | `50Mi` | +| `registryPrepopulate.waitRegistry.resources.requests.cpu` | Wait registry container cpu request | `500m` | +| `registryPrepopulate.waitRegistry.resources.requests.memory` | Wait registry container memory request | `200Mi` | | `registryPrepopulate.waitRegistry.resources.limits.memory` | Wait registry container memory limit | `400Mi` | | `registryPrepopulate.pause.resources.requests.cpu` | Pause container cpu request | `50m` | | `registryPrepopulate.pause.resources.requests.memory` | Pause container memory request | `64Mi` | diff --git a/charts/substra-backend/resources-dev.yaml b/charts/substra-backend/resources-dev.yaml new file mode 100644 index 000000000..ac14bfce0 --- /dev/null +++ b/charts/substra-backend/resources-dev.yaml @@ -0,0 +1,94 @@ +server: + resources: + requests: + cpu: "10m" + memory: "16Mi" +worker: + resources: + requests: + cpu: "10m" + memory: "16Mi" + computePod: + resources: + requests: + cpu: "10m" + memory: "16Mi" + events: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +schedulerWorker: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +scheduler: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +builder: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +api: + events: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +kaniko: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +registryPrepopulate: + waitRegistry: + resources: + requests: + cpu: "10m" + memory: "16Mi" + pause: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +postgresql: + primary: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +redis: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +docker-registry: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +minio: + resources: + requests: + cpu: "10m" + memory: "16Mi" + +localstack: + resources: + requests: + cpu: "10m" + memory: "16Mi" \ No newline at end of file diff --git a/charts/substra-backend/values.yaml b/charts/substra-backend/values.yaml index c628eb247..943a47108 100644 --- a/charts/substra-backend/values.yaml +++ b/charts/substra-backend/values.yaml @@ -179,8 +179,8 @@ server: ## resources: requests: - cpu: "200m" - memory: "512Mi" + cpu: "1000m" + memory: "6Gi" limits: cpu: "2000m" memory: "12Gi" @@ -313,8 +313,8 @@ worker: ## resources: requests: - cpu: "200m" - memory: "512Mi" + cpu: "1000m" + memory: "4Gi" limits: cpu: "2000m" memory: "8Gi" @@ -361,8 +361,8 @@ worker: ## resources: requests: - cpu: "500m" - memory: "512Mi" + cpu: "1000m" + memory: "1Gi" limits: memory: "64Gi" events: @@ -388,8 +388,8 @@ worker: ## resources: requests: - memory: "50Mi" - cpu: "100m" + memory: "200Mi" + cpu: "500m" limits: memory: "400Mi" cpu: "500m" @@ -462,8 +462,8 @@ schedulerWorker: ## resources: requests: - cpu: "50m" - memory: "50Mi" + cpu: "250m" + memory: "200Mi" limits: cpu: "250m" memory: "400Mi" @@ -505,8 +505,8 @@ scheduler: ## resources: requests: - cpu: "50m" - memory: "50Mi" + cpu: "250m" + memory: "200Mi" limits: cpu: "250m" memory: "400Mi" @@ -582,8 +582,8 @@ builder: ## resources: requests: - cpu: "200m" - memory: "512Mi" + cpu: "2000m" + memory: "4Gi" limits: cpu: "2000m" memory: "8Gi" @@ -675,8 +675,8 @@ api: ## resources: requests: - memory: "50Mi" - cpu: "100m" + memory: "200Mi" + cpu: "500m" limits: memory: "400Mi" cpu: "500m" @@ -764,8 +764,8 @@ kaniko: ## resources: requests: - cpu: "500m" - memory: "256Mi" + cpu: "1000m" + memory: "4Gi" limits: memory: "32Gi" ## @param kaniko.mirror If set to `true` pull base images from the local registry. @@ -829,8 +829,8 @@ registryPrepopulate: waitRegistry: resources: requests: - memory: "50Mi" - cpu: "100m" + memory: "200Mi" + cpu: "500m" limits: memory: "400Mi" ## @param registryPrepopulate.pause.resources.requests.cpu Pause container cpu request @@ -947,8 +947,8 @@ postgresql: - ALL resources: requests: - cpu: "50m" - memory: "256Mi" + cpu: "1000m" + memory: "2Gi" limits: cpu: "1000m" memory: "4Gi" @@ -968,8 +968,8 @@ redis: redis: 6379 resources: requests: - cpu: "50m" - memory: "256Mi" + cpu: "500m" + memory: "512Mi" limits: cpu: "500m" memory: "1024Mi" @@ -988,14 +988,14 @@ docker-registry: storage: filesystem persistence: enabled: true - size: 50Gi + size: 10Gi deleteEnabled: true service: type: NodePort resources: requests: - cpu: "100m" - memory: "256Mi" + cpu: "500m" + memory: "16Gi" limits: cpu: "500m" memory: "64Gi" @@ -1021,8 +1021,8 @@ minio: - ALL resources: requests: - cpu: "100m" - memory: "1Gi" + cpu: "500m" + memory: "16Gi" limits: cpu: "1000m" memory: "64Gi" @@ -1036,8 +1036,8 @@ localstack: nodePort: "" resources: requests: - cpu: "50m" - memory: "1Gi" + cpu: "500m" + memory: "16Gi" limits: cpu: "500m" memory: "64Gi" diff --git a/skaffold.yaml b/skaffold.yaml index 0ebe1cd42..c9b13eeac 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -35,6 +35,7 @@ deploy: chartPath: charts/substra-backend valuesFiles: - examples/values/backend-org-1.yaml + - charts/substra-backend/resources-dev.yaml namespace: org-1 setValues: _: _ @@ -59,6 +60,7 @@ deploy: chartPath: charts/substra-backend valuesFiles: - examples/values/backend-org-2.yaml + - charts/substra-backend/resources-dev.yaml namespace: org-2 setValues: _: _ @@ -194,6 +196,7 @@ profiles: chartPath: charts/substra-backend valuesFiles: - examples/values/backend-org-3.yaml + - charts/substra-backend/resources-dev.yaml namespace: org-3 setValues: _: _