Do an proper editable install of mounted Python packages

.github/workflows/main.yml

@@ -32,6 +32,18 @@ jobs:
     - name: Start demo
       run: |
         ./run_demo.sh --exit-when-done
+    - name: Debugging information
+      run: |
+        export KUBECONFIG=$PWD/.demo/kube.conf
+        .demo/kubectl get pods
+        for pod_name in $(.demo/kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
+          echo "${pod_name}"
+          .demo/kubectl describe pod/"${pod_name}" || true
+          for container_name in $(.demo/kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do
+            echo $pod_name $container_name
+            .demo/kubectl logs "${pod_name}" -c "${container_name}" || true
+          done
+        done
     - name: Check for success
       run: |
         if [ ! -f ".demo/.success" ]; then
@@ -49,9 +61,24 @@ jobs:
       run: |
         git clone https://github.com/DIRACGrid/diracx.git /tmp/diracx
         git clone https://github.com/DIRACGrid/DIRAC.git /tmp/DIRAC
+        # HACK: Make sure the container can write to the sources to put the egg-info
+        # HACK: If someone runs into this locally we can work on a better solution
+        chmod 777 /tmp/diracx/src /tmp/DIRAC/src
     - name: Start demo
       run: |
         ./run_demo.sh --exit-when-done /tmp/diracx /tmp/DIRAC
+    - name: Debugging information
+      run: |
+        export KUBECONFIG=$PWD/.demo/kube.conf
+        .demo/kubectl get pods
+        for pod_name in $(.demo/kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
+          echo "${pod_name}"
+          .demo/kubectl describe pod/"${pod_name}" || true
+          for container_name in $(.demo/kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do
+            echo $pod_name $container_name
+            .demo/kubectl logs "${pod_name}" -c "${container_name}" || true
+          done
+        done
     - name: Check for success
       run: |
         if [ ! -f ".demo/.success" ]; then

README.md

@@ -68,8 +68,7 @@ TODO
 | autoscaling.minReplicas | int | `1` |  |
 | autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
 | developer.enabled | bool | `true` |  |
-| developer.modulesToMount | list | `[]` |  |
-| developer.pythonSitePackagesDir | string | `"/opt/conda/lib/python3.11/site-packages"` |  |
+| developer.modulesToInstall | list | `[]` |  |
 | developer.sourcePath | string | `"/diracx_source"` |  |
 | dex."https.enabled" | bool | `false` |  |
 | dex.config.enablePasswordDB | bool | `true` |  |

demo/values.tpl.yaml

@@ -1,5 +1,5 @@
 developer:
-  modulesToMount: {{ modules_to_mount }}
+  modulesToInstall: {{ modules_to_mount }}
 
 diracx:
   settings:

diracx/templates/deployment.yaml

@@ -18,6 +18,7 @@ spec:
           {{- toYaml . | nindent 8 }}
         {{- end }}
         checksum/settings: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
+        checksum/entrypoint: {{ include (print $.Template.BasePath "/diracx-container-entrypoint.yaml") . | sha256sum }}
       labels:
         {{- include "diracx.selectorLabels" . | nindent 8 }}
     spec:
@@ -37,16 +38,21 @@ spec:
           emptyDir:
           sizeLimit: 5Mi
           {{- end }}
+        {{- if and .Values.developer.enabled .Values.developer.modulesToInstall }}
         - name: diracx-code-mount
           persistentVolumeClaim:
             claimName: pvc-diracx-code
+        {{- end }}
+        - name: container-entrypoint
+          configMap:
+            name: diracx-container-entrypoint
         - name: signing-key-mount
           emptyDir:
           sizeLimit: 5Mi
       initContainers:
       - name: init-cs
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-        command: [/dockerMicroMambaEntrypoint.sh]
+        command: ["bash", "/entrypoint.sh"]
         args:
           - "dirac"
           - "internal"
@@ -59,12 +65,20 @@ spec:
           - "--idp-client-id={{ .id }}"
           {{- end }}
         volumeMounts:
-            - mountPath: /cs_store
-              name: cs-store-mount
-              readOnly: false
+          - mountPath: /cs_store
+            name: cs-store-mount
+            readOnly: false
+          {{- if and .Values.developer.enabled .Values.developer.modulesToInstall }}
+          - mountPath: {{ .Values.developer.sourcePath }}
+            name: diracx-code-mount
+            readOnly: true
+          {{- end }}
+          - name: container-entrypoint
+            mountPath: /entrypoint.sh
+            subPath: entrypoint.sh
       - name: init-cs-user
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-        command: [/dockerMicroMambaEntrypoint.sh]
+        command: ["bash", "/entrypoint.sh"]
         args:
           - "dirac"
           - "internal"
@@ -74,9 +88,17 @@ spec:
           - "--user-group=admin"
           - "--sub=EgVsb2NhbA"
         volumeMounts:
-            - mountPath: /cs_store
-              name: cs-store-mount
-              readOnly: false
+          - mountPath: /cs_store
+            name: cs-store-mount
+            readOnly: false
+          {{- if and .Values.developer.enabled .Values.developer.modulesToInstall }}
+          - mountPath: {{ .Values.developer.sourcePath }}
+            name: diracx-code-mount
+            readOnly: true
+          {{- end }}
+          - name: container-entrypoint
+            mountPath: /entrypoint.sh
+            subPath: entrypoint.sh
       - name: init-signing-key
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
         command: [/dockerMicroMambaEntrypoint.sh]
@@ -89,24 +111,26 @@ spec:
           - "-mPEM"
           - "-f/signing-key/rs256.key"
         volumeMounts:
-            - mountPath: /signing-key/
-              name: signing-key-mount
-              readOnly: false
+          - mountPath: /signing-key/
+            name: signing-key-mount
+            readOnly: false
       {{- if .Values.diracx.manageSQLSchema }}
       - name: create-sql-db-schema
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-        command: [/dockerMicroMambaEntrypoint.sh]
+        command: ["bash", "/entrypoint.sh"]
         args: ["python", "-m", "diracx.db", "init-sql"]
         volumeMounts:
-          {{- range $moduleName := .Values.developer.modulesToMount }}
-          - mountPath: {{ $.Values.developer.pythonSitePackagesDir }}/{{ $moduleName }}
+          {{- if and .Values.developer.enabled .Values.developer.modulesToInstall }}
+          - mountPath: {{ .Values.developer.sourcePath }}
             name: diracx-code-mount
             readOnly: true
-            subPath: {{ $moduleName }}
           {{- end }}
+          - name: container-entrypoint
+            mountPath: /entrypoint.sh
+            subPath: entrypoint.sh
         envFrom:
-        - secretRef:
-            name: diracx-init-mysql-secrets
+          - secretRef:
+              name: diracx-init-mysql-secrets
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}
@@ -126,33 +150,40 @@ spec:
             httpGet:
               path: /docs/
               port: http
-          {{- if .Values.developer.enabled }}
+          command: ["bash", "/entrypoint.sh"]
           args:
-            {{- range $moduleName := .Values.developer.modulesToMount }}
-            - "--reload-dir={{ $.Values.developer.pythonSitePackagesDir }}/{{ $moduleName }}"
-            {{- end }}
+            - uvicorn
+            - --factory
+            - diracx.routers:create_app
+            - --host=0.0.0.0
+            - --port=8000
+          {{- if .Values.developer.enabled }}
             - "--reload"
             - "--proxy-headers"
+            {{- if .Values.developer.modulesToInstall }}
+            - "--reload-dir={{ .Values.developer.sourcePath }}"
+            {{- end }}
           {{- end }}
-
           volumeMounts:
             - mountPath: /cs_store
               name: cs-store-mount
               readOnly: true
             - mountPath: /signing-key/
               name: signing-key-mount
               readOnly: true
-            {{- range $moduleName := .Values.developer.modulesToMount }}
-            - mountPath: {{ $.Values.developer.pythonSitePackagesDir }}/{{ $moduleName }}
+            {{- if and .Values.developer.enabled .Values.developer.modulesToInstall }}
+            - mountPath: {{ .Values.developer.sourcePath }}
               name: diracx-code-mount
               readOnly: true
-              subPath: {{ $moduleName }}
             {{- end }}
+            - name: container-entrypoint
+              mountPath: /entrypoint.sh
+              subPath: entrypoint.sh
           envFrom:
-          # - configMapRef:
-          #     name: diracx-env-config
-          - secretRef:
-              name: diracx-secrets
+            # - configMapRef:
+            #     name: diracx-env-config
+            - secretRef:
+                name: diracx-secrets
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}

diracx/templates/diracx-container-entrypoint.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: diracx-container-entrypoint
+data:
+  entrypoint.sh: |
+    #!/bin/bash
+    set -euo pipefail
+    IFS=$'\n\t'
+
+    ulimit -n 8192
+    eval "$(micromamba shell hook --shell=posix)" && micromamba activate base
+
+    {{ if .Values.developer.modulesToInstall }}
+    pip install {{- range $moduleName := .Values.developer.modulesToInstall }} -e {{ $.Values.developer.sourcePath }}/{{ $moduleName }} {{- end }}
+    {{- end }}
+
+    exec "$@"

diracx/values.yaml

@@ -53,8 +53,7 @@ developer:
   enabled: true
   # Path from which to mount source of DIRACX
   sourcePath: /diracx_source
-  pythonSitePackagesDir: /opt/conda/lib/python3.11/site-packages
-  modulesToMount: []
+  modulesToInstall: []
   # If set, mount the CS stored localy instead of initializing a default one
   # localCSPath: /local_cs_store
 

run_demo.sh

@@ -98,9 +98,9 @@ done
 declare -a pkg_dirs=()
 declare -a pkg_names=()
 for src_dir in "$@"; do
+  pkg_dirs+=("${src_dir}")
   # shellcheck disable=SC2044
   for pkg_dir in $(find "$src_dir/src" -mindepth 2 -maxdepth 2 -type f -name '__init__.py'); do
-    pkg_dirs+=("$(dirname "${pkg_dir}")")
     pkg_name="$(basename "$(dirname "${pkg_dir}")")"
 
     # Check for the presence of $pkg_name in pkg_names array