Use MySQL

README.md

@@ -52,6 +52,7 @@ TODO
 
 | Repository | Name | Version |
 |------------|------|---------|
+| https://charts.bitnami.com/bitnami/ | mysql | 9.11.0 |
 | https://charts.dexidp.io/ | dex | 0.14.2 |
 | https://charts.min.io/ | minio | 5.0.11 |
 | https://opensearch-project.github.io/helm-charts/ | opensearch | 2.13.1 |
@@ -92,10 +93,11 @@ TODO
 | dex.service.ports.http.port | int | `8000` |  |
 | dex.service.type | string | `"NodePort"` |  |
 | diracx.csVolumeName | string | `"pv-cs-store"` |  |
+| diracx.manageSQLSchema | bool | `true` |  |
+| diracx.mysqlDatabases[0] | string | `"AuthDB"` |  |
+| diracx.mysqlDatabases[1] | string | `"JobDB"` |  |
+| diracx.mysqlDatabases[2] | string | `"JobLoggingDB"` |  |
 | diracx.settings.DIRACX_CONFIG_BACKEND_URL | string | `"git+file:///cs_store/initialRepo"` |  |
-| diracx.settings.DIRACX_DB_URL_AUTHDB | string | `"sqlite+aiosqlite:///:memory:"` |  |
-| diracx.settings.DIRACX_DB_URL_JOBDB | string | `"sqlite+aiosqlite:///:memory:"` |  |
-| diracx.settings.DIRACX_DB_URL_JOBLOGGINGDB | string | `"sqlite+aiosqlite:///:memory:"` |  |
 | diracx.settings.DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS | string | `"[\"http://anything:8000/docs/oauth2-redirect\"]"` |  |
 | diracx.settings.DIRACX_SERVICE_AUTH_TOKEN_KEY | string | `"file:///signing-key/rs256.key"` |  |
 | fullnameOverride | string | `""` |  |
@@ -122,6 +124,12 @@ TODO
 | minio.rootPassword | string | `"rootpass123"` |  |
 | minio.rootUser | string | `"rootuser"` |  |
 | minio.service.type | string | `"NodePort"` |  |
+| mysql.auth.createDatabase | bool | `false` |  |
+| mysql.auth.password | string | `"mysqlDevInsecure"` |  |
+| mysql.auth.rootPassword | string | `"mysqlRootDevInsecure"` |  |
+| mysql.auth.username | string | `"sqldiracx"` |  |
+| mysql.enabled | bool | `true` |  |
+| mysql.initdbScriptsConfigMap | string | `"mysql-init-diracx-dbs"` |  |
 | nameOverride | string | `""` | type=kubernetes.io/dockerconfigjson imagePullSecrets:   - name: regcred |
 | nodeSelector | object | `{}` |  |
 | opensearch.config."cluster.routing.allocation.disk.threshold_enabled" | string | `"true"` |  |

diracx/Chart.lock

@@ -8,5 +8,8 @@ dependencies:
 - name: minio
   repository: https://charts.min.io/
   version: 5.0.11
-digest: sha256:94139415b05d11d7b7d9a37a66fba19dec3243f967357360b8e8bbd0feb6c4cc
-generated: "2023-06-30T13:02:06.088719+02:00"
+- name: mysql
+  repository: https://charts.bitnami.com/bitnami/
+  version: 9.11.0
+digest: sha256:7e4213564b48cd4dbd66776499a16bc137a3b7dca8f93ab35d1a91fbeab19bcc
+generated: "2023-09-08T04:46:28.916039+02:00"

diracx/Chart.yaml

@@ -36,3 +36,8 @@ dependencies:
   version: 5.0.11
   repository: https://charts.min.io/
   condition: minio.enabled
+
+- name: mysql
+  version: 9.11.0
+  repository: https://charts.bitnami.com/bitnami/
+  condition: mysql.enabled

diracx/templates/deployment.yaml

@@ -92,6 +92,22 @@ spec:
             - mountPath: /signing-key/
               name: signing-key-mount
               readOnly: false
+      {{- if .Values.diracx.manageSQLSchema }}
+      - name: create-sql-db-schema
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+        command: [/dockerMicroMambaEntrypoint.sh]
+        args: ["python", "-m", "diracx.db", "init-sql"]
+        volumeMounts:
+          {{- range $moduleName := .Values.developer.modulesToMount }}
+          - mountPath: {{ $.Values.developer.pythonSitePackagesDir }}/{{ $moduleName }}
+            name: diracx-code-mount
+            readOnly: true
+            subPath: {{ $moduleName }}
+          {{- end }}
+        envFrom:
+        - secretRef:
+            name: diracx-init-mysql-secrets
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:

diracx/templates/mysql-init-dbs.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql-init-diracx-dbs
+data:
+  # Create the databases for DiracX and grant privileges
+  init-diracx-dbs.sql: |
+{{- range $dbName := .Values.diracx.mysqlDatabases }}
+    CREATE DATABASE IF NOT EXISTS `{{ $dbName }}`;
+    GRANT SELECT, INSERT, UPDATE, DELETE, INDEX, CREATE TEMPORARY TABLES, LOCK TABLES ON {{ $dbName }}.* TO '{{ $.Values.mysql.auth.username }}'@'%';
+{{- end }}

diracx/templates/secrets.yaml

@@ -3,4 +3,22 @@ kind: Secret
 metadata:
   name: diracx-secrets
 stringData:
-{{ .Values.diracx.settings | toYaml | indent 4 }}
+{{ .Values.diracx.settings | toYaml | indent 2 }}
+{{- if and .Values.developer.enabled .Values.mysql.enabled }}
+  {{- range $dbName := .Values.diracx.mysqlDatabases }}
+  DIRACX_DB_URL_{{ $dbName | upper }}: "mysql+aiomysql://{{ $.Values.mysql.auth.username }}:{{ $.Values.mysql.auth.password }}@{{ $.Release.Name }}-mysql:3306/{{ $dbName }}"
+  {{- end }}
+{{- end }}
+---
+{{- if .Values.diracx.manageSQLSchema }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: diracx-init-mysql-secrets
+stringData:
+{{- if and .Values.developer.enabled .Values.mysql.enabled }}
+  {{- range $dbName := .Values.diracx.mysqlDatabases }}
+  DIRACX_DB_URL_{{ $dbName | upper }}: "mysql+aiomysql://root:{{ $.Values.mysql.auth.rootPassword }}@{{ $.Release.Name }}-mysql:3306/{{ $dbName }}"
+  {{- end }}
+{{- end }}
+{{- end }}

diracx/values.yaml

@@ -65,12 +65,15 @@ diracx:
     # This corresponds to the basic dirac.cfg
     # which must be present on all the servers
     DIRACX_CONFIG_BACKEND_URL: "git+file:///cs_store/initialRepo"
-    DIRACX_DB_URL_AUTHDB: "sqlite+aiosqlite:///:memory:"
-    # mysql+aiomysql://
-    DIRACX_DB_URL_JOBDB: "sqlite+aiosqlite:///:memory:"
-    DIRACX_DB_URL_JOBLOGGINGDB: "sqlite+aiosqlite:///:memory:"
     DIRACX_SERVICE_AUTH_TOKEN_KEY: "file:///signing-key/rs256.key"
     DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS: '["http://anything:8000/docs/oauth2-redirect"]'
+  # Should DiracX include an init container which manages the SQL DB schema?
+  manageSQLSchema: true
+  # Which DiracX DBs are used?
+  mysqlDatabases:
+    - AuthDB
+    - JobDB
+    - JobLoggingDB
 
 ##########################
 
@@ -160,6 +163,17 @@ dex:
 
 ##########################
 
+mysql:
+  enabled: true
+  auth:
+    rootPassword: mysqlRootDevInsecure
+    username: sqldiracx
+    password: mysqlDevInsecure
+    createDatabase: false
+  initdbScriptsConfigMap: mysql-init-diracx-dbs
+
+##########################
+
 service:
   type: ClusterIP
   port: 8000