fix: add account operator does not synchronise content

backend/organization/management/commands/base_sync.py

@@ -0,0 +1,74 @@
+import abc
+import argparse
+
+import pydantic
+from django.core.management.base import BaseCommand
+from django.db import IntegrityError
+from django.db import models
+
+
+class Element(pydantic.BaseModel):
+    key: str
+    password: str
+
+
+class BaseSyncCommand(BaseCommand, abc.ABC):
+    help = "Sync base"
+    model: models.Model
+    model_name: str
+    field_key: str
+
+    def add_arguments(self, parser):
+        parser.add_argument("path", type=argparse.FileType())
+
+    def handle(self, *args, **options):
+        file = options["path"]
+
+        existing_keys = set(self.model.objects.values_list(self.field_key, flat=True))
+
+        for line in file:
+            line_trimmed = line.removesuffix("\n")
+            element = self.parse_line(line_trimmed)
+            self.handle_element(element)
+            existing_keys.discard(element.key)
+
+        self.delete(existing_keys)
+
+    def parse_line(self, line: str) -> Element:
+        (key, password) = line.rsplit(" ", maxsplit=1)
+
+        return Element(key=key, password=password)
+
+    def handle_element(self, element: Element) -> None:
+        try:
+            self.create(element)
+        except IntegrityError:
+            self.update_password(element)
+
+    def get(self, key: str) -> models.Model:
+        parameters = {self.field_key: key}
+        return self.model.objects.get(**parameters)
+
+    def create(self, element: Element) -> models.Model:
+        parameters = {
+            self.field_key: element.key,
+            "password": element.password,
+        }
+        model = self.model.objects.create(**parameters)
+        self.stdout.write(f"{self.model_name} created: {element.key}")
+        return model
+
+    def update_password(self, element: Element) -> models.Model:
+        model = self.get(element.key)
+        model.set_password(element.password)
+        model.save()
+        self.stdout.write(f"{self.model_name} updated: {element.key}")
+        return model
+
+    def delete(self, discarded_keys: set[str]) -> None:
+        # Using instance delete instead of queryset to activate foreign keys actions
+        for key in discarded_keys:
+            model = self.get(key)
+            model.delete()
+
+            self.stdout.write(f"{self.model_name} deleted: {key}")

backend/organization/management/commands/sync_incoming_organizations.py

@@ -0,0 +1,10 @@
+from organization.models import IncomingOrganization
+
+from .base_sync import BaseSyncCommand
+
+
+class Command(BaseSyncCommand):
+    help = "Sync incoming organizations"
+    model = IncomingOrganization
+    model_name = "Incoming organization"
+    field_key = "organization_id"

backend/organization/management/commands/sync_outgoing_organizations.py

@@ -0,0 +1,29 @@
+from django.db import models
+
+from organization.models import OutgoingOrganization
+
+from .base_sync import BaseSyncCommand
+from .base_sync import Element
+
+
+class Command(BaseSyncCommand):
+    help = "Sync outgoing organizations"
+    model = OutgoingOrganization
+    model_name = "Outgoing organization"
+    field_key = "organization_id"
+
+    # Password saved as clear text
+    def update_password(self, element: Element) -> models.Model:
+        model = self.get(element.key)
+        model.secret = element.password
+        model.save()
+        self.stdout.write(f"{self.model_name} updated: {element.key}")
+        return model
+
+    def create(self, element: Element) -> models.Model:
+        model = self.model.objects.create(
+            organization_id=element.key,
+            secret=element.password,
+        )
+        self.stdout.write(f"{self.model_name} created: {element.key}")
+        return model

backend/users/management/commands/sync_users.py

@@ -0,0 +1,57 @@
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.contrib.auth.password_validation import validate_password
+from django.core.exceptions import ValidationError
+from django.db import models
+
+from organization.management.commands.base_sync import BaseSyncCommand
+from organization.management.commands.base_sync import Element
+from users.models import UserChannel
+from users.views.user import _validate_role
+
+
+class UserElement(Element):
+    channel: str
+
+
+class Command(BaseSyncCommand):
+    help = "Sync users"
+    model_name = "User"
+    field_key = "username"
+
+    def __init__(self, *args, **kwargs):
+        self.model = get_user_model()
+        super().__init__(*args, **kwargs)
+
+    def parse_line(self, line: str) -> UserElement:
+        (key, password, channel) = line.split(" ")
+
+        return UserElement(key=key, password=password, channel=channel)
+
+    # Validate password, use specific function for creation and add UserChannel
+    def create(self, element: UserElement) -> models.Model:
+        try:
+            validate_password(element.password, self.model(username=element.key))
+        except ValidationError as err:
+            self.stderr.write("\n".join(err.messages))
+            return
+
+        user = self.model.objects.create_user(username=element.key, password=element.password)
+        self.stdout.write(f"{self.model_name} created: {element.key}, password={element.password}")
+        UserChannel.objects.create(user=user, channel_name=element.channel, role=_validate_role("ADMIN"))
+        self.stdout.write(f"User channel created: {element.key}")
+        return user
+
+    # Update password and user channel
+    def update_password(self, element: UserElement) -> models.Model:
+        user = super().update_password(element)
+        user_channel = UserChannel.objects.get(user=user)
+        user_channel.channel_name = element.channel
+        user_channel.save()
+        self.stdout.write(f"User channel updated: {element.key}")
+        return user
+
+    def delete(self, discarded_keys: set[str]) -> None:
+        # Prevent the deletion of virtual users
+        discarded_keys.difference_update(settings.VIRTUAL_USERNAMES.values())
+        super().delete(discarded_keys)

changes/999.added

@@ -0,0 +1 @@
+django commands `sync_users`,  `sync_incoming_organizations` and `sync_outgoing_organizations` that can add, update and remove corresponding entities.

charts/substra-backend/CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 
 <!-- towncrier release notes start -->
+## [26.12.0] - 2024-09-30
+
+# Changed
+
+`job-migrations` now relies on `sync_incoming_organizations`, `sync_outgoing_organizations` and `sync_users`.
+
 ## [26.11.0] - 2024-09-17
 
 # Added

charts/substra-backend/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: substra-backend
 home: https://github.com/Substra
-version: 26.11.0
+version: 26.12.0
 appVersion: 0.48.0
 kubeVersion: '>= 1.19.0-0'
 description: Main package for Substra

charts/substra-backend/templates/job-migrations.yaml

@@ -49,28 +49,13 @@ spec:
             ./manage.py migrate
 
             ## Users
-            while read -r user_password_channel; do
-                read user password channel <<< "$user_password_channel"
-                ./manage.py add_user "$user" "$password" "$channel"
-            done < /accounts/users
+            ./manage.py sync_users /accounts/users
 
             ## IncomingOrganization
-            while IFS= read -r org_password; do
-              # Extract the password as the last word
-              password=$(echo "$org_password" | awk '{print $NF}')
-              # Extract the username by removing the last word
-              orgname=$(echo "$org_password" | sed 's/ [^ ]*$//')
-              ./manage.py create_incoming_organization "$orgname" "$password"
-            done < /accounts/incoming_organizations
+            ./manage.py sync_incoming_organizations /accounts/incoming_organizations
 
             ## OutgoingOrganization
-            while IFS= read -r org_password; do
-              # Extract the password as the last word
-              password=$(echo "$org_password" | awk '{print $NF}')
-              # Extract the username by removing the last word
-              orgname=$(echo "$org_password" | sed 's/ [^ ]*$//')
-              ./manage.py create_outgoing_organization "$orgname" "$password"
-            done < /accounts/outgoing_organizations
+            ./manage.py sync_outgoing_organizations  /accounts/outgoing_organizations
         envFrom:
           - configMapRef:
               name: {{ include "substra.fullname" . }}-orchestrator