feat: Backend for Migrating Legacy Libraries (wip)

cms/djangoapps/contentstore/views/library.py

@@ -177,29 +177,41 @@ def _list_libraries(request):
         org - The organization used to filter libraries
         text_search - The string used to filter libraries by searching in title, id or org
     """
+    from openedx.core.djangoapps.content_libraries.models import ContentLibraryMigration
     org = request.GET.get('org', '')
     text_search = request.GET.get('text_search', '').lower()
 
     if org:
         libraries = modulestore().get_libraries(org=org)
     else:
         libraries = modulestore().get_libraries()
-
-    lib_info = [
-        {
+    lib_info = {}
+    for lib in libraries:
+        if not (
+            text_search in lib.display_name.lower() or
+            text_search in lib.context_key.org.lower() or
+            text_search in lib.context_key.library.lower()
+        ):
+            continue
+        if not has_studio_read_access(request.user, lib.context_key):
+            continue
+        lib_info[lib.context_key] = {
             "display_name": lib.display_name,
-            "library_key": str(lib.location.library_key),
+            "library_key": str(lib.context_key),
+            "migrated_to": None,
+        }
+        try:
+            migration = ContentLibraryMigration.objects.select_related(
+                "target", "target__learning_package", "target_collection"
+            ).get(source_key=lib.context_key)
+        except ContentLibraryMigration.DoesNotExist:
+            continue
+        lib_info["migrated_to"] = {
+            "library_key": str(migration.target.library_key),
+            "display_name": str(migration.target.learning_packge.title),
+            "collection_key": str(migration.target_collection.key) if migration.target_collection else None,
+            "collection_display_name": str(migration.target_collection.key) if migration.target_collection else None,
         }
-        for lib in libraries
-        if (
-            (
-                text_search in lib.display_name.lower() or
-                text_search in lib.location.library_key.org.lower() or
-                text_search in lib.location.library_key.library.lower()
-            ) and
-            has_studio_read_access(request.user, lib.location.library_key)
-        )
-    ]
     return JsonResponse(lib_info)
 
 

cms/lib/xblock/upstream_sync.py

@@ -146,16 +146,24 @@ def get_for_block(cls, downstream: XBlock) -> t.Self:
         If link exists, is supported, and is followable, returns UpstreamLink.
         Otherwise, raises an UpstreamLinkException.
         """
-        if not downstream.upstream:
-            raise NoUpstream()
+        from xmodule.library_content_block import LegacyLibraryContentBlock
         if not isinstance(downstream.usage_key.context_key, CourseKey):
             raise BadDownstream(_("Cannot update content because it does not belong to a course."))
         if downstream.has_children:
             raise BadDownstream(_("Updating content with children is not yet supported."))
-        try:
-            upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream)
-        except InvalidKeyError as exc:
-            raise BadUpstream(_("Reference to linked library item is malformed")) from exc
+
+        upstream_key: LibraryUsageLocatorV2
+        if downstream.upstream:
+            try:
+                upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream)
+            except InvalidKeyError as exc:
+                raise BadUpstream(_("Reference to linked library item is malformed")) from exc
+        elif issubclass(XBlock.load_class(downstream.parent.block_type), LegacyLibraryContentBlock):
+            parent: LegacyLibraryContentBlock = downstream.get_parent()
+            upstream_key = parent.get_migrated_upstream_for_child(downstream.usage_key.block_id)
+        else:
+            raise NoUpstream()
+
         downstream_type = downstream.usage_key.block_type
         if upstream_key.block_type != downstream_type:
             # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match.
@@ -199,6 +207,7 @@ def sync_from_upstream(downstream: XBlock, user: User) -> None:
     _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=False)
     _update_non_customizable_fields(upstream=upstream, downstream=downstream)
     _update_tags(upstream=upstream, downstream=downstream)
+    downstream.upstream = str(upstream.usage_key)  # @@TODO explain
     downstream.upstream_version = link.version_available
 
 
@@ -212,6 +221,7 @@ def fetch_customizable_fields(*, downstream: XBlock, user: User, upstream: XBloc
     if not upstream:
         _link, upstream = _load_upstream_link_and_block(downstream, user)
     _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=True)
+    downstream.upstream = str(upstream.usage_key)  # @@TODO explain
 
 
 def _load_upstream_link_and_block(downstream: XBlock, user: User) -> tuple[UpstreamLink, XBlock]:

openedx/core/djangoapps/content_libraries/admin.py

@@ -2,7 +2,9 @@
 Admin site for content libraries
 """
 from django.contrib import admin
-from .models import ContentLibrary, ContentLibraryPermission
+from .models import (
+    ContentLibrary, ContentLibraryPermission, ContentLibraryMigration, ContentLibraryBlockMigration
+)
 
 
 class ContentLibraryPermissionInline(admin.TabularInline):
@@ -39,3 +41,22 @@ def get_readonly_fields(self, request, obj=None):
             return ["library_key", "org", "slug"]
         else:
             return ["library_key", ]
+
+
+class ContentLibraryBlockMigrationInline(admin.TabularInline):
+    """
+    Django admin UI for content library block migrations
+    """
+    model = ContentLibraryBlockMigration
+    list_display = ("library_migration", "block_type", "source_block_id", "target_block_id")
+
+
+@admin.register(ContentLibraryMigration)
+class ContentLibraryMigrationAdmin(admin.ModelAdmin):
+    """
+    Django admin UI for content library migrations
+    """
+    list_display = ("source_key", "target", "target_collection")
+    inlines = (ContentLibraryBlockMigrationInline,)
+
+

openedx/core/djangoapps/content_libraries/management/commands/migrate_legacy_library.py

@@ -0,0 +1,49 @@
+"""
+Implements ./manage.py cms migrate_legacy_library
+"""
+import logging
+
+from django.core.management import BaseCommand
+
+from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
+from openedx.core.djangoapps.content_libraries.migration_api import migrate_legacy_library
+
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    @TODO
+    """
+
+    def add_arguments(self, parser):
+        """
+        Add arguments to the argument parser.
+        """
+        parser.add_argument(
+            'legacy_library',
+            type=LibraryLocator.from_string,
+        )
+        parser.add_argument(
+            'new_library',
+            type=LibraryLocatorV2.from_string,
+        )
+        parser.add_argument(
+            'collection',
+            type=str,
+        )
+
+    def handle(  # pylint: disable=arguments-differ
+        self,
+        legacy_library: LibraryLocator,
+        new_library: LibraryLocatorV2,
+        collection: str | None,
+        **kwargs,
+    ) -> None:
+        """
+        Handle the command.
+        """
+        from django.contrib.auth.models import User
+        user = User.objects.filter(is_superuser=True)[0]
+        migrate_legacy_library(legacy_library, new_library, collection_slug=collection, user=user)

openedx/core/djangoapps/content_libraries/migration_api.py

@@ -0,0 +1,133 @@
+"""
+@@TODO
+"""
+from __future__ import annotations
+
+from django.contrib.auth.models import AbstractUser as User
+from django.db import transaction
+from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
+from openedx_learning.api.authoring import add_to_collection, get_collection
+from openedx_learning.api.authoring_models import PublishableEntity, Component
+from openedx_tagging.core.tagging.api import tag_object
+from openedx_tagging.core.tagging.models import Taxonomy
+from organizations.models import Organization
+from xblock.fields import Scope
+
+from openedx.core.djangoapps.xblock.api import load_block
+from openedx.core.djangoapps.content_libraries.api import create_library_block
+from xmodule.util.keys import BlockKey
+from xmodule.modulestore.django import modulestore
+
+from .models import ContentLibrary, ContentLibraryMigration, ContentLibraryBlockMigration
+
+
+def migrate_legacy_library(
+    source_key: LibraryLocator,
+    target_key: LibraryLocatorV2,
+    *,
+    collection_slug: str | None,
+    user: User,
+    tags_to_add: dict[Taxonomy, list[str]] | None = None,
+) -> None:
+    """
+    Migrate a v1 (legacy) library into a v2 (learning core) library, optionally within a collection.
+
+    Use a single transaction so that if any step fails, nothing happens.
+
+    @@TODO handle or document various exceptions
+    @@TODO tags
+    """
+    source = modulestore().get_library(source_key)
+    target = ContentLibrary.objects.get(org=Organization.objects.get(short_name=target_key.org), slug=target_key.slug)
+    assert target.learning_package_id
+    collection = get_collection(target.learning_package_id, collection_slug) if collection_slug else None
+
+    # We need to be careful not to conflict with any existing block keys in the target library.
+    # This is unlikely to happen, since legacy library block ids are genreally randomly-generated GUIDs.
+    # Howevever, there are a couple scenarios where it could arise:
+    #  * An instance has two legacy libraries which were imported from the same source legacy library (and thus share
+    #    block GUIDs) which the author now wants to merge together into one big new library.
+    #  * A library was imported from handcrafted OLX, and thus has human-readable block IDs which are liable to overlap.
+    # When there is conflict, we'll append "-1" to the end of the id (or "-2", "-3", etc., until we find a free ID).
+    all_target_block_keys: set[BlockKey] = {
+        BlockKey(*block_type_and_id)
+        for block_type_and_id
+        in Component.objects.filter(
+            learning_package=target.learning_package,
+            component_type__namespace="xblock.v1",
+        ).values_list("component_type__name", "local_key")
+    }
+
+    # We also need to be careful not to conflict with other block IDs which we are moving in from the *source* library
+    # This is very unlikely, but it could happen if, for example:
+    # * the source library has a problem "foo", and
+    # * the target library also has a problem "foo", and
+    # * the source library ALSO has a problem "foo-1", thus
+    # * the source library's "foo" must be moved to the target as "foo-2".
+    all_source_block_keys: set[BlockKey] = {
+        BlockKey.from_usage_key(child_key)
+        for child_key in source.children
+    }
+
+    target_block_entity_keys: set[str] = set()
+
+    with transaction.atomic():
+        migration = ContentLibraryMigration.objects.create(
+            source_key=source_key,
+            target=target,
+            target_collection=collection,
+        )
+
+        for source_block in source.get_children():
+            block_type: str = source_block.usage_key.block_type
+
+            # Determine an available block_id...
+            target_block_key = BlockKey(block_type, source_block.usage_key.block_id)
+            if target_block_key in all_target_block_keys:
+                suffix = 0
+                while target_block_key in all_target_block_keys | all_source_block_keys:
+                    suffix += 1
+                    target_block_key = BlockKey(block_type, f"{source_block.usage_key.block_id}-{suffix}")
+
+            # Create the block in the v2 library
+            target_block_meta = create_library_block(
+                library_key=target_key,
+                block_type=block_type,
+                definition_id=target_block_key.id,
+                user_id=user.id,
+            )
+            target_block_entity_keys.add(f"xblock.v1:{block_type}:{target_block_key.id}")
+
+            # Copy its content over from the v1 library
+            target_block = load_block(target_block_meta.usage_key, user)
+            for field_name, field in source_block.__class__.fields.items():
+                if field.scope not in [Scope.settings, Scope.content]:
+                    continue
+                if not hasattr(target_block, field_name):
+                    continue
+                source_value = getattr(source_block, field_name)
+                if getattr(target_block, field_name) != source_value:
+                    setattr(target_block, field_name, source_value)
+            target_block.save()
+
+            # If requested, add tags
+            for taxonomy, taxonomy_tags in (tags_to_add or {}).items():
+                tag_object(str(target_block_meta.usage_key), taxonomy, taxonomy_tags)
+
+            # Make a record of the migration
+            ContentLibraryBlockMigration.objects.create(
+                library_migration=migration,
+                block_type=block_type,
+                source_block_id=source_block.usage_key.block_id,
+                target_block_id=target_block_key.id,
+            )
+
+        # If requested, add to a collection, and add tags
+        if collection_slug:
+            add_to_collection(
+                target.learning_package_id,
+                collection_slug,
+                PublishableEntity.objects.filter(
+                    key__in=target_block_entity_keys,
+                ),
+            )

openedx/core/djangoapps/content_libraries/migrations/0012_contentlibrarymigration_contentlibraryblockmigration.py

@@ -0,0 +1,38 @@
+# Generated by Django 4.2.16 on 2024-11-01 19:07
+
+from django.db import migrations, models
+import django.db.models.deletion
+import opaque_keys.edx.django.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('oel_collections', '0005_alter_collection_options_alter_collection_enabled'),
+        ('content_libraries', '0011_remove_contentlibrary_bundle_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ContentLibraryMigration',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('source_key', opaque_keys.edx.django.models.LearningContextKeyField(max_length=255, unique=True)),
+                ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='content_libraries.contentlibrary')),
+                ('target_collection', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_collections.collection')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ContentLibraryBlockMigration',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('block_type', models.SlugField()),
+                ('source_block_id', models.SlugField()),
+                ('target_block_id', models.SlugField()),
+                ('library_migration', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='block_migrations', to='content_libraries.contentlibrarymigration')),
+            ],
+            options={
+                'unique_together': {('library_migration', 'block_type', 'source_block_id')},
+            },
+        ),
+    ]