diff --git a/cms/djangoapps/contentstore/views/library.py b/cms/djangoapps/contentstore/views/library.py index 92e4329c2f94..7e3f2c67fa16 100644 --- a/cms/djangoapps/contentstore/views/library.py +++ b/cms/djangoapps/contentstore/views/library.py @@ -177,6 +177,7 @@ def _list_libraries(request): org - The organization used to filter libraries text_search - The string used to filter libraries by searching in title, id or org """ + from openedx.core.djangoapps.content_libraries.models import ContentLibraryMigration org = request.GET.get('org', '') text_search = request.GET.get('text_search', '').lower() @@ -184,22 +185,33 @@ def _list_libraries(request): libraries = modulestore().get_libraries(org=org) else: libraries = modulestore().get_libraries() - - lib_info = [ - { + lib_info = {} + for lib in libraries: + if not ( + text_search in lib.display_name.lower() or + text_search in lib.context_key.org.lower() or + text_search in lib.context_key.library.lower() + ): + continue + if not has_studio_read_access(request.user, lib.context_key): + continue + lib_info[lib.context_key] = { "display_name": lib.display_name, - "library_key": str(lib.location.library_key), + "library_key": str(lib.context_key), + "migrated_to": None, + } + try: + migration = ContentLibraryMigration.objects.select_related( + "target", "target__learning_package", "target_collection" + ).get(source_key=lib.context_key) + except ContentLibraryMigration.DoesNotExist: + continue + lib_info["migrated_to"] = { + "library_key": str(migration.target.library_key), + "display_name": str(migration.target.learning_packge.title), + "collection_key": str(migration.target_collection.key) if migration.target_collection else None, + "collection_display_name": str(migration.target_collection.key) if migration.target_collection else None, } - for lib in libraries - if ( - ( - text_search in lib.display_name.lower() or - text_search in lib.location.library_key.org.lower() or - text_search in lib.location.library_key.library.lower() - ) and - has_studio_read_access(request.user, lib.location.library_key) - ) - ] return JsonResponse(lib_info) diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 0d95931ce29d..ca545259dd1f 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -146,16 +146,24 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: If link exists, is supported, and is followable, returns UpstreamLink. Otherwise, raises an UpstreamLinkException. """ - if not downstream.upstream: - raise NoUpstream() + from xmodule.library_content_block import LegacyLibraryContentBlock if not isinstance(downstream.usage_key.context_key, CourseKey): raise BadDownstream(_("Cannot update content because it does not belong to a course.")) if downstream.has_children: raise BadDownstream(_("Updating content with children is not yet supported.")) - try: - upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) - except InvalidKeyError as exc: - raise BadUpstream(_("Reference to linked library item is malformed")) from exc + + upstream_key: LibraryUsageLocatorV2 + if downstream.upstream: + try: + upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) + except InvalidKeyError as exc: + raise BadUpstream(_("Reference to linked library item is malformed")) from exc + elif issubclass(XBlock.load_class(downstream.parent.block_type), LegacyLibraryContentBlock): + parent: LegacyLibraryContentBlock = downstream.get_parent() + upstream_key = parent.get_migrated_upstream_for_child(downstream.usage_key.block_id) + else: + raise NoUpstream() + downstream_type = downstream.usage_key.block_type if upstream_key.block_type != downstream_type: # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. @@ -199,6 +207,7 @@ def sync_from_upstream(downstream: XBlock, user: User) -> None: _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=False) _update_non_customizable_fields(upstream=upstream, downstream=downstream) _update_tags(upstream=upstream, downstream=downstream) + downstream.upstream = str(upstream.usage_key) # @@TODO explain downstream.upstream_version = link.version_available @@ -212,6 +221,7 @@ def fetch_customizable_fields(*, downstream: XBlock, user: User, upstream: XBloc if not upstream: _link, upstream = _load_upstream_link_and_block(downstream, user) _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=True) + downstream.upstream = str(upstream.usage_key) # @@TODO explain def _load_upstream_link_and_block(downstream: XBlock, user: User) -> tuple[UpstreamLink, XBlock]: diff --git a/openedx/core/djangoapps/content_libraries/admin.py b/openedx/core/djangoapps/content_libraries/admin.py index f84cac7f62e2..49d8b87a81a9 100644 --- a/openedx/core/djangoapps/content_libraries/admin.py +++ b/openedx/core/djangoapps/content_libraries/admin.py @@ -2,7 +2,9 @@ Admin site for content libraries """ from django.contrib import admin -from .models import ContentLibrary, ContentLibraryPermission +from .models import ( + ContentLibrary, ContentLibraryPermission, ContentLibraryMigration, ContentLibraryBlockMigration +) class ContentLibraryPermissionInline(admin.TabularInline): @@ -39,3 +41,22 @@ def get_readonly_fields(self, request, obj=None): return ["library_key", "org", "slug"] else: return ["library_key", ] + + +class ContentLibraryBlockMigrationInline(admin.TabularInline): + """ + Django admin UI for content library block migrations + """ + model = ContentLibraryBlockMigration + list_display = ("library_migration", "block_type", "source_block_id", "target_block_id") + + +@admin.register(ContentLibraryMigration) +class ContentLibraryMigrationAdmin(admin.ModelAdmin): + """ + Django admin UI for content library migrations + """ + list_display = ("source_key", "target", "target_collection") + inlines = (ContentLibraryBlockMigrationInline,) + + diff --git a/openedx/core/djangoapps/content_libraries/management/commands/migrate_legacy_library.py b/openedx/core/djangoapps/content_libraries/management/commands/migrate_legacy_library.py new file mode 100644 index 000000000000..0e799bb75411 --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/management/commands/migrate_legacy_library.py @@ -0,0 +1,49 @@ +""" +Implements ./manage.py cms migrate_legacy_library +""" +import logging + +from django.core.management import BaseCommand + +from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 +from openedx.core.djangoapps.content_libraries.migration_api import migrate_legacy_library + + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + @TODO + """ + + def add_arguments(self, parser): + """ + Add arguments to the argument parser. + """ + parser.add_argument( + 'legacy_library', + type=LibraryLocator.from_string, + ) + parser.add_argument( + 'new_library', + type=LibraryLocatorV2.from_string, + ) + parser.add_argument( + 'collection', + type=str, + ) + + def handle( # pylint: disable=arguments-differ + self, + legacy_library: LibraryLocator, + new_library: LibraryLocatorV2, + collection: str | None, + **kwargs, + ) -> None: + """ + Handle the command. + """ + from django.contrib.auth.models import User + user = User.objects.filter(is_superuser=True)[0] + migrate_legacy_library(legacy_library, new_library, collection_slug=collection, user=user) diff --git a/openedx/core/djangoapps/content_libraries/migration_api.py b/openedx/core/djangoapps/content_libraries/migration_api.py new file mode 100644 index 000000000000..13bf1b3a6364 --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/migration_api.py @@ -0,0 +1,133 @@ +""" +@@TODO +""" +from __future__ import annotations + +from django.contrib.auth.models import AbstractUser as User +from django.db import transaction +from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 +from openedx_learning.api.authoring import add_to_collection, get_collection +from openedx_learning.api.authoring_models import PublishableEntity, Component +from openedx_tagging.core.tagging.api import tag_object +from openedx_tagging.core.tagging.models import Taxonomy +from organizations.models import Organization +from xblock.fields import Scope + +from openedx.core.djangoapps.xblock.api import load_block +from openedx.core.djangoapps.content_libraries.api import create_library_block +from xmodule.util.keys import BlockKey +from xmodule.modulestore.django import modulestore + +from .models import ContentLibrary, ContentLibraryMigration, ContentLibraryBlockMigration + + +def migrate_legacy_library( + source_key: LibraryLocator, + target_key: LibraryLocatorV2, + *, + collection_slug: str | None, + user: User, + tags_to_add: dict[Taxonomy, list[str]] | None = None, +) -> None: + """ + Migrate a v1 (legacy) library into a v2 (learning core) library, optionally within a collection. + + Use a single transaction so that if any step fails, nothing happens. + + @@TODO handle or document various exceptions + @@TODO tags + """ + source = modulestore().get_library(source_key) + target = ContentLibrary.objects.get(org=Organization.objects.get(short_name=target_key.org), slug=target_key.slug) + assert target.learning_package_id + collection = get_collection(target.learning_package_id, collection_slug) if collection_slug else None + + # We need to be careful not to conflict with any existing block keys in the target library. + # This is unlikely to happen, since legacy library block ids are genreally randomly-generated GUIDs. + # Howevever, there are a couple scenarios where it could arise: + # * An instance has two legacy libraries which were imported from the same source legacy library (and thus share + # block GUIDs) which the author now wants to merge together into one big new library. + # * A library was imported from handcrafted OLX, and thus has human-readable block IDs which are liable to overlap. + # When there is conflict, we'll append "-1" to the end of the id (or "-2", "-3", etc., until we find a free ID). + all_target_block_keys: set[BlockKey] = { + BlockKey(*block_type_and_id) + for block_type_and_id + in Component.objects.filter( + learning_package=target.learning_package, + component_type__namespace="xblock.v1", + ).values_list("component_type__name", "local_key") + } + + # We also need to be careful not to conflict with other block IDs which we are moving in from the *source* library + # This is very unlikely, but it could happen if, for example: + # * the source library has a problem "foo", and + # * the target library also has a problem "foo", and + # * the source library ALSO has a problem "foo-1", thus + # * the source library's "foo" must be moved to the target as "foo-2". + all_source_block_keys: set[BlockKey] = { + BlockKey.from_usage_key(child_key) + for child_key in source.children + } + + target_block_entity_keys: set[str] = set() + + with transaction.atomic(): + migration = ContentLibraryMigration.objects.create( + source_key=source_key, + target=target, + target_collection=collection, + ) + + for source_block in source.get_children(): + block_type: str = source_block.usage_key.block_type + + # Determine an available block_id... + target_block_key = BlockKey(block_type, source_block.usage_key.block_id) + if target_block_key in all_target_block_keys: + suffix = 0 + while target_block_key in all_target_block_keys | all_source_block_keys: + suffix += 1 + target_block_key = BlockKey(block_type, f"{source_block.usage_key.block_id}-{suffix}") + + # Create the block in the v2 library + target_block_meta = create_library_block( + library_key=target_key, + block_type=block_type, + definition_id=target_block_key.id, + user_id=user.id, + ) + target_block_entity_keys.add(f"xblock.v1:{block_type}:{target_block_key.id}") + + # Copy its content over from the v1 library + target_block = load_block(target_block_meta.usage_key, user) + for field_name, field in source_block.__class__.fields.items(): + if field.scope not in [Scope.settings, Scope.content]: + continue + if not hasattr(target_block, field_name): + continue + source_value = getattr(source_block, field_name) + if getattr(target_block, field_name) != source_value: + setattr(target_block, field_name, source_value) + target_block.save() + + # If requested, add tags + for taxonomy, taxonomy_tags in (tags_to_add or {}).items(): + tag_object(str(target_block_meta.usage_key), taxonomy, taxonomy_tags) + + # Make a record of the migration + ContentLibraryBlockMigration.objects.create( + library_migration=migration, + block_type=block_type, + source_block_id=source_block.usage_key.block_id, + target_block_id=target_block_key.id, + ) + + # If requested, add to a collection, and add tags + if collection_slug: + add_to_collection( + target.learning_package_id, + collection_slug, + PublishableEntity.objects.filter( + key__in=target_block_entity_keys, + ), + ) diff --git a/openedx/core/djangoapps/content_libraries/migrations/0012_contentlibrarymigration_contentlibraryblockmigration.py b/openedx/core/djangoapps/content_libraries/migrations/0012_contentlibrarymigration_contentlibraryblockmigration.py new file mode 100644 index 000000000000..0dc5efb747a6 --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/migrations/0012_contentlibrarymigration_contentlibraryblockmigration.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.16 on 2024-11-01 19:07 + +from django.db import migrations, models +import django.db.models.deletion +import opaque_keys.edx.django.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_collections', '0005_alter_collection_options_alter_collection_enabled'), + ('content_libraries', '0011_remove_contentlibrary_bundle_uuid_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ContentLibraryMigration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source_key', opaque_keys.edx.django.models.LearningContextKeyField(max_length=255, unique=True)), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='content_libraries.contentlibrary')), + ('target_collection', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_collections.collection')), + ], + ), + migrations.CreateModel( + name='ContentLibraryBlockMigration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('block_type', models.SlugField()), + ('source_block_id', models.SlugField()), + ('target_block_id', models.SlugField()), + ('library_migration', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='block_migrations', to='content_libraries.contentlibrarymigration')), + ], + options={ + 'unique_together': {('library_migration', 'block_type', 'source_block_id')}, + }, + ), + ] diff --git a/openedx/core/djangoapps/content_libraries/models.py b/openedx/core/djangoapps/content_libraries/models.py index 4a210223cc29..aafa1cff22c9 100644 --- a/openedx/core/djangoapps/content_libraries/models.py +++ b/openedx/core/djangoapps/content_libraries/models.py @@ -46,17 +46,17 @@ from django.utils.translation import gettext_lazy as _ from opaque_keys.edx.django.models import CourseKeyField -from opaque_keys.edx.locator import LibraryLocatorV2 +from opaque_keys.edx.locator import LibraryUsageLocatorV2, LibraryLocatorV2, LibraryCollectionLocator from pylti1p3.contrib.django import DjangoDbToolConf from pylti1p3.contrib.django import DjangoMessageLaunch from pylti1p3.contrib.django.lti1p3_tool_config.models import LtiTool from pylti1p3.grade import Grade -from opaque_keys.edx.django.models import UsageKeyField from openedx.core.djangoapps.content_libraries.constants import ( LICENSE_OPTIONS, ALL_RIGHTS_RESERVED, ) -from openedx_learning.api.authoring_models import LearningPackage +from opaque_keys.edx.django.models import LearningContextKeyField, UsageKeyField +from openedx_learning.api.authoring_models import LearningPackage, Collection from organizations.models import Organization # lint-amnesty, pylint: disable=wrong-import-order from .apps import ContentLibrariesConfig @@ -223,6 +223,60 @@ def __str__(self): return f"ContentLibraryPermission ({self.access_level} for {who})" +class ContentLibraryMigration(models.Model): + """ + Record of a legacy (v1) content library that has been migrated into a new (v2) content library. + """ + source_key = LearningContextKeyField(unique=True, max_length=255) + target = models.ForeignKey(ContentLibrary, on_delete=models.CASCADE) + target_collection = models.ForeignKey(Collection, on_delete=models.SET_NULL, null=True) + + @property + def target_key(self) -> LibraryLocatorV2: + return self.target.library_key + + @property + def target_library_collection_key(self) -> LibraryCollectionLocator | None: + return ( + LibraryCollectionLocator(self.target_key, self.target_collection.key) + if self.target_collection + else None + ) + + def __str__(self) -> str: + return f"{self.source_key} -> {self.target_library_collection_key or self.target_key}" + + +class ContentLibraryBlockMigration(models.Model): + """ + Record of a legacy (v1) content library block that has been migrated into a new (v) content library block. + """ + library_migration = models.ForeignKey( + ContentLibraryMigration, on_delete=models.CASCADE, related_name="block_migrations" + ) + block_type = models.SlugField() + source_block_id = models.SlugField() + target_block_id = models.SlugField() + + @property + def source_usage_key(self) -> BlockUsageLocator: + return self.library_migration.source_key.make_usage_key(self.block_type, self.source_block_id) + + @property + def target_usage_key(self) -> LibraryUsageLocatorV2: + return LibraryUsageLocatorV2( + self.library_migration.target_key, + self.block_type, + self.target_block_id, + ) + + def __str__(self): + return f"{self.source_usage_key} -> {self.target_usage_key}" + + class Meta: + unique_together = [('library_migration', 'block_type', 'source_block_id')] + + class ContentLibraryBlockImportTask(models.Model): """ Model of a task to import blocks from an external source (e.g. modulestore). diff --git a/xmodule/library_content_block.py b/xmodule/library_content_block.py index 52e33108027c..93cb540a365b 100644 --- a/xmodule/library_content_block.py +++ b/xmodule/library_content_block.py @@ -104,6 +104,56 @@ class LegacyLibraryContentBlock(ItemBankMixin, XModuleToXBlockMixin, XBlock): scope=Scope.settings, ) + from opaque_keys.edx.locator import LibraryUsageLocatorV2 + + def get_migrated_upstream_for_child(self, child_block_id: str) -> LibraryUsageLocatorV2: + """ + @@TODO + """ + from openedx.core.djangoapps.content_libraries.models import ContentLibraryMigration + from cms.lib.xblock.upstream_sync import NoUpstream, BadUpstream + from xmodule.util.keys import BlockKey, derive_key + from opaque_keys import InvalidKeyError + if not self.source_library_id: + raise NoUpstream() + try: + source_library_key = self.source_library_key + except InvalidKeyError as exc: + raise BadUpstream("TODO") from exc + try: + library_migration = ContentLibraryMigration.objects.get(source_key=source_library_key) + except ContentLibraryMigration.DoesNotExist as exc: + # Source v1 library has not (yet?) been migrated to a v2 library. + # OR, we are on an instance that doesn't have the source v1 library. + raise NoUpstream() from exc + + # In order identify the new v2 library block, we need to know the v1 library block that this child came from. + # Unfortunately, there's no straightforward mapping from these children back to their v1 library source blocks. + # (ModuleStore does have a get_original_usage function that inspects edit_info, but we can't count on + # that always working, particularly if this block's course was imported from another instance.) + # Howver, we can work around this by just looping through every block in the legacy library, and testing to see + # if it's our source block. + + # So, for each block in the migrated legacy library... + for block_migration in library_migration.block_migrations.all(): + + # IF we were to have used the legacy library block as a legacy library_content block child, + # then what would its block_id be? + derived_child_block_id = derive_key( + source=block_migration.source_usage_key, + dest_parent=BlockKey.from_usage_key(self.usage_key), + ).block_id + + # If that derived block_id matches the child_block_id we're after, then we've found our legacy library + # source block! So, just return the usage key of the v2 library block that it's been migrated to. + if child_block_id == derived_child_block_id: + return block_migration.target_usage_key + + # The v1 library was migrated to a v2 library, but this particular child was never migrated to said v2 library. + # This can happen if a legacy LC block child was derived from a v1 library block which was later deleted from + # said v1 library, and the legacy LC block never synced the update which deleted said child. + raise NoUpstream() + @property def source_library_key(self): """