From 6598abbb6b2a07f2c4ecaf278ef109027e0e32b3 Mon Sep 17 00:00:00 2001 From: Jesper Hodge <19345795+jesperhodge@users.noreply.github.com> Date: Mon, 31 Jul 2023 13:37:00 -0400 Subject: [PATCH] Studio content api videos (#32803) * refactor: extract methods to video_storage_handlers * refactor: move private functions * refactor: move functions to videos_storage_handlers * refactor: asset_storage_handlers * feat: add video api views * feat: add video urls * feat: add mock videos post * refactor: mock video upload url * fix: json extraction * fix: url pattern for video deletion * fix: video url views * fix: lint * fix: lint * fix: tests * fix: tests * fix: tests * Feat studio content api transcripts (#32858) * feat: add transcript endpoints feat: add transcript upload endpoint, check that transcripts for deletion exist fix: remove transcript credentials view cause out of scope * fix: lint * feat: TNL-10897 fix destroy() args to kwargs bug --------- Co-authored-by: Bernard Szabo --------- Co-authored-by: Bernard Szabo --- .../contentstore/rest_api/v1/urls.py | 28 + .../rest_api/v1/views/__init__.py | 2 + .../contentstore/rest_api/v1/views/assets.py | 6 +- .../rest_api/v1/views/transcripts.py | 62 ++ .../contentstore/rest_api/v1/views/videos.py | 159 +++ .../contentstore/rest_api/v1/views/xblock.py | 6 +- cms/djangoapps/contentstore/toggles.py | 19 + .../transcript_storage_handlers.py | 265 +++++ .../contentstore/video_storage_handlers.py | 902 ++++++++++++++++++ .../views/tests/test_transcript_settings.py | 34 +- .../contentstore/views/tests/test_videos.py | 30 +- .../contentstore/views/transcript_settings.py | 203 +--- .../contentstore/views/transcripts_ajax.py | 2 +- cms/djangoapps/contentstore/views/videos.py | 792 ++------------- cms/urls.py | 2 +- 15 files changed, 1552 insertions(+), 960 deletions(-) create mode 100644 cms/djangoapps/contentstore/rest_api/v1/views/transcripts.py create mode 100644 cms/djangoapps/contentstore/rest_api/v1/views/videos.py create mode 100644 cms/djangoapps/contentstore/transcript_storage_handlers.py create mode 100644 cms/djangoapps/contentstore/video_storage_handlers.py diff --git a/cms/djangoapps/contentstore/rest_api/v1/urls.py b/cms/djangoapps/contentstore/rest_api/v1/urls.py index dcd2a688a90..b3def50e8e7 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/urls.py +++ b/cms/djangoapps/contentstore/rest_api/v1/urls.py @@ -13,10 +13,14 @@ ProctoringErrorsView, xblock, assets, + videos, + transcripts, ) app_name = 'v1' +VIDEO_ID_PATTERN = r'(?:(?P[-\w]+))' + urlpatterns = [ re_path( fr'^proctored_exam_settings/{COURSE_ID_PATTERN}$', @@ -51,4 +55,28 @@ fr'^file_assets/{settings.COURSE_ID_PATTERN}/{settings.ASSET_KEY_PATTERN}?$', assets.AssetsView.as_view(), name='studio_content_assets' ), + re_path( + fr'^videos/uploads/{settings.COURSE_ID_PATTERN}/{VIDEO_ID_PATTERN}?$', + videos.VideosView.as_view(), name='studio_content_videos_uploads' + ), + re_path( + fr'^videos/images/{settings.COURSE_ID_PATTERN}/{VIDEO_ID_PATTERN}?$', + videos.VideoImagesView.as_view(), name='studio_content_videos_images' + ), + re_path( + fr'^videos/encodings/{settings.COURSE_ID_PATTERN}$', + videos.VideoEncodingsDownloadView.as_view(), name='studio_content_videos_encodings' + ), + re_path( + r'^videos/features/$', + videos.VideoFeaturesView.as_view(), name='studio_content_videos_features' + ), + re_path( + fr'^videos/upload_link/{settings.COURSE_ID_PATTERN}$', + videos.UploadLinkView.as_view(), name='studio_content_videos_upload_link' + ), + re_path( + fr'^video_transcripts/{settings.COURSE_ID_PATTERN}$', + transcripts.TranscriptView.as_view(), name='studio_content_video_transcripts' + ), ] diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/__init__.py b/cms/djangoapps/contentstore/rest_api/v1/views/__init__.py index 1de2139d6f9..34d7c0bfb67 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/__init__.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/__init__.py @@ -6,3 +6,5 @@ from .proctoring import ProctoredExamSettingsView, ProctoringErrorsView from .settings import CourseSettingsView from .xblock import XblockView +from .assets import AssetsView +from .videos import VideosView diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/assets.py b/cms/djangoapps/contentstore/rest_api/v1/views/assets.py index 4a726a1c043..3508020a85b 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/assets.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/assets.py @@ -1,4 +1,6 @@ -# lint-amnesty, pylint: disable=missing-module-docstring +""" +Public rest API endpoints for the Studio Content API Assets. +""" import logging from rest_framework.generics import RetrieveUpdateDestroyAPIView, CreateAPIView from django.views.decorators.csrf import csrf_exempt @@ -19,7 +21,7 @@ @view_auth_classes() class AssetsView(DeveloperErrorViewMixin, RetrieveUpdateDestroyAPIView, CreateAPIView): """ - public rest API endpoint for the Studio Content API. + public rest API endpoints for the Studio Content API Assets. course_key: required argument, needed to authorize course authors and identify the asset. asset_key_string: required argument, needed to identify the asset. """ diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/transcripts.py b/cms/djangoapps/contentstore/rest_api/v1/views/transcripts.py new file mode 100644 index 00000000000..af23a81fb86 --- /dev/null +++ b/cms/djangoapps/contentstore/rest_api/v1/views/transcripts.py @@ -0,0 +1,62 @@ +""" +Public rest API endpoints for the Studio Content API video assets. +""" +import logging +from rest_framework.generics import ( + CreateAPIView, + RetrieveAPIView, + DestroyAPIView +) +from django.views.decorators.csrf import csrf_exempt +from django.http import Http404 + +from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes +from common.djangoapps.util.json_request import expect_json_in_class_view + +from ....api import course_author_access_required + +from cms.djangoapps.contentstore.transcript_storage_handlers import ( + upload_transcript, + delete_video_transcript_or_404, + handle_transcript_download, +) +import cms.djangoapps.contentstore.toggles as contentstore_toggles + +log = logging.getLogger(__name__) +toggles = contentstore_toggles + + +@view_auth_classes() +class TranscriptView(DeveloperErrorViewMixin, CreateAPIView, RetrieveAPIView, DestroyAPIView): + """ + public rest API endpoints for the Studio Content API video transcripts. + course_key: required argument, needed to authorize course authors and identify the video. + edx_video_id: optional query parameter, needed to identify the transcript. + language_code: optional query parameter, needed to identify the transcript. + """ + + def dispatch(self, request, *args, **kwargs): + if not toggles.use_studio_content_api(): + raise Http404 + return super().dispatch(request, *args, **kwargs) + + @csrf_exempt + @course_author_access_required + @expect_json_in_class_view + def create(self, request, course_key_string): # pylint: disable=arguments-differ + return upload_transcript(request) + + @course_author_access_required + def retrieve(self, request, course_key_string): # pylint: disable=arguments-differ + """ + Get a video transcript. edx_video_id and language_code query parameters are required. + """ + return handle_transcript_download(request) + + @course_author_access_required + def destroy(self, request, course_key_string): # pylint: disable=arguments-differ + """ + Delete a video transcript. edx_video_id and language_code query parameters are required. + """ + + return delete_video_transcript_or_404(request) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/videos.py b/cms/djangoapps/contentstore/rest_api/v1/views/videos.py new file mode 100644 index 00000000000..46282b2eb61 --- /dev/null +++ b/cms/djangoapps/contentstore/rest_api/v1/views/videos.py @@ -0,0 +1,159 @@ +""" +Public rest API endpoints for the Studio Content API video assets. +""" +import logging +from rest_framework.generics import ( + CreateAPIView, + RetrieveAPIView, + DestroyAPIView +) +from django.views.decorators.csrf import csrf_exempt +from django.http import Http404 + +from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes +from common.djangoapps.util.json_request import expect_json_in_class_view + +from ....api import course_author_access_required + +from cms.djangoapps.contentstore.video_storage_handlers import ( + handle_videos, + get_video_encodings_download, + handle_video_images, + enabled_video_features, + handle_generate_video_upload_link +) +import cms.djangoapps.contentstore.toggles as contentstore_toggles + +log = logging.getLogger(__name__) +toggles = contentstore_toggles + + +@view_auth_classes() +class VideosView(DeveloperErrorViewMixin, CreateAPIView, RetrieveAPIView, DestroyAPIView): + """ + public rest API endpoints for the Studio Content API video assets. + course_key: required argument, needed to authorize course authors and identify the video. + video_id: required argument, needed to identify the video. + """ + + def dispatch(self, request, *args, **kwargs): + # TODO: probably want to refactor this to a decorator. + """ + The dispatch method of a View class handles HTTP requests in general + and calls other methods to handle specific HTTP methods. + We use this to raise a 404 if the content api is disabled. + """ + if not toggles.use_studio_content_api(): + raise Http404 + return super().dispatch(request, *args, **kwargs) + + @csrf_exempt + @course_author_access_required + @expect_json_in_class_view + def create(self, request, course_key): # pylint: disable=arguments-differ + return handle_videos(request, course_key.html_id()) + + @course_author_access_required + def retrieve(self, request, course_key, edx_video_id=None): # pylint: disable=arguments-differ + return handle_videos(request, course_key.html_id(), edx_video_id) + + @course_author_access_required + @expect_json_in_class_view + def destroy(self, request, course_key, edx_video_id): # pylint: disable=arguments-differ + return handle_videos(request, course_key.html_id(), edx_video_id) + + +@view_auth_classes() +class VideoImagesView(DeveloperErrorViewMixin, CreateAPIView): + """ + public rest API endpoint for uploading a video image. + course_key: required argument, needed to authorize course authors and identify the video. + video_id: required argument, needed to identify the video. + """ + + def dispatch(self, request, *args, **kwargs): + # TODO: probably want to refactor this to a decorator. + """ + The dispatch method of a View class handles HTTP requests in general + and calls other methods to handle specific HTTP methods. + We use this to raise a 404 if the content api is disabled. + """ + if not toggles.use_studio_content_api(): + raise Http404 + return super().dispatch(request, *args, **kwargs) + + @csrf_exempt + @course_author_access_required + @expect_json_in_class_view + def create(self, request, course_key, edx_video_id=None): # pylint: disable=arguments-differ + return handle_video_images(request, course_key.html_id(), edx_video_id) + + +@view_auth_classes() +class VideoEncodingsDownloadView(DeveloperErrorViewMixin, RetrieveAPIView): + """ + public rest API endpoint providing a CSV report containing the encoded video URLs for video uploads. + course_key: required argument, needed to authorize course authors and identify relevant videos. + """ + + def dispatch(self, request, *args, **kwargs): + # TODO: probably want to refactor this to a decorator. + """ + The dispatch method of a View class handles HTTP requests in general + and calls other methods to handle specific HTTP methods. + We use this to raise a 404 if the content api is disabled. + """ + if not toggles.use_studio_content_api(): + raise Http404 + return super().dispatch(request, *args, **kwargs) + + @csrf_exempt + @course_author_access_required + def retrieve(self, request, course_key): # pylint: disable=arguments-differ + return get_video_encodings_download(request, course_key.html_id()) + + +@view_auth_classes() +class VideoFeaturesView(DeveloperErrorViewMixin, RetrieveAPIView): + """ + public rest API endpoint providing a list of enabled video features. + """ + + def dispatch(self, request, *args, **kwargs): + # TODO: probably want to refactor this to a decorator. + """ + The dispatch method of a View class handles HTTP requests in general + and calls other methods to handle specific HTTP methods. + We use this to raise a 404 if the content api is disabled. + """ + if not toggles.use_studio_content_api(): + raise Http404 + return super().dispatch(request, *args, **kwargs) + + @csrf_exempt + def retrieve(self, request): # pylint: disable=arguments-differ + return enabled_video_features(request) + + +@view_auth_classes() +class UploadLinkView(DeveloperErrorViewMixin, CreateAPIView): + """ + public rest API endpoint providing a list of enabled video features. + """ + + def dispatch(self, request, *args, **kwargs): + # TODO: probably want to refactor this to a decorator. + """ + The dispatch method of a View class handles HTTP requests in general + and calls other methods to handle specific HTTP methods. + We use this to raise a 404 if the content api is disabled. + """ + if not toggles.use_studio_content_api(): + raise Http404 + return super().dispatch(request, *args, **kwargs) + + @csrf_exempt + @course_author_access_required + @expect_json_in_class_view + def create(self, request, course_key): # pylint: disable=arguments-differ + return handle_generate_video_upload_link(request, course_key.html_id()) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/xblock.py b/cms/djangoapps/contentstore/rest_api/v1/views/xblock.py index 94a87981d39..2da54811d4b 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/xblock.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/xblock.py @@ -1,4 +1,6 @@ -# lint-amnesty, pylint: disable=missing-module-docstring +""" +Public rest API endpoints for the Studio Content API. +""" import logging from rest_framework.generics import RetrieveUpdateDestroyAPIView, CreateAPIView from django.views.decorators.csrf import csrf_exempt @@ -20,7 +22,7 @@ @view_auth_classes() class XblockView(DeveloperErrorViewMixin, RetrieveUpdateDestroyAPIView, CreateAPIView): """ - public rest API endpoint for the Studio Content API. + Public rest API endpoints for the Studio Content API. course_key: required argument, needed to authorize course authors. usage_key_string (optional): xblock identifier, for example in the form of "block-v1:+type@+block@" diff --git a/cms/djangoapps/contentstore/toggles.py b/cms/djangoapps/contentstore/toggles.py index 4f84e7bf9e3..7a02efc0787 100644 --- a/cms/djangoapps/contentstore/toggles.py +++ b/cms/djangoapps/contentstore/toggles.py @@ -497,6 +497,25 @@ def use_new_course_team_page(course_key): return ENABLE_NEW_STUDIO_COURSE_TEAM_PAGE.is_enabled(course_key) +# .. toggle_name: contentstore.mock_video_uploads +# .. toggle_implementation: WaffleFlag +# .. toggle_default: False +# .. toggle_description: This flag mocks contentstore video uploads for local development, if you don't have access to AWS +# .. toggle_use_cases: open_edx +# .. toggle_creation_date: 2023-7-25 +# .. toggle_tickets: TNL-10897 +# .. toggle_warning: +MOCK_VIDEO_UPLOADS = WaffleFlag( + f'{CONTENTSTORE_NAMESPACE}.mock_video_uploads', __name__) + + +def use_mock_video_uploads(): + """ + Returns a boolean if video uploads should be mocked for local development + """ + return MOCK_VIDEO_UPLOADS.is_enabled() + + # .. toggle_name: contentstore.default_enable_flexible_peer_openassessments # .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False diff --git a/cms/djangoapps/contentstore/transcript_storage_handlers.py b/cms/djangoapps/contentstore/transcript_storage_handlers.py new file mode 100644 index 00000000000..b2a682f429a --- /dev/null +++ b/cms/djangoapps/contentstore/transcript_storage_handlers.py @@ -0,0 +1,265 @@ +""" +Business logic for video transcripts. +""" + + +import logging +import os + +from django.core.files.base import ContentFile +from django.http import HttpResponse, HttpResponseNotFound +from django.utils.translation import gettext as _ +from edxval.api import ( + create_or_update_video_transcript, + delete_video_transcript as delete_video_transcript_source_function, + get_3rd_party_transcription_plans, + get_available_transcript_languages, + get_video_transcript_data, + update_transcript_credentials_state_for_org, + get_video_transcript +) +from opaque_keys.edx.keys import CourseKey + +from common.djangoapps.util.json_request import JsonResponse +from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag +from openedx.core.djangoapps.video_pipeline.api import update_3rd_party_transcription_service_credentials +from xmodule.video_block.transcripts_utils import Transcript, TranscriptsGenerationException # lint-amnesty, pylint: disable=wrong-import-order + +from .toggles import use_mock_video_uploads +from .video_storage_handlers import TranscriptProvider + +LOGGER = logging.getLogger(__name__) + + +class TranscriptionProviderErrorType: + """ + Transcription provider's error types enumeration. + """ + INVALID_CREDENTIALS = 1 + + +def validate_transcript_credentials(provider, **credentials): + """ + Validates transcript credentials. + + Validations: + Providers must be either 3PlayMedia or Cielo24. + In case of: + 3PlayMedia - 'api_key' and 'api_secret_key' are required. + Cielo24 - 'api_key' and 'username' are required. + + It ignores any extra/unrelated parameters passed in credentials and + only returns the validated ones. + """ + error_message, validated_credentials = '', {} + valid_providers = list(get_3rd_party_transcription_plans().keys()) + if provider in valid_providers: + must_have_props = [] + if provider == TranscriptProvider.THREE_PLAY_MEDIA: + must_have_props = ['api_key', 'api_secret_key'] + elif provider == TranscriptProvider.CIELO24: + must_have_props = ['api_key', 'username'] + + missing = [ + must_have_prop for must_have_prop in must_have_props if must_have_prop not in list(credentials.keys()) # lint-amnesty, pylint: disable=consider-iterating-dictionary + ] + if missing: + error_message = '{missing} must be specified.'.format(missing=' and '.join(missing)) + return error_message, validated_credentials + + validated_credentials.update({ + prop: credentials[prop] for prop in must_have_props + }) + else: + error_message = f'Invalid Provider {provider}.' + + return error_message, validated_credentials + + +def handle_transcript_credentials(request, course_key_string): + """ + JSON view handler to update the transcript organization credentials. + + Arguments: + request: WSGI request object + course_key_string: A course identifier to extract the org. + + Returns: + - A 200 response if credentials are valid and successfully updated in edx-video-pipeline. + - A 404 response if transcript feature is not enabled for this course. + - A 400 if credentials do not pass validations, hence not updated in edx-video-pipeline. + """ + course_key = CourseKey.from_string(course_key_string) + if not VideoTranscriptEnabledFlag.feature_enabled(course_key): + return HttpResponseNotFound() + + provider = request.json.pop('provider') + error_message, validated_credentials = validate_transcript_credentials(provider=provider, **request.json) + if error_message: + response = JsonResponse({'error': error_message}, status=400) + else: + # Send the validated credentials to edx-video-pipeline and video-encode-manager + credentials_payload = dict(validated_credentials, org=course_key.org, provider=provider) + error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload) + # Send appropriate response based on whether credentials were updated or not. + if is_updated: + # Cache credentials state in edx-val. + update_transcript_credentials_state_for_org(org=course_key.org, provider=provider, exists=is_updated) + response = JsonResponse(status=200) + else: + # Error response would contain error types and the following + # error type is received from edx-video-pipeline whenever we've + # got invalid credentials for a provider. Its kept this way because + # edx-video-pipeline doesn't support i18n translations yet. + error_type = error_response.get('error_type') + if error_type == TranscriptionProviderErrorType.INVALID_CREDENTIALS: + error_message = _('The information you entered is incorrect.') + + response = JsonResponse({'error': error_message}, status=400) + + return response + + +def handle_transcript_download(request): + """ + JSON view handler to download a transcript. + + Arguments: + request: WSGI request object + + Returns: + - A 200 response with SRT transcript file attached. + - A 400 if there is a validation error. + - A 404 if there is no such transcript. + """ + missing = [attr for attr in ['edx_video_id', 'language_code'] if attr not in request.GET] + if missing: + return JsonResponse( + {'error': _('The following parameters are required: {missing}.').format(missing=', '.join(missing))}, + status=400 + ) + + edx_video_id = request.GET['edx_video_id'] + language_code = request.GET['language_code'] + transcript = get_video_transcript_data(video_id=edx_video_id, language_code=language_code) + if transcript: + name_and_extension = os.path.splitext(transcript['file_name']) + basename, file_format = name_and_extension[0], name_and_extension[1][1:] + transcript_filename = f'{basename}.{Transcript.SRT}' + transcript_content = Transcript.convert( + content=transcript['content'], + input_format=file_format, + output_format=Transcript.SRT + ) + # Construct an HTTP response + response = HttpResponse(transcript_content, content_type=Transcript.mime_types[Transcript.SRT]) + response['Content-Disposition'] = f'attachment; filename="{transcript_filename}"' + else: + response = HttpResponseNotFound() + + return response + + +def _create_or_update_video_transcript(**kwargs): + if use_mock_video_uploads(): + return True + + return create_or_update_video_transcript(**kwargs) + + +def upload_transcript(request): + """ + Upload a transcript file + + Arguments: + request: A WSGI request object + + Transcript file in SRT format + """ + edx_video_id = request.POST['edx_video_id'] + language_code = request.POST['language_code'] + new_language_code = request.POST['new_language_code'] + transcript_file = request.FILES['file'] + try: + # Convert SRT transcript into an SJSON format + # and upload it to S3. + sjson_subs = Transcript.convert( + content=transcript_file.read().decode('utf-8'), + input_format=Transcript.SRT, + output_format=Transcript.SJSON + ).encode() + _create_or_update_video_transcript( + video_id=edx_video_id, + language_code=language_code, + metadata={ + 'provider': TranscriptProvider.CUSTOM, + 'file_format': Transcript.SJSON, + 'language_code': new_language_code + }, + file_data=ContentFile(sjson_subs), + ) + response = JsonResponse(status=201) + except (TranscriptsGenerationException, UnicodeDecodeError): + LOGGER.error("Unable to update transcript on edX video %s for language %s", edx_video_id, new_language_code) + response = JsonResponse( + {'error': _('There is a problem with this transcript file. Try to upload a different file.')}, + status=400 + ) + finally: + LOGGER.info("Updated transcript on edX video %s for language %s", edx_video_id, new_language_code) + return response + + +def validate_transcript_upload_data(data, files): + """ + Validates video transcript file. + Arguments: + data: A request's data part. + files: A request's files part. + Returns: + None or String + If there is error returns error message otherwise None. + """ + error = None + # Validate the must have attributes - this error is unlikely to be faced by common users. + must_have_attrs = ['edx_video_id', 'language_code', 'new_language_code'] + missing = [attr for attr in must_have_attrs if attr not in data] + if missing: + error = _('The following parameters are required: {missing}.').format(missing=', '.join(missing)) + elif ( + data['language_code'] != data['new_language_code'] and + data['new_language_code'] in get_available_transcript_languages(video_id=data['edx_video_id']) + ): + error = _('A transcript with the "{language_code}" language code already exists.'.format( # lint-amnesty, pylint: disable=translation-of-non-string + language_code=data['new_language_code'] + )) + elif 'file' not in files: + error = _('A transcript file is required.') + + return error + + +def delete_video_transcript(video_id=None, language_code=None): + return delete_video_transcript_source_function(video_id=video_id, language_code=language_code) + + +def delete_video_transcript_or_404(request): + """ + Delete a video transcript or return 404 if it doesn't exist. + """ + missing = [attr for attr in ['edx_video_id', 'language_code'] if attr not in request.GET] + if missing: + return JsonResponse( + {'error': _('The following parameters are required: {missing}.').format(missing=', '.join(missing))}, + status=400 + ) + + video_id = request.GET.get('edx_video_id') + language_code = request.GET.get('language_code') + + if not get_video_transcript(video_id=video_id, language_code=language_code): + return HttpResponseNotFound() + + delete_video_transcript(video_id=video_id, language_code=language_code) + + return JsonResponse(status=200) diff --git a/cms/djangoapps/contentstore/video_storage_handlers.py b/cms/djangoapps/contentstore/video_storage_handlers.py new file mode 100644 index 00000000000..d1a7c55ac6f --- /dev/null +++ b/cms/djangoapps/contentstore/video_storage_handlers.py @@ -0,0 +1,902 @@ +""" +Views related to the video upload feature +""" + + +import codecs +import csv +import io +import json +import logging +from contextlib import closing +from datetime import datetime, timedelta +from uuid import uuid4 +from boto.s3.connection import S3Connection +from boto import s3 +from django.conf import settings +from django.contrib.staticfiles.storage import staticfiles_storage +from django.http import FileResponse, HttpResponseNotFound +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_noop +from edx_toggles.toggles import WaffleSwitch +from edxval.api import ( + SortDirection, + VideoSortField, + create_or_update_transcript_preferences, + create_video, + get_3rd_party_transcription_plans, + get_available_transcript_languages, + get_video_transcript_url, + get_transcript_credentials_state_for_org, + get_transcript_preferences, + get_videos_for_course, + remove_transcript_preferences, + remove_video_for_course, + update_video_image, + update_video_status +) +from opaque_keys.edx.keys import CourseKey +from pytz import UTC +from rest_framework import status as rest_status +from rest_framework.response import Response + +from common.djangoapps.edxmako.shortcuts import render_to_response +from common.djangoapps.util.json_request import JsonResponse +from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag +from openedx.core.djangoapps.video_config.toggles import PUBLIC_VIDEO_SHARE +from openedx.core.djangoapps.video_pipeline.config.waffle import ( + DEPRECATE_YOUTUBE, + ENABLE_DEVSTACK_VIDEO_UPLOADS, +) +from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag +from xmodule.video_block.transcripts_utils import Transcript # lint-amnesty, pylint: disable=wrong-import-order + +from .models import VideoUploadConfig +from .toggles import use_new_video_uploads_page, use_mock_video_uploads +from .utils import reverse_course_url, get_video_uploads_url +from .video_utils import validate_video_image +from .views.course import get_course_and_check_access + +LOGGER = logging.getLogger(__name__) + +# Waffle switches namespace for videos +WAFFLE_NAMESPACE = 'videos' + +# Waffle switch for enabling/disabling video image upload feature +VIDEO_IMAGE_UPLOAD_ENABLED = WaffleSwitch( # lint-amnesty, pylint: disable=toggle-missing-annotation + f'{WAFFLE_NAMESPACE}.video_image_upload_enabled', __name__ +) + +# Waffle flag namespace for studio +WAFFLE_STUDIO_FLAG_NAMESPACE = 'studio' + +ENABLE_VIDEO_UPLOAD_PAGINATION = CourseWaffleFlag( # lint-amnesty, pylint: disable=toggle-missing-annotation + f'{WAFFLE_STUDIO_FLAG_NAMESPACE}.enable_video_upload_pagination', __name__ +) +# Default expiration, in seconds, of one-time URLs used for uploading videos. +KEY_EXPIRATION_IN_SECONDS = 86400 + +VIDEO_SUPPORTED_FILE_FORMATS = { + '.mp4': 'video/mp4', + '.mov': 'video/quicktime', +} + +VIDEO_UPLOAD_MAX_FILE_SIZE_GB = 5 + +# maximum time for video to remain in upload state +MAX_UPLOAD_HOURS = 24 + +VIDEOS_PER_PAGE = 100 + + +class TranscriptProvider: + """ + Transcription Provider Enumeration + """ + CIELO24 = 'Cielo24' + THREE_PLAY_MEDIA = '3PlayMedia' + CUSTOM = 'Custom' + + +class StatusDisplayStrings: + """ + A class to map status strings as stored in VAL to display strings for the + video upload page + """ + + # Translators: This is the status of an active video upload + _UPLOADING = gettext_noop("Uploading") + # Translators: This is the status for a video that the servers are currently processing + _IN_PROGRESS = gettext_noop("In Progress") + # Translators: This is the status for a video that the servers have successfully processed + _COMPLETE = gettext_noop("Ready") + # Translators: This is the status for a video that is uploaded completely + _UPLOAD_COMPLETED = gettext_noop("Uploaded") + # Translators: This is the status for a video that the servers have failed to process + _FAILED = gettext_noop("Failed") + # Translators: This is the status for a video that is cancelled during upload by user + _CANCELLED = gettext_noop("Cancelled") + # Translators: This is the status for a video which has failed + # due to being flagged as a duplicate by an external or internal CMS + _DUPLICATE = gettext_noop("Failed Duplicate") + # Translators: This is the status for a video which has duplicate token for youtube + _YOUTUBE_DUPLICATE = gettext_noop("YouTube Duplicate") + # Translators: This is the status for a video for which an invalid + # processing token was provided in the course settings + _INVALID_TOKEN = gettext_noop("Invalid Token") + # Translators: This is the status for a video that was included in a course import + _IMPORTED = gettext_noop("Imported") + # Translators: This is the status for a video that is in an unknown state + _UNKNOWN = gettext_noop("Unknown") + # Translators: This is the status for a video that is having its transcription in progress on servers + _TRANSCRIPTION_IN_PROGRESS = gettext_noop("Transcription in Progress") + # Translators: This is the status for a video whose transcription is complete + _TRANSCRIPT_READY = gettext_noop("Transcript Ready") + # Translators: This is the status for a video whose transcription job was failed for some languages + _PARTIAL_FAILURE = gettext_noop("Partial Failure") + # Translators: This is the status for a video whose transcription job has failed altogether + _TRANSCRIPT_FAILED = gettext_noop("Transcript Failed") + + _STATUS_MAP = { + "upload": _UPLOADING, + "ingest": _IN_PROGRESS, + "transcode_queue": _IN_PROGRESS, + "transcode_active": _IN_PROGRESS, + "file_delivered": _COMPLETE, + "file_complete": _COMPLETE, + "upload_completed": _UPLOAD_COMPLETED, + "file_corrupt": _FAILED, + "pipeline_error": _FAILED, + "upload_failed": _FAILED, + "s3_upload_failed": _FAILED, + "upload_cancelled": _CANCELLED, + "duplicate": _DUPLICATE, + "youtube_duplicate": _YOUTUBE_DUPLICATE, + "invalid_token": _INVALID_TOKEN, + "imported": _IMPORTED, + "transcription_in_progress": _TRANSCRIPTION_IN_PROGRESS, + "transcript_ready": _TRANSCRIPT_READY, + "partial_failure": _PARTIAL_FAILURE, + # TODO: Add a related unit tests when the VAL update is part of platform + "transcript_failed": _TRANSCRIPT_FAILED, + } + + @staticmethod + def get(val_status): + """Map a VAL status string to a localized display string""" + # pylint: disable=translation-of-non-string + return _(StatusDisplayStrings._STATUS_MAP.get(val_status, StatusDisplayStrings._UNKNOWN)) + + +def handle_videos(request, course_key_string, edx_video_id=None): + """ + Restful handler for video uploads. + + GET + html: return an HTML page to display previous video uploads and allow + new ones + json: return json representing the videos that have been uploaded and + their statuses + POST + json: generate new video upload urls, for example upload urls for S3 buckets. To upload the video, you should + make a PUT request to the returned upload_url values. This can happen on the frontend, MFE, + or client side - it is not implemented in the backend. + Example payload: + { + "files": [{ + "file_name": "video.mp4", + "content_type": "video/mp4" + }] + } + Returns (JSON): + { + "files": [{ + "file_name": "video.mp4", + "upload_url": "http://example.com/put_video" + }] + } + DELETE + soft deletes a video for particular course + """ + course = _get_and_validate_course(course_key_string, request.user) + + if (not course and not use_mock_video_uploads()): + return HttpResponseNotFound() + + if request.method == "GET": + if "application/json" in request.META.get("HTTP_ACCEPT", ""): + return videos_index_json(course) + pagination_conf = _generate_pagination_configuration(course_key_string, request) + return videos_index_html(course, pagination_conf) + elif request.method == "DELETE": + remove_video_for_course(course_key_string, edx_video_id) + return JsonResponse() + else: + if is_status_update_request(request.json): + return send_video_status_update(request.json) + elif _is_pagination_context_update_request(request): + return _update_pagination_context(request) + + data, status = videos_post(course, request) + return JsonResponse(data, status=status) + + +def handle_generate_video_upload_link(request, course_key_string): + """ + API for creating a video upload. Returns an edx_video_id and a presigned URL that can be used + to upload the video to AWS S3. + """ + course = _get_and_validate_course(course_key_string, request.user) + if not course: + return Response(data='Course Not Found', status=rest_status.HTTP_400_BAD_REQUEST) + + data, status = videos_post(course, request) + return Response(data, status=status) + + +def handle_video_images(request, course_key_string, edx_video_id=None): + """Function to handle image files""" + + # respond with a 404 if image upload is not enabled. + if not VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled() and not use_mock_video_uploads(): + return HttpResponseNotFound() + + if 'file' not in request.FILES: + return JsonResponse({'error': _('An image file is required.')}, status=400) + + image_file = request.FILES['file'] + error = validate_video_image(image_file) + if error: + return JsonResponse({'error': error}, status=400) + + with closing(image_file): + image_url = update_video_image(edx_video_id, course_key_string, image_file, image_file.name) + LOGGER.info( + 'VIDEOS: Video image uploaded for edx_video_id [%s] in course [%s]', edx_video_id, course_key_string + ) + + return JsonResponse({'image_url': image_url}) + + +def check_video_images_upload_enabled(request): + """Function to check if images can be uploaded""" + # respond with a false if image upload is not enabled. + if not VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(): + return JsonResponse({'allowThumbnailUpload': False}) + + return JsonResponse({'allowThumbnailUpload': True}) + + +def enabled_video_features(request): + """ Return a dict with info about which video features are enabled """ + + features = { + 'allowThumbnailUpload': VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(), + 'videoSharingEnabled': PUBLIC_VIDEO_SHARE.is_enabled(), + } + return JsonResponse(features) + + +def validate_transcript_preferences(provider, cielo24_fidelity, cielo24_turnaround, + three_play_turnaround, video_source_language, preferred_languages): + """ + Validate 3rd Party Transcription Preferences. + + Arguments: + provider: Transcription provider + cielo24_fidelity: Cielo24 transcription fidelity. + cielo24_turnaround: Cielo24 transcription turnaround. + three_play_turnaround: 3PlayMedia transcription turnaround. + video_source_language: Source/Speech language of the videos that are going to be submitted to the Providers. + preferred_languages: list of language codes. + + Returns: + validated preferences or a validation error. + """ + error, preferences = None, {} + + # validate transcription providers + transcription_plans = get_3rd_party_transcription_plans() + if provider in list(transcription_plans.keys()): # lint-amnesty, pylint: disable=consider-iterating-dictionary + + # Further validations for providers + if provider == TranscriptProvider.CIELO24: + + # Validate transcription fidelity + if cielo24_fidelity in transcription_plans[provider]['fidelity']: + + # Validate transcription turnaround + if cielo24_turnaround not in transcription_plans[provider]['turnaround']: + error = f'Invalid cielo24 turnaround {cielo24_turnaround}.' + return error, preferences + + # Validate transcription languages + supported_languages = transcription_plans[provider]['fidelity'][cielo24_fidelity]['languages'] + if video_source_language not in supported_languages: + error = f'Unsupported source language {video_source_language}.' + return error, preferences + + if not preferred_languages or not set(preferred_languages) <= set(supported_languages.keys()): + error = f'Invalid languages {preferred_languages}.' + return error, preferences + + # Validated Cielo24 preferences + preferences = { + 'video_source_language': video_source_language, + 'cielo24_fidelity': cielo24_fidelity, + 'cielo24_turnaround': cielo24_turnaround, + 'preferred_languages': preferred_languages, + } + else: + error = f'Invalid cielo24 fidelity {cielo24_fidelity}.' + elif provider == TranscriptProvider.THREE_PLAY_MEDIA: + + # Validate transcription turnaround + if three_play_turnaround not in transcription_plans[provider]['turnaround']: + error = f'Invalid 3play turnaround {three_play_turnaround}.' + return error, preferences + + # Validate transcription languages + valid_translations_map = transcription_plans[provider]['translations'] + if video_source_language not in list(valid_translations_map.keys()): + error = f'Unsupported source language {video_source_language}.' + return error, preferences + + valid_target_languages = valid_translations_map[video_source_language] + if not preferred_languages or not set(preferred_languages) <= set(valid_target_languages): + error = f'Invalid languages {preferred_languages}.' + return error, preferences + + # Validated 3PlayMedia preferences + preferences = { + 'three_play_turnaround': three_play_turnaround, + 'video_source_language': video_source_language, + 'preferred_languages': preferred_languages, + } + else: + error = f'Invalid provider {provider}.' + + return error, preferences + + +def handle_transcript_preferences(request, course_key_string): + """ + JSON view handler to post the transcript preferences. + + Arguments: + request: WSGI request object + course_key_string: string for course key + + Returns: valid json response or 400 with error message + """ + course_key = CourseKey.from_string(course_key_string) + is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course_key) + if not is_video_transcript_enabled: + return HttpResponseNotFound() + if request.method == 'POST': + data = request.json + provider = data.get('provider') + error, preferences = validate_transcript_preferences( + provider=provider, + cielo24_fidelity=data.get('cielo24_fidelity', ''), + cielo24_turnaround=data.get('cielo24_turnaround', ''), + three_play_turnaround=data.get('three_play_turnaround', ''), + video_source_language=data.get('video_source_language'), + preferred_languages=list(map(str, data.get('preferred_languages', []))) + ) + if error: + response = JsonResponse({'error': error}, status=400) + else: + preferences.update({'provider': provider}) + transcript_preferences = create_or_update_transcript_preferences(course_key_string, **preferences) + response = JsonResponse({'transcript_preferences': transcript_preferences}, status=200) + + return response + elif request.method == 'DELETE': + remove_transcript_preferences(course_key_string) + return JsonResponse() + + +def get_video_encodings_download(request, course_key_string): + """ + Returns a CSV report containing the encoded video URLs for video uploads + in the following format: + + Video ID,Name,Status,Profile1 URL,Profile2 URL + aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4 + """ + course = _get_and_validate_course(course_key_string, request.user) + + if not course: + return HttpResponseNotFound() + + def get_profile_header(profile): + """Returns the column header string for the given profile's URLs""" + # Translators: This is the header for a CSV file column + # containing URLs for video encodings for the named profile + # (e.g. desktop, mobile high quality, mobile low quality) + return _("{profile_name} URL").format(profile_name=profile) + + profile_whitelist = VideoUploadConfig.get_profile_whitelist() + videos, __ = _get_videos(course) + videos = list(videos) + name_col = _("Name") + duration_col = _("Duration") + added_col = _("Date Added") + video_id_col = _("Video ID") + status_col = _("Status") + profile_cols = [get_profile_header(profile) for profile in profile_whitelist] + + def make_csv_dict(video): + """ + Makes a dictionary suitable for writing CSV output. This involves + extracting the required items from the original video dict and + converting all keys and values to UTF-8 encoded string objects, + because the CSV module doesn't play well with unicode objects. + """ + # Translators: This is listed as the duration for a video that has not + # yet reached the point in its processing by the servers where its + # duration is determined. + duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending") + ret = dict( + [ + (name_col, video["client_video_id"]), + (duration_col, duration_val), + (added_col, video["created"].isoformat()), + (video_id_col, video["edx_video_id"]), + (status_col, video["status"]), + ] + + [ + (get_profile_header(encoded_video["profile"]), encoded_video["url"]) + for encoded_video in video["encoded_videos"] + if encoded_video["profile"] in profile_whitelist + ] + ) + return dict(ret.items()) + + # Write csv to bytes-like object. We need a separate writer and buffer as the csv + # writer writes str and the FileResponse expects a bytes files. + buffer = io.BytesIO() + buffer_writer = codecs.getwriter("utf-8")(buffer) + writer = csv.DictWriter( + buffer_writer, + [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols, + dialect=csv.excel + ) + writer.writeheader() + for video in videos: + writer.writerow(make_csv_dict(video)) + buffer.seek(0) + + # Translators: This is the suggested filename when downloading the URL + # listing for videos uploaded through Studio + filename = _("{course}_video_urls").format(course=course.id.course) + ".csv" + return FileResponse(buffer, as_attachment=True, filename=filename, content_type="text/csv") + + +def _get_and_validate_course(course_key_string, user): + """ + Given a course key, return the course if it exists, the given user has + access to it, and it is properly configured for video uploads + """ + course_key = CourseKey.from_string(course_key_string) + + # For now, assume all studio users that have access to the course can upload videos. + # In the future, we plan to add a new org-level role for video uploaders. + course = get_course_and_check_access(course_key, user) + + if ( + settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and + getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and + course and + course.video_pipeline_configured + ): + return course + else: + return None + + +def convert_video_status(video, is_video_encodes_ready=False): + """ + Convert status of a video. Status can be converted to one of the following: + + * FAILED if video is in `upload` state for more than 24 hours + * `YouTube Duplicate` if status is `invalid_token` + * user-friendly video status + """ + now = datetime.now(video.get('created', datetime.now().replace(tzinfo=UTC)).tzinfo) + + if video['status'] == 'upload' and (now - video['created']) > timedelta(hours=MAX_UPLOAD_HOURS): + new_status = 'upload_failed' + status = StatusDisplayStrings.get(new_status) + message = 'Video with id [{}] is still in upload after [{}] hours, setting status to [{}]'.format( + video['edx_video_id'], MAX_UPLOAD_HOURS, new_status + ) + send_video_status_update([ + { + 'edxVideoId': video['edx_video_id'], + 'status': new_status, + 'message': message + } + ]) + elif video['status'] == 'invalid_token': + status = StatusDisplayStrings.get('youtube_duplicate') + elif is_video_encodes_ready: + status = StatusDisplayStrings.get('file_complete') + else: + status = StatusDisplayStrings.get(video['status']) + + return status + + +def _get_videos(course, pagination_conf=None): + """ + Retrieves the list of videos from VAL corresponding to this course. + """ + videos, pagination_context = get_videos_for_course( + str(course.id), + VideoSortField.created, + SortDirection.desc, + pagination_conf + ) + videos = list(videos) + + # This is required to see if edx video pipeline is enabled while converting the video status. + course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token') + transcription_statuses = ['transcription_in_progress', 'transcript_ready', 'partial_failure', 'transcript_failed'] + + # convert VAL's status to studio's Video Upload feature status. + for video in videos: + # If we are using "new video workflow" and status is in `transcription_statuses` then video encodes are ready. + # This is because Transcription starts once all the encodes are complete except for YT, but according to + # "new video workflow" YT is disabled as well as deprecated. So, Its precise to say that the Transcription + # starts once all the encodings are complete *for the new video workflow*. + is_video_encodes_ready = not course_video_upload_token and (video['status'] in transcription_statuses) + # Update with transcript languages + video['transcripts'] = get_available_transcript_languages(video_id=video['edx_video_id']) + video['transcription_status'] = ( + StatusDisplayStrings.get(video['status']) if is_video_encodes_ready else '' + ) + video['transcript_urls'] = {} + for language_code in video['transcripts']: + video['transcript_urls'][language_code] = get_video_transcript_url( + video_id=video['edx_video_id'], + language_code=language_code, + ) + # Convert the video status. + video['status'] = convert_video_status(video, is_video_encodes_ready) + + return videos, pagination_context + + +def _get_default_video_image_url(): + """ + Returns default video image url + """ + return staticfiles_storage.url(settings.VIDEO_IMAGE_DEFAULT_FILENAME) + + +def _get_index_videos(course, pagination_conf=None): + """ + Returns the information about each video upload required for the video list + """ + course_id = str(course.id) + attrs = [ + 'edx_video_id', 'client_video_id', 'created', 'duration', + 'status', 'courses', 'transcripts', 'transcription_status', + 'transcript_urls', 'error_description' + ] + + def _get_values(video): + """ + Get data for predefined video attributes. + """ + values = {} + for attr in attrs: + if attr == 'courses': + course = [c for c in video['courses'] if course_id in c] + (__, values['course_video_image_url']), = list(course[0].items()) + else: + values[attr] = video[attr] + + return values + + videos, pagination_context = _get_videos(course, pagination_conf) + return [_get_values(video) for video in videos], pagination_context + + +def get_all_transcript_languages(): + """ + Returns all possible languages for transcript. + """ + third_party_transcription_languages = {} + transcription_plans = get_3rd_party_transcription_plans() + cielo_fidelity = transcription_plans[TranscriptProvider.CIELO24]['fidelity'] + + # Get third party transcription languages. + third_party_transcription_languages.update(transcription_plans[TranscriptProvider.THREE_PLAY_MEDIA]['languages']) + third_party_transcription_languages.update(cielo_fidelity['MECHANICAL']['languages']) + third_party_transcription_languages.update(cielo_fidelity['PREMIUM']['languages']) + third_party_transcription_languages.update(cielo_fidelity['PROFESSIONAL']['languages']) + + all_languages_dict = dict(settings.ALL_LANGUAGES, **third_party_transcription_languages) + # Return combined system settings and 3rd party transcript languages. + all_languages = [] + for key, value in sorted(all_languages_dict.items(), key=lambda k_v: k_v[1]): + all_languages.append({ + 'language_code': key, + 'language_text': value + }) + return all_languages + + +def videos_index_html(course, pagination_conf=None): + """ + Returns an HTML page to display previous video uploads and allow new ones + """ + is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id) + previous_uploads, pagination_context = _get_index_videos(course, pagination_conf) + context = { + 'context_course': course, + 'image_upload_url': reverse_course_url('video_images_handler', str(course.id)), + 'video_handler_url': reverse_course_url('videos_handler', str(course.id)), + 'encodings_download_url': reverse_course_url('video_encodings_download', str(course.id)), + 'default_video_image_url': _get_default_video_image_url(), + 'previous_uploads': previous_uploads, + 'concurrent_upload_limit': settings.VIDEO_UPLOAD_PIPELINE.get('CONCURRENT_UPLOAD_LIMIT', 0), + 'video_supported_file_formats': list(VIDEO_SUPPORTED_FILE_FORMATS.keys()), + 'video_upload_max_file_size': VIDEO_UPLOAD_MAX_FILE_SIZE_GB, + 'video_image_settings': { + 'video_image_upload_enabled': VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(), + 'max_size': settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MAX_BYTES'], + 'min_size': settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MIN_BYTES'], + 'max_width': settings.VIDEO_IMAGE_MAX_WIDTH, + 'max_height': settings.VIDEO_IMAGE_MAX_HEIGHT, + 'supported_file_formats': settings.VIDEO_IMAGE_SUPPORTED_FILE_FORMATS + }, + 'is_video_transcript_enabled': is_video_transcript_enabled, + 'active_transcript_preferences': None, + 'transcript_credentials': None, + 'transcript_available_languages': get_all_transcript_languages(), + 'video_transcript_settings': { + 'transcript_download_handler_url': reverse('transcript_download_handler'), + 'transcript_upload_handler_url': reverse('transcript_upload_handler'), + 'transcript_delete_handler_url': reverse_course_url('transcript_delete_handler', str(course.id)), + 'trancript_download_file_format': Transcript.SRT + }, + 'pagination_context': pagination_context + } + + if is_video_transcript_enabled: + context['video_transcript_settings'].update({ + 'transcript_preferences_handler_url': reverse_course_url( + 'transcript_preferences_handler', + str(course.id) + ), + 'transcript_credentials_handler_url': reverse_course_url( + 'transcript_credentials_handler', + str(course.id) + ), + 'transcription_plans': get_3rd_party_transcription_plans(), + }) + context['active_transcript_preferences'] = get_transcript_preferences(str(course.id)) + # Cached state for transcript providers' credentials (org-specific) + context['transcript_credentials'] = get_transcript_credentials_state_for_org(course.id.org) + if use_new_video_uploads_page(course.id): + return redirect(get_video_uploads_url(course.id)) + return render_to_response('videos_index.html', context) + + +def videos_index_json(course): + """ + Returns JSON in the following format: + { + 'videos': [{ + 'edx_video_id': 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa', + 'client_video_id': 'video.mp4', + 'created': '1970-01-01T00:00:00Z', + 'duration': 42.5, + 'status': 'upload', + 'course_video_image_url': 'https://video/images/1234.jpg' + }] + } + """ + index_videos, __ = _get_index_videos(course) + return JsonResponse({"videos": index_videos}, status=200) + + +def videos_post(course, request): + """ + Input (JSON): + { + "files": [{ + "file_name": "video.mp4", + "content_type": "video/mp4" + }] + } + + Returns (JSON): + { + "files": [{ + "file_name": "video.mp4", + "upload_url": "http://example.com/put_video" + }] + } + + The returned array corresponds exactly to the input array. + """ + + if use_mock_video_uploads(): + return {'files': [{'file_name': 'video.mp4', 'upload_url': 'http://example.com/put_video'}]}, 200 + + error = None + data = request.json + if 'files' not in data: + error = "Request object is not JSON or does not contain 'files'" + elif any( + 'file_name' not in file or 'content_type' not in file + for file in data['files'] + ): + error = "Request 'files' entry does not contain 'file_name' and 'content_type'" + elif any( + file['content_type'] not in list(VIDEO_SUPPORTED_FILE_FORMATS.values()) + for file in data['files'] + ): + error = "Request 'files' entry contain unsupported content_type" + + if error: + return {'error': error}, 400 + + bucket = storage_service_bucket() + req_files = data['files'] + resp_files = [] + + for req_file in req_files: + file_name = req_file['file_name'] + + try: + file_name.encode('ascii') + except UnicodeEncodeError: + error_msg = 'The file name for %s must contain only ASCII characters.' % file_name + return {'error': error_msg}, 400 + + edx_video_id = str(uuid4()) + key = storage_service_key(bucket, file_name=edx_video_id) + + metadata_list = [ + ('client_video_id', file_name), + ('course_key', str(course.id)), + ] + + course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token') + + # Only include `course_video_upload_token` if youtube has not been deprecated + # for this course. + if not DEPRECATE_YOUTUBE.is_enabled(course.id) and course_video_upload_token: + metadata_list.append(('course_video_upload_token', course_video_upload_token)) + + is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id) + if is_video_transcript_enabled: + transcript_preferences = get_transcript_preferences(str(course.id)) + if transcript_preferences is not None: + metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences))) + + for metadata_name, value in metadata_list: + key.set_metadata(metadata_name, value) + upload_url = key.generate_url( + KEY_EXPIRATION_IN_SECONDS, + 'PUT', + headers={'Content-Type': req_file['content_type']} + ) + + # persist edx_video_id in VAL + create_video({ + 'edx_video_id': edx_video_id, + 'status': 'upload', + 'client_video_id': file_name, + 'duration': 0, + 'encoded_videos': [], + 'courses': [str(course.id)] + }) + + resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id}) + + return {'files': resp_files}, 200 + + +def storage_service_bucket(): + """ + Returns an S3 bucket for video upload. + """ + if ENABLE_DEVSTACK_VIDEO_UPLOADS.is_enabled(): + params = { + 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, + 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY, + 'security_token': settings.AWS_SECURITY_TOKEN + + } + else: + params = { + 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, + 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY + } + + conn = S3Connection(**params) + + # We don't need to validate our bucket, it requires a very permissive IAM permission + # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys() + # meaning it would need ListObjects on the whole bucket, not just the path used in each + # environment (since we share a single bucket for multiple deployments in some configurations) + return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False) + + +def storage_service_key(bucket, file_name): + """ + Returns an S3 key to the given file in the given bucket. + """ + key_name = "{}/{}".format( + settings.VIDEO_UPLOAD_PIPELINE.get("ROOT_PATH", ""), + file_name + ) + return s3.key.Key(bucket, key_name) + + +def send_video_status_update(updates): + """ + Update video status in edx-val. + """ + for update in updates: + update_video_status(update.get('edxVideoId'), update.get('status')) + LOGGER.info( + 'VIDEOS: Video status update with id [%s], status [%s] and message [%s]', + update.get('edxVideoId'), + update.get('status'), + update.get('message') + ) + + return JsonResponse() + + +def is_status_update_request(request_data): + """ + Returns True if `request_data` contains status update else False. + """ + return any('status' in update for update in request_data) + + +def _generate_pagination_configuration(course_key_string, request): + """ + Returns pagination configuration + """ + course_key = CourseKey.from_string(course_key_string) + if not ENABLE_VIDEO_UPLOAD_PAGINATION.is_enabled(course_key): + return None + return { + 'page_number': request.GET.get('page', 1), + 'videos_per_page': request.session.get("VIDEOS_PER_PAGE", VIDEOS_PER_PAGE) + } + + +def _is_pagination_context_update_request(request): + """ + Checks if request contains `videos_per_page` + """ + return request.POST.get('id', '') == "videos_per_page" + + +def _update_pagination_context(request): + """ + Updates session with posted value + """ + error_msg = _('A non zero positive integer is expected') + try: + videos_per_page = int(request.POST.get('value')) + if videos_per_page <= 0: + return JsonResponse({'error': error_msg}, status=500) + except ValueError: + return JsonResponse({'error': error_msg}, status=500) + + request.session['VIDEOS_PER_PAGE'] = videos_per_page + return JsonResponse() diff --git a/cms/djangoapps/contentstore/views/tests/test_transcript_settings.py b/cms/djangoapps/contentstore/views/tests/test_transcript_settings.py index be4b920d497..4edea5665e2 100644 --- a/cms/djangoapps/contentstore/views/tests/test_transcript_settings.py +++ b/cms/djangoapps/contentstore/views/tests/test_transcript_settings.py @@ -12,13 +12,15 @@ from edxval import api from cms.djangoapps.contentstore.tests.utils import CourseTestCase +from cms.djangoapps.contentstore.transcript_storage_handlers import ( + TranscriptionProviderErrorType, + validate_transcript_credentials +) from cms.djangoapps.contentstore.utils import reverse_course_url from common.djangoapps.student.roles import CourseStaffRole from openedx.core.djangoapps.profile_images.tests.helpers import make_image_file from openedx.core.djangoapps.oauth_dispatch.jwt import create_jwt_for_user -from ..transcript_settings import TranscriptionProviderErrorType, validate_transcript_credentials - @ddt.ddt @patch( @@ -94,7 +96,7 @@ def test_404_with_feature_disabled(self): ) ) @ddt.unpack - @patch('cms.djangoapps.contentstore.views.transcript_settings.update_3rd_party_transcription_service_credentials') + @patch('cms.djangoapps.contentstore.transcript_storage_handlers.update_3rd_party_transcription_service_credentials') def test_transcript_credentials_handler(self, request_payload, update_credentials_response, expected_status_code, expected_response, mock_update_credentials): """ @@ -211,7 +213,7 @@ def test_405_with_not_allowed_request_method(self): response = self.client.post(self.view_url, content_type='application/json') self.assertEqual(response.status_code, 405) - @patch('cms.djangoapps.contentstore.views.transcript_settings.get_video_transcript_data') + @patch('cms.djangoapps.contentstore.transcript_storage_handlers.get_video_transcript_data') def test_transcript_download_handler(self, mock_get_video_transcript_data): """ Tests that transcript download handler works as expected. @@ -303,9 +305,9 @@ def test_405_with_not_allowed_request_method(self): response = self.client.get(self.view_url, content_type='application/json') self.assertEqual(response.status_code, 405) - @patch('cms.djangoapps.contentstore.views.transcript_settings.create_or_update_video_transcript') + @patch('cms.djangoapps.contentstore.transcript_storage_handlers.create_or_update_video_transcript') @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en']), ) def test_transcript_upload_handler(self, mock_create_or_update_video_transcript): @@ -370,7 +372,7 @@ def test_transcript_upload_handler(self, mock_create_or_update_video_transcript) ) @ddt.unpack @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en']), ) def test_transcript_upload_handler_missing_attrs(self, request_payload, expected_error_message): @@ -383,7 +385,7 @@ def test_transcript_upload_handler_missing_attrs(self, request_payload, expected self.assertEqual(json.loads(response.content.decode('utf-8'))['error'], expected_error_message) @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en', 'es']) ) def test_transcript_upload_handler_existing_transcript(self): @@ -405,7 +407,7 @@ def test_transcript_upload_handler_existing_transcript(self): ) @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en']), ) def test_transcript_upload_handler_with_image(self): @@ -432,7 +434,7 @@ def test_transcript_upload_handler_with_image(self): ) @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en']), ) def test_transcript_upload_handler_with_invalid_transcript(self): @@ -588,9 +590,9 @@ def test_405_with_not_allowed_request_method(self): response = self.client.get(self.view_url, content_type='application/json') self.assertEqual(response.status_code, 405) - @patch('cms.djangoapps.contentstore.views.transcript_settings.create_or_update_video_transcript') + @patch('cms.djangoapps.contentstore.transcript_storage_handlers.create_or_update_video_transcript') @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en']), ) def test_transcript_upload_handler(self, mock_create_or_update_video_transcript): @@ -655,7 +657,7 @@ def test_transcript_upload_handler(self, mock_create_or_update_video_transcript) ) @ddt.unpack @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en']), ) def test_transcript_upload_handler_missing_attrs(self, request_payload, expected_error_message): @@ -668,7 +670,7 @@ def test_transcript_upload_handler_missing_attrs(self, request_payload, expected self.assertEqual(json.loads(response.content.decode('utf-8'))['error'], expected_error_message) @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en', 'es']) ) def test_transcript_upload_handler_existing_transcript(self): @@ -690,7 +692,7 @@ def test_transcript_upload_handler_existing_transcript(self): ) @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en']), ) def test_transcript_upload_handler_with_image(self): @@ -717,7 +719,7 @@ def test_transcript_upload_handler_with_image(self): ) @patch( - 'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages', + 'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages', Mock(return_value=['en']), ) def test_transcript_upload_handler_with_invalid_transcript(self): diff --git a/cms/djangoapps/contentstore/views/tests/test_videos.py b/cms/djangoapps/contentstore/views/tests/test_videos.py index 44d1ee177b9..c0f48073ad7 100644 --- a/cms/djangoapps/contentstore/views/tests/test_videos.py +++ b/cms/djangoapps/contentstore/views/tests/test_videos.py @@ -43,11 +43,15 @@ ENABLE_VIDEO_UPLOAD_PAGINATION, KEY_EXPIRATION_IN_SECONDS, VIDEO_IMAGE_UPLOAD_ENABLED, - PUBLIC_VIDEO_SHARE, - StatusDisplayStrings, - TranscriptProvider, +) +from cms.djangoapps.contentstore.video_storage_handlers import ( _get_default_video_image_url, - convert_video_status, storage_service_bucket, storage_service_key + TranscriptProvider, + StatusDisplayStrings, + convert_video_status, + storage_service_bucket, + storage_service_key, + PUBLIC_VIDEO_SHARE ) @@ -210,7 +214,7 @@ class VideoUploadPostTestsMixin: """ @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') @patch('boto.s3.key.Key') - @patch('cms.djangoapps.contentstore.views.videos.S3Connection') + @patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection') def test_post_success(self, mock_conn, mock_key): files = [ { @@ -467,7 +471,7 @@ def test_get_html_paginated(self): @override_settings(AWS_ACCESS_KEY_ID="test_key_id", AWS_SECRET_ACCESS_KEY="test_secret") @patch("boto.s3.key.Key") - @patch("cms.djangoapps.contentstore.views.videos.S3Connection") + @patch("cms.djangoapps.contentstore.video_storage_handlers.S3Connection") @ddt.data( ( [ @@ -529,7 +533,7 @@ def test_video_supported_file_formats(self, files, expected_status, mock_conn, m self.assertEqual(response['error'], "Request 'files' entry contain unsupported content_type") @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') - @patch('cms.djangoapps.contentstore.views.videos.S3Connection') + @patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection') def test_upload_with_non_ascii_charaters(self, mock_conn): """ Test that video uploads throws error message when file name contains special characters. @@ -552,7 +556,7 @@ def test_upload_with_non_ascii_charaters(self, mock_conn): @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret', AWS_SECURITY_TOKEN='token') @patch('boto.s3.key.Key') - @patch('cms.djangoapps.contentstore.views.videos.S3Connection') + @patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection') @override_waffle_flag(ENABLE_DEVSTACK_VIDEO_UPLOADS, active=True) def test_devstack_upload_connection(self, mock_conn, mock_key): files = [{'file_name': 'first.mp4', 'content_type': 'video/mp4'}] @@ -580,7 +584,7 @@ def test_devstack_upload_connection(self, mock_conn, mock_key): ) @patch('boto.s3.key.Key') - @patch('cms.djangoapps.contentstore.views.videos.S3Connection') + @patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection') def test_send_course_to_vem_pipeline(self, mock_conn, mock_key): """ Test that uploads always go to VEM S3 bucket by default. @@ -610,7 +614,7 @@ def test_send_course_to_vem_pipeline(self, mock_conn, mock_key): @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') @patch('boto.s3.key.Key') - @patch('cms.djangoapps.contentstore.views.videos.S3Connection') + @patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection') @ddt.data( { 'global_waffle': True, @@ -770,7 +774,7 @@ def assert_video_status(self, url, edx_video_id, status): # Test should fail if video not found self.assertEqual(True, False, 'Invalid edx_video_id') - @patch('cms.djangoapps.contentstore.views.videos.LOGGER') + @patch('cms.djangoapps.contentstore.video_storage_handlers.LOGGER') def test_video_status_update_request(self, mock_logger): """ Verifies that video status update request works as expected. @@ -1447,8 +1451,8 @@ def test_remove_transcript_preferences_not_found(self): @ddt.unpack @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') @patch('boto.s3.key.Key') - @patch('cms.djangoapps.contentstore.views.videos.S3Connection') - @patch('cms.djangoapps.contentstore.views.videos.get_transcript_preferences') + @patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection') + @patch('cms.djangoapps.contentstore.video_storage_handlers.get_transcript_preferences') def test_transcript_preferences_metadata(self, transcript_preferences, is_video_transcript_enabled, mock_transcript_preferences, mock_conn, mock_key): """ diff --git a/cms/djangoapps/contentstore/views/transcript_settings.py b/cms/djangoapps/contentstore/views/transcript_settings.py index 40846a2ec3c..776d3432b5e 100644 --- a/cms/djangoapps/contentstore/views/transcript_settings.py +++ b/cms/djangoapps/contentstore/views/transcript_settings.py @@ -4,32 +4,23 @@ import logging -import os from django.contrib.auth.decorators import login_required -from django.core.files.base import ContentFile -from django.http import HttpResponse, HttpResponseNotFound -from django.utils.translation import gettext as _ +from django.http import HttpResponseNotFound from django.views.decorators.http import require_GET, require_http_methods, require_POST -from edxval.api import ( - create_or_update_video_transcript, - delete_video_transcript, - get_3rd_party_transcription_plans, - get_available_transcript_languages, - get_video_transcript_data, - update_transcript_credentials_state_for_org -) from opaque_keys.edx.keys import CourseKey from rest_framework.decorators import api_view +from cms.djangoapps.contentstore.transcript_storage_handlers import ( + validate_transcript_upload_data, + upload_transcript, + delete_video_transcript, + handle_transcript_credentials, + handle_transcript_download, +) from common.djangoapps.student.auth import has_studio_write_access from common.djangoapps.util.json_request import JsonResponse, expect_json -from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag -from openedx.core.djangoapps.video_pipeline.api import update_3rd_party_transcription_service_credentials from openedx.core.lib.api.view_utils import view_auth_classes -from xmodule.video_block.transcripts_utils import Transcript, TranscriptsGenerationException # lint-amnesty, pylint: disable=wrong-import-order - -from .videos import TranscriptProvider __all__ = [ 'transcript_credentials_handler', @@ -42,51 +33,6 @@ LOGGER = logging.getLogger(__name__) -class TranscriptionProviderErrorType: - """ - Transcription provider's error types enumeration. - """ - INVALID_CREDENTIALS = 1 - - -def validate_transcript_credentials(provider, **credentials): - """ - Validates transcript credentials. - - Validations: - Providers must be either 3PlayMedia or Cielo24. - In case of: - 3PlayMedia - 'api_key' and 'api_secret_key' are required. - Cielo24 - 'api_key' and 'username' are required. - - It ignores any extra/unrelated parameters passed in credentials and - only returns the validated ones. - """ - error_message, validated_credentials = '', {} - valid_providers = list(get_3rd_party_transcription_plans().keys()) - if provider in valid_providers: - must_have_props = [] - if provider == TranscriptProvider.THREE_PLAY_MEDIA: - must_have_props = ['api_key', 'api_secret_key'] - elif provider == TranscriptProvider.CIELO24: - must_have_props = ['api_key', 'username'] - - missing = [ - must_have_prop for must_have_prop in must_have_props if must_have_prop not in list(credentials.keys()) # lint-amnesty, pylint: disable=consider-iterating-dictionary - ] - if missing: - error_message = '{missing} must be specified.'.format(missing=' and '.join(missing)) - return error_message, validated_credentials - - validated_credentials.update({ - prop: credentials[prop] for prop in must_have_props - }) - else: - error_message = f'Invalid Provider {provider}.' - - return error_message, validated_credentials - - @expect_json @login_required @require_POST @@ -103,35 +49,7 @@ def transcript_credentials_handler(request, course_key_string): - A 404 response if transcript feature is not enabled for this course. - A 400 if credentials do not pass validations, hence not updated in edx-video-pipeline. """ - course_key = CourseKey.from_string(course_key_string) - if not VideoTranscriptEnabledFlag.feature_enabled(course_key): - return HttpResponseNotFound() - - provider = request.json.pop('provider') - error_message, validated_credentials = validate_transcript_credentials(provider=provider, **request.json) - if error_message: - response = JsonResponse({'error': error_message}, status=400) - else: - # Send the validated credentials to edx-video-pipeline and video-encode-manager - credentials_payload = dict(validated_credentials, org=course_key.org, provider=provider) - error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload) - # Send appropriate response based on whether credentials were updated or not. - if is_updated: - # Cache credentials state in edx-val. - update_transcript_credentials_state_for_org(org=course_key.org, provider=provider, exists=is_updated) - response = JsonResponse(status=200) - else: - # Error response would contain error types and the following - # error type is received from edx-video-pipeline whenever we've - # got invalid credentials for a provider. Its kept this way because - # edx-video-pipeline doesn't support i18n translations yet. - error_type = error_response.get('error_type') - if error_type == TranscriptionProviderErrorType.INVALID_CREDENTIALS: - error_message = _('The information you entered is incorrect.') - - response = JsonResponse({'error': error_message}, status=400) - - return response + return handle_transcript_credentials(request, course_key_string) @login_required @@ -148,112 +66,17 @@ def transcript_download_handler(request): - A 400 if there is a validation error. - A 404 if there is no such transcript. """ - missing = [attr for attr in ['edx_video_id', 'language_code'] if attr not in request.GET] - if missing: - return JsonResponse( - {'error': _('The following parameters are required: {missing}.').format(missing=', '.join(missing))}, - status=400 - ) - - edx_video_id = request.GET['edx_video_id'] - language_code = request.GET['language_code'] - transcript = get_video_transcript_data(video_id=edx_video_id, language_code=language_code) - if transcript: - name_and_extension = os.path.splitext(transcript['file_name']) - basename, file_format = name_and_extension[0], name_and_extension[1][1:] - transcript_filename = f'{basename}.{Transcript.SRT}' - transcript_content = Transcript.convert( - content=transcript['content'], - input_format=file_format, - output_format=Transcript.SRT - ) - # Construct an HTTP response - response = HttpResponse(transcript_content, content_type=Transcript.mime_types[Transcript.SRT]) - response['Content-Disposition'] = f'attachment; filename="{transcript_filename}"' - else: - response = HttpResponseNotFound() - - return response - - -def upload_transcript(request): - """ - Upload a transcript file - - Arguments: - request: A WSGI request object - - Transcript file in SRT format - """ - edx_video_id = request.POST['edx_video_id'] - language_code = request.POST['language_code'] - new_language_code = request.POST['new_language_code'] - transcript_file = request.FILES['file'] - try: - # Convert SRT transcript into an SJSON format - # and upload it to S3. - sjson_subs = Transcript.convert( - content=transcript_file.read().decode('utf-8'), - input_format=Transcript.SRT, - output_format=Transcript.SJSON - ).encode() - create_or_update_video_transcript( - video_id=edx_video_id, - language_code=language_code, - metadata={ - 'provider': TranscriptProvider.CUSTOM, - 'file_format': Transcript.SJSON, - 'language_code': new_language_code - }, - file_data=ContentFile(sjson_subs), - ) - response = JsonResponse(status=201) - except (TranscriptsGenerationException, UnicodeDecodeError): - LOGGER.error("Unable to update transcript on edX video %s for language %s", edx_video_id, new_language_code) - response = JsonResponse( - {'error': _('There is a problem with this transcript file. Try to upload a different file.')}, - status=400 - ) - finally: - LOGGER.info("Updated transcript on edX video %s for language %s", edx_video_id, new_language_code) - return response - - -def validate_transcript_upload_data(data, files): - """ - Validates video transcript file. - Arguments: - data: A request's data part. - files: A request's files part. - Returns: - None or String - If there is error returns error message otherwise None. - """ - error = None - # Validate the must have attributes - this error is unlikely to be faced by common users. - must_have_attrs = ['edx_video_id', 'language_code', 'new_language_code'] - missing = [attr for attr in must_have_attrs if attr not in data] - if missing: - error = _('The following parameters are required: {missing}.').format(missing=', '.join(missing)) - elif ( - data['language_code'] != data['new_language_code'] and - data['new_language_code'] in get_available_transcript_languages(video_id=data['edx_video_id']) - ): - error = _('A transcript with the "{language_code}" language code already exists.'.format( # lint-amnesty, pylint: disable=translation-of-non-string - language_code=data['new_language_code'] - )) - elif 'file' not in files: - error = _('A transcript file is required.') - - return error + return handle_transcript_download(request) +# New version of this transcript upload API in contentstore/rest_api/transcripts.py +# Keeping the old API for backward compatibility @api_view(['POST']) @view_auth_classes() @expect_json def transcript_upload_api(request): """ - API View for uploading transcript files. + (Old) API View for uploading transcript files. Arguments: request: A WSGI request object diff --git a/cms/djangoapps/contentstore/views/transcripts_ajax.py b/cms/djangoapps/contentstore/views/transcripts_ajax.py index fd272e23509..61ea4da6150 100644 --- a/cms/djangoapps/contentstore/views/transcripts_ajax.py +++ b/cms/djangoapps/contentstore/views/transcripts_ajax.py @@ -21,7 +21,7 @@ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import UsageKey -from cms.djangoapps.contentstore.views.videos import TranscriptProvider +from cms.djangoapps.contentstore.video_storage_handlers import TranscriptProvider from common.djangoapps.student.auth import has_course_author_access from common.djangoapps.util.json_request import JsonResponse from xmodule.contentstore.content import StaticContent # lint-amnesty, pylint: disable=wrong-import-order diff --git a/cms/djangoapps/contentstore/views/videos.py b/cms/djangoapps/contentstore/views/videos.py index 7627c1e3f5a..e1bdfa1cde1 100644 --- a/cms/djangoapps/contentstore/views/videos.py +++ b/cms/djangoapps/contentstore/views/videos.py @@ -3,65 +3,35 @@ """ -import codecs -import csv -import io -import json import logging -from contextlib import closing -from datetime import datetime, timedelta -from uuid import uuid4 -from boto.s3.connection import S3Connection -from boto import s3 -from django.conf import settings from django.contrib.auth.decorators import login_required -from django.contrib.staticfiles.storage import staticfiles_storage -from django.http import FileResponse, HttpResponseNotFound -from django.shortcuts import redirect -from django.urls import reverse -from django.utils.translation import gettext as _ -from django.utils.translation import gettext_noop from django.views.decorators.http import require_GET, require_http_methods, require_POST from edx_toggles.toggles import WaffleSwitch -from edxval.api import ( - SortDirection, - VideoSortField, - create_or_update_transcript_preferences, - create_video, - get_3rd_party_transcription_plans, - get_available_transcript_languages, - get_video_transcript_url, - get_transcript_credentials_state_for_org, - get_transcript_preferences, - get_videos_for_course, - remove_transcript_preferences, - remove_video_for_course, - update_video_image, - update_video_status -) -from opaque_keys.edx.keys import CourseKey -from pytz import UTC -from rest_framework import status as rest_status from rest_framework.decorators import api_view -from rest_framework.response import Response - -from common.djangoapps.edxmako.shortcuts import render_to_response -from common.djangoapps.util.json_request import JsonResponse, expect_json -from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag -from openedx.core.djangoapps.video_config.toggles import PUBLIC_VIDEO_SHARE -from openedx.core.djangoapps.video_pipeline.config.waffle import ( - DEPRECATE_YOUTUBE, - ENABLE_DEVSTACK_VIDEO_UPLOADS, + +from cms.djangoapps.contentstore.video_storage_handlers import ( + handle_videos, + handle_generate_video_upload_link, + handle_video_images, + check_video_images_upload_enabled, + enabled_video_features, + handle_transcript_preferences, + get_video_encodings_download, + validate_transcript_preferences as validate_transcript_preferences_source_function, + convert_video_status as convert_video_status_source_function, + get_all_transcript_languages as get_all_transcript_languages_source_function, + videos_index_html as videos_index_html_source_function, + videos_index_json as videos_index_json_source_function, + videos_post as videos_post_source_function, + storage_service_bucket as storage_service_bucket_source_function, + storage_service_key as storage_service_key_source_function, + send_video_status_update as send_video_status_update_source_function, + is_status_update_request as is_status_update_request_source_function, ) + +from common.djangoapps.util.json_request import expect_json from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag from openedx.core.lib.api.view_utils import view_auth_classes -from xmodule.video_block.transcripts_utils import Transcript # lint-amnesty, pylint: disable=wrong-import-order - -from ..models import VideoUploadConfig -from ..toggles import use_new_video_uploads_page -from ..utils import reverse_course_url, get_video_uploads_url -from ..video_utils import validate_video_image -from .course import get_course_and_check_access __all__ = [ 'videos_handler', @@ -105,85 +75,6 @@ VIDEOS_PER_PAGE = 100 -class TranscriptProvider: - """ - Transcription Provider Enumeration - """ - CIELO24 = 'Cielo24' - THREE_PLAY_MEDIA = '3PlayMedia' - CUSTOM = 'Custom' - - -class StatusDisplayStrings: - """ - A class to map status strings as stored in VAL to display strings for the - video upload page - """ - - # Translators: This is the status of an active video upload - _UPLOADING = gettext_noop("Uploading") - # Translators: This is the status for a video that the servers are currently processing - _IN_PROGRESS = gettext_noop("In Progress") - # Translators: This is the status for a video that the servers have successfully processed - _COMPLETE = gettext_noop("Ready") - # Translators: This is the status for a video that is uploaded completely - _UPLOAD_COMPLETED = gettext_noop("Uploaded") - # Translators: This is the status for a video that the servers have failed to process - _FAILED = gettext_noop("Failed") - # Translators: This is the status for a video that is cancelled during upload by user - _CANCELLED = gettext_noop("Cancelled") - # Translators: This is the status for a video which has failed - # due to being flagged as a duplicate by an external or internal CMS - _DUPLICATE = gettext_noop("Failed Duplicate") - # Translators: This is the status for a video which has duplicate token for youtube - _YOUTUBE_DUPLICATE = gettext_noop("YouTube Duplicate") - # Translators: This is the status for a video for which an invalid - # processing token was provided in the course settings - _INVALID_TOKEN = gettext_noop("Invalid Token") - # Translators: This is the status for a video that was included in a course import - _IMPORTED = gettext_noop("Imported") - # Translators: This is the status for a video that is in an unknown state - _UNKNOWN = gettext_noop("Unknown") - # Translators: This is the status for a video that is having its transcription in progress on servers - _TRANSCRIPTION_IN_PROGRESS = gettext_noop("Transcription in Progress") - # Translators: This is the status for a video whose transcription is complete - _TRANSCRIPT_READY = gettext_noop("Transcript Ready") - # Translators: This is the status for a video whose transcription job was failed for some languages - _PARTIAL_FAILURE = gettext_noop("Partial Failure") - # Translators: This is the status for a video whose transcription job has failed altogether - _TRANSCRIPT_FAILED = gettext_noop("Transcript Failed") - - _STATUS_MAP = { - "upload": _UPLOADING, - "ingest": _IN_PROGRESS, - "transcode_queue": _IN_PROGRESS, - "transcode_active": _IN_PROGRESS, - "file_delivered": _COMPLETE, - "file_complete": _COMPLETE, - "upload_completed": _UPLOAD_COMPLETED, - "file_corrupt": _FAILED, - "pipeline_error": _FAILED, - "upload_failed": _FAILED, - "s3_upload_failed": _FAILED, - "upload_cancelled": _CANCELLED, - "duplicate": _DUPLICATE, - "youtube_duplicate": _YOUTUBE_DUPLICATE, - "invalid_token": _INVALID_TOKEN, - "imported": _IMPORTED, - "transcription_in_progress": _TRANSCRIPTION_IN_PROGRESS, - "transcript_ready": _TRANSCRIPT_READY, - "partial_failure": _PARTIAL_FAILURE, - # TODO: Add a related unit tests when the VAL update is part of platform - "transcript_failed": _TRANSCRIPT_FAILED, - } - - @staticmethod - def get(val_status): - """Map a VAL status string to a localized display string""" - # pylint: disable=translation-of-non-string - return _(StatusDisplayStrings._STATUS_MAP.get(val_status, StatusDisplayStrings._UNKNOWN)) - - @expect_json @login_required @require_http_methods(("GET", "POST", "DELETE")) @@ -199,31 +90,17 @@ def videos_handler(request, course_key_string, edx_video_id=None): POST json: create a new video upload; the actual files should not be provided to this endpoint but rather PUT to the respective upload_url values - contained in the response + contained in the response. Example payload: + { + "files": [{ + "file_name": "video.mp4", + "content_type": "video/mp4" + }] + } DELETE soft deletes a video for particular course """ - course = _get_and_validate_course(course_key_string, request.user) - - if not course: - return HttpResponseNotFound() - - if request.method == "GET": - if "application/json" in request.META.get("HTTP_ACCEPT", ""): - return videos_index_json(course) - pagination_conf = _generate_pagination_configuration(course_key_string, request) - return videos_index_html(course, pagination_conf) - elif request.method == "DELETE": - remove_video_for_course(course_key_string, edx_video_id) - return JsonResponse() - else: - if is_status_update_request(request.json): - return send_video_status_update(request.json) - elif _is_pagination_context_update_request(request): - return _update_pagination_context(request) - - data, status = videos_post(course, request) - return JsonResponse(data, status=status) + return handle_videos(request, course_key_string, edx_video_id) @api_view(['POST']) @@ -234,12 +111,7 @@ def generate_video_upload_link_handler(request, course_key_string): API for creating a video upload. Returns an edx_video_id and a presigned URL that can be used to upload the video to AWS S3. """ - course = _get_and_validate_course(course_key_string, request.user) - if not course: - return Response(data='Course Not Found', status=rest_status.HTTP_400_BAD_REQUEST) - - data, status = videos_post(course, request) - return Response(data, status=status) + return handle_generate_video_upload_link(request, course_key_string) @expect_json @@ -247,131 +119,31 @@ def generate_video_upload_link_handler(request, course_key_string): @require_POST def video_images_handler(request, course_key_string, edx_video_id=None): """Function to handle image files""" - - # respond with a 404 if image upload is not enabled. - if not VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(): - return HttpResponseNotFound() - - if 'file' not in request.FILES: - return JsonResponse({'error': _('An image file is required.')}, status=400) - - image_file = request.FILES['file'] - error = validate_video_image(image_file) - if error: - return JsonResponse({'error': error}, status=400) - - with closing(image_file): - image_url = update_video_image(edx_video_id, course_key_string, image_file, image_file.name) - LOGGER.info( - 'VIDEOS: Video image uploaded for edx_video_id [%s] in course [%s]', edx_video_id, course_key_string - ) - - return JsonResponse({'image_url': image_url}) + return handle_video_images(request, course_key_string, edx_video_id) @login_required @require_GET def video_images_upload_enabled(request): """Function to check if images can be uploaded""" - # respond with a false if image upload is not enabled. - if not VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(): - return JsonResponse({'allowThumbnailUpload': False}) - - return JsonResponse({'allowThumbnailUpload': True}) + return check_video_images_upload_enabled(request) @login_required @require_GET def get_video_features(request): """ Return a dict with info about which video features are enabled """ - - features = { - 'allowThumbnailUpload': VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(), - 'videoSharingEnabled': PUBLIC_VIDEO_SHARE.is_enabled(), - } - return JsonResponse(features) + return enabled_video_features(request) def validate_transcript_preferences(provider, cielo24_fidelity, cielo24_turnaround, three_play_turnaround, video_source_language, preferred_languages): """ - Validate 3rd Party Transcription Preferences. - - Arguments: - provider: Transcription provider - cielo24_fidelity: Cielo24 transcription fidelity. - cielo24_turnaround: Cielo24 transcription turnaround. - three_play_turnaround: 3PlayMedia transcription turnaround. - video_source_language: Source/Speech language of the videos that are going to be submitted to the Providers. - preferred_languages: list of language codes. - - Returns: - validated preferences or a validation error. + Exposes helper method without breaking existing bindings/dependencies """ - error, preferences = None, {} - - # validate transcription providers - transcription_plans = get_3rd_party_transcription_plans() - if provider in list(transcription_plans.keys()): # lint-amnesty, pylint: disable=consider-iterating-dictionary - - # Further validations for providers - if provider == TranscriptProvider.CIELO24: - - # Validate transcription fidelity - if cielo24_fidelity in transcription_plans[provider]['fidelity']: - - # Validate transcription turnaround - if cielo24_turnaround not in transcription_plans[provider]['turnaround']: - error = f'Invalid cielo24 turnaround {cielo24_turnaround}.' - return error, preferences - - # Validate transcription languages - supported_languages = transcription_plans[provider]['fidelity'][cielo24_fidelity]['languages'] - if video_source_language not in supported_languages: - error = f'Unsupported source language {video_source_language}.' - return error, preferences - - if not preferred_languages or not set(preferred_languages) <= set(supported_languages.keys()): - error = f'Invalid languages {preferred_languages}.' - return error, preferences - - # Validated Cielo24 preferences - preferences = { - 'video_source_language': video_source_language, - 'cielo24_fidelity': cielo24_fidelity, - 'cielo24_turnaround': cielo24_turnaround, - 'preferred_languages': preferred_languages, - } - else: - error = f'Invalid cielo24 fidelity {cielo24_fidelity}.' - elif provider == TranscriptProvider.THREE_PLAY_MEDIA: - - # Validate transcription turnaround - if three_play_turnaround not in transcription_plans[provider]['turnaround']: - error = f'Invalid 3play turnaround {three_play_turnaround}.' - return error, preferences - - # Validate transcription languages - valid_translations_map = transcription_plans[provider]['translations'] - if video_source_language not in list(valid_translations_map.keys()): - error = f'Unsupported source language {video_source_language}.' - return error, preferences - - valid_target_languages = valid_translations_map[video_source_language] - if not preferred_languages or not set(preferred_languages) <= set(valid_target_languages): - error = f'Invalid languages {preferred_languages}.' - return error, preferences - - # Validated 3PlayMedia preferences - preferences = { - 'three_play_turnaround': three_play_turnaround, - 'video_source_language': video_source_language, - 'preferred_languages': preferred_languages, - } - else: - error = f'Invalid provider {provider}.' - - return error, preferences + return validate_transcript_preferences_source_function(provider, cielo24_fidelity, cielo24_turnaround, + three_play_turnaround, video_source_language, + preferred_languages) @expect_json @@ -387,32 +159,7 @@ def transcript_preferences_handler(request, course_key_string): Returns: valid json response or 400 with error message """ - course_key = CourseKey.from_string(course_key_string) - is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course_key) - if not is_video_transcript_enabled: - return HttpResponseNotFound() - if request.method == 'POST': - data = request.json - provider = data.get('provider') - error, preferences = validate_transcript_preferences( - provider=provider, - cielo24_fidelity=data.get('cielo24_fidelity', ''), - cielo24_turnaround=data.get('cielo24_turnaround', ''), - three_play_turnaround=data.get('three_play_turnaround', ''), - video_source_language=data.get('video_source_language'), - preferred_languages=list(map(str, data.get('preferred_languages', []))) - ) - if error: - response = JsonResponse({'error': error}, status=400) - else: - preferences.update({'provider': provider}) - transcript_preferences = create_or_update_transcript_preferences(course_key_string, **preferences) - response = JsonResponse({'transcript_preferences': transcript_preferences}, status=200) - - return response - elif request.method == 'DELETE': - remove_transcript_preferences(course_key_string) - return JsonResponse() + return handle_transcript_preferences(request, course_key_string) @login_required @@ -425,492 +172,67 @@ def video_encodings_download(request, course_key_string): Video ID,Name,Status,Profile1 URL,Profile2 URL aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4 """ - course = _get_and_validate_course(course_key_string, request.user) - - if not course: - return HttpResponseNotFound() - - def get_profile_header(profile): - """Returns the column header string for the given profile's URLs""" - # Translators: This is the header for a CSV file column - # containing URLs for video encodings for the named profile - # (e.g. desktop, mobile high quality, mobile low quality) - return _("{profile_name} URL").format(profile_name=profile) - - profile_whitelist = VideoUploadConfig.get_profile_whitelist() - videos, __ = _get_videos(course) - videos = list(videos) - name_col = _("Name") - duration_col = _("Duration") - added_col = _("Date Added") - video_id_col = _("Video ID") - status_col = _("Status") - profile_cols = [get_profile_header(profile) for profile in profile_whitelist] - - def make_csv_dict(video): - """ - Makes a dictionary suitable for writing CSV output. This involves - extracting the required items from the original video dict and - converting all keys and values to UTF-8 encoded string objects, - because the CSV module doesn't play well with unicode objects. - """ - # Translators: This is listed as the duration for a video that has not - # yet reached the point in its processing by the servers where its - # duration is determined. - duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending") - ret = dict( - [ - (name_col, video["client_video_id"]), - (duration_col, duration_val), - (added_col, video["created"].isoformat()), - (video_id_col, video["edx_video_id"]), - (status_col, video["status"]), - ] + - [ - (get_profile_header(encoded_video["profile"]), encoded_video["url"]) - for encoded_video in video["encoded_videos"] - if encoded_video["profile"] in profile_whitelist - ] - ) - return dict(ret.items()) - - # Write csv to bytes-like object. We need a separate writer and buffer as the csv - # writer writes str and the FileResponse expects a bytes files. - buffer = io.BytesIO() - buffer_writer = codecs.getwriter("utf-8")(buffer) - writer = csv.DictWriter( - buffer_writer, - [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols, - dialect=csv.excel - ) - writer.writeheader() - for video in videos: - writer.writerow(make_csv_dict(video)) - buffer.seek(0) - - # Translators: This is the suggested filename when downloading the URL - # listing for videos uploaded through Studio - filename = _("{course}_video_urls").format(course=course.id.course) + ".csv" - return FileResponse(buffer, as_attachment=True, filename=filename, content_type="text/csv") - - -def _get_and_validate_course(course_key_string, user): - """ - Given a course key, return the course if it exists, the given user has - access to it, and it is properly configured for video uploads - """ - course_key = CourseKey.from_string(course_key_string) - - # For now, assume all studio users that have access to the course can upload videos. - # In the future, we plan to add a new org-level role for video uploaders. - course = get_course_and_check_access(course_key, user) - - if ( - settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and - getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and - course and - course.video_pipeline_configured - ): - return course - else: - return None + return get_video_encodings_download(request, course_key_string) def convert_video_status(video, is_video_encodes_ready=False): """ - Convert status of a video. Status can be converted to one of the following: - - * FAILED if video is in `upload` state for more than 24 hours - * `YouTube Duplicate` if status is `invalid_token` - * user-friendly video status - """ - now = datetime.now(video.get('created', datetime.now().replace(tzinfo=UTC)).tzinfo) - - if video['status'] == 'upload' and (now - video['created']) > timedelta(hours=MAX_UPLOAD_HOURS): - new_status = 'upload_failed' - status = StatusDisplayStrings.get(new_status) - message = 'Video with id [{}] is still in upload after [{}] hours, setting status to [{}]'.format( - video['edx_video_id'], MAX_UPLOAD_HOURS, new_status - ) - send_video_status_update([ - { - 'edxVideoId': video['edx_video_id'], - 'status': new_status, - 'message': message - } - ]) - elif video['status'] == 'invalid_token': - status = StatusDisplayStrings.get('youtube_duplicate') - elif is_video_encodes_ready: - status = StatusDisplayStrings.get('file_complete') - else: - status = StatusDisplayStrings.get(video['status']) - - return status - - -def _get_videos(course, pagination_conf=None): - """ - Retrieves the list of videos from VAL corresponding to this course. - """ - videos, pagination_context = get_videos_for_course( - str(course.id), - VideoSortField.created, - SortDirection.desc, - pagination_conf - ) - videos = list(videos) - - # This is required to see if edx video pipeline is enabled while converting the video status. - course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token') - transcription_statuses = ['transcription_in_progress', 'transcript_ready', 'partial_failure', 'transcript_failed'] - - # convert VAL's status to studio's Video Upload feature status. - for video in videos: - # If we are using "new video workflow" and status is in `transcription_statuses` then video encodes are ready. - # This is because Transcription starts once all the encodes are complete except for YT, but according to - # "new video workflow" YT is disabled as well as deprecated. So, Its precise to say that the Transcription - # starts once all the encodings are complete *for the new video workflow*. - is_video_encodes_ready = not course_video_upload_token and (video['status'] in transcription_statuses) - # Update with transcript languages - video['transcripts'] = get_available_transcript_languages(video_id=video['edx_video_id']) - video['transcription_status'] = ( - StatusDisplayStrings.get(video['status']) if is_video_encodes_ready else '' - ) - video['transcript_urls'] = {} - for language_code in video['transcripts']: - video['transcript_urls'][language_code] = get_video_transcript_url( - video_id=video['edx_video_id'], - language_code=language_code, - ) - # Convert the video status. - video['status'] = convert_video_status(video, is_video_encodes_ready) - - return videos, pagination_context - - -def _get_default_video_image_url(): + Exposes helper method without breaking existing bindings/dependencies """ - Returns default video image url - """ - return staticfiles_storage.url(settings.VIDEO_IMAGE_DEFAULT_FILENAME) - - -def _get_index_videos(course, pagination_conf=None): - """ - Returns the information about each video upload required for the video list - """ - course_id = str(course.id) - attrs = [ - 'edx_video_id', 'client_video_id', 'created', 'duration', - 'status', 'courses', 'transcripts', 'transcription_status', - 'transcript_urls', 'error_description' - ] - - def _get_values(video): - """ - Get data for predefined video attributes. - """ - values = {} - for attr in attrs: - if attr == 'courses': - course = [c for c in video['courses'] if course_id in c] - (__, values['course_video_image_url']), = list(course[0].items()) - else: - values[attr] = video[attr] - - return values - - videos, pagination_context = _get_videos(course, pagination_conf) - return [_get_values(video) for video in videos], pagination_context + return convert_video_status_source_function(video, is_video_encodes_ready) def get_all_transcript_languages(): """ - Returns all possible languages for transcript. + Exposes helper method without breaking existing bindings/dependencies """ - third_party_transcription_languages = {} - transcription_plans = get_3rd_party_transcription_plans() - cielo_fidelity = transcription_plans[TranscriptProvider.CIELO24]['fidelity'] - - # Get third party transcription languages. - third_party_transcription_languages.update(transcription_plans[TranscriptProvider.THREE_PLAY_MEDIA]['languages']) - third_party_transcription_languages.update(cielo_fidelity['MECHANICAL']['languages']) - third_party_transcription_languages.update(cielo_fidelity['PREMIUM']['languages']) - third_party_transcription_languages.update(cielo_fidelity['PROFESSIONAL']['languages']) - - all_languages_dict = dict(settings.ALL_LANGUAGES, **third_party_transcription_languages) - # Return combined system settings and 3rd party transcript languages. - all_languages = [] - for key, value in sorted(all_languages_dict.items(), key=lambda k_v: k_v[1]): - all_languages.append({ - 'language_code': key, - 'language_text': value - }) - return all_languages + return get_all_transcript_languages_source_function() def videos_index_html(course, pagination_conf=None): """ - Returns an HTML page to display previous video uploads and allow new ones + Exposes helper method without breaking existing bindings/dependencies """ - is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id) - previous_uploads, pagination_context = _get_index_videos(course, pagination_conf) - context = { - 'context_course': course, - 'image_upload_url': reverse_course_url('video_images_handler', str(course.id)), - 'video_handler_url': reverse_course_url('videos_handler', str(course.id)), - 'encodings_download_url': reverse_course_url('video_encodings_download', str(course.id)), - 'default_video_image_url': _get_default_video_image_url(), - 'previous_uploads': previous_uploads, - 'concurrent_upload_limit': settings.VIDEO_UPLOAD_PIPELINE.get('CONCURRENT_UPLOAD_LIMIT', 0), - 'video_supported_file_formats': list(VIDEO_SUPPORTED_FILE_FORMATS.keys()), - 'video_upload_max_file_size': VIDEO_UPLOAD_MAX_FILE_SIZE_GB, - 'video_image_settings': { - 'video_image_upload_enabled': VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(), - 'max_size': settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MAX_BYTES'], - 'min_size': settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MIN_BYTES'], - 'max_width': settings.VIDEO_IMAGE_MAX_WIDTH, - 'max_height': settings.VIDEO_IMAGE_MAX_HEIGHT, - 'supported_file_formats': settings.VIDEO_IMAGE_SUPPORTED_FILE_FORMATS - }, - 'is_video_transcript_enabled': is_video_transcript_enabled, - 'active_transcript_preferences': None, - 'transcript_credentials': None, - 'transcript_available_languages': get_all_transcript_languages(), - 'video_transcript_settings': { - 'transcript_download_handler_url': reverse('transcript_download_handler'), - 'transcript_upload_handler_url': reverse('transcript_upload_handler'), - 'transcript_delete_handler_url': reverse_course_url('transcript_delete_handler', str(course.id)), - 'trancript_download_file_format': Transcript.SRT - }, - 'pagination_context': pagination_context - } - - if is_video_transcript_enabled: - context['video_transcript_settings'].update({ - 'transcript_preferences_handler_url': reverse_course_url( - 'transcript_preferences_handler', - str(course.id) - ), - 'transcript_credentials_handler_url': reverse_course_url( - 'transcript_credentials_handler', - str(course.id) - ), - 'transcription_plans': get_3rd_party_transcription_plans(), - }) - context['active_transcript_preferences'] = get_transcript_preferences(str(course.id)) - # Cached state for transcript providers' credentials (org-specific) - context['transcript_credentials'] = get_transcript_credentials_state_for_org(course.id.org) - if use_new_video_uploads_page(course.id): - return redirect(get_video_uploads_url(course.id)) - return render_to_response('videos_index.html', context) + return videos_index_html_source_function(course, pagination_conf) def videos_index_json(course): """ - Returns JSON in the following format: - { - 'videos': [{ - 'edx_video_id': 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa', - 'client_video_id': 'video.mp4', - 'created': '1970-01-01T00:00:00Z', - 'duration': 42.5, - 'status': 'upload', - 'course_video_image_url': 'https://video/images/1234.jpg' - }] - } + Exposes helper method without breaking existing bindings/dependencies """ - index_videos, __ = _get_index_videos(course) - return JsonResponse({"videos": index_videos}, status=200) + return videos_index_json_source_function(course) def videos_post(course, request): """ - Input (JSON): - { - "files": [{ - "file_name": "video.mp4", - "content_type": "video/mp4" - }] - } - - Returns (JSON): - { - "files": [{ - "file_name": "video.mp4", - "upload_url": "http://example.com/put_video" - }] - } - - The returned array corresponds exactly to the input array. + Exposes helper method without breaking existing bindings/dependencies """ - error = None - data = request.json - if 'files' not in data: - error = "Request object is not JSON or does not contain 'files'" - elif any( - 'file_name' not in file or 'content_type' not in file - for file in data['files'] - ): - error = "Request 'files' entry does not contain 'file_name' and 'content_type'" - elif any( - file['content_type'] not in list(VIDEO_SUPPORTED_FILE_FORMATS.values()) - for file in data['files'] - ): - error = "Request 'files' entry contain unsupported content_type" - - if error: - return {'error': error}, 400 - - bucket = storage_service_bucket() - req_files = data['files'] - resp_files = [] - - for req_file in req_files: - file_name = req_file['file_name'] - - try: - file_name.encode('ascii') - except UnicodeEncodeError: - error_msg = 'The file name for %s must contain only ASCII characters.' % file_name - return {'error': error_msg}, 400 - - edx_video_id = str(uuid4()) - key = storage_service_key(bucket, file_name=edx_video_id) - - metadata_list = [ - ('client_video_id', file_name), - ('course_key', str(course.id)), - ] - - course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token') - - # Only include `course_video_upload_token` if youtube has not been deprecated - # for this course. - if not DEPRECATE_YOUTUBE.is_enabled(course.id) and course_video_upload_token: - metadata_list.append(('course_video_upload_token', course_video_upload_token)) - - is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id) - if is_video_transcript_enabled: - transcript_preferences = get_transcript_preferences(str(course.id)) - if transcript_preferences is not None: - metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences))) - - for metadata_name, value in metadata_list: - key.set_metadata(metadata_name, value) - upload_url = key.generate_url( - KEY_EXPIRATION_IN_SECONDS, - 'PUT', - headers={'Content-Type': req_file['content_type']} - ) - - # persist edx_video_id in VAL - create_video({ - 'edx_video_id': edx_video_id, - 'status': 'upload', - 'client_video_id': file_name, - 'duration': 0, - 'encoded_videos': [], - 'courses': [str(course.id)] - }) - - resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id}) - - return {'files': resp_files}, 200 + return videos_post_source_function(course, request) def storage_service_bucket(): """ - Returns an S3 bucket for video upload. + Exposes helper method without breaking existing bindings/dependencies """ - if ENABLE_DEVSTACK_VIDEO_UPLOADS.is_enabled(): - params = { - 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, - 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY, - 'security_token': settings.AWS_SECURITY_TOKEN - - } - else: - params = { - 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, - 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY - } - - conn = S3Connection(**params) - - # We don't need to validate our bucket, it requires a very permissive IAM permission - # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys() - # meaning it would need ListObjects on the whole bucket, not just the path used in each - # environment (since we share a single bucket for multiple deployments in some configurations) - return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False) + return storage_service_bucket_source_function() def storage_service_key(bucket, file_name): """ - Returns an S3 key to the given file in the given bucket. + Exposes helper method without breaking existing bindings/dependencies """ - key_name = "{}/{}".format( - settings.VIDEO_UPLOAD_PIPELINE.get("ROOT_PATH", ""), - file_name - ) - return s3.key.Key(bucket, key_name) + return storage_service_key_source_function(bucket, file_name) def send_video_status_update(updates): """ - Update video status in edx-val. + Exposes helper method without breaking existing bindings/dependencies """ - for update in updates: - update_video_status(update.get('edxVideoId'), update.get('status')) - LOGGER.info( - 'VIDEOS: Video status update with id [%s], status [%s] and message [%s]', - update.get('edxVideoId'), - update.get('status'), - update.get('message') - ) - - return JsonResponse() + return send_video_status_update_source_function(updates) def is_status_update_request(request_data): """ - Returns True if `request_data` contains status update else False. - """ - return any('status' in update for update in request_data) - - -def _generate_pagination_configuration(course_key_string, request): - """ - Returns pagination configuration - """ - course_key = CourseKey.from_string(course_key_string) - if not ENABLE_VIDEO_UPLOAD_PAGINATION.is_enabled(course_key): - return None - return { - 'page_number': request.GET.get('page', 1), - 'videos_per_page': request.session.get("VIDEOS_PER_PAGE", VIDEOS_PER_PAGE) - } - - -def _is_pagination_context_update_request(request): - """ - Checks if request contains `videos_per_page` - """ - return request.POST.get('id', '') == "videos_per_page" - - -def _update_pagination_context(request): - """ - Updates session with posted value + Exposes helper method without breaking existing bindings/dependencies """ - error_msg = _('A non zero positive integer is expected') - try: - videos_per_page = int(request.POST.get('value')) - if videos_per_page <= 0: - return JsonResponse({'error': error_msg}, status=500) - except ValueError: - return JsonResponse({'error': error_msg}, status=500) - - request.session['VIDEOS_PER_PAGE'] = videos_per_page - return JsonResponse() + return is_status_update_request_source_function(request_data) diff --git a/cms/urls.py b/cms/urls.py index eb34f241fd8..5631c3aa957 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -328,7 +328,7 @@ urlpatterns.extend(get_plugin_url_patterns(ProjectType.CMS)) -# Contentstore +# Contentstore REST APIs urlpatterns += [ path('api/contentstore/', include('cms.djangoapps.contentstore.rest_api.urls')) ]