add button for ingesting cedar templates from CEDAR workbench

admin/management/urls.py

@@ -16,4 +16,5 @@
     re_path(r'^ban_spam_regex', views.BanSpamByRegex.as_view(), name='ban_spam_regex'),
     re_path(r'^daily_reporters_go', views.DailyReportersGo.as_view(), name='daily_reporters_go'),
     re_path(r'^monthly_reporters_go', views.MonthlyReportersGo.as_view(), name='monthly_reporters_go'),
+    re_path(r'^ingest_cedar_metadata_templates', views.IngestCedarMetadataTemplates.as_view(), name='ingest_cedar_metadata_templates'),
 ]

admin/management/views.py

@@ -9,6 +9,7 @@
 from osf.management.commands.update_registration_schemas import update_registration_schemas
 from osf.management.commands.daily_reporters_go import daily_reporters_go
 from osf.management.commands.monthly_reporters_go import monthly_reporters_go
+from osf.management.commands.fetch_cedar_metadata_templates import ingest_cedar_metadata_templates
 from scripts.find_spammy_content import manage_spammy_content
 from django.urls import reverse
 from django.shortcuts import redirect
@@ -134,3 +135,9 @@ def post(self, request, *args, **kwargs):
         else:
             messages.success(request, 'Monthly reporters successfully went.')
         return redirect(reverse('management:commands'))
+
+class IngestCedarMetadataTemplates(ManagementCommandPermissionView):
+    def post(self, request):
+        ingest_cedar_metadata_templates()
+        messages.success(request, 'Cedar templates have been successfully imported from Cedar Workbench.')
+        return redirect(reverse('management:commands'))

admin/templates/management/commands.html

@@ -116,6 +116,20 @@ <h4><u>Monthly Reporters, Go!</u></h4>
                     </nav>
                 </form>
             </section>
+            <section>
+                <h4><u>Ingest Cedar Metadata templates</u></h4>
+                <p>
+                    Use this management command to ingest CEDAR metadata templates from CEDAR Workbench.
+                    Note: This requires CEDAR_API_HOST and CEDAR_API_KEY to be set for the environment.
+                </p>
+                <form method="post"
+                      action="{% url 'management:ingest_cedar_metadata_templates'%}">
+                    {% csrf_token %}
+                    <nav>
+                        <input class="btn btn-success" type="submit" value="Run" />
+                    </nav>
+                </form>
+            </section>
         </div>
     </section>
 {% endblock %}

osf/management/commands/fetch_cedar_metadata_templates.py

@@ -1,6 +1,7 @@
 import logging
 
 from django.core.management.base import BaseCommand
+from framework.celery_tasks import app as celery_app
 
 from osf.models import CedarMetadataTemplate
 from osf.external.cedar.client import CedarClient
@@ -12,56 +13,59 @@
 class Command(BaseCommand):
 
     def handle(self, *args, **kwargs):
+        ingest_cedar_metadata_templates()
 
+@celery_app.task(name='management.commands.ingest_cedar_metadata_templates')
+def ingest_cedar_metadata_templates():
+    try:
+        ids = CedarClient().retrieve_all_template_ids()
+    except CedarClientError as e:
+        logger.error(f'Unable to retrieve all cedar template ids: e={e.reason}')
+        return
+
+    fetched = set()
+    newly_added = set()
+    unchanged = set()
+    updated = set()
+    failed = set()
+    for cedar_id in ids:
         try:
-            ids = CedarClient().retrieve_all_template_ids()
+            template = CedarClient().retrieve_template_by_id(cedar_id)
         except CedarClientError as e:
-            logger.error(f'Unable to retrieve all cedar template ids: e={e.reason}')
-            return
-
-        fetched = set()
-        newly_added = set()
-        unchanged = set()
-        updated = set()
-        failed = set()
-        for cedar_id in ids:
-            try:
-                template = CedarClient().retrieve_template_by_id(cedar_id)
-            except CedarClientError as e:
-                logger.error(f'Unable to retrieve the cedar template: id={cedar_id}, e={e.reason}')
-                failed.add(cedar_id)
-                continue
-            else:
-                fetched.add(cedar_id)
-            schema_name = template['schema:name']
-            pav_last_updated_on = template['pav:lastUpdatedOn']
-            existing_versions = CedarMetadataTemplate.objects.filter(cedar_id=cedar_id)
-            if existing_versions:
-                latest_version = existing_versions.order_by('-template_version').first()
-                if pav_last_updated_on == latest_version.template['pav:lastUpdatedOn']:
-                    unchanged.add(cedar_id)
-                else:
-                    # New version should be inactive
-                    CedarMetadataTemplate.objects.create(
-                        schema_name=schema_name,
-                        template=template,
-                        cedar_id=cedar_id,
-                        active=False,
-                        template_version=latest_version.template_version + 1
-                    )
-                    updated.add(cedar_id)
+            logger.error(f'Unable to retrieve the cedar template: id={cedar_id}, e={e.reason}')
+            failed.add(cedar_id)
+            continue
+        else:
+            fetched.add(cedar_id)
+        schema_name = template['schema:name']
+        pav_last_updated_on = template['pav:lastUpdatedOn']
+        existing_versions = CedarMetadataTemplate.objects.filter(cedar_id=cedar_id)
+        if existing_versions:
+            latest_version = existing_versions.order_by('-template_version').first()
+            if pav_last_updated_on == latest_version.template['pav:lastUpdatedOn']:
+                unchanged.add(cedar_id)
             else:
-                # Initial version should be active
+                # New version should be inactive
                 CedarMetadataTemplate.objects.create(
                     schema_name=schema_name,
                     template=template,
                     cedar_id=cedar_id,
-                    template_version=1
+                    active=False,
+                    template_version=latest_version.template_version + 1
                 )
-                newly_added.add(cedar_id)
+                updated.add(cedar_id)
+        else:
+            # Initial version should be active
+            CedarMetadataTemplate.objects.create(
+                schema_name=schema_name,
+                template=template,
+                cedar_id=cedar_id,
+                template_version=1
+            )
+            newly_added.add(cedar_id)
 
-        logger.error(f'failed ({len(failed)}): {failed}')
-        logger.info(f'fetched ({len(fetched)}): {fetched}')
-        logger.info(f'\tnewly_added ({len(newly_added)}): {newly_added}')
-        logger.info(f'\tunchanged ({len(unchanged)}): {unchanged}')
-        logger.info(f'\tupdated ({len(updated)}): {updated}')
+    logger.error(f'failed ({len(failed)}): {failed}')
+    logger.info(f'fetched ({len(fetched)}): {fetched}')
+    logger.info(f'\tnewly_added ({len(newly_added)}): {newly_added}')
+    logger.info(f'\tunchanged ({len(unchanged)}): {unchanged}')
+    logger.info(f'\tupdated ({len(updated)}): {updated}')

website/settings/defaults.py

@@ -443,6 +443,7 @@ class CeleryConfig:
         'osf.management.commands.spam_metrics',
         'osf.management.commands.daily_reporters_go',
         'osf.management.commands.monthly_reporters_go',
+        'osf.management.commands.ingest_cedar_metadata_templates',
     }
 
     med_pri_modules = {