From c4fe494bb15bbf3a00a24bd42610ca255a9a9508 Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Mon, 29 Jan 2024 11:28:42 -0500 Subject: [PATCH] add button for ingesting cedar templates from CEDAR workbench --- admin/management/urls.py | 1 + admin/management/views.py | 7 ++ admin/templates/management/commands.html | 14 +++ .../fetch_cedar_metadata_templates.py | 90 ++++++++++--------- website/settings/defaults.py | 1 + 5 files changed, 70 insertions(+), 43 deletions(-) diff --git a/admin/management/urls.py b/admin/management/urls.py index 1905cd03814..168441b5181 100644 --- a/admin/management/urls.py +++ b/admin/management/urls.py @@ -16,4 +16,5 @@ re_path(r'^ban_spam_regex', views.BanSpamByRegex.as_view(), name='ban_spam_regex'), re_path(r'^daily_reporters_go', views.DailyReportersGo.as_view(), name='daily_reporters_go'), re_path(r'^monthly_reporters_go', views.MonthlyReportersGo.as_view(), name='monthly_reporters_go'), + re_path(r'^ingest_cedar_metadata_templates', views.IngestCedarMetadataTemplates.as_view(), name='ingest_cedar_metadata_templates'), ] diff --git a/admin/management/views.py b/admin/management/views.py index 7e151df1d54..4323d7fd429 100644 --- a/admin/management/views.py +++ b/admin/management/views.py @@ -9,6 +9,7 @@ from osf.management.commands.update_registration_schemas import update_registration_schemas from osf.management.commands.daily_reporters_go import daily_reporters_go from osf.management.commands.monthly_reporters_go import monthly_reporters_go +from osf.management.commands.fetch_cedar_metadata_templates import ingest_cedar_metadata_templates from scripts.find_spammy_content import manage_spammy_content from django.urls import reverse from django.shortcuts import redirect @@ -134,3 +135,9 @@ def post(self, request, *args, **kwargs): else: messages.success(request, 'Monthly reporters successfully went.') return redirect(reverse('management:commands')) + +class IngestCedarMetadataTemplates(ManagementCommandPermissionView): + def post(self, request): + ingest_cedar_metadata_templates() + messages.success(request, 'Cedar templates have been successfully imported from Cedar Workbench.') + return redirect(reverse('management:commands')) diff --git a/admin/templates/management/commands.html b/admin/templates/management/commands.html index 319b3a1545d..269ead1bd3d 100644 --- a/admin/templates/management/commands.html +++ b/admin/templates/management/commands.html @@ -116,6 +116,20 @@

Monthly Reporters, Go!

+
+

Ingest Cedar Metadata templates

+

+ Use this management command to ingest CEDAR metadata templates from CEDAR Workbench. + Note: This requires CEDAR_API_HOST and CEDAR_API_KEY to be set for the environment. +

+
+ {% csrf_token %} + +
+
{% endblock %} diff --git a/osf/management/commands/fetch_cedar_metadata_templates.py b/osf/management/commands/fetch_cedar_metadata_templates.py index 68b6e457bad..6663fe67072 100644 --- a/osf/management/commands/fetch_cedar_metadata_templates.py +++ b/osf/management/commands/fetch_cedar_metadata_templates.py @@ -1,6 +1,7 @@ import logging from django.core.management.base import BaseCommand +from framework.celery_tasks import app as celery_app from osf.models import CedarMetadataTemplate from osf.external.cedar.client import CedarClient @@ -12,56 +13,59 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): + ingest_cedar_metadata_templates() +@celery_app.task(name='management.commands.ingest_cedar_metadata_templates') +def ingest_cedar_metadata_templates(): + try: + ids = CedarClient().retrieve_all_template_ids() + except CedarClientError as e: + logger.error(f'Unable to retrieve all cedar template ids: e={e.reason}') + return + + fetched = set() + newly_added = set() + unchanged = set() + updated = set() + failed = set() + for cedar_id in ids: try: - ids = CedarClient().retrieve_all_template_ids() + template = CedarClient().retrieve_template_by_id(cedar_id) except CedarClientError as e: - logger.error(f'Unable to retrieve all cedar template ids: e={e.reason}') - return - - fetched = set() - newly_added = set() - unchanged = set() - updated = set() - failed = set() - for cedar_id in ids: - try: - template = CedarClient().retrieve_template_by_id(cedar_id) - except CedarClientError as e: - logger.error(f'Unable to retrieve the cedar template: id={cedar_id}, e={e.reason}') - failed.add(cedar_id) - continue - else: - fetched.add(cedar_id) - schema_name = template['schema:name'] - pav_last_updated_on = template['pav:lastUpdatedOn'] - existing_versions = CedarMetadataTemplate.objects.filter(cedar_id=cedar_id) - if existing_versions: - latest_version = existing_versions.order_by('-template_version').first() - if pav_last_updated_on == latest_version.template['pav:lastUpdatedOn']: - unchanged.add(cedar_id) - else: - # New version should be inactive - CedarMetadataTemplate.objects.create( - schema_name=schema_name, - template=template, - cedar_id=cedar_id, - active=False, - template_version=latest_version.template_version + 1 - ) - updated.add(cedar_id) + logger.error(f'Unable to retrieve the cedar template: id={cedar_id}, e={e.reason}') + failed.add(cedar_id) + continue + else: + fetched.add(cedar_id) + schema_name = template['schema:name'] + pav_last_updated_on = template['pav:lastUpdatedOn'] + existing_versions = CedarMetadataTemplate.objects.filter(cedar_id=cedar_id) + if existing_versions: + latest_version = existing_versions.order_by('-template_version').first() + if pav_last_updated_on == latest_version.template['pav:lastUpdatedOn']: + unchanged.add(cedar_id) else: - # Initial version should be active + # New version should be inactive CedarMetadataTemplate.objects.create( schema_name=schema_name, template=template, cedar_id=cedar_id, - template_version=1 + active=False, + template_version=latest_version.template_version + 1 ) - newly_added.add(cedar_id) + updated.add(cedar_id) + else: + # Initial version should be active + CedarMetadataTemplate.objects.create( + schema_name=schema_name, + template=template, + cedar_id=cedar_id, + template_version=1 + ) + newly_added.add(cedar_id) - logger.error(f'failed ({len(failed)}): {failed}') - logger.info(f'fetched ({len(fetched)}): {fetched}') - logger.info(f'\tnewly_added ({len(newly_added)}): {newly_added}') - logger.info(f'\tunchanged ({len(unchanged)}): {unchanged}') - logger.info(f'\tupdated ({len(updated)}): {updated}') + logger.error(f'failed ({len(failed)}): {failed}') + logger.info(f'fetched ({len(fetched)}): {fetched}') + logger.info(f'\tnewly_added ({len(newly_added)}): {newly_added}') + logger.info(f'\tunchanged ({len(unchanged)}): {unchanged}') + logger.info(f'\tupdated ({len(updated)}): {updated}') diff --git a/website/settings/defaults.py b/website/settings/defaults.py index 443a26e9397..001e52f0a3c 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -443,6 +443,7 @@ class CeleryConfig: 'osf.management.commands.spam_metrics', 'osf.management.commands.daily_reporters_go', 'osf.management.commands.monthly_reporters_go', + 'osf.management.commands.ingest_cedar_metadata_templates', } med_pri_modules = {