OSFMAP Search Improvements (#10386)

* Update metadata vocabularies
* Support DataCite schema version 4.4
* Specify OSFMAP

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
   hooks:
   - id: double-quote-string-fixer
   - id: trailing-whitespace
-    exclude: website/static/vendor/*
+    exclude: "^(website/static/vendor/|osf/metadata/definitions/.*\\.xsd)"
   - id: flake8
     additional_dependencies: ["flake8==3.6.0", "flake8-mutable==1.2.0"]
 - repo: https://github.com/pre-commit/mirrors-jshint

admin/nodes/views.py

@@ -508,9 +508,7 @@ class NodeReindexShare(NodeMixin, View):
 
     def post(self, request, *args, **kwargs):
         node = self.get_object()
-        if settings.SHARE_ENABLED:
-            update_share(node)
-
+        update_share(node)
         update_admin_log(
             user_id=self.request.user.id,
             object_id=node._id,

admin/preprint_providers/views.py

@@ -384,7 +384,7 @@ def get(self, request, *args, **kwargs):
         except ValidationError as e:
             messages.error(request, e.message)
 
-        return redirect(reverse_lazy('preprint_providers:share_source', kwargs={'preprint_provider_id': provider.id}))
+        return redirect(reverse_lazy('preprint_providers:detail', kwargs={'preprint_provider_id': provider.id}))
 
 class SubjectDynamicUpdateView(PermissionRequiredMixin, View):
     permission_required = 'osf.change_preprintprovider'

admin/preprints/views.py

@@ -42,7 +42,7 @@
     UNFLAG_SPAM,
 )
 
-from website import search, settings
+from website import search
 
 
 class PreprintMixin(PermissionRequiredMixin):
@@ -141,8 +141,7 @@ class PreprintReindexShare(PreprintMixin, View):
 
     def post(self, request, *args, **kwargs):
         preprint = self.get_object()
-        if settings.SHARE_ENABLED:
-            update_share(preprint)
+        update_share(preprint)
         update_admin_log(
             user_id=self.request.user.id,
             object_id=preprint._id,

admin_tests/nodes/test_views.py

@@ -1,10 +1,8 @@
 import datetime as dt
 import pytest
-import json
 import mock
 import pytz
 import datetime
-import responses
 
 from osf.models import AdminLogEntry, NodeLog, AbstractNode
 from admin.nodes.views import (
@@ -22,6 +20,7 @@
     RemoveStuckRegistrationsView
 )
 from admin_tests.utilities import setup_log_view, setup_view
+from api_tests.share._utils import mock_update_share
 from website import settings
 from nose import tools as nt
 from django.utils import timezone
@@ -265,32 +264,19 @@ def test_reindex_node_share(self):
         count = AdminLogEntry.objects.count()
         view = NodeReindexShare()
         view = setup_log_view(view, self.request, guid=self.node._id)
-        with mock.patch('api.share.utils.settings.SHARE_ENABLED', True):
-            with mock.patch('api.share.utils.settings.SHARE_API_TOKEN', 'mock-api-token'):
-                with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
-                    rsps.add(responses.POST, 'https://share.osf.io/api/v2/normalizeddata/')
-                    view.post(self.request)
-                    data = json.loads(rsps.calls[-1].request.body.decode())
-
-                    share_graph = data['data']['attributes']['data']['@graph']
-                    identifier_node = next(n for n in share_graph if n['@type'] == 'workidentifier')
-                    assert identifier_node['creative_work']['@type'] == 'project'
-                    nt.assert_equal(AdminLogEntry.objects.count(), count + 1)
+        with mock_update_share() as _shmock:
+            view.post(self.request)
+            assert _shmock.called_once_with(self.node._id)
+        nt.assert_equal(AdminLogEntry.objects.count(), count + 1)
 
     def test_reindex_registration_share(self):
         count = AdminLogEntry.objects.count()
         view = NodeReindexShare()
         view = setup_log_view(view, self.request, guid=self.registration._id)
-        with mock.patch('api.share.utils.settings.SHARE_ENABLED', True):
-            with mock.patch('api.share.utils.settings.SHARE_API_TOKEN', 'mock-api-token'):
-                with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
-                    rsps.add(responses.POST, 'https://share.osf.io/api/v2/normalizeddata/')
-                    view.post(self.request)
-                    data = json.loads(rsps.calls[-1].request.body.decode())
-
-                    assert any(graph for graph in data['data']['attributes']['data']['@graph']
-                               if graph['@type'] == self.registration.provider.share_publish_type.lower())
-                    nt.assert_equal(AdminLogEntry.objects.count(), count + 1)
+        with mock_update_share() as _shmock:
+            view.post(self.request)
+            assert _shmock.called_once_with(self.registration._id)
+        nt.assert_equal(AdminLogEntry.objects.count(), count + 1)
 
     @mock.patch('website.search.search.update_node')
     def test_reindex_node_elastic(self, mock_update_node):

admin_tests/preprints/test_views.py

@@ -1,7 +1,5 @@
 import pytest
 import mock
-import json
-import responses
 
 from django.test import RequestFactory
 from django.urls import reverse
@@ -24,6 +22,7 @@
 from osf.models.spam import SpamStatus
 from osf.utils.workflows import DefaultStates, RequestTypes
 
+from api_tests.share._utils import expect_ingest_request
 from admin_tests.utilities import setup_view, setup_log_view
 
 from admin.preprints import views
@@ -334,22 +333,17 @@ def test_change_preprint_provider_subjects_change_permissions(self, plain_view,
 @pytest.mark.enable_implicit_clean
 class TestPreprintReindex:
 
-    def test_reindex_preprint_share(self, preprint, req):
+    def test_reindex_preprint_share(self, preprint, req, mock_share_requests):
         preprint.provider.access_token = 'totally real access token I bought from a guy wearing a trenchcoat in the summer'
         preprint.provider.save()
 
         count = AdminLogEntry.objects.count()
         view = views.PreprintReindexShare()
         view = setup_log_view(view, req, guid=preprint._id)
 
-        with mock.patch('api.share.utils.settings.SHARE_ENABLED', True):
-            with mock.patch('api.share.utils.settings.SHARE_API_TOKEN', 'mock-api-token'):
-                with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
-                    rsps.add(responses.POST, 'https://share.osf.io/api/v2/normalizeddata/')
-                    view.post(req)
-                    data = json.loads(rsps.calls[0].request.body.decode())
-                    assert data['data']['type'] == 'NormalizedData'
-                    assert AdminLogEntry.objects.count() == count + 1
+        with expect_ingest_request(mock_share_requests, preprint._id, token=preprint.provider.access_token):
+            view.post(req)
+        assert AdminLogEntry.objects.count() == count + 1
 
     @mock.patch('website.search.search.update_preprint')
     def test_reindex_preprint_elastic(self, mock_update_search, preprint, req):

admin_tests/registration_providers/test_views.py

@@ -144,9 +144,9 @@ def view(self, req, provider):
         view = views.ShareSourceRegistrationProvider()
         return setup_view(view, req, registration_provider_id=provider.id)
 
-    def test_share_source(self, mock_share, view, provider, req):
-        mock_share.reset()
-        mock_share.add(
+    def test_share_source(self, mock_share_responses, view, provider, req):
+        mock_share_responses.reset()
+        mock_share_responses.add(
             responses.POST,
             f'{settings.SHARE_URL}api/v2/sources/',
             json.dumps(
@@ -172,9 +172,9 @@ def test_share_source(self, mock_share, view, provider, req):
         assert provider.access_token == 'test access token'
 
     @mock.patch.object(settings, 'SHARE_PROVIDER_PREPEND', 'testenv')
-    def test_share_source_prefix(self, mock_share, view, provider, req):
-        mock_share.reset()
-        mock_share.add(
+    def test_share_source_prefix(self, mock_share_responses, view, provider, req):
+        mock_share_responses.reset()
+        mock_share_responses.add(
             responses.POST,
             f'{settings.SHARE_URL}api/v2/sources/',
             json.dumps(

api/base/storage.py

@@ -3,7 +3,7 @@
 from storages.backends.gcloud import GoogleCloudStorage
 from future.moves.urllib.parse import urljoin
 
-from website.settings import DOMAIN
+from website.settings import INTERNAL_DOMAIN
 
 
 class DevFileSystemStorage(FileSystemStorage):
@@ -14,7 +14,7 @@ def url(self, name):
         url = filepath_to_uri(name)
         if url is not None:
             url = url.lstrip('/')
-        url = urljoin(DOMAIN, url)
+        url = urljoin(INTERNAL_DOMAIN, url)
         return urljoin(self.base_url, url)
 
 class RequestlessURLGoogleCloudStorage(GoogleCloudStorage):