Merged dspace-cris-2023_02_x into task/dspace-cris-2023_02_x/LM-46

dspace-api/src/main/java/org/dspace/app/deduplication/service/impl/SolrDedupServiceImpl.java

@@ -679,11 +679,11 @@ public void optimize() {
                 return;
             }
             long start = System.currentTimeMillis();
-            System.out.println("SOLR Search Optimize -- Process Started:" + start);
+            System.out.println("SOLR Dedup Optimize -- Process Started:" + start);
             getSolr().optimize();
             long finish = System.currentTimeMillis();
-            System.out.println("SOLR Search Optimize -- Process Finished:" + finish);
-            System.out.println("SOLR Search Optimize -- Total time taken:" + (finish - start) + " (ms).");
+            System.out.println("SOLR Dedup Optimize -- Process Finished:" + finish);
+            System.out.println("SOLR Dedup Optimize -- Total time taken:" + (finish - start) + " (ms).");
         } catch (SolrServerException sse) {
             System.err.println(sse.getMessage());
         } catch (IOException ioe) {

dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java

@@ -85,7 +85,7 @@ public void internalRun() throws Exception {
         timeUntilReindex = getTimeUntilReindex();
         maxTime = getMaxTime();
 
-        Context context = new Context();
+        Context context = new Context(Context.Mode.READ_ONLY);
 
         try {
             context.turnOffAuthorisationSystem();
@@ -128,6 +128,7 @@ private void performStatusUpdate(Context context) throws SearchServiceException,
                     if (indexableObject.isPresent()) {
                         logDebugAndOut("Item exists in DB, updating solr document");
                         updateItem(context, indexableObject.get());
+                        context.uncacheEntity(indexableObject.get().getIndexedObject());
                     } else {
                         logDebugAndOut("Item doesn't exist in DB, removing solr document");
                         removeItem(context, uniqueId);

dspace-api/src/main/java/org/dspace/app/suggestion/OAIREPublicationLoaderRunnable.java

@@ -73,7 +73,7 @@ public void internalRun() throws Exception {
 
         for (Item researcher : researchers) {
 
-            oairePublicationLoader.importAuthorRecords(context, researcher);
+            oairePublicationLoader.importRecords(context, researcher);
         }
 
     }

dspace-api/src/main/java/org/dspace/app/suggestion/SolrSuggestionProvider.java

@@ -13,6 +13,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.dspace.content.Item;
 import org.dspace.content.service.ItemService;
 import org.dspace.core.Context;
 import org.dspace.external.model.ExternalDataObject;
@@ -137,4 +138,31 @@ public void flagRelatedSuggestionsAsProcessed(Context context, ExternalDataObjec
      */
     protected abstract boolean isExternalDataObjectPotentiallySuggested(Context context,
             ExternalDataObject externalDataObject);
+
+    /**
+     * Save a List of ImportRecord into Solr.
+     * ImportRecord will be translate into a SolrDocument by the method translateImportRecordToSolrDocument.
+     *
+     * @param context the DSpace Context
+     * @param item a DSpace Item
+     * @throws SolrServerException
+     * @throws IOException
+     */
+    public abstract void importRecords(Context context, Item item) throws Exception;
+
+    /**
+     * Save a List of ImportRecord into Solr.
+     * ImportRecord will be translate into a SolrDocument by the method translateImportRecordToSolrDocument.
+     *
+     * @param context the DSpace Context
+     * @param query the query to be run
+     * @throws SolrServerException
+     * @throws IOException
+     */
+    public abstract void importRecords(Context context, String query) throws Exception;
+
+    public boolean isSupportingQuery() {
+        return false;
+    }
+
 }

dspace-api/src/main/java/org/dspace/app/suggestion/SolrSuggestionStorageService.java

@@ -188,4 +188,21 @@ Suggestion findUnprocessedSuggestion(Context context, String source, UUID target
      * @throws IOException
      */
     SuggestionTarget findTarget(Context context, String source, UUID target) throws SolrServerException, IOException;
+
+    /**
+     * Find all the unprocessed suggestions related to the given source and have score
+     * greater than or equal to given score.
+     * @param  context             the DSpace Context
+     * @param  source              the source name
+     * @param  score               the score
+     * @param  pageSize            the page size
+     * @param  offset              the page offset
+     * @param  ascending           true to retrieve the suggestions ordered by score
+     *                             ascending
+     * @return                     the found suggestions
+     * @throws SolrServerException
+     * @throws IOException
+     */
+    List<Suggestion> findAllUnprocessedSuggestionsBySourceAndScore(Context context, String source, String score,
+        int pageSize, long offset, boolean ascending) throws SolrServerException, IOException;
 }

dspace-api/src/main/java/org/dspace/app/suggestion/SolrSuggestionStorageServiceImpl.java

@@ -22,6 +22,7 @@
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Logger;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrQuery.SortClause;
@@ -53,6 +54,9 @@ public class SolrSuggestionStorageServiceImpl implements SolrSuggestionStorageSe
 
     protected SolrClient solrSuggestionClient;
 
+    private static final Logger log =
+        org.apache.logging.log4j.LogManager.getLogger(SolrSuggestionStorageServiceImpl.class);
+
     @Autowired
     private ItemService itemService;
 
@@ -73,29 +77,67 @@ protected SolrClient getSolr() {
     @Override
     public void addSuggestion(Suggestion suggestion, boolean force, boolean commit)
             throws SolrServerException, IOException {
-        if (force || !exist(suggestion)) {
-            Gson gson = new Gson();
-            SolrInputDocument document = new SolrInputDocument();
-            document.addField(SOURCE, suggestion.getSource());
-            String suggestionFullID = suggestion.getID();
-            document.addField(SUGGESTION_FULLID, suggestionFullID);
-            document.addField(SUGGESTION_ID, suggestionFullID.split(":", 3)[2]);
-            document.addField(TARGET_ID, suggestion.getTarget().getID().toString());
-            document.addField(DISPLAY, suggestion.getDisplay());
-            document.addField(TITLE, getFirstValue(suggestion, "dc", "title", null));
-            document.addField(DATE, getFirstValue(suggestion, "dc", "date", "issued"));
-            document.addField(CONTRIBUTORS, getAllValues(suggestion, "dc", "contributor", "author"));
-            document.addField(ABSTRACT, getFirstValue(suggestion, "dc", "description", "abstract"));
-            document.addField(CATEGORY, getAllValues(suggestion, "dc", "source", null));
-            document.addField(EXTERNAL_URI, suggestion.getExternalSourceUri());
-            document.addField(SCORE, suggestion.getScore());
-            document.addField(PROCESSED, false);
-            document.addField(EVIDENCES, gson.toJson(suggestion.getEvidences()));
-            getSolr().add(document);
-            if (commit) {
-                getSolr().commit();
+        try {
+            if (force || !exist(suggestion)) {
+                Gson gson = new Gson();
+                SolrInputDocument document = new SolrInputDocument();
+                document.addField(SOURCE, suggestion.getSource());
+                String suggestionFullID = suggestion.getID();
+                document.addField(SUGGESTION_FULLID, suggestionFullID);
+                document.addField(SUGGESTION_ID, suggestionFullID.split(":", 3)[2]);
+                document.addField(TARGET_ID, suggestion.getTarget().getID().toString());
+                document.addField(DISPLAY, suggestion.getDisplay());
+                document.addField(TITLE, getFirstValue(suggestion, "dc", "title", null));
+                document.addField(DATE, getFirstValue(suggestion, "dc", "date", "issued"));
+                document.addField(CONTRIBUTORS, getAllValues(suggestion, "dc", "contributor", "author"));
+                document.addField(ABSTRACT, getFirstValue(suggestion, "dc", "description", "abstract"));
+                document.addField(CATEGORY, getAllValues(suggestion, "dc", "source", null));
+                document.addField(EXTERNAL_URI, suggestion.getExternalSourceUri());
+                document.addField(SCORE, suggestion.getScore());
+                document.addField(PROCESSED, false);
+                document.addField(EVIDENCES, gson.toJson(suggestion.getEvidences()));
+                getSolr().add(document);
+                if (commit) {
+                    getSolr().commit();
+                }
+            }
+        } catch (Exception e) {
+            log.error(e);
+        }
+    }
+
+    @Override
+    public List<Suggestion> findAllUnprocessedSuggestionsBySourceAndScore(Context context, String source, String score,
+        int pageSize, long offset, boolean ascending) throws SolrServerException, IOException {
+
+        SolrQuery solrQuery = new SolrQuery();
+        solrQuery.setRows(pageSize);
+        solrQuery.setStart((int) offset);
+        solrQuery.setQuery("*:*");
+        solrQuery.addFilterQuery(
+            SOURCE + ":" + source,
+            SCORE  + ":[ " + score + " TO * ]",
+            PROCESSED + ":false");
+
+        if (ascending) {
+            solrQuery.addSort(SortClause.asc("trust"));
+        } else {
+            solrQuery.addSort(SortClause.desc("trust"));
+        }
+
+        solrQuery.addSort(SortClause.desc("date"));
+        solrQuery.addSort(SortClause.asc("suggestion_id"));
+        solrQuery.addSort(SortClause.asc("title"));
+
+        QueryResponse response = getSolr().query(solrQuery);
+        List<Suggestion> suggestions = new ArrayList<Suggestion>();
+        for (SolrDocument solrDoc : response.getResults()) {
+            Suggestion suggestion = convertSolrDoc(context, solrDoc, source);
+            if (suggestion != null) {
+                suggestions.add(suggestion);
             }
         }
+        return suggestions;
     }
 
     @Override

dspace-api/src/main/java/org/dspace/app/suggestion/oaire/OAIREPublicationLoader.java

@@ -35,16 +35,16 @@
  */
 public class OAIREPublicationLoader extends SolrSuggestionProvider {
 
-    private List<String> names;
+    protected List<String> names;
 
-    private ExternalDataProvider primaryProvider;
+    protected ExternalDataProvider primaryProvider;
 
-    private List<ExternalDataProvider> otherProviders;
+    protected List<ExternalDataProvider> otherProviders;
 
     @Autowired
-    private ConfigurationService configurationService;
+    protected ConfigurationService configurationService;
 
-    private List<EvidenceScorer> pipeline;
+    protected List<EvidenceScorer> pipeline;
 
     public void setPrimaryProvider(ExternalDataProvider primaryProvider) {
         this.primaryProvider = primaryProvider;
@@ -103,8 +103,9 @@ public List<Suggestion> reduceAndTransform(Item researcher, List<ExternalDataObj
      * @throws SolrServerException
      * @throws IOException
      */
-    public void importAuthorRecords(Context context, Item researcher)
-            throws SolrServerException, IOException {
+    @Override
+    public void importRecords(Context context, Item researcher)
+            throws Exception {
         List<ExternalDataObject> metadata = getImportRecords(researcher);
         List<Suggestion> records = reduceAndTransform(researcher, metadata);
         for (Suggestion record : records) {
@@ -214,7 +215,7 @@ private boolean isDuplicate(ExternalDataObject dto, List<ExternalDataObject> imp
      * @param researcher DSpace item
      * @return list of metadata values
      */
-    private List<String> searchMetadataValues(Item researcher) {
+    public List<String> searchMetadataValues(Item researcher) {
         List<String> authors = new ArrayList<String>();
         for (String name : names) {
             String value = itemService.getMetadata(researcher, name);
@@ -237,4 +238,9 @@ protected boolean isExternalDataObjectPotentiallySuggested(Context context, Exte
         }
     }
 
+    @Override
+    public void importRecords(Context context, String query) throws Exception {
+        throw new UnsupportedOperationException("This operation is not supported by OAIRE loader");
+    }
+
 }

dspace-api/src/main/java/org/dspace/app/suggestion/orcid/OrcidPublicationLoader.java

@@ -62,6 +62,11 @@ public void importWorks(Context context, Item profile, String orcid) throws Solr
         solrSuggestionStorageService.commit();
     }
 
+    @Override
+    public void importRecords(Context context, Item researcher) throws Exception {
+        importWorks(context, researcher, itemService.getMetadata(researcher, "person.identifier.orcid"));
+    }
+
     private List<Suggestion> convertToSuggestions(Item profile, List<ExternalDataObject> externalDataObjects) {
         return externalDataObjects.stream()
             .map(externalDataObject -> convertToSuggestion(profile, externalDataObject))
@@ -124,4 +129,9 @@ public void setProvider(ExternalDataProvider provider) {
         this.provider = provider;
     }
 
+    @Override
+    public void importRecords(Context context, String query) throws Exception {
+        throw new UnsupportedOperationException("This operation is not supported by orcid loader");
+    }
+
 }

dspace-api/src/main/java/org/dspace/app/util/AuthorizeUtil.java

@@ -628,12 +628,23 @@ public static boolean authorizeNewAccountRegistration(Context context, HttpServl
             // actually expected to be returning true.
             // For example the LDAP canSelfRegister will return true due to auto-register, while that
             // does not imply a new user can register explicitly
-            return AuthenticateServiceFactory.getInstance().getAuthenticationService()
-                                             .allowSetPassword(context, request, null);
+            return authorizePasswordChange(context, request);
         }
         return false;
     }
 
+    /**
+     * This method will return a boolean indicating whether the current user is allowed to reset the password
+     * or not
+     *
+     * @return          A boolean indicating whether the current user can reset its password or not
+     * @throws SQLException If something goes wrong
+     */
+    public static boolean authorizeForgotPassword() {
+        return DSpaceServicesFactory.getInstance().getConfigurationService()
+                             .getBooleanProperty("user.forgot-password", true);
+    }
+
     /**
      * This method will return a boolean indicating whether it's allowed to update the password for the EPerson
      * with the given email and canLogin property
@@ -647,15 +658,27 @@ public static boolean authorizeUpdatePassword(Context context, String email) {
             if (eperson != null && eperson.canLogIn()) {
                 HttpServletRequest request = new DSpace().getRequestService().getCurrentRequest()
                                                          .getHttpServletRequest();
-                return AuthenticateServiceFactory.getInstance().getAuthenticationService()
-                                                 .allowSetPassword(context, request, null);
+                return authorizePasswordChange(context, request);
             }
         } catch (SQLException e) {
             log.error("Something went wrong trying to retrieve EPerson for email: " + email, e);
         }
         return false;
     }
 
+    /**
+     * Checks if the current configuration has at least one password based authentication method
+     *
+     * @param context Dspace Context
+     * @param request Current Request
+     * @return True if the password change is enabled
+     * @throws SQLException
+     */
+    protected static boolean authorizePasswordChange(Context context, HttpServletRequest request) throws SQLException {
+        return AuthenticateServiceFactory.getInstance().getAuthenticationService()
+                                         .allowSetPassword(context, request, null);
+    }
+
     /**
      * This method checks if the community Admin can manage accounts
      *

dspace-api/src/main/java/org/dspace/app/util/DCInputsReader.java

@@ -18,6 +18,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -876,7 +877,7 @@ private List<String> getSubmissionFormMetadata(Collection collection, boolean gr
     public List<String> getLanguagesForMetadata(Collection collection, String metadata, boolean group)
         throws DCInputsReaderException {
 
-        return getAllInputsByCollection(collection)
+        Optional<DCInput> dcInputMetadata = getAllInputsByCollection(collection)
             .filter(dcInput -> group ? isGroupType(dcInput) : !isGroupType(dcInput))
             .filter(dcInput -> {
                 try {
@@ -887,9 +888,11 @@ public List<String> getLanguagesForMetadata(Collection collection, String metada
                     throw new RuntimeException(e);
                 }
             })
-            .findFirst()
-            .orElseThrow(() -> new DCInputsReaderException("No DCInput found for the metadata field " + metadata))
-            .getAllLanguageValues();
+            .findFirst();
+        if (dcInputMetadata.isPresent()) {
+            dcInputMetadata.get().getAllLanguageValues();
+        }
+        return List.of();
 
     }