From a9e120f3d434ece061dcaf7371822b9201eac655 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 17 Jun 2024 13:02:33 +0200 Subject: [PATCH 1/4] feat: new parameter "fromdate" to evaluate items only from certain date --- .../app/mediafilter/MediaFilterScript.java | 15 +++++++++++++-- .../MediaFilterScriptConfiguration.java | 2 ++ .../mediafilter/MediaFilterServiceImpl.java | 18 ++++++++++++++++++ .../service/MediaFilterService.java | 3 +++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java index 5fbbebbb28c..71c0d6e14dd 100644 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java +++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java @@ -7,6 +7,7 @@ */ package org.dspace.app.mediafilter; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -37,8 +38,9 @@ * MFM: -v verbose outputs all extracted text to STDOUT; -f force forces all * bitstreams to be processed, even if they have been before; -n noindex does not * recreate index after processing bitstreams; -i [identifier] limits processing - * scope to a community, collection or item; and -m [max] limits processing to a - * maximum number of items. + * scope to a community, collection or item; -m [max] limits processing to a + * maximum number of items; -fd [fromdate] takes only items starting from this date, + * filtering by last_modified in the item table. */ public class MediaFilterScript extends DSpaceRunnable { @@ -60,6 +62,7 @@ public class MediaFilterScript extends DSpaceRunnable> filterFormats = new HashMap<>(); + private LocalDate fromDate = null; public MediaFilterScriptConfiguration getScriptConfiguration() { return new DSpace().getServiceManager() @@ -112,6 +115,10 @@ public void setup() throws ParseException { skipIds = commandLine.getOptionValues('s'); } + if (commandLine.hasOption('f')) { + fromDate = LocalDate.parse(commandLine.getOptionValue('f')); + } + } @@ -215,6 +222,10 @@ public void internalRun() throws Exception { mediaFilterService.setSkipList(Arrays.asList(skipIds)); } + if (fromDate != null) { + mediaFilterService.setFromDate(fromDate); + } + Context c = null; try { diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java index 7465fa6e127..9f62f19e836 100644 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java @@ -52,6 +52,8 @@ public Options getOptions() { .build(); options.addOption(pluginOption); + options.addOption("fd", "fromdate", true, "Process only item from specified last modified date"); + Option skipOption = Option.builder("s") .longOpt("skip") .hasArg() diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java index a6ba9fde88d..512b8f803b9 100644 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java @@ -9,8 +9,11 @@ import java.io.InputStream; import java.sql.SQLException; +import java.time.LocalDate; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -93,6 +96,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB protected boolean isVerbose = false; protected boolean isQuiet = false; protected boolean isForce = false; // default to not forced + protected LocalDate fromDate = null; protected MediaFilterServiceImpl() { @@ -120,6 +124,15 @@ public void applyFiltersAllItems(Context context) throws Exception { for (Community topLevelCommunity : topLevelCommunities) { applyFiltersCommunity(context, topLevelCommunity); } + } else if (fromDate != null) { + Iterator itemIterator = + itemService.findByLastModifiedSince( + context, + Date.from(fromDate.atStartOfDay(ZoneId.systemDefault()).toInstant()) + ); + while (itemIterator.hasNext() && processed < max2Process) { + applyFiltersItem(context, itemIterator.next()); + } } else { //otherwise, just find every item and process Iterator itemIterator = itemService.findAll(context); @@ -588,4 +601,9 @@ public void setFilterFormats(Map> filterFormats) { public void setLogHandler(DSpaceRunnableHandler handler) { this.handler = handler; } + + @Override + public void setFromDate(LocalDate fromDate) { + this.fromDate = fromDate; + } } diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java index bc92ff52109..30e6dba42f0 100644 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java +++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java @@ -8,6 +8,7 @@ package org.dspace.app.mediafilter.service; import java.sql.SQLException; +import java.time.LocalDate; import java.util.List; import java.util.Map; @@ -149,4 +150,6 @@ public void updatePoliciesOfDerivativeBitstreams(Context context, Item item, Bit * @param handler */ public void setLogHandler(DSpaceRunnableHandler handler); + + public void setFromDate(LocalDate fromDate); } From 2f6b7f3ee4b0c40d215730dd2e86f4492a455e77 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 17 Jun 2024 13:57:39 +0200 Subject: [PATCH 2/4] refactor: changed short parameter fd to d --- .../main/java/org/dspace/app/mediafilter/MediaFilterScript.java | 2 +- .../dspace/app/mediafilter/MediaFilterScriptConfiguration.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java index 71c0d6e14dd..e76feada033 100644 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java +++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java @@ -115,7 +115,7 @@ public void setup() throws ParseException { skipIds = commandLine.getOptionValues('s'); } - if (commandLine.hasOption('f')) { + if (commandLine.hasOption('d')) { fromDate = LocalDate.parse(commandLine.getOptionValue('f')); } diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java index 9f62f19e836..c9f61292d61 100644 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java @@ -52,7 +52,7 @@ public Options getOptions() { .build(); options.addOption(pluginOption); - options.addOption("fd", "fromdate", true, "Process only item from specified last modified date"); + options.addOption("d", "fromdate", true, "Process only item from specified last modified date"); Option skipOption = Option.builder("s") .longOpt("skip") From 3fd88b867eb9bbd8b3621904ecac5b845fc7346e Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 17 Jun 2024 16:39:02 +0200 Subject: [PATCH 3/4] refactor: changed short parameter fd to d --- .../main/java/org/dspace/app/mediafilter/MediaFilterScript.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java index e76feada033..7f022f38b31 100644 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java +++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java @@ -116,7 +116,7 @@ public void setup() throws ParseException { } if (commandLine.hasOption('d')) { - fromDate = LocalDate.parse(commandLine.getOptionValue('f')); + fromDate = LocalDate.parse(commandLine.getOptionValue('d')); } From 9c7b20ff5721e2886acd7186ba4df310c9efcf07 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 4 Sep 2024 15:31:28 +0200 Subject: [PATCH 4/4] fix: changed parameter of HQL query --- .../src/main/java/org/dspace/content/dao/impl/ItemDAOImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemDAOImpl.java b/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemDAOImpl.java index 3be39f1788f..bd042648384 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemDAOImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemDAOImpl.java @@ -441,7 +441,7 @@ public int countItems(Context context, List collections, boolean inc public Iterator findByLastModifiedSince(Context context, Date since) throws SQLException { Query query = createQuery(context, - "SELECT i.id FROM Item i WHERE last_modified > :last_modified ORDER BY id"); + "SELECT i.id FROM Item i WHERE lastModified > :last_modified ORDER BY id"); query.setParameter("last_modified", since, TemporalType.TIMESTAMP); @SuppressWarnings("unchecked") List uuids = query.getResultList();