Generate risk incidences when Siegfried outputs warnings on format id…

…entification (#3263) * SiegfriedPlugin generates risk incidences for warnings. Signed-off-by: sugarylump <[email protected]> * Use correct file id when running Siegfried on representations Signed-off-by: sugarylump <[email protected]> --------- Signed-off-by: sugarylump <[email protected]>
keeps · Aug 1, 2024 · 0d0c112 · 0d0c112
1 parent 9f8f57a
commit 0d0c112
Show file tree

Hide file tree

Showing 8 changed files with 241 additions and 57 deletions.
diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java
@@ -1423,6 +1423,7 @@ public enum OrchestratorType {
   public static final String PLUGIN_PARAMS_FORMAT = "parameter.format";
   public static final String PLUGIN_PARAMS_FORMAT_VERSION = "parameter.format_version";
   public static final String PLUGIN_PARAMS_PRONOM = "parameter.pronom";
+  public static final String PLUGIN_PARAMS_CLEAR_INCIDENCES = "parameter.clear_incidences";
 
   public static final String PLUGIN_CATEGORY_CONVERSION = "conversion";
   public static final String PLUGIN_CATEGORY_CHARACTERIZATION = "characterization";
@@ -1573,6 +1574,9 @@ public enum OrchestratorType {
   public static final String RISK_INCIDENCE_FILE_PATH_COMPUTED_SEPARATOR = "/";
   public static final String RISK_INCIDENCE_FILE_EXTENSION = ".json";
 
+  /* Risk Ids */
+  public static final String RISK_ID_SIEGFRIED_IDENTIFICATION_WARNING = "urn:siegfried:r1";
+
   /* Representation information */
   public static final String REPRESENTATION_INFORMATION_ID = "id";
   public static final String REPRESENTATION_INFORMATION_NAME = "name";
@@ -1703,6 +1707,7 @@ public enum OrchestratorType {
 
   /* Siegfriend payload fields */
   public static final String SIEGFRIED_PAYLOAD_MATCHES = "matches";
+  public static final String SIEGFRIED_PAYLOAD_MATCH_WARNING = "warning";
   public static final String SIEGFRIED_PAYLOAD_MATCH_NS = "ns";
   public static final String SIEGFRIED_PAYLOAD_MATCH_NS_PRONOM = "pronom";
   public static final String SIEGFRIED_PAYLOAD_MATCH_MIMETYPE = "mime";
@@ -1745,6 +1750,7 @@ public enum OrchestratorType {
   public static final String PRESERVATION_REGISTRY_MIME = "mime";
 
   public static final String PRESERVATION_FORMAT_NOTE_MANUAL = "manual";
+  public static final String PRESERVATION_FORMAT_NOTE_SIEGFRIED_WARNING = "SIEGFRIED WARNING";
 
   public static final String PREMIS_RELATIONSHIP_TYPE_STRUCTURAL = "structural";
   public static final String PREMIS_RELATIONSHIP_SUBTYPE_HASPART = "hasPart";

diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java
@@ -49,7 +49,15 @@ public Filter getFilter() {
     return filter;
   }
 
+  public void setFilter(Filter filter) {
+    this.filter = filter;
+  }
+
   public boolean isOnlyActive() {
     return onlyActive;
   }
+
+  public void setOnlyActive(boolean onlyActive) {
+    this.onlyActive = onlyActive;
+  }
 }
diff --git a/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java b/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java
@@ -280,6 +280,46 @@ private static ExtensionComplexType getTechnicalMetadata(gov.loc.premis.v3.File
     return extensionComplexType;
   }
 
+  public static List<String> getFormatNotes(ModelService model, String aipId, String representationId,
+    List<String> fileDirectoryPath, String fileId, String username) {
+    Binary premisBin = null;
+
+    try {
+      try {
+        premisBin = model.retrievePreservationFile(aipId, representationId, fileDirectoryPath, fileId);
+      } catch (NotFoundException e) {
+        LOGGER.debug("PREMIS object skeleton does not exist yet. Creating PREMIS object!");
+        List<String> algorithms = RodaCoreFactory.getFixityAlgorithms();
+
+        if (fileId == null) {
+          PremisSkeletonPluginUtils.createPremisSkeletonOnRepresentation(model, aipId, representationId, algorithms,
+            username);
+        } else {
+          File file = model.retrieveFile(aipId, representationId, fileDirectoryPath, fileId);
+          PremisSkeletonPluginUtils.createPremisSkeletonOnFile(model, file, algorithms, username);
+        }
+
+        premisBin = model.retrievePreservationFile(aipId, representationId, fileDirectoryPath, fileId);
+        LOGGER.debug("PREMIS object skeleton created");
+      }
+      gov.loc.premis.v3.File premisFile = binaryToFile(premisBin.getContent(), false);
+      ObjectCharacteristicsComplexType objectCharacteristics;
+      if (premisFile.getObjectIdentifier() != null && !premisFile.getObjectIdentifier().isEmpty()) {
+        objectCharacteristics = premisFile.getObjectCharacteristics().get(0);
+        if (objectCharacteristics.getFormat() != null && !objectCharacteristics.getFormat().isEmpty()) {
+          for (FormatComplexType format : objectCharacteristics.getFormat()) {
+            if (format.getFormatNote() != null) {
+              return format.getFormatNote();
+            }
+          }
+        }
+      }
+    } catch (RODAException | IOException e) {
+      LOGGER.error("PREMIS could not be checked due to an error", e);
+    }
+    return new ArrayList<>();
+  }
+
   public static boolean formatWasManuallyModified(ModelService model, String aipId, String representationId,
     List<String> fileDirectoryPath, String fileId, String username) {
     Binary premisBin = null;
@@ -1059,14 +1099,6 @@ public static void updateFormatPreservationMetadata(ModelService model, String a
           PremisSkeletonPluginUtils.createPremisSkeletonOnRepresentation(model, aipId, representationId, algorithms,
             username);
         } else {
-          // File file;
-          // if (shallow) {
-          // file = model.retrieveFileInsideManifest(aipId, representationId,
-          // fileDirectoryPath, fileId);
-          // } else {
-          // file = model.retrieveFile(aipId, representationId, fileDirectoryPath,
-          // fileId);
-          // }
           File file = model.retrieveFile(aipId, representationId, fileDirectoryPath, fileId);
           PremisSkeletonPluginUtils.createPremisSkeletonOnFile(model, file, algorithms, username);
         }

diff --git a/.../roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java b/.../roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java
@@ -150,7 +150,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic
                 if (aip.getRepresentations() != null && !aip.getRepresentations().isEmpty()) {
                   for (Representation representation : aip.getRepresentations()) {
                     LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId());
-                    List<LinkingIdentifier> newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model,
+                    List<LinkingIdentifier> newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index,
                       representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual);
                     if (!newSources.isEmpty()) {
                       sources.addAll(newSources);
@@ -180,7 +180,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic
               if (aip.getRepresentations() != null && !aip.getRepresentations().isEmpty()) {
                 for (Representation representation : aip.getRepresentations()) {
                   LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId());
-                  List<LinkingIdentifier> newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model,
+                  List<LinkingIdentifier> newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index,
                     representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual);
                   if (!newSources.isEmpty()) {
                     sources.addAll(newSources);
@@ -203,7 +203,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic
               }
             }
           } catch (PluginException | NotFoundException | GenericException | RequestNotValidException
-            | AuthorizationDeniedException e) {
+            | AuthorizationDeniedException | AlreadyExistsException e) {
             LOGGER.error("Error running Siegfried {}: {}", aip.getId(), e.getMessage(), e);
 
             jobPluginInfo.incrementObjectsProcessedWithFailure();
@@ -225,7 +225,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic
               for (Representation representation : filteredList) {
                 try {
                   LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId());
-                  sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, representation,
+                  sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation,
                     cachedJob.getId(), cachedJob.getUsername(), overwriteManual));
                   if (sources.isEmpty()) {
                     state = PluginState.SKIPPED;
@@ -283,7 +283,8 @@ public Report executeOnRepresentation(IndexService index, ModelService model, St
       PluginHelper.updatePartialJobReport(this, model, reportItem, false, cachedJob);
       LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), representation.getAipId());
       try {
-        sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, representation, cachedJob.getId(),
+        sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation,
+          cachedJob.getId(),
           cachedJob.getUsername(), overwriteManual));
         if (sources.isEmpty()) {
           jobPluginInfo.incrementObjectsProcessedWithSkipped();
@@ -295,7 +296,7 @@ public Report executeOnRepresentation(IndexService index, ModelService model, St
           model.notifyRepresentationUpdated(representation).failOnError();
         }
       } catch (PluginException | NotFoundException | GenericException | RequestNotValidException
-        | AuthorizationDeniedException e) {
+        | AuthorizationDeniedException | AlreadyExistsException e) {
         LOGGER.error("Error running Siegfried {}: {}", representation.getAipId(), e.getMessage(), e);
 
         jobPluginInfo.incrementObjectsProcessedWithFailure();
@@ -331,7 +332,8 @@ public Report executeOnFile(IndexService index, ModelService model, StorageServi
         file.getAipId());
 
       try {
-        sources.addAll(SiegfriedPluginUtils.runSiegfriedOnFile(model, file, cachedJob.getUsername(), overwriteManual));
+        sources.addAll(
+          SiegfriedPluginUtils.runSiegfriedOnFile(model, index, file, cachedJob.getUsername(), overwriteManual));
         if (sources.isEmpty()) {
           jobPluginInfo.incrementObjectsProcessedWithSkipped();
           reportItem.setPluginState(PluginState.SKIPPED)
@@ -341,7 +343,7 @@ public Report executeOnFile(IndexService index, ModelService model, StorageServi
           reportItem.setPluginState(PluginState.SUCCESS);
         }
       } catch (PluginException | NotFoundException | GenericException | RequestNotValidException
-        | AuthorizationDeniedException e) {
+        | AuthorizationDeniedException | AlreadyExistsException e) {
         LOGGER.error("Error running Siegfried on file {}: {}", file.getId(), e.getMessage(), e);
 
         jobPluginInfo.incrementObjectsProcessedWithFailure();