Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generate risk incidences when Siegfried outputs warnings on format identification #3263

Merged
merged 2 commits into from
Aug 1, 2024
Merged

Generate risk incidences when Siegfried outputs warnings on format identification #3263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
43d0e72
SiegfriedPlugin generates risk incidences for warnings.
SugaryLump Jul 19, 2024
354465d
Use correct file id when running Siegfried on representations
SugaryLump Aug 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1422,6 +1422,7 @@ public enum OrchestratorType {
public static final String PLUGIN_PARAMS_FORMAT = "parameter.format";
public static final String PLUGIN_PARAMS_FORMAT_VERSION = "parameter.format_version";
public static final String PLUGIN_PARAMS_PRONOM = "parameter.pronom";
public static final String PLUGIN_PARAMS_CLEAR_INCIDENCES = "parameter.clear_incidences";

public static final String PLUGIN_CATEGORY_CONVERSION = "conversion";
public static final String PLUGIN_CATEGORY_CHARACTERIZATION = "characterization";
Expand Down Expand Up @@ -1572,6 +1573,9 @@ public enum OrchestratorType {
public static final String RISK_INCIDENCE_FILE_PATH_COMPUTED_SEPARATOR = "/";
public static final String RISK_INCIDENCE_FILE_EXTENSION = ".json";

/* Risk Ids */
public static final String RISK_ID_SIEGFRIED_IDENTIFICATION_WARNING = "urn:siegfried:r1";

/* Representation information */
public static final String REPRESENTATION_INFORMATION_ID = "id";
public static final String REPRESENTATION_INFORMATION_NAME = "name";
Expand Down Expand Up @@ -1702,6 +1706,7 @@ public enum OrchestratorType {

/* Siegfriend payload fields */
public static final String SIEGFRIED_PAYLOAD_MATCHES = "matches";
public static final String SIEGFRIED_PAYLOAD_MATCH_WARNING = "warning";
public static final String SIEGFRIED_PAYLOAD_MATCH_NS = "ns";
public static final String SIEGFRIED_PAYLOAD_MATCH_NS_PRONOM = "pronom";
public static final String SIEGFRIED_PAYLOAD_MATCH_MIMETYPE = "mime";
Expand Down Expand Up @@ -1744,6 +1749,7 @@ public enum OrchestratorType {
public static final String PRESERVATION_REGISTRY_MIME = "mime";

public static final String PRESERVATION_FORMAT_NOTE_MANUAL = "manual";
public static final String PRESERVATION_FORMAT_NOTE_SIEGFRIED_WARNING = "SIEGFRIED WARNING";

public static final String PREMIS_RELATIONSHIP_TYPE_STRUCTURAL = "structural";
public static final String PREMIS_RELATIONSHIP_SUBTYPE_HASPART = "hasPart";
Expand Down
8 changes: 8 additions & 0 deletions roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,15 @@ public Filter getFilter() {
return filter;
}

public void setFilter(Filter filter) {
this.filter = filter;
}

public boolean isOnlyActive() {
return onlyActive;
}

public void setOnlyActive(boolean onlyActive) {
this.onlyActive = onlyActive;
}
}
48 changes: 40 additions & 8 deletions roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,46 @@ private static ExtensionComplexType getTechnicalMetadata(gov.loc.premis.v3.File
return extensionComplexType;
}

public static List<String> getFormatNotes(ModelService model, String aipId, String representationId,
List<String> fileDirectoryPath, String fileId, String username) {
Binary premisBin = null;

try {
try {
premisBin = model.retrievePreservationFile(aipId, representationId, fileDirectoryPath, fileId);
} catch (NotFoundException e) {
LOGGER.debug("PREMIS object skeleton does not exist yet. Creating PREMIS object!");
List<String> algorithms = RodaCoreFactory.getFixityAlgorithms();

if (fileId == null) {
PremisSkeletonPluginUtils.createPremisSkeletonOnRepresentation(model, aipId, representationId, algorithms,
username);
} else {
File file = model.retrieveFile(aipId, representationId, fileDirectoryPath, fileId);
PremisSkeletonPluginUtils.createPremisSkeletonOnFile(model, file, algorithms, username);
}

premisBin = model.retrievePreservationFile(aipId, representationId, fileDirectoryPath, fileId);
LOGGER.debug("PREMIS object skeleton created");
}
gov.loc.premis.v3.File premisFile = binaryToFile(premisBin.getContent(), false);
ObjectCharacteristicsComplexType objectCharacteristics;
if (premisFile.getObjectIdentifier() != null && !premisFile.getObjectIdentifier().isEmpty()) {
objectCharacteristics = premisFile.getObjectCharacteristics().get(0);
if (objectCharacteristics.getFormat() != null && !objectCharacteristics.getFormat().isEmpty()) {
for (FormatComplexType format : objectCharacteristics.getFormat()) {
if (format.getFormatNote() != null) {
return format.getFormatNote();
}
}
}
}
} catch (RODAException | IOException e) {
LOGGER.error("PREMIS could not be checked due to an error", e);
}
return new ArrayList<>();
}

public static boolean formatWasManuallyModified(ModelService model, String aipId, String representationId,
List<String> fileDirectoryPath, String fileId, String username) {
Binary premisBin = null;
Expand Down Expand Up @@ -1059,14 +1099,6 @@ public static void updateFormatPreservationMetadata(ModelService model, String a
PremisSkeletonPluginUtils.createPremisSkeletonOnRepresentation(model, aipId, representationId, algorithms,
username);
} else {
// File file;
// if (shallow) {
// file = model.retrieveFileInsideManifest(aipId, representationId,
// fileDirectoryPath, fileId);
// } else {
// file = model.retrieveFile(aipId, representationId, fileDirectoryPath,
// fileId);
// }
File file = model.retrieveFile(aipId, representationId, fileDirectoryPath, fileId);
PremisSkeletonPluginUtils.createPremisSkeletonOnFile(model, file, algorithms, username);
}
Expand Down
18 changes: 10 additions & 8 deletions .../roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic
if (aip.getRepresentations() != null && !aip.getRepresentations().isEmpty()) {
for (Representation representation : aip.getRepresentations()) {
LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId());
List<LinkingIdentifier> newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model,
List<LinkingIdentifier> newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index,
representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual);
if (!newSources.isEmpty()) {
sources.addAll(newSources);
Expand Down Expand Up @@ -180,7 +180,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic
if (aip.getRepresentations() != null && !aip.getRepresentations().isEmpty()) {
for (Representation representation : aip.getRepresentations()) {
LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId());
List<LinkingIdentifier> newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model,
List<LinkingIdentifier> newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index,
representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual);
if (!newSources.isEmpty()) {
sources.addAll(newSources);
Expand All @@ -203,7 +203,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic
}
}
} catch (PluginException | NotFoundException | GenericException | RequestNotValidException
| AuthorizationDeniedException e) {
| AuthorizationDeniedException | AlreadyExistsException e) {
LOGGER.error("Error running Siegfried {}: {}", aip.getId(), e.getMessage(), e);

jobPluginInfo.incrementObjectsProcessedWithFailure();
Expand All @@ -225,7 +225,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic
for (Representation representation : filteredList) {
try {
LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId());
sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, representation,
sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation,
cachedJob.getId(), cachedJob.getUsername(), overwriteManual));
if (sources.isEmpty()) {
state = PluginState.SKIPPED;
Expand Down Expand Up @@ -283,7 +283,8 @@ public Report executeOnRepresentation(IndexService index, ModelService model, St
PluginHelper.updatePartialJobReport(this, model, reportItem, false, cachedJob);
LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), representation.getAipId());
try {
sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, representation, cachedJob.getId(),
sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation,
cachedJob.getId(),
cachedJob.getUsername(), overwriteManual));
if (sources.isEmpty()) {
jobPluginInfo.incrementObjectsProcessedWithSkipped();
Expand All @@ -295,7 +296,7 @@ public Report executeOnRepresentation(IndexService index, ModelService model, St
model.notifyRepresentationUpdated(representation).failOnError();
}
} catch (PluginException | NotFoundException | GenericException | RequestNotValidException
| AuthorizationDeniedException e) {
| AuthorizationDeniedException | AlreadyExistsException e) {
LOGGER.error("Error running Siegfried {}: {}", representation.getAipId(), e.getMessage(), e);

jobPluginInfo.incrementObjectsProcessedWithFailure();
Expand Down Expand Up @@ -331,7 +332,8 @@ public Report executeOnFile(IndexService index, ModelService model, StorageServi
file.getAipId());

try {
sources.addAll(SiegfriedPluginUtils.runSiegfriedOnFile(model, file, cachedJob.getUsername(), overwriteManual));
sources.addAll(
SiegfriedPluginUtils.runSiegfriedOnFile(model, index, file, cachedJob.getUsername(), overwriteManual));
if (sources.isEmpty()) {
jobPluginInfo.incrementObjectsProcessedWithSkipped();
reportItem.setPluginState(PluginState.SKIPPED)
Expand All @@ -341,7 +343,7 @@ public Report executeOnFile(IndexService index, ModelService model, StorageServi
reportItem.setPluginState(PluginState.SUCCESS);
}
} catch (PluginException | NotFoundException | GenericException | RequestNotValidException
| AuthorizationDeniedException e) {
| AuthorizationDeniedException | AlreadyExistsException e) {
LOGGER.error("Error running Siegfried on file {}: {}", file.getId(), e.getMessage(), e);

jobPluginInfo.incrementObjectsProcessedWithFailure();
Expand Down
Loading
Loading