Skip to content

Commit

Permalink
Merge branch 'ELTER-20-B2SHARE-import' into 'develop'
Browse files Browse the repository at this point in the history 
Add B2shareImportService

See merge request eip/catalogue!535
  • Loading branch information
@rodscott
rodscott committed Aug 16, 2023
2 parents 0b2f38f + 6382a34 commit 15d2cff
Show file tree
Hide file tree
Showing 8 changed files with 1,065 additions and 87 deletions.
10 changes: 10 additions & 0 deletions java/src/main/java/uk/ac/ceh/gateway/catalogue/config/DevelopmentUserStoreConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,16 @@ public CatalogueUser sitesMetadataImport() throws UsernameAlreadyTakenException
return user;
}

@Bean
public CatalogueUser b2shareMetadataImport() throws UsernameAlreadyTakenException {
val user = new CatalogueUser()
.setUsername("B2SHARE metadata import")
.setEmail("[email protected]");
addUserToGroup(user, ELTER_EDITOR, ELTER_PUBLISHER);
userStore().addUser(user, "password");
return user;
}

@Bean
public CatalogueUser erammpEditor() throws UsernameAlreadyTakenException {
val user = new CatalogueUser()
Expand Down
476 changes: 476 additions & 0 deletions java/src/main/java/uk/ac/ceh/gateway/catalogue/elter/B2shareImportService.java
View file

Large diffs are not rendered by default.

84 changes: 84 additions & 0 deletions java/src/main/java/uk/ac/ceh/gateway/catalogue/elter/ElterDocument.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
Expand All @@ -16,6 +17,7 @@
import uk.ac.ceh.gateway.catalogue.indexing.solr.WellKnownText;
import uk.ac.ceh.gateway.catalogue.model.*;

import java.time.LocalDate;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -71,6 +73,88 @@ public class ElterDocument extends AbstractMetadataDocument implements WellKnown
private String importId;
private ZonedDateTime importLastModified;

public void importDataciteJson(JsonNode inputJson){
// create document from Datacite API JSON

// don't currently need other parts of the response
inputJson = inputJson.get("data").get("attributes");

// ensure title is set to something
JsonNode jsonTitles = inputJson.get("titles");
int numTitles = jsonTitles.size();
if (numTitles == 0){
this.setTitle("TITLE MISSING");
}
else {
this.setTitle(jsonTitles.get(0).get("title").asText());
ArrayList<String> alternativeTitles = new ArrayList<>();
for (int i = 1; i < numTitles; i++){
alternativeTitles.add(jsonTitles.get(i).get("title").asText());
}
this.setAlternateTitles(alternativeTitles);
}
// description
StringBuilder descriptionBuilder = new StringBuilder();
for (Iterator<JsonNode> iter = inputJson.get("descriptions").iterator(); iter.hasNext(); ) {
JsonNode node = iter.next();
if (descriptionBuilder.length() > 0){
descriptionBuilder.append("\n\n");
}
JsonNode descriptionTypeNode = node.get("descriptionType");
if (descriptionTypeNode != null) {
String descriptionType = descriptionTypeNode.asText();
if (! descriptionType.equals("Other")) {
descriptionBuilder.append(descriptionType + ": ");
}
}
descriptionBuilder.append(node.get("description").asText().strip());
}
this.setDescription(descriptionBuilder.toString());
// authors
JsonNode jsonCreators = inputJson.get("creators").path(0);
if (! jsonCreators.isMissingNode()){
ResponsibleParty documentCreators = ResponsibleParty.builder()
.individualName(jsonCreators.get("name").asText())
.organisationName("Unknown")
.role("author")
.build();
ArrayList<ResponsibleParty> list1 = new ArrayList<>();
list1.add(documentCreators);
this.setResponsibleParties(list1);
}
// onlineresources
ArrayList<OnlineResource> list2 = new ArrayList<>();
list2.add(
OnlineResource.builder()
.url(inputJson.get("url").asText())
.name("View record")
.description("View record at this link")
.function("information")
.build()
);
this.setOnlineResources(list2);
// reference dates
// the timestamp parsing is extremely dubious but we need working code NOW for the Frankfurt meeting.
// TBF using LocalDate is totally broken anyway so it all needs redoing at some point.
this.setDatasetReferenceDate(
DatasetReferenceDate.builder()
.creationDate(LocalDate.parse(inputJson.get("created").asText().substring(0,10)))
.publicationDate(LocalDate.parse(inputJson.get("published").asText().substring(0,4) + "-01-01"))
.creationDate(LocalDate.parse(inputJson.get("created").asText().substring(0,10)))
.build()
);
// fixed stuff
this.setAccessLimitation(
AccessLimitation.builder()
.value("no limitations to public access")
.code("Available")
.uri("http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/noLimitations")
.build()
);
this.setDataLevel("Level 0");
this.setType("signpost");
}

@Override
public Set<Relationship> getRelationships() {
val relations = Optional.ofNullable(super.getRelationships())
Expand Down
91 changes: 4 additions & 87 deletions java/src/main/java/uk/ac/ceh/gateway/catalogue/elter/LinkedDocumentRetrievalService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@
import com.fasterxml.jackson.databind.ObjectMapper;

import java.net.URL;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand All @@ -26,11 +23,6 @@
import org.springframework.web.client.RestClientResponseException;
import org.springframework.web.client.RestTemplate;

import uk.ac.ceh.gateway.catalogue.gemini.AccessLimitation;
import uk.ac.ceh.gateway.catalogue.gemini.DatasetReferenceDate;
import uk.ac.ceh.gateway.catalogue.gemini.OnlineResource;
import uk.ac.ceh.gateway.catalogue.model.ResponsibleParty;

@Profile("server:elter")
@Slf4j
@Service
Expand Down Expand Up @@ -64,87 +56,12 @@ public ElterDocument get(String url) {
// call Datacite
String dataciteRecordUrl = dataciteApiRoot + "/" + inputDoi;
log.info("GET {}", dataciteRecordUrl);
JsonNode jsonRecordAttributes = objectMapper.readTree(new URL(dataciteRecordUrl)).get("data").get("attributes");
JsonNode dataciteJson = objectMapper.readTree(new URL(dataciteRecordUrl));

// create document from Datacite response
// create and return ElterDocument
ElterDocument document = new ElterDocument();
// ensure title is set to something
JsonNode jsonTitles = jsonRecordAttributes.get("titles");
int numTitles = jsonTitles.size();
if (numTitles == 0){
document.setTitle("TITLE MISSING");
}
else {
document.setTitle(jsonTitles.get(0).get("title").asText());
ArrayList<String> alternativeTitles = new ArrayList<>();
for (int i = 1; i < numTitles; i++){
alternativeTitles.add(jsonTitles.get(i).get("title").asText());
}
document.setAlternateTitles(alternativeTitles);
}
// description
StringBuilder descriptionBuilder = new StringBuilder();
for (Iterator<JsonNode> iter = jsonRecordAttributes.get("descriptions").iterator(); iter.hasNext(); ) {
JsonNode node = iter.next();
if (descriptionBuilder.length() > 0){
descriptionBuilder.append("\n\n");
}
JsonNode descriptionTypeNode = node.get("descriptionType");
if (descriptionTypeNode != null) {
String descriptionType = descriptionTypeNode.asText();
if (! descriptionType.equals("Other")) {
descriptionBuilder.append(descriptionType + ": ");
}
}
descriptionBuilder.append(node.get("description").asText().strip());
}
document.setDescription(descriptionBuilder.toString());
// authors
JsonNode jsonCreators = jsonRecordAttributes.get("creators").path(0);
if (! jsonCreators.isMissingNode()){
ResponsibleParty documentCreators = ResponsibleParty.builder()
.individualName(jsonCreators.get("name").asText())
.organisationName("Unknown")
.role("author")
.build();
ArrayList<ResponsibleParty> list1 = new ArrayList<>();
list1.add(documentCreators);
document.setResponsibleParties(list1);
}
// onlineresources
ArrayList<OnlineResource> list2 = new ArrayList<>();
list2.add(
OnlineResource.builder()
.url(jsonRecordAttributes.get("url").asText())
.name("View record")
.description("View record at this link")
.function("information")
.build()
);
document.setOnlineResources(list2);
// reference dates
// the timestamp parsing is extremely dubious but we need working code NOW for the Frankfurt meeting.
// TBF using LocalDate is totally broken anyway so it all needs redoing at some point.
document.setDatasetReferenceDate(
DatasetReferenceDate.builder()
.creationDate(LocalDate.parse(jsonRecordAttributes.get("created").asText().substring(0,10)))
.publicationDate(LocalDate.parse(jsonRecordAttributes.get("published").asText().substring(0,4) + "-01-01"))
.creationDate(LocalDate.parse(jsonRecordAttributes.get("created").asText().substring(0,10)))
.build()
);
// import ID - should equal input doi but can't be too careful
document.setImportId(jsonRecordAttributes.get("doi").asText());
// fixed stuff
document.setAccessLimitation(
AccessLimitation.builder()
.value("no limitations to public access")
.code("Available")
.uri("http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/noLimitations")
.build()
);
document.setDataLevel("Level 0");
document.setType("signpost");

document.importDataciteJson(dataciteJson);
document.setImportId(inputDoi);
return document;
}
else {
Expand Down
1 change: 1 addition & 0 deletions java/src/main/resources/application.properties
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# suppress inspection "UnusedProperty" for whole file
# suppress inspection "HttpUrlsUsage" for whole file
# suppress inspection "SpringBootApplicationProperties" for whole file
b2share.api=https://b2share.eudat.eu/api/records/?q=community:d952913c-451e-4b5c-817e-d578dc8a4469&size=500
crowd.address=https://crowd.ceh.ac.uk/crowd/rest/usermanagement/latest
crowd.username=eip-ro
data.repository.location=/var/ceh-catalogue/datastore
Expand Down
Loading

0 comments on commit 15d2cff

Please sign in to comment.