Merge pull request fli-iam#2308 from fli-iam/develop

develop to master

docker-compose/database/db-changes/datasets/0050_examination_update_bug

@@ -0,0 +1,8 @@
+# List of datasets not correctly updated
+
+UPDATE dataset d SET d.subject_id =
+    (SELECT e.subject_id FROM examination e WHERE e.id =
+        (SELECT examination_id FROM dataset_acquisition da WHERE da.id = d.dataset_acquisition_id))
+WHERE d.subject_id !=
+    (SELECT e.subject_id from examination e WHERE e.id =
+        (SELECT examination_id from dataset_acquisition da WHERE da.id = d.dataset_acquisition_id));

docker-compose/database/oneshot-updates/correct_Cohort_VHD_datasets_naming.sql

@@ -0,0 +1,6 @@
+UPDATE dataset_metadata mtd SET mtd.name = mtd.comment
+WHERE mtd.id IN (
+SELECT ds.updated_metadata_id FROM dataset ds 
+INNER JOIN dataset_acquisition acq ON ds.dataset_acquisition_id = acq.id
+INNER JOIN examination ex ON acq.examination_id = ex.id AND ex.study_id = 66
+INNER JOIN dataset_property prop ON ds.id = prop.dataset_id AND prop.name = 'volume.name');

docker-compose/database/oneshot-updates/tag_ofsep_organs.sql

@@ -1,21 +1,42 @@
+-- Remove all volume.organ dataset_tags
+
+DELETE FROM datasets.dataset_tag
+WHERE study_tag_id IN (SELECT id FROM datasets.study_tag WHERE name LIKE 'volume.organ:%');
+
+-- Remove all volume.organ study_tags in datasets and studies MS
+
+DELETE FROM studies.study_tag
+WHERE name LIKE 'volume.organ:%';
+
+DELETE FROM datasets.study_tag
+WHERE name LIKE 'volume.organ:%';
+
+-- Correct study_id of dataset_processings
+
+UPDATE dataset_processing dp SET dp.study_id = (SELECT e.study_id FROM examination e
+INNER JOIN dataset_acquisition da ON da.examination_id = e.id
+INNER JOIN dataset ds ON ds.dataset_acquisition_id = da.id
+WHERE ds.id = (SELECT iddp.dataset_id FROM input_of_dataset_processing iddp WHERE dp.id = iddp.processing_id LIMIT 1))
+WHERE dp.parent_id IS NOT NULL;
+
 -- Create study_tags in studies for all relevant studies
 
 INSERT INTO studies.study_tag (color, name, study_id)
 SELECT DISTINCT '#e74c3c', 'volume.organ:brain', proc.study_id 
 FROM datasets.dataset_property prop
-INNER JOIN datasets.dataset_processing proc ON proc.id = prop.dataset_processing_id
+INNER JOIN datasets.dataset_processing proc ON proc.parent_id = prop.dataset_processing_id
 WHERE prop.name = 'volume.organ' AND prop.value = 'brain';
 
 INSERT INTO studies.study_tag (color, name, study_id)
 SELECT DISTINCT '#27ae60', 'volume.organ:spine', proc.study_id 
 FROM datasets.dataset_property prop
-INNER JOIN datasets.dataset_processing proc ON proc.id = prop.dataset_processing_id
+INNER JOIN datasets.dataset_processing proc ON proc.parent_id = prop.dataset_processing_id
 WHERE prop.name = 'volume.organ' AND prop.value = 'spine';
 
 INSERT INTO studies.study_tag (color, name, study_id)
 SELECT DISTINCT '#198bda', 'volume.organ:null', proc.study_id 
 FROM datasets.dataset_property prop
-INNER JOIN datasets.dataset_processing proc ON proc.id = prop.dataset_processing_id
+INNER JOIN datasets.dataset_processing proc ON proc.parent_id = prop.dataset_processing_id
 WHERE prop.name = 'volume.organ' AND prop.value = 'null';
 
 -- Copy studies study_tags to datasets study_tags
@@ -26,23 +47,23 @@ FROM studies.study_tag;
 
 -- Create dataset_tag for all relevant dataset
 
-INSERT INTO dataset_tag (dataset_id, study_tag_id)
+INSERT INTO datasets.dataset_tag (dataset_id, study_tag_id)
 SELECT DISTINCT prop.dataset_id, tag.id 
-FROM dataset_property prop
-INNER JOIN dataset_processing proc ON proc.id = prop.dataset_processing_id
-INNER JOIN study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:brain'
+FROM datasets.dataset_property prop
+INNER JOIN datasets.dataset_processing proc ON proc.id = prop.dataset_processing_id
+INNER JOIN datasets.study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:brain'
 WHERE prop.name = 'volume.organ' AND prop.value = 'brain';
 
-INSERT INTO dataset_tag (dataset_id, study_tag_id)
+INSERT INTO datasets.dataset_tag (dataset_id, study_tag_id)
 SELECT DISTINCT prop.dataset_id, tag.id 
-FROM dataset_property prop
-INNER JOIN dataset_processing proc ON proc.id = prop.dataset_processing_id
-INNER JOIN study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:spine'
+FROM datasets.dataset_property prop
+INNER JOIN datasets.dataset_processing proc ON proc.parent_id = prop.dataset_processing_id
+INNER JOIN datasets.study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:spine'
 WHERE prop.name = 'volume.organ' AND prop.value = 'spine';
 
-INSERT INTO dataset_tag (dataset_id, study_tag_id)
+INSERT INTO datasets.dataset_tag (dataset_id, study_tag_id)
 SELECT DISTINCT prop.dataset_id, tag.id
-FROM dataset_property prop
-INNER JOIN dataset_processing proc ON proc.id = prop.dataset_processing_id
-INNER JOIN study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:null'
+FROM datasets.dataset_property prop
+INNER JOIN datasets.dataset_processing proc ON proc.parent_id = prop.dataset_processing_id
+INNER JOIN datasets.study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:null'
 WHERE prop.name = 'volume.organ' AND prop.value = 'null';

shanoir-ng-back/pom.xml

@@ -47,7 +47,7 @@ along with this program. If not, see https://www.gnu.org/licenses/gpl-3.0.html
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<java.version>17</java.version>
+		<java.version>21</java.version>
 		<mapstruct.version>1.5.3.Final</mapstruct.version>
 		<keycloak.version>23.0.5</keycloak.version>
 		<dcm4che.version>5.31.0</dcm4che.version>

shanoir-ng-datasets/pom.xml

@@ -27,7 +27,7 @@
 	<description>Dataset microservice for Shanoir NG</description>
 
 	<properties>
-		<solrj.version>9.2.1</solrj.version>
+		<solrj.version>9.4.1</solrj.version>
 		<spring.data.solr.version>4.3.15</spring.data.solr.version>		
 	</properties>
 

shanoir-ng-datasets/src/main/java/org/shanoir/ng/configuration/amqp/RabbitMQDatasetsService.java

@@ -17,7 +17,6 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.persistence.EntityManager;
-import org.apache.solr.client.solrj.SolrServerException;
 import org.shanoir.ng.bids.service.BIDSService;
 import org.shanoir.ng.dataset.dto.StudyStorageVolumeDTO;
 import org.shanoir.ng.dataset.model.Dataset;
@@ -58,11 +57,9 @@
 import org.springframework.transaction.annotation.Isolation;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
-import org.springframework.util.CollectionUtils;
 
 import java.io.IOException;
 import java.util.*;
-import java.util.stream.Collectors;
 
 /**
  * RabbitMQ configuration.
@@ -186,14 +183,8 @@ public String receiveStudyUpdate(final String studyAsString) {
 			}
 
 			studyService.updateStudy(updated, current);
-
-			List<Long> subjectIds = current.getSubjectStudyList()
-					.stream().map(subStu ->
-							subStu.getSubject().getId()
-					).collect(Collectors.toList()
-				);
-
-			solrService.updateSubjects(subjectIds);
+
+			solrService.updateStudyAsync(current.getId());
 
 		} catch (Exception ex) {
 			LOG.error("An error occured while processing study update", ex);
@@ -227,11 +218,6 @@ public boolean receiveSubjectNameUpdate(final String subjectStr) {
 			if (su.getId() == null) throw new IllegalStateException("The entity should must have an id ! Received string : \"" + subjectStr + "\"");
 			subjectRepository.save(su);
 
-			// Update solr references
-			List<Long> subjectIdList = new ArrayList<Long>();
-			subjectIdList.add(su.getId());
-			solrService.updateSubjects(subjectIdList);
-
 			// Update BIDS
 			Set<Long> studyIds = new HashSet<>();
 
@@ -241,6 +227,12 @@ public boolean receiveSubjectNameUpdate(final String subjectStr) {
 			for (Study stud : studyRepository.findAllById(studyIds)) {
 				bidsService.deleteBidsFolder(stud.getId(), stud.getName());
 			}
+
+			// Update solr references
+			List<Long> subjectIdList = new ArrayList<Long>();
+			subjectIdList.add(su.getId());
+			solrService.updateSubjectsAsync(subjectIdList);
+
 			return true;
 		} catch (Exception e) {
 			throw new AmqpRejectAndDontRequeueException(RABBIT_MQ_ERROR, e);

shanoir-ng-datasets/src/main/java/org/shanoir/ng/dataset/model/Dataset.java

@@ -133,7 +133,7 @@ public abstract class Dataset extends AbstractEntity {
 	@OneToOne(cascade = CascadeType.ALL)
 	private DatasetMetadata updatedMetadata;
 
-	@ManyToMany
+	@ManyToMany(fetch = FetchType.LAZY)
 	@JoinTable(name = "DATASET_TAG", joinColumns = @JoinColumn(name = "DATASET_ID"), inverseJoinColumns = @JoinColumn(name = "STUDY_TAG_ID"))
 	private List<StudyTag> tags;
 

shanoir-ng-datasets/src/main/java/org/shanoir/ng/dataset/repository/DatasetRepository.java

@@ -45,7 +45,17 @@ public interface DatasetRepository extends PagingAndSortingRepository<Dataset, L
 	Iterable<Dataset> findByDatasetAcquisition_Examination_Study_Id(Long studyId);
 
 	int countByDatasetAcquisition_Examination_Study_Id(Long studyId);
-
+
+	@Query(value = "SELECT ds.id FROM dataset ds " +
+			"INNER JOIN dataset_acquisition acq ON ds.dataset_acquisition_id = acq.id " +
+			"INNER JOIN examination ex ON acq.examination_id = ex.id " +
+			"WHERE ex.study_id = ?1", nativeQuery = true)
+	List<Long> findIdsByStudyId(Long studyId);
+
+	@Query(value = "SELECT ds.id FROM dataset ds " +
+			"WHERE ds.subject_id IN (?1)", nativeQuery = true)
+	List<Long> findIdsBySubjectIdIn(List<Long> subjectIds);
+
 	Iterable<Dataset> findByDatasetAcquisitionId(Long acquisitionId);
 
 	Iterable<Dataset> findBydatasetAcquisitionStudyCardId(Long studycardId);

shanoir-ng-datasets/src/main/java/org/shanoir/ng/dicom/WADOURLHandler.java

@@ -1,10 +1,10 @@
 package org.shanoir.ng.dicom;
 
+import org.springframework.stereotype.Component;
+
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.springframework.stereotype.Component;
-
 /**
  * This class manages WADO-URLs, either WADO-RS or WADO-URI.
  * 
@@ -27,12 +27,13 @@ public String convertWADO_URI_TO_WADO_RS(String url) {
 				"wado\\?requestType=WADO", "rs",
 				"&studyUID=", "/studies/",
 				"&seriesUID=", "/series/",
+				"&objectUID=", "/instances/",
 				"&contentType=application/dicom", ""
 		};
         for (int i = 0; i < patterns.length; i += 2) {
             Pattern pattern = Pattern.compile(patterns[i]);
             Matcher matcher = pattern.matcher(url);
-            url = matcher.replaceAll(patterns[i + 1]);
+			url = matcher.replaceAll(patterns[i + 1]);
         }
         return url;
     }

shanoir-ng-datasets/src/main/java/org/shanoir/ng/dicom/web/DICOMWebApiController.java

@@ -5,7 +5,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.servlet.http.HttpServletRequest;
-import org.apache.solr.common.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.shanoir.ng.dicom.web.service.DICOMWebService;
 import org.shanoir.ng.examination.model.Examination;
 import org.shanoir.ng.examination.service.ExaminationService;

shanoir-ng-datasets/src/main/java/org/shanoir/ng/examination/service/ExaminationServiceImpl.java

@@ -24,10 +24,9 @@
 import org.shanoir.ng.datasetacquisition.service.DatasetAcquisitionService;
 import org.shanoir.ng.examination.model.Examination;
 import org.shanoir.ng.examination.repository.ExaminationRepository;
+import org.shanoir.ng.processing.model.DatasetProcessing;
 import org.shanoir.ng.shared.configuration.RabbitMQConfiguration;
-import org.shanoir.ng.shared.event.ShanoirEvent;
 import org.shanoir.ng.shared.event.ShanoirEventService;
-import org.shanoir.ng.shared.event.ShanoirEventType;
 import org.shanoir.ng.shared.exception.EntityNotFoundException;
 import org.shanoir.ng.shared.exception.ErrorModel;
 import org.shanoir.ng.shared.exception.RestServiceException;
@@ -244,7 +243,22 @@ private Examination updateExaminationValues(final Examination examinationDb, fin
 				}
 			}
 		}
-		examinationDb.setSubject(examination.getSubject());
+		// If we updated the subject, we need to update datasets and processed datasets too
+		if (!examination.getSubject().getId().equals(examinationDb.getSubject().getId())) {
+			examinationDb.setSubject(examination.getSubject());
+			for (DatasetAcquisition acquisition : examinationDb.getDatasetAcquisitions()) {
+				for (Dataset ds : acquisition.getDatasets()) {
+					ds.setSubjectId(examination.getSubject().getId());
+					// Processed datasets
+					for (DatasetProcessing processing : ds.getProcessings()) {
+						for (Dataset procDs : processing.getOutputDatasets()) {
+							procDs.setSubjectId(examination.getSubject().getId());
+						}
+					}
+				}
+			}
+		}
+
 		examinationDb.setCenterId(examination.getCenterId());
 		examinationDb.setComment(examination.getComment());
 		examinationDb.setExaminationDate(examination.getExaminationDate());

shanoir-ng-datasets/src/main/java/org/shanoir/ng/shared/model/Study.java

@@ -22,6 +22,7 @@
 import org.shanoir.ng.tag.model.Tag;
 
 import java.util.List;
+import java.util.Set;
 
 /**
  * @author yyao
@@ -56,7 +57,7 @@ public class Study extends IdName {
     private List<Examination> examinations;
 
 	@OneToMany(fetch = FetchType.LAZY, mappedBy = "study", cascade = CascadeType.ALL, orphanRemoval = true)
-	private List<StudyTag> studyTags;
+	private Set<StudyTag> studyTags;
 
 
 	/**
@@ -140,11 +141,11 @@ public void setExaminations(List<Examination> examinations) {
         this.examinations = examinations;
     }
 
-	public List<StudyTag> getStudyTags() {
+	public Set<StudyTag> getStudyTags() {
 		return studyTags;
 	}
 
-	public void setStudyTags(List<StudyTag> studyTags) {
+	public void setStudyTags(Set<StudyTag> studyTags) {
 		this.studyTags = studyTags;
 	}
 }

shanoir-ng-datasets/src/main/java/org/shanoir/ng/solr/repository/ShanoirMetadataRepositoryImpl.java

@@ -304,14 +304,14 @@ public class ShanoirMetadataRepositoryImpl implements ShanoirMetadataRepositoryC
 
 	public static final String SUBJECT_TAG_QUERY = "SELECT d.id AS dataset_id, tag.name AS tag" +
 			" FROM dataset d" +
-			" LEFT JOIN subject_study substu ON d.subject_id = substu.subject_id" +
-			" LEFT JOIN subject_study_tag substutag ON substu.id = substutag.subject_study_id" +
-			" LEFT JOIN tag ON substutag.tags_id = tag.id";
+			" INNER JOIN subject_study substu ON d.subject_id = substu.subject_id" +
+			" INNER JOIN subject_study_tag substutag ON substu.id = substutag.subject_study_id" +
+			" INNER JOIN tag ON substutag.tags_id = tag.id";
 
 	public static final String STUDY_TAG_QUERY = "SELECT d.id AS dataset_id, tag.name AS tag" +
 			" FROM dataset d " +
-			" LEFT JOIN dataset_tag dstag ON d.id = dstag.dataset_id " +
-			" LEFT JOIN study_tag tag ON dstag.study_tag_id = tag.id";
+			" INNER JOIN dataset_tag dstag ON d.id = dstag.dataset_id " +
+			" INNER JOIN study_tag tag ON dstag.study_tag_id = tag.id";
 
 	@PersistenceContext
 	private EntityManager em;

shanoir-ng-datasets/src/main/java/org/shanoir/ng/solr/service/SolrService.java

@@ -20,14 +20,13 @@
 package org.shanoir.ng.solr.service;
 
 import org.apache.solr.client.solrj.SolrServerException;
-import org.shanoir.ng.shared.event.ShanoirEvent;
 import org.shanoir.ng.shared.exception.RestServiceException;
 import org.shanoir.ng.solr.model.ShanoirSolrDocument;
 import org.shanoir.ng.solr.model.ShanoirSolrQuery;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.solr.core.query.result.SolrResultPage;
-import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.security.access.prepost.PreAuthorize;
 
 import java.io.IOException;
@@ -60,6 +59,10 @@ public interface SolrService {
 
 	void updateDatasets(List<Long> datasetIds) throws SolrServerException, IOException;
 
-	void updateSubjects(List<Long> subjectIds) throws SolrServerException, IOException;
+	void updateDatasetsAsync(List<Long> datasetIds) throws SolrServerException, IOException;
+
+	void updateSubjectsAsync(List<Long> subjectIds) throws SolrServerException, IOException;
+
+	void updateStudyAsync(Long studyId) throws SolrServerException, IOException;
 
 }