Merge pull request #80 from nitin-ebi/recover-state

EVA-3474 : Recover continuous id blocks one by one

accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/generators/monotonic/MonotonicAccessionGenerator.java

@@ -27,9 +27,9 @@
 import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService;
 import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.MonotonicDatabaseService;
 import uk.ac.ebi.ampt2d.commons.accession.utils.ExponentialBackOff;
-import uk.ac.ebi.ampt2d.commons.accession.utils.exceptions.ExponentialBackOffMaxRetriesRuntimeException;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -52,7 +52,15 @@ public MonotonicAccessionGenerator(String categoryId,
                                        ContiguousIdBlockService contiguousIdBlockService,
                                        MonotonicDatabaseService databaseService) {
         this(categoryId, applicationInstanceId, contiguousIdBlockService);
-        recoverState(databaseService.getAccessionsInRanges(getAvailableRanges()));
+        // As we are going through the available ranges and at the same time we are also going to manipulate/update them
+        // Need to make a copy of the original for iteration to avoid ConcurrentModificationException
+        MonotonicRangePriorityQueue copyOfAvailableRanges = new MonotonicRangePriorityQueue();
+        for (MonotonicRange range : getAvailableRanges()) {
+            copyOfAvailableRanges.offer(range);
+        }
+        for (MonotonicRange monotonicRange : copyOfAvailableRanges) {
+            recoverState(databaseService.getAccessionsInRanges(Collections.singletonList(monotonicRange)));
+        }
     }
 
     public MonotonicAccessionGenerator(String categoryId,

accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/persistence/jpa/monotonic/service/ContiguousIdBlockService.java

@@ -48,6 +48,7 @@ public ContiguousIdBlockService(ContiguousIdBlockRepository repository, Map<Stri
     @Transactional
     public void save(Iterable<ContiguousIdBlock> blocks) {
         repository.saveAll(blocks);
+        entityManager.flush();
     }
 
     @Transactional(isolation = Isolation.SERIALIZABLE)

accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/service/BasicSpringDataRepositoryMonotonicDatabaseService.java

@@ -17,7 +17,6 @@
  */
 package uk.ac.ebi.ampt2d.commons.accession.service;
 
-import uk.ac.ebi.ampt2d.commons.accession.core.DatabaseService;
 import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper;
 import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRange;
 import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.MonotonicDatabaseService;
@@ -31,6 +30,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Basic implementation of {@link MonotonicDatabaseService} that requires a Spring Data repository that extends
@@ -62,18 +62,18 @@ public BasicSpringDataRepositoryMonotonicDatabaseService(
 
     @Override
     public long[] getAccessionsInRanges(Collection<MonotonicRange> ranges) {
-        List<AccessionProjection<Long>> accessionsInRanges = new ArrayList<>();
+        List<Long> accessionsInDB = new ArrayList<>();
         for (MonotonicRange potentiallyBigRange : ranges) {
             for (MonotonicRange range : ensureRangeMaxSize(potentiallyBigRange, MAX_RANGE_SIZE)) {
-                accessionsInRanges.addAll(
+                List<AccessionProjection<Long>> accessionsInRange =
                         repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(range.getStart(),
-                                                                                            range.getEnd()));
+                                range.getEnd());
+
+                accessionsInDB.addAll(accessionsInRange.stream().map(ap -> ap.getAccession()).collect(Collectors.toList()));
             }
         }
-        long[] accessionArray = new long[accessionsInRanges.size()];
-        for (int i = 0; i < accessionsInRanges.size(); i++) {
-            accessionArray[i] = accessionsInRanges.get(i).getAccession();
-        }
+
+        long[] accessionArray = accessionsInDB.stream().mapToLong(l -> l).toArray();
         return accessionArray;
     }
 

accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java

@@ -28,16 +28,21 @@
 import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper;
 import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper;
 import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator;
+import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRange;
+import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRangePriorityQueue;
 import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction;
+import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock;
 import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService;
 import uk.ac.ebi.ampt2d.commons.accession.service.BasicSpringDataRepositoryMonotonicDatabaseService;
 import uk.ac.ebi.ampt2d.test.configuration.TestMonotonicDatabaseServiceTestConfiguration;
 import uk.ac.ebi.ampt2d.test.models.TestModel;
 import uk.ac.ebi.ampt2d.test.persistence.TestMonotonicEntity;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.LongStream;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
@@ -62,6 +67,47 @@ public void testUnknownCategory() throws AccessionCouldNotBeGeneratedException {
                         .getOrCreate(getObjectsForAccessionsInRange(1, 10));
     }
 
+    @Test
+    public void testRecoverState() {
+        String categoryId = "eva_2";
+        String instanceId2 = "test-instance_2";
+
+        // create 3 un-complete contiguous id blocks of size 10
+        // block-1 : (100 to 109), block-2 : (110 to 119), block-3 : (120 to 129)
+        List<ContiguousIdBlock> uncompletedBlocks = new ArrayList<>();
+        uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 100, 10));
+        uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 110, 10));
+        uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 120, 10));
+        contiguousIdBlockService.save(uncompletedBlocks);
+
+        assertEquals(3, contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).size());
+
+        // create and save accessions in db (100 to 124)
+        List<AccessionWrapper<TestModel, String, Long>> accessions = LongStream.range(100l, 125l)
+                .boxed()
+                .map(longAcc -> new AccessionWrapper<>(longAcc, "hash" + longAcc, TestModel.of("test-obj-" + longAcc)))
+                .collect(Collectors.toList());
+        databaseService.save(accessions);
+
+        // run recover blocks
+        MonotonicAccessionGenerator generator = getGenerator(categoryId, instanceId2);
+
+        // As we have already saved accessions in db from 100 to 124, the status should be
+        // block-1 (100 to 109) : fully complete
+        // block-2 (110 to 119) : fully complete
+        // block-3 (120 to 124) : partially complete
+        assertEquals(1, contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).size());
+        ContiguousIdBlock uncompletedBlock = contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).get(0);
+        assertEquals(120l, uncompletedBlock.getFirstValue());
+        assertEquals(129l, uncompletedBlock.getLastValue());
+        assertEquals(124l, uncompletedBlock.getLastCommitted());
+
+        MonotonicRangePriorityQueue availableRanges = generator.getAvailableRanges();
+        assertEquals(1, availableRanges.size());
+        assertEquals(125l, availableRanges.peek().getStart());
+        assertEquals(129l, availableRanges.peek().getEnd());
+    }
+
     @Test
     public void testAlternateRangesWithDifferentGenerators() throws AccessionCouldNotBeGeneratedException {
         /* blockStartValue= 0, blockSize= 10 , nextBlockInterval= 20