Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

EVA-3474 : Recover continuous id blocks one by one #80

Merged
merged 2 commits into from
Jan 10, 2024
Merged

EVA-3474 : Recover continuous id blocks one by one #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ad090ca
recovering continuous id blocks one by one
nitin-ebi Jan 9, 2024
5c6382c
test available ranges after recovery
nitin-ebi Jan 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions .../uk/ac/ebi/ampt2d/commons/accession/generators/monotonic/MonotonicAccessionGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService;
import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.MonotonicDatabaseService;
import uk.ac.ebi.ampt2d.commons.accession.utils.ExponentialBackOff;
import uk.ac.ebi.ampt2d.commons.accession.utils.exceptions.ExponentialBackOffMaxRetriesRuntimeException;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;

Expand All @@ -52,7 +52,15 @@ public MonotonicAccessionGenerator(String categoryId,
ContiguousIdBlockService contiguousIdBlockService,
MonotonicDatabaseService databaseService) {
this(categoryId, applicationInstanceId, contiguousIdBlockService);
recoverState(databaseService.getAccessionsInRanges(getAvailableRanges()));
// As we are going through the available ranges and at the same time we are also going to manipulate/update them
// Need to make a copy of the original for iteration to avoid ConcurrentModificationException
MonotonicRangePriorityQueue copyOfAvailableRanges = new MonotonicRangePriorityQueue();
for (MonotonicRange range : getAvailableRanges()) {
copyOfAvailableRanges.offer(range);
}
for (MonotonicRange monotonicRange : copyOfAvailableRanges) {
recoverState(databaseService.getAccessionsInRanges(Collections.singletonList(monotonicRange)));
}
}

public MonotonicAccessionGenerator(String categoryId,
Expand Down
1 change: 1 addition & 0 deletions .../ampt2d/commons/accession/persistence/jpa/monotonic/service/ContiguousIdBlockService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public ContiguousIdBlockService(ContiguousIdBlockRepository repository, Map<Stri
@Transactional
public void save(Iterable<ContiguousIdBlock> blocks) {
repository.saveAll(blocks);
entityManager.flush();
}

@Transactional(isolation = Isolation.SERIALIZABLE)
Expand Down
16 changes: 8 additions & 8 deletions ...i/ampt2d/commons/accession/service/BasicSpringDataRepositoryMonotonicDatabaseService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
*/
package uk.ac.ebi.ampt2d.commons.accession.service;

import uk.ac.ebi.ampt2d.commons.accession.core.DatabaseService;
import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper;
import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRange;
import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.MonotonicDatabaseService;
Expand All @@ -31,6 +30,7 @@
import java.util.Collection;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;

/**
* Basic implementation of {@link MonotonicDatabaseService} that requires a Spring Data repository that extends
Expand Down Expand Up @@ -62,18 +62,18 @@ public BasicSpringDataRepositoryMonotonicDatabaseService(

@Override
public long[] getAccessionsInRanges(Collection<MonotonicRange> ranges) {
List<AccessionProjection<Long>> accessionsInRanges = new ArrayList<>();
List<Long> accessionsInDB = new ArrayList<>();
for (MonotonicRange potentiallyBigRange : ranges) {
for (MonotonicRange range : ensureRangeMaxSize(potentiallyBigRange, MAX_RANGE_SIZE)) {
accessionsInRanges.addAll(
List<AccessionProjection<Long>> accessionsInRange =
repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(range.getStart(),
range.getEnd()));
range.getEnd());

accessionsInDB.addAll(accessionsInRange.stream().map(ap -> ap.getAccession()).collect(Collectors.toList()));
}
}
long[] accessionArray = new long[accessionsInRanges.size()];
for (int i = 0; i < accessionsInRanges.size(); i++) {
accessionArray[i] = accessionsInRanges.get(i).getAccession();
}

long[] accessionArray = accessionsInDB.stream().mapToLong(l -> l).toArray();
return accessionArray;
}

Expand Down
46 changes: 46 additions & 0 deletions .../ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,21 @@
import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper;
import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper;
import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator;
import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRange;
import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRangePriorityQueue;
import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction;
import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock;
import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService;
import uk.ac.ebi.ampt2d.commons.accession.service.BasicSpringDataRepositoryMonotonicDatabaseService;
import uk.ac.ebi.ampt2d.test.configuration.TestMonotonicDatabaseServiceTestConfiguration;
import uk.ac.ebi.ampt2d.test.models.TestModel;
import uk.ac.ebi.ampt2d.test.persistence.TestMonotonicEntity;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.LongStream;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
Expand All @@ -62,6 +67,47 @@ public void testUnknownCategory() throws AccessionCouldNotBeGeneratedException {
.getOrCreate(getObjectsForAccessionsInRange(1, 10));
}

@Test
public void testRecoverState() {
String categoryId = "eva_2";
String instanceId2 = "test-instance_2";

// create 3 un-complete contiguous id blocks of size 10
// block-1 : (100 to 109), block-2 : (110 to 119), block-3 : (120 to 129)
List<ContiguousIdBlock> uncompletedBlocks = new ArrayList<>();
uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 100, 10));
uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 110, 10));
uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 120, 10));
contiguousIdBlockService.save(uncompletedBlocks);

assertEquals(3, contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).size());

// create and save accessions in db (100 to 124)
List<AccessionWrapper<TestModel, String, Long>> accessions = LongStream.range(100l, 125l)
.boxed()
.map(longAcc -> new AccessionWrapper<>(longAcc, "hash" + longAcc, TestModel.of("test-obj-" + longAcc)))
.collect(Collectors.toList());
databaseService.save(accessions);

// run recover blocks
MonotonicAccessionGenerator generator = getGenerator(categoryId, instanceId2);

// As we have already saved accessions in db from 100 to 124, the status should be
// block-1 (100 to 109) : fully complete
// block-2 (110 to 119) : fully complete
// block-3 (120 to 124) : partially complete
assertEquals(1, contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).size());
ContiguousIdBlock uncompletedBlock = contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).get(0);
assertEquals(120l, uncompletedBlock.getFirstValue());
assertEquals(129l, uncompletedBlock.getLastValue());
assertEquals(124l, uncompletedBlock.getLastCommitted());

MonotonicRangePriorityQueue availableRanges = generator.getAvailableRanges();
assertEquals(1, availableRanges.size());
assertEquals(125l, availableRanges.peek().getStart());
assertEquals(129l, availableRanges.peek().getEnd());
}

@Test
public void testAlternateRangesWithDifferentGenerators() throws AccessionCouldNotBeGeneratedException {
/* blockStartValue= 0, blockSize= 10 , nextBlockInterval= 20
Expand Down
Loading