Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EVA-3260 update block allocation strategy #83

Merged
merged 10 commits into from
Mar 21, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,10 @@ public void merge(ACCESSION accessionOrigin, ACCESSION mergeInto, String reason)
dbService.merge(accessionOrigin, mergeInto, reason);
}

public void shutDownAccessioning() {
accessionGenerator.shutDownAccessionGenerator();
}

protected AccessionGenerator<MODEL, ACCESSION> getAccessionGenerator() {
return accessionGenerator;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package uk.ac.ebi.ampt2d.commons.accession.core.exceptions;

public class AccessionGeneratorShutDownException extends RuntimeException {
public AccessionGeneratorShutDownException(String msg) {
super(msg);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,4 +50,9 @@ <HASH> List<AccessionWrapper<MODEL, HASH, ACCESSION>> generateAccessions(Map<HAS
* @param response DB response
*/
void postSave(SaveResponse<ACCESSION> response);

/**
* This method should be used to shut-down the generator and release resources
*/
void shutDownAccessionGenerator();
}
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,8 @@ public static <MODEL> SingleAccessionGenerator<MODEL, String> ofSHA1AccessionGen
return ofHashAccessionGenerator(summaryFunction, new SHA1HashingFunction());
}

@Override
public void shutDownAccessionGenerator() {
// Do nothing - no resources to release
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -168,4 +168,14 @@ public Set<ContiguousIdBlock> recoverState(long[] committedElements) throws Acce
this.availableRanges.addAll(newAvailableRanges);
return doCommit(committedElements);
}

public List<ContiguousIdBlock> getAssignedBlocks(){
return assignedBlocks.stream().collect(Collectors.toList());
}

public void shutDownBlockManager() {
assignedBlocks.clear();
availableRanges.clear();
generatedAccessions.clear();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import uk.ac.ebi.ampt2d.commons.accession.block.initialization.BlockInitializationException;
import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException;
import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionGeneratorShutDownException;
import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionIsNotPendingException;
import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper;
import uk.ac.ebi.ampt2d.commons.accession.core.models.SaveResponse;
Expand Down Expand Up @@ -47,6 +48,8 @@ public class MonotonicAccessionGenerator<MODEL> implements AccessionGenerator<MO
private final String applicationInstanceId;
private final ContiguousIdBlockService blockService;

private Boolean SHUTDOWN = Boolean.FALSE;

public MonotonicAccessionGenerator(String categoryId,
String applicationInstanceId,
ContiguousIdBlockService contiguousIdBlockService,
Expand Down Expand Up @@ -88,7 +91,7 @@ private static BlockManager initializeBlockManager(ContiguousIdBlockService bloc
assertBlockParametersAreInitialized(blockService, categoryId);
BlockManager blockManager = new BlockManager();
List<ContiguousIdBlock> uncompletedBlocks = blockService
.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId,
.reserveUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId,
applicationInstanceId);
//Insert as available ranges
for (ContiguousIdBlock block : uncompletedBlocks) {
Expand Down Expand Up @@ -116,6 +119,7 @@ private void recoverState(long[] committedElements) throws AccessionIsNotPending

public synchronized long[] generateAccessions(int numAccessionsToGenerate)
throws AccessionCouldNotBeGeneratedException {
checkAccessionGeneratorNotShutDown();
long[] accessions = new long[numAccessionsToGenerate];
reserveNewBlocksUntilSizeIs(numAccessionsToGenerate);

Expand Down Expand Up @@ -147,20 +151,24 @@ private synchronized void reserveNewBlock(String categoryId, String instanceId)
}

public synchronized void commit(long... accessions) throws AccessionIsNotPendingException {
checkAccessionGeneratorNotShutDown();
blockService.save(blockManager.commit(accessions));
}

public synchronized void release(long... accessions) throws AccessionIsNotPendingException {
checkAccessionGeneratorNotShutDown();
blockManager.release(accessions);
}

public synchronized MonotonicRangePriorityQueue getAvailableRanges() {
checkAccessionGeneratorNotShutDown();
return blockManager.getAvailableRanges();
}

@Override
public <HASH> List<AccessionWrapper<MODEL, HASH, Long>> generateAccessions(Map<HASH, MODEL> messages)
throws AccessionCouldNotBeGeneratedException {
checkAccessionGeneratorNotShutDown();
long[] accessions = generateAccessions(messages.size());
int i = 0;
List<AccessionWrapper<MODEL, HASH, Long>> accessionedModels = new ArrayList<>();
Expand All @@ -174,8 +182,27 @@ public <HASH> List<AccessionWrapper<MODEL, HASH, Long>> generateAccessions(Map<H

@Override
public synchronized void postSave(SaveResponse<Long> response) {
checkAccessionGeneratorNotShutDown();
commit(response.getSavedAccessions().stream().mapToLong(l -> l).toArray());
release(response.getSaveFailedAccessions().stream().mapToLong(l -> l).toArray());
}

public void shutDownAccessionGenerator(){
List<ContiguousIdBlock> blockList = blockManager.getAssignedBlocks();
blockList.stream().forEach(block -> block.releaseReserved());
blockService.save(blockList);
blockManager.shutDownBlockManager();
SHUTDOWN = Boolean.TRUE;
}

/**
* Before doing any operation on Accession Generator, we need to make sure it has not been shut down.
* We should make the check by calling this method as the first thing in all public methods of this class
*/
private void checkAccessionGeneratorNotShutDown(){
if(SHUTDOWN){
throw new AccessionGeneratorShutDownException("Accession Generator has been shut down and is no longer available");
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import javax.persistence.Index;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import java.time.LocalDateTime;

/**
* This class represents a block allocated by an application instance, in a monotonic sequence associated with a
Expand Down Expand Up @@ -63,6 +64,10 @@ public class ContiguousIdBlock implements Comparable<ContiguousIdBlock> {

private long lastCommitted;

private Boolean reserved;

private LocalDateTime createdTimestamp;

// Create / update dates

ContiguousIdBlock() {
Expand All @@ -75,6 +80,8 @@ public ContiguousIdBlock(String categoryId, String applicationInstanceId, long f
this.firstValue = firstValue;
this.lastValue = firstValue + size - 1;
this.lastCommitted = firstValue - 1;
this.reserved = Boolean.TRUE;
this.createdTimestamp = LocalDateTime.now();
}

/**
Expand Down Expand Up @@ -143,6 +150,26 @@ public long getLastValue() {
return lastValue;
}

public Boolean isReserved() {
return reserved == Boolean.TRUE;
apriltuesday marked this conversation as resolved.
Show resolved Hide resolved
}

public Boolean isNotReserved() {
return reserved == Boolean.FALSE;
}

public void markAsReserved() {
this.reserved = Boolean.TRUE;
}

public void releaseReserved() {
this.reserved = Boolean.FALSE;
}

public boolean isFull() {
return lastCommitted == lastValue;
}

public boolean isNotFull() {
return lastCommitted != lastValue;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ public ContiguousIdBlockService(ContiguousIdBlockRepository repository, Map<Stri

@Transactional
public void save(Iterable<ContiguousIdBlock> blocks) {
// release block if full
blocks.forEach(block -> {if (block.isFull()) {block.releaseReserved();}});
repository.saveAll(blocks);
entityManager.flush();
}
Expand Down Expand Up @@ -74,12 +76,22 @@ public BlockParameters getBlockParameters(String categoryId) {
return categoryBlockInitializations.get(categoryId);
}

@Transactional(readOnly = true)
public List<ContiguousIdBlock> getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(
@Transactional(isolation = Isolation.SERIALIZABLE)
public List<ContiguousIdBlock> reserveUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(
apriltuesday marked this conversation as resolved.
Show resolved Hide resolved
String categoryId, String applicationInstanceId) {
try (Stream<ContiguousIdBlock> reservedBlocksOfThisInstance = repository
.findAllByCategoryIdAndApplicationInstanceIdOrderByLastValueAsc(categoryId, applicationInstanceId)) {
return reservedBlocksOfThisInstance.filter(ContiguousIdBlock::isNotFull).collect(Collectors.toList());
List<ContiguousIdBlock> blocksList = reservedBlocksOfThisInstance
.filter(block -> block.isNotFull() && block.isNotReserved()).collect(Collectors.toList());

blocksList.stream().forEach(block -> block.markAsReserved());
save(blocksList);

return blocksList;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I'm not mistaken, this is the core of the process safe block reservation process. I feel like we should describe in a comment why/how this works.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe it makes more sense to describe this at the level of the class rather than this method.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added description on top of the class in the form of Javadoc to explain the behavior

}
}

public ContiguousIdBlockRepository getRepository() {
apriltuesday marked this conversation as resolved.
Show resolved Hide resolved
return repository;
}
}
Loading
Loading