Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Defect/download failure fix #371

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions score-client/src/main/conf/logback.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-->
<configuration debug="false">
<configuration debug="true">

<property name="log.dir" value="${logging.path:-/tmp/score-client/logs}" />
<property name="log.name" value="client" />
Expand All @@ -36,7 +36,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
</encoder>
</appender>

<root level="INFO">
<root level="DEBUG">
<appender-ref ref="FILE" />
</root>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

Expand All @@ -60,6 +62,8 @@ public enum OutputLayout {
BUNDLE, FILENAME, ID
}

private Logger session = LoggerFactory.getLogger("session");

/**
* Options
*/
Expand Down Expand Up @@ -148,7 +152,8 @@ private List<String> getIdsFromManifest(ManifestResource manifestResource) throw
val manifest = manifestService.getDownloadManifest(manifestResource);

validateManifest(manifest);

session.info("Download Command Class Parameters : manifest.getEntries() ::::"+manifest.getEntries());
terminal.println("Download Command Class Parameters : manifest.getEntries() ::::"+manifest.getEntries());
val entries = manifest.getEntries();
if (entries.isEmpty()) {
throw new BadManifestException(format("Manifest '%s' is empty", manifestResource));
Expand All @@ -165,7 +170,10 @@ private List<String> getIdsFromAnalysis(String programId, String analysisId) {
private int downloadObjects(List<String> objectIds) throws IOException {
// Entities are defined in Meta service
val entities = resolveEntities(objectIds);

terminal
.printLine()
.println("Downloading objects size ::::"+objectIds.size())
.printLine();
if (!verifyLocalAvailableSpace(entities)) {
return FAILURE_STATUS;
}
Expand All @@ -178,6 +186,14 @@ private int downloadObjects(List<String> objectIds) throws IOException {
if (layout != OutputLayout.ID) {
entitySet = filterEntities(entities);
}
terminal
.printLine()
.println("Filtered Entites ::::"+entitySet.size())
.printLine();
terminal
.printLine()
.println("Force value ::::"+force)
.printLine();

// If --force is specified, delete all the old files first, so that we can resume later if the new downloads fail.
if (force) {
Expand All @@ -200,7 +216,10 @@ private int downloadObjects(List<String> objectIds) throws IOException {
val builder = DownloadRequest.builder();
val request = builder.outputDir(outputDir).entity(entity).objectId(entity.getId()).offset(offset).length(length)
.validate(validate).build();

terminal
.printLine()
.println("Request payload ::::"+request.toString())
.printLine();
// only try to re-download a file that exists if --force was specified on the command line
val target = getLayoutTarget(entity);
if (target.exists()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import com.beust.jcommander.Parameters;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
Expand All @@ -43,6 +45,7 @@
@Profile("!kf")
public class UploadCommand extends RepositoryAccessCommand {

private Logger session = LoggerFactory.getLogger("session");
/**
* Options.
*/
Expand Down Expand Up @@ -90,6 +93,8 @@ public int execute() throws Exception {
uploadFile(objectId, file, checksum);
}
} else {
session.info("Upload Command Class Parameters : file ::::"+file+" objectId ::::::"+objectId+" manifestResource ::::::"+manifestResource+" md5 ::::"+md5);
terminal.println("Upload Command Class Parameters : file ::::"+file+" objectId ::::::"+objectId+" manifestResource ::::::"+manifestResource+" md5 ::::"+md5);
checkParameter(file != null, "--file must be specified if --object-id is specified");
checkParameter(md5 != null, "--md5 must be specified if --object-id is specified");
uploadFile(objectId, file, md5);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,8 @@ private void validateParms() {
terminal.println("When --stdout specified, output format forced to SAM.");
}
}

session.info("View Command Class parameters : objectId ::::"+objectId+" sequenceFile ::::"+sequenceFile+" manifestResource ::::"+manifestResource);
terminal.println("View Command Class parameters : objectId ::::"+objectId+" sequenceFile ::::"+sequenceFile+" manifestResource ::::"+manifestResource);
checkParameter(objectId != null || sequenceFile != null || manifestResource != null,
"One of --object-id, --input-file or --manifest must be specified. " +
"For CRAM files, please additionally include --reference-file.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ private void resumeIfPossible(DownloadRequest request, boolean checksum)
log.debug("Attempting to resume download for {}", request.toString());
ObjectSpecification spec = null;
try {
terminal
.printLine()
.println("Request payload in Download Service ::::"+request.toString())
.printLine();
spec = downloadStateStore.getProgress(request.getOutputDir(), request.getObjectId());
} catch (NotRetryableException e) {
log.info("New download: {} because {}", request.getObjectId(), e.getMessage());
Expand Down