Skip to content

Commit

Permalink
Switch compression to zstdmt
Browse files Browse the repository at this point in the history
Seems to be about 6x faster than gzip, but now bottlenecked on
JuliaIO/Tar.jl#33.
  • Loading branch information
Keno committed May 1, 2020
1 parent 6d358f2 commit e160dbd
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 3 deletions.
1 change: 1 addition & 0 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4"
rr_jll = "e86bdf43-55f7-5ea2-9fd0-e7daa2c0f2b4"

[compat]
Expand Down
2 changes: 1 addition & 1 deletion S3Vendor_go/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func vendor(req events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse,
os.Unsetenv("AWS_SESSION_TOKEN")

currentTime := time.Now()
fname := fmt.Sprintf("reports/%s-%s.tar.gz", currentTime.Format("2006-01-02T15-04-05"), user.GetLogin())
fname := fmt.Sprintf("reports/%s-%s.tar.zstd", currentTime.Format("2006-01-02T15-04-05"), user.GetLogin())

awsSession := session.New()
svc := sts.New(awsSession)
Expand Down
7 changes: 5 additions & 2 deletions src/BugReporting.jl
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
module BugReporting

using rr_jll
using Zstd_jll
using HTTP, JSON
using AWSCore, AWSS3
using Tar
Expand Down Expand Up @@ -28,7 +29,7 @@ end

function rr_pack(trace_directory)
check_rr_available()

trace_directory = normalize_inner_trace(trace_directory)
rr() do rr_path
run(`$rr_path pack $(trace_directory)`)
Expand Down Expand Up @@ -153,7 +154,9 @@ function upload_rr_trace(trace_directory)
aws = AWSCore.aws_config(creds = creds, region="us-east-1")

# Tar it up
proc = open(`gzip -`, "r+")
proc = zstdmt() do zstdp
open(`$zstdp -`, "r+")
end

t = @async begin try
upload = s3_begin_multipart_upload(aws, TRACE_BUCKET, s3creds["UPLOAD_PATH"])
Expand Down

0 comments on commit e160dbd

Please sign in to comment.