Skip to content

Commit

Permalink
feat: add better handler for part size (#214)
Browse files Browse the repository at this point in the history
* feat: add better handler for part size


fix: use local file 


fix: try with another path


fix: use bytes 


chore: go back


go back readme


goback


goback


goback

* chore: better handling

* fix: typo readme

* chore: wrong comparaison

* fix: typo
  • Loading branch information
erwanlpfr authored Jun 2, 2023
1 parent bcffe0b commit 5ea9a7c
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 3 deletions.
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,15 @@ You can populate below template according to your requirements and use it as you

# AWS_STORAGE_CLASS="GLACIER"

# Setting this variable will change the S3 default part size for the copy step.
# This value is useful when you want to upload large files.
# NB : While using Scaleway as S3 provider, be aware that the parts counter is set to 1.000.
# While Minio uses a hard coded value to 10.000. As a workaround, try to set a higher value.
# Defaults to "16" (MB) if unset (from minio), you can set this value according to your needs.
# The unit is in MB and an integer.

# AWS_PART_SIZE=16

# You can also backup files to any WebDAV server:

# The URL of the remote WebDAV server
Expand Down
1 change: 1 addition & 0 deletions cmd/backup/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type Config struct {
AwsSecretAccessKey string `split_words:"true"`
AwsSecretAccessKeyFile string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"`
AwsPartSize int64 `split_words:"true"`
BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
BackupFilenameExpand bool `split_words:"true"`
Expand Down
1 change: 1 addition & 0 deletions cmd/backup/script.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ func newScript() (*script, error) {
BucketName: s.c.AwsS3BucketName,
StorageClass: s.c.AwsStorageClass,
CACert: s.c.AwsEndpointCACert.Cert,
PartSize: s.c.AwsPartSize,
}
if s3Backend, err := s3.NewStorageBackend(s3Config, logFunc); err != nil {
return nil, err
Expand Down
26 changes: 23 additions & 3 deletions internal/storage/s3/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"crypto/x509"
"errors"
"fmt"
"os"
"path"
"path/filepath"
"time"
Expand All @@ -22,6 +23,7 @@ type s3Storage struct {
client *minio.Client
bucket string
storageClass string
partSize int64
}

// Config contains values that define the configuration of a S3 backend.
Expand All @@ -35,6 +37,7 @@ type Config struct {
RemotePath string
BucketName string
StorageClass string
PartSize int64
CACert *x509.Certificate
}

Expand Down Expand Up @@ -89,6 +92,7 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error
client: mc,
bucket: opts.BucketName,
storageClass: opts.StorageClass,
partSize: opts.PartSize,
}, nil
}

Expand All @@ -100,16 +104,32 @@ func (v *s3Storage) Name() string {
// Copy copies the given file to the S3/Minio storage backend.
func (b *s3Storage) Copy(file string) error {
_, name := path.Split(file)

if _, err := b.client.FPutObject(context.Background(), b.bucket, filepath.Join(b.DestinationPath, name), file, minio.PutObjectOptions{
putObjectOptions := minio.PutObjectOptions{
ContentType: "application/tar+gzip",
StorageClass: b.storageClass,
}); err != nil {
}

if b.partSize > 0 {
srcFileInfo, err := os.Stat(file)
if err != nil {
return fmt.Errorf("(*s3Storage).Copy: error reading the local file: %w", err)
}

_, partSize, _, err := minio.OptimalPartInfo(srcFileInfo.Size(), uint64(b.partSize*1024*1024))
if err != nil {
return fmt.Errorf("(*s3Storage).Copy: error computing the optimal s3 part size: %w", err)
}

putObjectOptions.PartSize = uint64(partSize)
}

if _, err := b.client.FPutObject(context.Background(), b.bucket, filepath.Join(b.DestinationPath, name), file, putObjectOptions); err != nil {
if errResp := minio.ToErrorResponse(err); errResp.Message != "" {
return fmt.Errorf("(*s3Storage).Copy: error uploading backup to remote storage: [Message]: '%s', [Code]: %s, [StatusCode]: %d", errResp.Message, errResp.Code, errResp.StatusCode)
}
return fmt.Errorf("(*s3Storage).Copy: error uploading backup to remote storage: %w", err)
}

b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to bucket `%s`.", file, b.bucket)

return nil
Expand Down

0 comments on commit 5ea9a7c

Please sign in to comment.