From e81f2a00060608f09a774c9219a4aaae5c4627ae Mon Sep 17 00:00:00 2001
From: Mikubill <31246794+Mikubill@users.noreply.github.com>
Date: Wed, 21 Sep 2022 15:09:50 +0000
Subject: [PATCH] serve s3 as a proxy server
---
backend/webdav/webdav.go | 6 +-
cmd/serve/s3/backend.go | 143 ++++++----
cmd/serve/s3/list.go | 9 +-
cmd/serve/s3/logger.go | 2 +-
cmd/serve/s3/pager.go | 2 +-
cmd/serve/s3/s3_test.go | 356 ++++++++++++++++++++++++
cmd/serve/s3/server.go | 4 +-
cmd/serve/s3/testdata/webdav-tests.conf | 6 +
cmd/serve/s3/utils.go | 2 +-
go.mod | 11 +-
go.sum | 22 +-
11 files changed, 469 insertions(+), 94 deletions(-)
create mode 100644 cmd/serve/s3/testdata/webdav-tests.conf
diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go
index e8f83f66a4a67..31caefdc4dc29 100644
--- a/backend/webdav/webdav.go
+++ b/backend/webdav/webdav.go
@@ -470,7 +470,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if opt.User != "" || opt.Pass != "" {
f.srv.SetUserPass(opt.User, opt.Pass)
} else if opt.BearerToken != "" {
- f.setBearerToken(opt.BearerToken)
+ f.SetBearerToken(opt.BearerToken)
} else if f.opt.BearerTokenCommand != "" {
err = f.fetchAndSetBearerToken()
if err != nil {
@@ -512,7 +512,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// sets the BearerToken up
-func (f *Fs) setBearerToken(token string) {
+func (f *Fs) SetBearerToken(token string) {
f.opt.BearerToken = token
f.srv.SetHeader("Authorization", "Bearer "+token)
}
@@ -570,7 +570,7 @@ func (f *Fs) fetchAndSetBearerToken() error {
if err != nil {
return err
}
- f.setBearerToken(token)
+ f.SetBearerToken(token)
return nil
}
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index 4e88e418f80e0..e69926ad272c6 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -10,10 +10,12 @@ import (
"strings"
"sync"
- "github.com/Mikubill/gofakes3"
+ "github.com/JankariTech/gofakes3"
"github.com/ncw/swift/v2"
+ "github.com/rclone/rclone/backend/webdav"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/vfs"
+ "github.com/rclone/rclone/vfs/vfsflags"
)
var (
@@ -25,22 +27,36 @@ var (
// backend for gofakes3
type s3Backend struct {
opt *Options
- vfs *vfs.VFS
+ s *Server
meta *sync.Map
}
// newBackend creates a new SimpleBucketBackend.
-func newBackend(vfs *vfs.VFS, opt *Options) gofakes3.Backend {
+func newBackend(opt *Options, s *Server) gofakes3.Backend {
return &s3Backend{
- vfs: vfs,
opt: opt,
+ s: s,
meta: new(sync.Map),
}
}
+func (b *s3Backend) setAuthForWebDAV(accessKey string) *vfs.VFS {
+ // new VFS
+ if _, ok := b.s.f.(*webdav.Fs); ok {
+ info, name, remote, config, _ := fs.ConfigFs(b.s.f.Name() + ":")
+ f, _ := info.NewFs(context.Background(), name+accessKey, remote, config)
+ vf := vfs.New(f, &vfsflags.Opt)
+ vf.Fs().(*webdav.Fs).SetBearerToken(accessKey)
+ return vf
+ }
+ return vfs.New(b.s.f, &vfsflags.Opt)
+}
+
// ListBuckets always returns the default bucket.
-func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) {
- dirEntries, err := getDirEntries("/", b.vfs)
+func (b *s3Backend) ListBuckets(accessKey string) ([]gofakes3.BucketInfo, error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ dirEntries, err := getDirEntries("/", vf)
if err != nil {
return nil, err
}
@@ -59,8 +75,10 @@ func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) {
}
// ListBucket lists the objects in the given bucket.
-func (b *s3Backend) ListBucket(bucket string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
- _, err := b.vfs.Stat(bucket)
+func (b *s3Backend) ListBucket(accessKey string, bucket string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ _, err := vf.Stat(bucket)
if err != nil {
return nil, gofakes3.BucketNotFound(bucket)
}
@@ -79,7 +97,7 @@ func (b *s3Backend) ListBucket(bucket string, prefix *gofakes3.Prefix, page gofa
response := gofakes3.NewObjectList()
path, remaining := prefixParser(prefix)
- err = b.entryListR(bucket, path, remaining, prefix.HasDelimiter, response)
+ err = b.entryListR(vf, bucket, path, remaining, prefix.HasDelimiter, response)
if err == gofakes3.ErrNoSuchKey {
// AWS just returns an empty list
response = gofakes3.NewObjectList()
@@ -93,14 +111,16 @@ func (b *s3Backend) ListBucket(bucket string, prefix *gofakes3.Prefix, page gofa
// HeadObject returns the fileinfo for the given object name.
//
// Note that the metadata is not supported yet.
-func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object, error) {
- _, err := b.vfs.Stat(bucketName)
+func (b *s3Backend) HeadObject(accessKey string, bucketName, objectName string) (*gofakes3.Object, error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ _, err := vf.Stat(bucketName)
if err != nil {
return nil, gofakes3.BucketNotFound(bucketName)
}
fp := path.Join(bucketName, objectName)
- node, err := b.vfs.Stat(fp)
+ node, err := vf.Stat(fp)
if err != nil {
return nil, gofakes3.KeyNotFound(objectName)
}
@@ -140,14 +160,16 @@ func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object,
}
// GetObject fetchs the object from the filesystem.
-func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
- _, err = b.vfs.Stat(bucketName)
+func (b *s3Backend) GetObject(accessKey string, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ _, err = vf.Stat(bucketName)
if err != nil {
return nil, gofakes3.BucketNotFound(bucketName)
}
fp := path.Join(bucketName, objectName)
- node, err := b.vfs.Stat(fp)
+ node, err := vf.Stat(fp)
if err != nil {
return nil, gofakes3.KeyNotFound(objectName)
}
@@ -214,20 +236,22 @@ func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofak
}
// TouchObject creates or updates meta on specified object.
-func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
- _, err = b.vfs.Stat(fp)
+func (b *s3Backend) TouchObject(accessKey string, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ _, err = vf.Stat(fp)
if err == vfs.ENOENT {
- f, err := b.vfs.Create(fp)
+ f, err := vf.Create(fp)
if err != nil {
return result, err
}
_ = f.Close()
- return b.TouchObject(fp, meta)
+ return b.TouchObject(accessKey, fp, meta)
} else if err != nil {
return result, err
}
- _, err = b.vfs.Stat(fp)
+ _, err = vf.Stat(fp)
if err != nil {
return result, err
}
@@ -237,7 +261,7 @@ func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofak
if val, ok := meta["X-Amz-Meta-Mtime"]; ok {
ti, err := swift.FloatStringToTime(val)
if err == nil {
- return result, b.vfs.Chtimes(fp, ti, ti)
+ return result, vf.Chtimes(fp, ti, ti)
}
// ignore error since the file is successfully created
}
@@ -245,7 +269,7 @@ func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofak
if val, ok := meta["mtime"]; ok {
ti, err := swift.FloatStringToTime(val)
if err == nil {
- return result, b.vfs.Chtimes(fp, ti, ti)
+ return result, vf.Chtimes(fp, ti, ti)
}
// ignore error since the file is successfully created
}
@@ -255,30 +279,33 @@ func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofak
// PutObject creates or overwrites the object with the given name.
func (b *s3Backend) PutObject(
+ accessKey string,
bucketName, objectName string,
meta map[string]string,
input io.Reader, size int64,
) (result gofakes3.PutObjectResult, err error) {
- _, err = b.vfs.Stat(bucketName)
+ vf := b.setAuthForWebDAV(accessKey)
+
+ _, err = vf.Stat(bucketName)
if err != nil {
return result, gofakes3.BucketNotFound(bucketName)
}
fp := path.Join(bucketName, objectName)
objectDir := path.Dir(fp)
- // _, err = db.fs.Stat(objectDir)
+ // _, err = b.fs.Stat(objectDir)
// if err == vfs.ENOENT {
// fs.Errorf(objectDir, "PutObject failed: path not found")
// return result, gofakes3.KeyNotFound(objectName)
// }
if objectDir != "." {
- if err := mkdirRecursive(objectDir, b.vfs); err != nil {
+ if err := mkdirRecursive(objectDir, vf); err != nil {
return result, err
}
}
- f, err := b.vfs.Create(fp)
+ f, err := vf.Create(fp)
if err != nil {
return result, err
}
@@ -286,17 +313,17 @@ func (b *s3Backend) PutObject(
if _, err := io.Copy(f, input); err != nil {
// remove file when i/o error occurred (FsPutErr)
_ = f.Close()
- _ = b.vfs.Remove(fp)
+ _ = vf.Remove(fp)
return result, err
}
if err := f.Close(); err != nil {
// remove file when close error occurred (FsPutErr)
- _ = b.vfs.Remove(fp)
+ _ = vf.Remove(fp)
return result, err
}
- _, err = b.vfs.Stat(fp)
+ _, err = vf.Stat(fp)
if err != nil {
return result, err
}
@@ -306,7 +333,7 @@ func (b *s3Backend) PutObject(
if val, ok := meta["X-Amz-Meta-Mtime"]; ok {
ti, err := swift.FloatStringToTime(val)
if err == nil {
- return result, b.vfs.Chtimes(fp, ti, ti)
+ return result, vf.Chtimes(fp, ti, ti)
}
// ignore error since the file is successfully created
}
@@ -314,7 +341,7 @@ func (b *s3Backend) PutObject(
if val, ok := meta["mtime"]; ok {
ti, err := swift.FloatStringToTime(val)
if err == nil {
- return result, b.vfs.Chtimes(fp, ti, ti)
+ return result, vf.Chtimes(fp, ti, ti)
}
// ignore error since the file is successfully created
}
@@ -323,9 +350,11 @@ func (b *s3Backend) PutObject(
}
// DeleteMulti deletes multiple objects in a single request.
-func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
+func (b *s3Backend) DeleteMulti(accessKey string, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
for _, object := range objects {
- if err := b.deleteObject(bucketName, object); err != nil {
+ if err := b.deleteObject(vf, bucketName, object); err != nil {
fs.Errorf("serve s3", "delete object failed: %v", err)
result.Error = append(result.Error, gofakes3.ErrorResult{
Code: gofakes3.ErrInternal,
@@ -343,13 +372,15 @@ func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result go
}
// DeleteObject deletes the object with the given name.
-func (b *s3Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) {
- return result, b.deleteObject(bucketName, objectName)
+func (b *s3Backend) DeleteObject(accessKey string, bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ return result, b.deleteObject(vf, bucketName, objectName)
}
// deleteObject deletes the object from the filesystem.
-func (b *s3Backend) deleteObject(bucketName, objectName string) error {
- _, err := b.vfs.Stat(bucketName)
+func (b *s3Backend) deleteObject(vf *vfs.VFS, bucketName, objectName string) error {
+ _, err := vf.Stat(bucketName)
if err != nil {
return gofakes3.BucketNotFound(bucketName)
}
@@ -357,18 +388,20 @@ func (b *s3Backend) deleteObject(bucketName, objectName string) error {
fp := path.Join(bucketName, objectName)
// S3 does not report an error when attemping to delete a key that does not exist, so
// we need to skip IsNotExist errors.
- if err := b.vfs.Remove(fp); err != nil && !os.IsNotExist(err) {
+ if err := vf.Remove(fp); err != nil && !os.IsNotExist(err) {
return err
}
// FIXME: unsafe operation
- rmdirRecursive(fp, b.vfs)
+ rmdirRecursive(fp, vf)
return nil
}
// CreateBucket creates a new bucket.
-func (b *s3Backend) CreateBucket(name string) error {
- _, err := b.vfs.Stat(name)
+func (b *s3Backend) CreateBucket(accessKey string, name string) error {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ _, err := vf.Stat(name)
if err != nil && err != vfs.ENOENT {
return gofakes3.ErrInternal
}
@@ -377,20 +410,22 @@ func (b *s3Backend) CreateBucket(name string) error {
return gofakes3.ErrBucketAlreadyExists
}
- if err := b.vfs.Mkdir(name, 0755); err != nil {
+ if err := vf.Mkdir(name, 0755); err != nil {
return gofakes3.ErrInternal
}
return nil
}
// DeleteBucket deletes the bucket with the given name.
-func (b *s3Backend) DeleteBucket(name string) error {
- _, err := b.vfs.Stat(name)
+func (b *s3Backend) DeleteBucket(accessKey string, name string) error {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ _, err := vf.Stat(name)
if err != nil {
return gofakes3.BucketNotFound(name)
}
- if err := b.vfs.Remove(name); err != nil {
+ if err := vf.Remove(name); err != nil {
return gofakes3.ErrBucketNotEmpty
}
@@ -398,8 +433,10 @@ func (b *s3Backend) DeleteBucket(name string) error {
}
// BucketExists checks if the bucket exists.
-func (b *s3Backend) BucketExists(name string) (exists bool, err error) {
- _, err = b.vfs.Stat(name)
+func (b *s3Backend) BucketExists(accessKey string, name string) (exists bool, err error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
+ _, err = vf.Stat(name)
if err != nil {
return false, nil
}
@@ -408,7 +445,9 @@ func (b *s3Backend) BucketExists(name string) (exists bool, err error) {
}
// CopyObject copy specified object from srcKey to dstKey.
-func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) {
+func (b *s3Backend) CopyObject(accessKey string, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) {
+ vf := b.setAuthForWebDAV(accessKey)
+
fp := path.Join(srcBucket, srcKey)
if srcBucket == dstBucket && srcKey == dstKey {
b.meta.Store(fp, meta)
@@ -425,15 +464,15 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta
return result, nil
}
- return result, b.vfs.Chtimes(fp, ti, ti)
+ return result, vf.Chtimes(fp, ti, ti)
}
- cStat, err := b.vfs.Stat(fp)
+ cStat, err := vf.Stat(fp)
if err != nil {
return
}
- c, err := b.GetObject(srcBucket, srcKey, nil)
+ c, err := b.GetObject(accessKey, srcBucket, srcKey, nil)
if err != nil {
return
}
@@ -450,7 +489,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta
meta["mtime"] = swift.TimeToFloatString(cStat.ModTime())
}
- _, err = b.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size)
+ _, err = b.PutObject(accessKey, dstBucket, dstKey, meta, c.Contents, c.Size)
if err != nil {
return
}
diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go
index d2373615242cb..f5639d0721052 100644
--- a/cmd/serve/s3/list.go
+++ b/cmd/serve/s3/list.go
@@ -4,13 +4,14 @@ import (
"path"
"strings"
- "github.com/Mikubill/gofakes3"
+ "github.com/JankariTech/gofakes3"
+ "github.com/rclone/rclone/vfs"
)
-func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {
+func (b *s3Backend) entryListR(vf *vfs.VFS, bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {
fp := path.Join(bucket, fdPath)
- dirEntries, err := getDirEntries(fp, b.vfs)
+ dirEntries, err := getDirEntries(fp, vf)
if err != nil {
return err
}
@@ -30,7 +31,7 @@ func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, resp
response.AddPrefix(gofakes3.URLEncode(objectPath))
continue
}
- err := b.entryListR(bucket, path.Join(fdPath, object), "", false, response)
+ err := b.entryListR(vf, bucket, path.Join(fdPath, object), "", false, response)
if err != nil {
return err
}
diff --git a/cmd/serve/s3/logger.go b/cmd/serve/s3/logger.go
index 8c9b3067bff17..62cbebd9c839c 100644
--- a/cmd/serve/s3/logger.go
+++ b/cmd/serve/s3/logger.go
@@ -3,7 +3,7 @@ package s3
import (
"fmt"
- "github.com/Mikubill/gofakes3"
+ "github.com/JankariTech/gofakes3"
"github.com/rclone/rclone/fs"
)
diff --git a/cmd/serve/s3/pager.go b/cmd/serve/s3/pager.go
index 1aa3c5e68b8c4..b2bcde75e2333 100644
--- a/cmd/serve/s3/pager.go
+++ b/cmd/serve/s3/pager.go
@@ -4,7 +4,7 @@ package s3
import (
"sort"
- "github.com/Mikubill/gofakes3"
+ "github.com/JankariTech/gofakes3"
)
// pager splits the object list into smulitply pages.
diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.go
index 2622d5fbcd0f0..f650a9ede5b87 100644
--- a/cmd/serve/s3/s3_test.go
+++ b/cmd/serve/s3/s3_test.go
@@ -6,18 +6,32 @@ package s3
import (
"bytes"
"context"
+<<<<<<< HEAD
"fmt"
"io"
+=======
+ "encoding/hex"
+ "fmt"
+ "io"
+ "math/rand"
+ "net/http"
+ "net/http/httptest"
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
"net/url"
"os"
"os/exec"
"path"
"strings"
+<<<<<<< HEAD
+=======
+ "sync"
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
"testing"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
+<<<<<<< HEAD
"github.com/rclone/rclone/fs/object"
_ "github.com/rclone/rclone/backend/local"
@@ -28,11 +42,24 @@ import (
"github.com/rclone/rclone/fstest"
httplib "github.com/rclone/rclone/lib/http"
"github.com/rclone/rclone/lib/random"
+=======
+ _ "github.com/rclone/rclone/backend/local"
+ "github.com/rclone/rclone/cmd/serve/servetest"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/config/configfile"
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/fstest"
+ httplib "github.com/rclone/rclone/lib/http"
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
+<<<<<<< HEAD
endpoint = "localhost:0"
)
@@ -40,18 +67,159 @@ const (
func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
keyid = random.String(16)
keysec = random.String(16)
+=======
+ endpoint = "localhost:0"
+ propfindResponseRoot = `
+
+
+ /
+
+
+ admin
+ Mon, 26 Jun 2023 04:17:38 GMT
+
+
+
+
+ HTTP/1.1 200 OK
+
+
+
+
+
+
+
+ HTTP/1.1 404 Not Found
+
+
+
+ /bucket/
+
+
+ bucket
+ Fri, 16 Jun 2023 11:11:32 GMT
+
+
+
+
+ HTTP/1.1 200 OK
+
+
+
+
+
+
+
+ HTTP/1.1 404 Not Found
+
+
+
+ /bucket2/
+
+
+ bucket2
+ Tue, 20 Jun 2023 04:00:56 GMT
+
+
+
+
+ HTTP/1.1 200 OK
+
+
+
+
+
+
+
+ HTTP/1.1 404 Not Found
+
+
+
+ /newbucket/
+
+
+ newbucket
+ Mon, 19 Jun 2023 07:38:24 GMT
+
+
+
+
+ HTTP/1.1 200 OK
+
+
+
+
+
+
+
+ HTTP/1.1 404 Not Found
+
+
+`
+ propfindEmptyBucketResponse = `
+
+
+
+ /bucket/
+
+
+ Wed, 28 Jun 2023 07:58:52 GMT
+
+
+
+ 0
+ -3
+ "649be83ccc047"
+
+ HTTP/1.1 200 OK
+
+
+`
+ propfindFileResponse = `
+
+
+ /bucket/%s
+
+
+ %d
+
+ "a78a2b6eebbe67dda2c898c23c164eb6"
+
+ HTTP/1.1 200 OK
+
+
+
+`
+)
+
+// Configure and serve the server
+func serveS3(f fs.Fs, keyid string, keysec string) (testURL string) {
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
serveropt := &Options{
HTTP: httplib.DefaultCfg(),
pathBucketMode: true,
hashName: "",
hashType: hash.None,
+<<<<<<< HEAD
authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)},
+=======
+ }
+ if keyid != "" && keysec != "" {
+ serveropt.authPair = []string{fmt.Sprintf("%s,%s", keyid, keysec)}
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
}
serveropt.HTTP.ListenAddr = []string{endpoint}
w, _ := newServer(context.Background(), f, serveropt)
router := w.Router()
+<<<<<<< HEAD
+=======
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
w.Bind(router)
w.Serve()
testURL = w.Server.URLs()[0]
@@ -59,21 +227,51 @@ func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
return
}
+<<<<<<< HEAD
+=======
+func RandString(n int) string {
+ src := rand.New(rand.NewSource(time.Now().UnixNano()))
+ b := make([]byte, (n+1)/2)
+
+ if _, err := src.Read(b); err != nil {
+ panic(err)
+ }
+
+ return hex.EncodeToString(b)[:n]
+}
+
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
// TestS3 runs the s3 server then runs the unit tests for the
// s3 remote against it.
func TestS3(t *testing.T) {
start := func(f fs.Fs) (configmap.Simple, func()) {
+<<<<<<< HEAD
testURL, keyid, keysec := serveS3(f)
// Config for the backend we'll use to connect to the server
config := configmap.Simple{
"type": "s3",
"provider": "Rclone",
"endpoint": testURL,
+=======
+ keyid := RandString(16)
+ keysec := RandString(16)
+ testURL := serveS3(f, keyid, keysec)
+ // Config for the backend we'll use to connect to the server
+ s3ServerConfig := configmap.Simple{
+ "type": "s3",
+ "provider": "Rclone",
+ "endpoint": testURL,
+ "list_url_encode": "true",
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
"access_key_id": keyid,
"secret_access_key": keysec,
}
+<<<<<<< HEAD
return config, func() {}
+=======
+ return s3ServerConfig, func() {}
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
}
RunS3UnitTests(t, "s3", start)
@@ -92,7 +290,11 @@ func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
assert.NoError(t, err)
f := fremote
+<<<<<<< HEAD
config, cleanup := start(f)
+=======
+ s3ServerConfig, cleanup := start(f)
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
defer cleanup()
// Change directory to run the tests
@@ -113,7 +315,11 @@ func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
if *fstest.Verbose {
args = append(args, "-verbose")
}
+<<<<<<< HEAD
remoteName := "serve" + name + ":"
+=======
+ remoteName := name + "test:"
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
args = append(args, "-remote", remoteName)
args = append(args, "-run", "^TestIntegration$")
args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries))
@@ -122,7 +328,11 @@ func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
// Configure the backend with environment variables
cmd.Env = os.Environ()
prefix := "RCLONE_CONFIG_" + strings.ToUpper(remoteName[:len(remoteName)-1]) + "_"
+<<<<<<< HEAD
for k, v := range config {
+=======
+ for k, v := range s3ServerConfig {
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v)
}
@@ -134,6 +344,138 @@ func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
assert.NoError(t, err, "Running "+name+" integration tests")
}
+<<<<<<< HEAD
+=======
+// prepare the test server and return a function to tidy it up afterwards
+func prepareWebDavFs(t *testing.T, handler http.HandlerFunc) (fs.Fs, func()) {
+ // Make the test server
+ ts := httptest.NewServer(handler)
+
+ _ = config.SetConfigPath("./testdata/webdav-tests.conf")
+
+ // Configure the remote
+ configfile.Install()
+ err := config.SetValueAndSave("webdavtest", "url", ts.URL)
+ assert.NoError(t, err)
+
+ // Instantiate the WebDAV server
+ info, name, remote, webdavServerConfig, _ := fs.ConfigFs("webdavtest:")
+ f, err := info.NewFs(context.Background(), name, remote, webdavServerConfig)
+
+ require.NoError(t, err)
+
+ return f, ts.Close
+}
+
+func TestForwardAccessKeyToWebDav(t *testing.T) {
+ keyid := RandString(16)
+ keysec := RandString(16)
+ expectedAuthHeader := "Bearer " + keyid
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ assert.Equal(t, expectedAuthHeader, r.Header.Get("Authorization"))
+ _, err := fmt.Fprint(w, propfindResponseRoot)
+ assert.NoError(t, err)
+ })
+ f, clean := prepareWebDavFs(t, handler)
+ defer clean()
+ endpoint := serveS3(f, keyid, keysec)
+ testURL, _ := url.Parse(endpoint)
+ minioClient, err := minio.New(testURL.Host, &minio.Options{
+ Creds: credentials.NewStaticV4(keyid, keysec, ""),
+ Secure: false,
+ })
+ assert.NoError(t, err)
+ buckets, err := minioClient.ListBuckets(context.Background())
+ assert.NoError(t, err)
+ assert.Equal(t, buckets[0].Name, "bucket")
+ assert.Equal(t, buckets[1].Name, "bucket2")
+}
+
+// when receiving multiple requests in parallel with different tokens we have to make sure the correct
+// Auth header is set for every request and there is no race-condition where parallel requests overwrite each
+// others headers
+// to test that case we send multiple PutObject requests with an object name that matches the S3 key
+// on the webdav side, we check if the object name is the same as the Auth Bearer token
+func TestForwardAccessKeyToWebDavParallelRequests(t *testing.T) {
+ webdavHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // the proxy will send different requests to check the path etc
+ if r.Method == "PROPFIND" {
+ if r.URL.Path == "/" {
+ _, err := fmt.Fprint(w, propfindResponseRoot)
+ assert.NoError(t, err)
+ } else if r.URL.Path == "/bucket/" {
+ _, err := fmt.Fprint(w, propfindEmptyBucketResponse)
+ assert.NoError(t, err)
+ } else {
+ // this is the main check, we want to make sure that the path belongs
+ // to the correct user
+ expectedKeyIdExtractedFromPath := path.Base(r.URL.Path)
+ assert.Equal(t, "Bearer "+expectedKeyIdExtractedFromPath, r.Header.Get("Authorization"))
+ _, err := fmt.Fprintf(w, propfindFileResponse, expectedKeyIdExtractedFromPath, 8)
+ assert.NoError(t, err)
+ }
+ } else if r.Method == "PUT" {
+ // this is the main check, we want to make sure that the path belongs
+ // to the correct user
+ expectedKeyIdExtractedFromPath := path.Base(r.URL.Path)
+ assert.Equal(t, "Bearer "+expectedKeyIdExtractedFromPath, r.Header.Get("Authorization"))
+ }
+ })
+ f, clean := prepareWebDavFs(t, webdavHandler)
+ defer clean()
+
+ keyids := []string{
+ "KeyOfUserAlice",
+ "KeyOfUserBob",
+ "KeyOfUserCarol",
+ "KeyOfUserDavid",
+ }
+ var wg sync.WaitGroup
+ wg.Add(len(keyids))
+
+ endpoint := serveS3(f, "", "")
+ testURL, _ := url.Parse(endpoint)
+
+ responseChannel := make(chan error)
+
+ for _, keyid := range keyids {
+ keyid := keyid
+ go func(responseChannel chan<- error) {
+ defer wg.Done()
+ minioClient, err := minio.New(testURL.Host, &minio.Options{
+ Creds: credentials.NewStaticV4(keyid, "does-not-matter-will-be-ignored-by-server", ""),
+ Secure: false,
+ })
+ if err != nil {
+ responseChannel <- err
+ return
+ }
+ buf := bytes.NewBufferString("contents")
+ uploadHash := hash.NewMultiHasher()
+ in := io.TeeReader(buf, uploadHash)
+ _, err = minioClient.PutObject(
+ context.Background(), "bucket", keyid, in, int64(buf.Len()),
+ minio.PutObjectOptions{},
+ )
+ if err != nil {
+ responseChannel <- err
+ return
+ }
+ }(responseChannel)
+ }
+
+ go func() {
+ wg.Wait()
+ close(responseChannel)
+ }()
+
+ for i := 0; i < len(keyids); i++ {
+ response := <-responseChannel
+ assert.NoError(t, response)
+ }
+}
+
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
// tests using the minio client
func TestEncodingWithMinioClient(t *testing.T) {
cases := []struct {
@@ -180,8 +522,14 @@ func TestEncodingWithMinioClient(t *testing.T) {
)
_, err = f.Put(context.Background(), in, obji)
assert.NoError(t, err)
+<<<<<<< HEAD
endpoint, keyid, keysec := serveS3(f)
+=======
+ keyid := RandString(16)
+ keysec := RandString(16)
+ endpoint := serveS3(f, keyid, keysec)
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
testURL, _ := url.Parse(endpoint)
minioClient, err := minio.New(testURL.Host, &minio.Options{
Creds: credentials.NewStaticV4(keyid, keysec, ""),
@@ -192,11 +540,19 @@ func TestEncodingWithMinioClient(t *testing.T) {
buckets, err := minioClient.ListBuckets(context.Background())
assert.NoError(t, err)
assert.Equal(t, buckets[0].Name, tt.bucket)
+<<<<<<< HEAD
objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
Recursive: true,
})
for object := range objects {
assert.Equal(t, path.Join(tt.path, tt.filename), object.Key)
+=======
+ s3objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
+ Recursive: true,
+ })
+ for s3object := range s3objects {
+ assert.Equal(t, path.Join(tt.path, tt.filename), s3object.Key)
+>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
}
})
}
diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go
index b9e9a75504daa..192c4d031d803 100644
--- a/cmd/serve/s3/server.go
+++ b/cmd/serve/s3/server.go
@@ -7,7 +7,7 @@ import (
"math/rand"
"net/http"
- "github.com/Mikubill/gofakes3"
+ "github.com/JankariTech/gofakes3"
"github.com/go-chi/chi/v5"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
@@ -51,7 +51,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
var newLogger logger
w.faker = gofakes3.New(
- newBackend(w.vfs, opt),
+ newBackend(opt, w),
gofakes3.WithHostBucket(!opt.pathBucketMode),
gofakes3.WithLogger(newLogger),
gofakes3.WithRequestID(rand.Uint64()),
diff --git a/cmd/serve/s3/testdata/webdav-tests.conf b/cmd/serve/s3/testdata/webdav-tests.conf
new file mode 100644
index 0000000000000..1758a12de00c2
--- /dev/null
+++ b/cmd/serve/s3/testdata/webdav-tests.conf
@@ -0,0 +1,6 @@
+[webdavtest]
+type = webdav
+url = http://127.0.0.1:40777
+vendor = nextcloud
+nextcloud_chunk_size = 0
+
diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.go
index c61283cc6bf4f..bee514d0cb7d5 100644
--- a/cmd/serve/s3/utils.go
+++ b/cmd/serve/s3/utils.go
@@ -9,7 +9,7 @@ import (
"path"
"strings"
- "github.com/Mikubill/gofakes3"
+ "github.com/JankariTech/gofakes3"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/vfs"
diff --git a/go.mod b/go.mod
index 14e3813df7d43..d61f4ca99b83f 100644
--- a/go.mod
+++ b/go.mod
@@ -9,8 +9,8 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.0
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
+ github.com/JankariTech/gofakes3 v0.0.0-20230622065030-1b69ffde3106
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd
- github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700
github.com/Unknwon/goconfig v1.0.0
github.com/a8m/tree v0.0.0-20230208161321-36ae24ddad15
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3
@@ -43,7 +43,6 @@ require (
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
github.com/mattn/go-colorable v0.1.13
github.com/mattn/go-runewidth v0.0.15
- github.com/minio/minio-go/v7 v7.0.63
github.com/mitchellh/go-homedir v1.1.0
github.com/moby/sys/mountinfo v0.6.2
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1
@@ -105,7 +104,6 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emersion/go-message v0.17.0 // indirect
github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
@@ -132,7 +130,6 @@ require (
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
- github.com/json-iterator/go v1.1.12 // indirect
github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
github.com/jtolio/noiseconn v0.0.0-20230621152802-afeab29449e0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
@@ -142,10 +139,6 @@ require (
github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
- github.com/minio/md5-simd v1.1.2 // indirect
- github.com/minio/sha256-simd v1.0.1 // indirect
- github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
- github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/errors v0.9.1 // indirect
@@ -155,7 +148,6 @@ require (
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
github.com/relvacode/iso8601 v1.3.0 // indirect
- github.com/rs/xid v1.5.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
@@ -177,7 +169,6 @@ require (
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
- gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
storj.io/common v0.0.0-20231027080355-b4cb1b0d728e // indirect
storj.io/drpc v0.0.33 // indirect
diff --git a/go.sum b/go.sum
index 2611131027412..51e1a963c9325 100644
--- a/go.sum
+++ b/go.sum
@@ -54,14 +54,14 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/JankariTech/gofakes3 v0.0.0-20230622065030-1b69ffde3106 h1:eUn1uYewfc26nWQP0b4BkqegoEtf4KJjt9O+nIs0Zyk=
+github.com/JankariTech/gofakes3 v0.0.0-20230622065030-1b69ffde3106/go.mod h1:TUu8ssPGUhHRypzW2cbsTJjVf84m0/WYPgK4ua1vBoo=
github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE=
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
-github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q=
-github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY=
github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I=
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug=
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo=
@@ -160,8 +160,6 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw
github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/emersion/go-message v0.17.0 h1:NIdSKHiVUx4qKqdd0HyJFD41cW8iFguM2XJnRZWQH04=
github.com/emersion/go-message v0.17.0/go.mod h1:/9Bazlb1jwUNB0npYYBsdJ2EMOiiyN3m5UVHbY7GoNw=
github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY=
@@ -265,7 +263,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -339,7 +336,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
github.com/josephspurrier/goversioninfo v1.4.0 h1:Puhl12NSHUSALHSuzYwPYQkqa2E1+7SrtAPJorKK0C8=
github.com/josephspurrier/goversioninfo v1.4.0/go.mod h1:JWzv5rKQr+MmW+LvM412ToT/IkYDZjaclF2pKDss8IY=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -355,7 +351,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@@ -393,23 +388,14 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
-github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
-github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
-github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ=
-github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
-github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 h1:nAjWYc03awJAjsozNehdGZsm5LP7AhLOvjgbS8zN1tk=
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0=
@@ -469,8 +455,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
-github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
@@ -930,8 +914,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
-gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY=
gopkg.in/validator.v2 v2.0.1/go.mod h1:lIUZBlB3Im4s/eYp39Ry/wkR02yOPhZ9IwIRBjuPuG8=