Skip to content

Commit

Permalink
fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@saw-jan
saw-jan committed Jan 10, 2024
1 parent e81f2a0 commit f539d1e
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 171 deletions.
224 changes: 71 additions & 153 deletions cmd/serve/s3/s3_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,43 +6,23 @@ package s3
import (
"bytes"
"context"
<<<<<<< HEAD
"fmt"
"io"
=======
"encoding/hex"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
"net/url"
"os"
"os/exec"
"path"
"strings"
<<<<<<< HEAD
=======
"sync"
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
"testing"
"time"

"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
<<<<<<< HEAD
"github.com/rclone/rclone/fs/object"

_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/cmd/serve/servetest"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fstest"
httplib "github.com/rclone/rclone/lib/http"
"github.com/rclone/rclone/lib/random"
=======
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/cmd/serve/servetest"
"github.com/rclone/rclone/fs"
Expand All @@ -53,21 +33,11 @@ import (
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
httplib "github.com/rclone/rclone/lib/http"
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

const (
<<<<<<< HEAD
endpoint = "localhost:0"
)

// Configure and serve the server
func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
keyid = random.String(16)
keysec = random.String(16)
=======
endpoint = "localhost:0"
propfindResponseRoot = `
<d:multistatus
Expand Down Expand Up @@ -198,37 +168,27 @@ func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {

// Configure and serve the server
func serveS3(f fs.Fs, keyid string, keysec string) (testURL string) {
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
serveropt := &Options{
HTTP: httplib.DefaultCfg(),
pathBucketMode: true,
hashName: "",
hashType: hash.None,
<<<<<<< HEAD
authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)},
=======
}
if keyid != "" && keysec != "" {
serveropt.authPair = []string{fmt.Sprintf("%s,%s", keyid, keysec)}
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
}

serveropt.HTTP.ListenAddr = []string{endpoint}
w, _ := newServer(context.Background(), f, serveropt)
router := w.Router()
<<<<<<< HEAD

=======
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
w.Bind(router)
w.Serve()
testURL = w.Server.URLs()[0]

return
}

<<<<<<< HEAD
=======
func RandString(n int) string {
src := rand.New(rand.NewSource(time.Now().UnixNano()))
b := make([]byte, (n+1)/2)
Expand All @@ -240,38 +200,24 @@ func RandString(n int) string {
return hex.EncodeToString(b)[:n]
}

>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
// TestS3 runs the s3 server then runs the unit tests for the
// s3 remote against it.
func TestS3(t *testing.T) {
start := func(f fs.Fs) (configmap.Simple, func()) {
<<<<<<< HEAD
testURL, keyid, keysec := serveS3(f)
// Config for the backend we'll use to connect to the server
config := configmap.Simple{
"type": "s3",
"provider": "Rclone",
"endpoint": testURL,
=======
keyid := RandString(16)
keysec := RandString(16)
testURL := serveS3(f, keyid, keysec)
// Config for the backend we'll use to connect to the server
s3ServerConfig := configmap.Simple{
config := configmap.Simple{
"type": "s3",
"provider": "Rclone",
"endpoint": testURL,
"list_url_encode": "true",
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
"access_key_id": keyid,
"secret_access_key": keysec,
}

<<<<<<< HEAD
return config, func() {}
=======
return s3ServerConfig, func() {}
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
}

RunS3UnitTests(t, "s3", start)
Expand All @@ -290,11 +236,7 @@ func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
assert.NoError(t, err)

f := fremote
<<<<<<< HEAD
config, cleanup := start(f)
=======
s3ServerConfig, cleanup := start(f)
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
defer cleanup()

// Change directory to run the tests
Expand All @@ -315,11 +257,7 @@ func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
if *fstest.Verbose {
args = append(args, "-verbose")
}
<<<<<<< HEAD
remoteName := "serve" + name + ":"
=======
remoteName := name + "test:"
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
args = append(args, "-remote", remoteName)
args = append(args, "-run", "^TestIntegration$")
args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries))
Expand All @@ -328,11 +266,7 @@ func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
// Configure the backend with environment variables
cmd.Env = os.Environ()
prefix := "RCLONE_CONFIG_" + strings.ToUpper(remoteName[:len(remoteName)-1]) + "_"
<<<<<<< HEAD
for k, v := range config {
=======
for k, v := range s3ServerConfig {
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v)
}

Expand All @@ -344,8 +278,76 @@ func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
assert.NoError(t, err, "Running "+name+" integration tests")
}

<<<<<<< HEAD
=======
// tests using the minio client
func TestEncodingWithMinioClient(t *testing.T) {
cases := []struct {
description string
bucket string
path string
filename string
expected string
}{
{
description: "weird file in bucket root",
bucket: "mybucket",
path: "",
filename: " file with w€r^d ch@r \\#~+§4%&'. txt ",
},
{
description: "weird file inside a weird folder",
bucket: "mybucket",
path: "ä#/नेपाल&/?/",
filename: " file with w€r^d ch@r \\#~+§4%&'. txt ",
},
}

for _, tt := range cases {
t.Run(tt.description, func(t *testing.T) {
fstest.Initialise()
f, _, clean, err := fstest.RandomRemote()
assert.NoError(t, err)
defer clean()
err = f.Mkdir(context.Background(), path.Join(tt.bucket, tt.path))
assert.NoError(t, err)

buf := bytes.NewBufferString("contents")
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)

obji := object.NewStaticObjectInfo(
path.Join(tt.bucket, tt.path, tt.filename),
time.Now(),
int64(buf.Len()),
true,
nil,
nil,
)
_, err = f.Put(context.Background(), in, obji)
assert.NoError(t, err)

keyid := RandString(16)
keysec := RandString(16)
endpoint := serveS3(f, keyid, keysec)
testURL, _ := url.Parse(endpoint)
minioClient, err := minio.New(testURL.Host, &minio.Options{
Creds: credentials.NewStaticV4(keyid, keysec, ""),
Secure: false,
})
assert.NoError(t, err)

buckets, err := minioClient.ListBuckets(context.Background())
assert.NoError(t, err)
assert.Equal(t, buckets[0].Name, tt.bucket)
s3objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
Recursive: true,
})
for s3object := range s3objects {
assert.Equal(t, path.Join(tt.path, tt.filename), s3object.Key)
}
})
}
}

// prepare the test server and return a function to tidy it up afterwards
func prepareWebDavFs(t *testing.T, handler http.HandlerFunc) (fs.Fs, func()) {
// Make the test server
Expand Down Expand Up @@ -474,87 +476,3 @@ func TestForwardAccessKeyToWebDavParallelRequests(t *testing.T) {
assert.NoError(t, response)
}
}

>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
// tests using the minio client
func TestEncodingWithMinioClient(t *testing.T) {
cases := []struct {
description string
bucket string
path string
filename string
expected string
}{
{
description: "weird file in bucket root",
bucket: "mybucket",
path: "",
filename: " file with w€r^d ch@r \\#~+§4%&'. txt ",
},
{
description: "weird file inside a weird folder",
bucket: "mybucket",
path: "ä#/नेपाल&/?/",
filename: " file with w€r^d ch@r \\#~+§4%&'. txt ",
},
}

for _, tt := range cases {
t.Run(tt.description, func(t *testing.T) {
fstest.Initialise()
f, _, clean, err := fstest.RandomRemote()
assert.NoError(t, err)
defer clean()
err = f.Mkdir(context.Background(), path.Join(tt.bucket, tt.path))
assert.NoError(t, err)

buf := bytes.NewBufferString("contents")
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)

obji := object.NewStaticObjectInfo(
path.Join(tt.bucket, tt.path, tt.filename),
time.Now(),
int64(buf.Len()),
true,
nil,
nil,
)
_, err = f.Put(context.Background(), in, obji)
assert.NoError(t, err)
<<<<<<< HEAD

endpoint, keyid, keysec := serveS3(f)
=======
keyid := RandString(16)
keysec := RandString(16)
endpoint := serveS3(f, keyid, keysec)
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
testURL, _ := url.Parse(endpoint)
minioClient, err := minio.New(testURL.Host, &minio.Options{
Creds: credentials.NewStaticV4(keyid, keysec, ""),
Secure: false,
})
assert.NoError(t, err)

buckets, err := minioClient.ListBuckets(context.Background())
assert.NoError(t, err)
assert.Equal(t, buckets[0].Name, tt.bucket)
<<<<<<< HEAD
objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
Recursive: true,
})
for object := range objects {
assert.Equal(t, path.Join(tt.path, tt.filename), object.Key)
=======
s3objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
Recursive: true,
})
for s3object := range s3objects {
assert.Equal(t, path.Join(tt.path, tt.filename), s3object.Key)
>>>>>>> 3c67b121f (serve s3: let rclone act as an S3 compatible server)
}
})
}

}
Loading

0 comments on commit f539d1e

Please sign in to comment.