diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go index 7e218f9851ffe..a2568305d5e2d 100644 --- a/cmd/serve/s3/s3.go +++ b/cmd/serve/s3/s3.go @@ -4,8 +4,8 @@ import ( "context" _ "embed" - "github.com/rclone/rclone/backend/webdav" "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/cmd/serve/proxy/proxyflags" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/hash" @@ -21,6 +21,7 @@ var DefaultOpt = Options{ hashName: "MD5", hashType: hash.MD5, noCleanup: false, + Auth: httplib.DefaultAuthCfg(), HTTP: httplib.DefaultCfg(), proxyMode: false, } @@ -32,8 +33,10 @@ const flagPrefix = "" func init() { flagSet := Command.Flags() + httplib.AddAuthFlagsPrefix(flagSet, flagPrefix, &Opt.Auth) httplib.AddHTTPFlagsPrefix(flagSet, flagPrefix, &Opt.HTTP) vfsflags.AddFlags(flagSet) + proxyflags.AddFlags(flagSet) flags.BoolVarP(flagSet, &Opt.pathBucketMode, "force-path-style", "", Opt.pathBucketMode, "If true use path style access if false use virtual hosted style (default true)", "") flags.StringVarP(flagSet, &Opt.hashName, "etag-hash", "", Opt.hashName, "Which hash to use for the ETag, or auto or blank for off", "") flags.StringArrayVarP(flagSet, &Opt.authPair, "auth-key", "", Opt.authPair, "Set key pair for v4 authorization: access_key_id,secret_access_key", "") @@ -53,16 +56,21 @@ var Command = &cobra.Command{ }, Use: "s3 remote:path", Short: `Serve remote:path over s3.`, - Long: serveS3Help + httplib.Help(flagPrefix) + vfs.Help, + Long: serveS3Help + httplib.Help(flagPrefix) + httplib.AuthHelp(flagPrefix) + vfs.Help, RunE: func(command *cobra.Command, args []string) error { - cmd.CheckArgs(1, 1, command, args) - f := cmd.NewFsSrc(args) - - if _, ok := f.(*webdav.Fs); !ok && Opt.proxyMode { - fs.Logf("serve s3", "--proxy-mode is supported only for 'webdav' provider") - return fs.ErrorNotImplemented + var f fs.Fs + if proxyflags.Opt.AuthProxy == "" { + cmd.CheckArgs(1, 1, command, args) + f = cmd.NewFsSrc(args) + } else { + cmd.CheckArgs(0, 0, command, args) } + // if _, ok := f.(*webdav.Fs); !ok && Opt.proxyMode { + // fs.Logf("serve s3", "--proxy-mode is supported only for 'webdav' provider") + // return fs.ErrorNotImplemented + // } + if Opt.hashName == "auto" { Opt.hashType = f.Hashes().GetOne() } else if Opt.hashName != "" { diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go index 152b4479b1a22..6f97911634135 100644 --- a/cmd/serve/s3/server.go +++ b/cmd/serve/s3/server.go @@ -3,8 +3,6 @@ package s3 import ( "context" - "crypto/md5" - "encoding/hex" "errors" "fmt" "math/rand" @@ -13,7 +11,8 @@ import ( "github.com/Mikubill/gofakes3" "github.com/Mikubill/gofakes3/signature" "github.com/go-chi/chi/v5" - "github.com/rclone/rclone/backend/webdav" + "github.com/rclone/rclone/cmd/serve/proxy" + "github.com/rclone/rclone/cmd/serve/proxy/proxyflags" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/hash" httplib "github.com/rclone/rclone/lib/http" @@ -35,6 +34,7 @@ type Options struct { hashType hash.Type authPair []string noCleanup bool + Auth httplib.AuthConfig HTTP httplib.Config proxyMode bool } @@ -42,10 +42,12 @@ type Options struct { // Server is a s3.FileSystem interface type Server struct { *httplib.Server + opt Options f fs.Fs _vfs *vfs.VFS // don't use directly, use getVFS faker *gofakes3.GoFakeS3 handler http.Handler + proxy *proxy.Proxy ctx context.Context // for global config } @@ -54,6 +56,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error w := &Server{ f: f, ctx: ctx, + opt: *opt, } var newLogger logger @@ -67,34 +70,62 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied ) - w.Server, err = httplib.NewServer(ctx, - httplib.WithConfig(opt.HTTP), - ) - if err != nil { - return nil, fmt.Errorf("failed to init server: %w", err) - } - w.handler = http.NewServeMux() w.handler = w.faker.Server() - if opt.proxyMode { - w.handler = proxyMiddleware(w.handler, w) + if proxyflags.Opt.AuthProxy != "" { + fmt.Println("proxy mode...") + w.proxy = proxy.New(ctx, &proxyflags.Opt) + // override auth + // w.opt.Auth.CustomAuthFn = w.auth + w.handler = middlewareAuthCustom(w.handler, w) } else { w._vfs = vfs.New(f, &vfsflags.Opt) + } - if len(opt.authPair) == 0 { - fs.Logf("serve s3", "No auth provided so allowing anonymous access") - } + if len(opt.authPair) == 0 { + fs.Logf("serve s3", "No auth provided so allowing anonymous access") + } + + w.Server, err = httplib.NewServer(ctx, + httplib.WithConfig(w.opt.HTTP), + httplib.WithAuth(w.opt.Auth), + ) + if err != nil { + return nil, fmt.Errorf("failed to init server: %w", err) } + + // if opt.proxyMode { + // w.handler = proxyMiddleware(w.handler, w) + // } else { + // w._vfs = vfs.New(f, &vfsflags.Opt) + // } + return w, nil } +// func (w *Server) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) { +// if w._vfs != nil { +// return w._vfs, nil +// } +// value := ctx.Value(ctxKeyId) +// if value == nil { +// return nil, errors.New("no VFS found in context") +// } +// VFS, ok := value.(*vfs.VFS) +// if !ok { +// return nil, fmt.Errorf("context value is not VFS: %#v", value) +// } +// return VFS, nil +// } + +// Gets the VFS in use for this request func (w *Server) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) { if w._vfs != nil { return w._vfs, nil } - value := ctx.Value(ctxKeyId) + value := httplib.CtxGetAuth(ctx) if value == nil { return nil, errors.New("no VFS found in context") } @@ -105,6 +136,17 @@ func (w *Server) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) { return VFS, nil } + +// auth does proxy authorization +func (w *Server) auth(accessKeyId string) (value interface{}, err error) { + fmt.Println("auth called...") + VFS, _, err := w.proxy.Call(accessKeyId, "", false) + if err != nil { + return nil, err + } + return VFS, err +} + // Bind register the handler to http.Router func (w *Server) Bind(router chi.Router) { router.Handle("/*", w.handler) @@ -116,17 +158,24 @@ func (w *Server) serve() error { return nil } -func proxyMiddleware(next http.Handler, ws *Server) http.Handler { +func middlewareAuthCustom(next http.Handler, ws *Server) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { accessKey, _ := parseAuthToken(r) + value, err := ws.auth(accessKey) + if err != nil { + fs.Infof(r.URL.Path, "%s: Auth failed: %v", r.RemoteAddr, err) + } - info, name, remote, config, _ := fs.ConfigFs(ws.f.Name() + ":") - newFs, _ := info.NewFs(r.Context(), name+stringToMd5Hash(accessKey), remote, config) - _vfs := vfs.New(newFs, &vfsflags.Opt) - _vfs.Fs().(*webdav.Fs).SetBearerToken(accessKey) + if value != nil { + r = r.WithContext(context.WithValue(r.Context(), ctxKeyId, value)) + } + // info, name, remote, config, _ := fs.ConfigFs(ws.f.Name() + ":") + // newFs, _ := info.NewFs(r.Context(), name+stringToMd5Hash(accessKey), remote, config) + // _vfs := vfs.New(newFs, &vfsflags.Opt) + // _vfs.Fs().(*webdav.Fs).SetBearerToken(accessKey) - ctx := context.WithValue(r.Context(), ctxKeyId, _vfs) - next.ServeHTTP(w, r.WithContext(ctx)) + // ctx := context.WithValue(r.Context(), ctxKeyId, _vfs) + next.ServeHTTP(w, r) }) } @@ -140,8 +189,8 @@ func parseAuthToken(r *http.Request) (accessKey string, error signature.ErrorCod return req.Credential.GetAccessKey(), signature.ErrNone } -func stringToMd5Hash(s string) string { - hasher := md5.New() - hasher.Write([]byte(s)) - return hex.EncodeToString(hasher.Sum(nil)) -} +// func stringToMd5Hash(s string) string { +// hasher := md5.New() +// hasher.Write([]byte(s)) +// return hex.EncodeToString(hasher.Sum(nil)) +// }