diff --git a/.gitignore b/.gitignore index d449a90..5f821b9 100644 --- a/.gitignore +++ b/.gitignore @@ -111,4 +111,5 @@ modules.xml # End of https://www.gitignore.io/api/go,jetbrains+all www/ -dist/ \ No newline at end of file +dist/ +testdata/ \ No newline at end of file diff --git a/README.md b/README.md index b876509..088a926 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ CLI Usage: [Docs](./docs/yeet.md) +## Features + +* Fast +* 0 setup +* Local fs support +* S3 support +* Redis-backed S3 support +* Cuckoo filter for Redis-S3 +* File watching (local only) + ## Docker ``` diff --git a/cache/hashmap.go b/cache/hashmap.go index e31ce66..b4a6c12 100644 --- a/cache/hashmap.go +++ b/cache/hashmap.go @@ -2,6 +2,7 @@ package cache import ( "github.com/Vilsol/yeet/source" + "github.com/Vilsol/yeet/utils" "github.com/cornelk/hashmap" "github.com/rs/zerolog/log" "github.com/spf13/viper" @@ -61,7 +62,7 @@ func (c *HashMapCache) Index() (int64, error) { c.data.Set(event.CleanPath, instance) if viper.GetBool("warmup") { - instance.Get(instance) + instance.Get(instance, nil) } log.Trace().Msgf("Added to cache: %s", event.CleanPath) @@ -92,11 +93,25 @@ func (c *HashMapCache) Store(path []byte, host []byte, instance *commonInstance) func (c *HashMapCache) Get(path []byte, host []byte) (string, io.Reader, int) { if !c.hosts { if instance, ok := c.data.Get(path); ok { - return instance.(*commonInstance).Get(instance.(*commonInstance)) + return instance.(*commonInstance).Get(instance.(*commonInstance), nil) } } else { - if instance, ok := c.data.Get(append(host, path...)); ok { - return instance.(*commonInstance).Get(instance.(*commonInstance)) + key := append(host, path...) + instance, ok := c.data.Get(key) + if !ok { + pathStr := utils.ByteSliceToString(path) + instance = &commonInstance{ + Instance: &CachedInstance{ + RelativePath: pathStr, + AbsolutePath: pathStr, + }, + Get: load(c), + } + c.data.Set(key, instance) + } + + if instance != nil { + return instance.(*commonInstance).Get(instance.(*commonInstance), host) } } diff --git a/cache/shared.go b/cache/shared.go index 4de362a..973db39 100644 --- a/cache/shared.go +++ b/cache/shared.go @@ -9,9 +9,7 @@ import ( "github.com/spf13/viper" "io" "path" - "reflect" "time" - "unsafe" ) func indexBase(c Cache) (int64, error) { @@ -25,10 +23,11 @@ func indexBase(c Cache) (int64, error) { } // Host is not used when indexing is supported - c.Store(unsafeGetBytes(cleanedPath), nil, instance) + c.Store(utils.UnsafeGetBytes(cleanedPath), nil, instance) if viper.GetBool("warmup") { - instance.Get(instance) + // Host is not used when indexing is supported + instance.Get(instance, nil) return int64(len(instance.Instance.Data)) } @@ -58,9 +57,13 @@ func index(source source.Source, f source.IndexFunc) (int64, error) { return totalSize, nil } -func load(c Cache) func(*commonInstance) (string, io.Reader, int) { - return func(instance *commonInstance) (string, io.Reader, int) { - hijacker := c.Source().Get(instance.Instance.AbsolutePath, nil) +func load(c Cache) func(*commonInstance, []byte) (string, io.Reader, int) { + return func(instance *commonInstance, host []byte) (string, io.Reader, int) { + hijacker := c.Source().Get(instance.Instance.AbsolutePath, host) + + if hijacker == nil { + return "", nil, 0 + } log.Debug().Msgf("Loaded file [%d][%s]: %s", hijacker.Size, hijacker.FileType(), instance.Instance.AbsolutePath) @@ -68,7 +71,7 @@ func load(c Cache) func(*commonInstance) (string, io.Reader, int) { instance.Instance.LoadTime = time.Now() instance.Instance.Data = hijacker.Buffer instance.Instance.ContentType = hijacker.FileType() - instance.Get = func(cache *commonInstance) (string, io.Reader, int) { + instance.Get = func(cache *commonInstance, _ []byte) (string, io.Reader, int) { return cache.Instance.ContentType, bytes.NewReader(cache.Instance.Data), len(cache.Instance.Data) } } @@ -94,13 +97,3 @@ func expiry(c Cache) { } }(c) } - -func unsafeGetBytes(s string) []byte { - return (*[0x7fff0000]byte)(unsafe.Pointer( - (*reflect.StringHeader)(unsafe.Pointer(&s)).Data), - )[:len(s):len(s)] -} - -func byteSliceToString(bs []byte) string { - return *(*string)(unsafe.Pointer(&bs)) -} diff --git a/cache/cache.go b/cache/types.go similarity index 87% rename from cache/cache.go rename to cache/types.go index 15977ba..d065ab7 100644 --- a/cache/cache.go +++ b/cache/types.go @@ -16,7 +16,7 @@ type CachedInstance struct { type commonInstance struct { Instance *CachedInstance - Get func(instance *commonInstance) (string, io.Reader, int) + Get func(instance *commonInstance, host []byte) (string, io.Reader, int) } type KeyValue struct { diff --git a/cmd/serve.go b/cmd/serve.go index 670a1b1..8b00f3c 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -7,6 +7,8 @@ import ( "time" ) +const botAgents = "bot|crawl|spider|external|meta|scrap|archive|discourse" + func init() { ServeCMD.PersistentFlags().String("host", "", "Hostname to bind the webserver") ServeCMD.PersistentFlags().Int("port", 8080, "Port to run the webserver on") @@ -22,6 +24,12 @@ func init() { ServeCMD.PersistentFlags().StringSliceP("paths", "p", []string{"./www"}, "Paths to serve on the webserver") ServeCMD.PersistentFlags().BoolP("watch", "w", false, "Watch filesystem for changes") + ServeCMD.PersistentFlags().String("tls-cert", "", "TLS Certificate file path") + ServeCMD.PersistentFlags().String("tls-key", "", "TLS Key file path") + + ServeCMD.PersistentFlags().String("bot-proxy", "", "Bot proxy URL") + ServeCMD.PersistentFlags().String("bot-agents", botAgents, "Bot User-Agent header regex") + _ = viper.BindPFlag("paths", ServeCMD.PersistentFlags().Lookup("paths")) _ = viper.BindPFlag("watch", ServeCMD.PersistentFlags().Lookup("watch")) @@ -36,6 +44,12 @@ func init() { _ = viper.BindPFlag("index.file", ServeCMD.PersistentFlags().Lookup("index-file")) + _ = viper.BindPFlag("tls.cert", ServeCMD.PersistentFlags().Lookup("tls-cert")) + _ = viper.BindPFlag("tls.key", ServeCMD.PersistentFlags().Lookup("tls-key")) + + _ = viper.BindPFlag("bot.proxy", ServeCMD.PersistentFlags().Lookup("bot-proxy")) + _ = viper.BindPFlag("bot.agents", ServeCMD.PersistentFlags().Lookup("bot-agents")) + RootCMD.AddCommand(ServeCMD) } diff --git a/cmd/serve/s3_redis.go b/cmd/serve/s3_redis.go new file mode 100644 index 0000000..b3f8e20 --- /dev/null +++ b/cmd/serve/s3_redis.go @@ -0,0 +1,63 @@ +package serve + +import ( + "github.com/Vilsol/yeet/cache" + "github.com/Vilsol/yeet/cmd" + "github.com/Vilsol/yeet/server" + "github.com/Vilsol/yeet/source" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func init() { + s3RedisCmd.Flags().String("network", "tcp", "The network type, either tcp or unix") + s3RedisCmd.Flags().StringP("address", "a", "localhost:6379", "host:port address of Redis") + s3RedisCmd.Flags().String("user", "", "Username of Redis") + s3RedisCmd.Flags().String("pass", "", "Password of Redis") + s3RedisCmd.Flags().Int("db", 0, "DB of Redis") + + _ = s3RedisCmd.MarkFlagRequired("address") + + _ = viper.BindPFlag("network", s3RedisCmd.Flags().Lookup("network")) + _ = viper.BindPFlag("address", s3RedisCmd.Flags().Lookup("address")) + _ = viper.BindPFlag("username", s3RedisCmd.Flags().Lookup("username")) + _ = viper.BindPFlag("password", s3RedisCmd.Flags().Lookup("password")) + _ = viper.BindPFlag("db", s3RedisCmd.Flags().Lookup("db")) + + cmd.ServeCMD.AddCommand(s3RedisCmd) +} + +var s3RedisCmd = &cobra.Command{ + Use: "s3-redis", + Short: "Serve Redis-backed S3 buckets", + RunE: func(cmd *cobra.Command, args []string) error { + if viper.GetBool("watch") { + return errors.New("watch is not supported for s3-redis") + } + + src, err := source.NewS3Redis( + viper.GetString("network"), + viper.GetString("address"), + viper.GetString("username"), + viper.GetString("password"), + viper.GetInt("db"), + ) + + if err != nil { + return err + } + + c, err := cache.NewHashMapCache(src, true) + + if err != nil { + return err + } + + if _, err := c.Index(); err != nil { + return err + } + + return server.Run(c) + }, +} diff --git a/docs/yeet.md b/docs/yeet.md index 745c751..51c47d5 100644 --- a/docs/yeet.md +++ b/docs/yeet.md @@ -15,4 +15,4 @@ yeet is an in-memory indexed static file webserver * [yeet serve](yeet_serve.md) - Serve files with yeet -###### Auto generated by spf13/cobra on 5-Jan-2022 +###### Auto generated by spf13/cobra on 7-Jan-2022 diff --git a/docs/yeet_serve.md b/docs/yeet_serve.md index 225b4b8..55153df 100644 --- a/docs/yeet_serve.md +++ b/docs/yeet_serve.md @@ -5,6 +5,8 @@ Serve files with yeet ### Options ``` + --bot-agents string Bot User-Agent header regex (default "bot|crawl|spider|external|meta|scrap|archive|discourse") + --bot-proxy string Bot proxy URL --expiry Use cache expiry --expiry-interval duration Interval between cache GC's (default 10m0s) --expiry-time duration Lifetime of a cache entry (default 1h0m0s) @@ -13,6 +15,8 @@ Serve files with yeet --index-file string The directory default index file (default "index.html") -p, --paths strings Paths to serve on the webserver (default [./www]) --port int Port to run the webserver on (default 8080) + --tls-cert string TLS Certificate file path + --tls-key string TLS Key file path --warmup Load all files into memory on startup -w, --watch Watch filesystem for changes ``` @@ -30,5 +34,6 @@ Serve files with yeet * [yeet](yeet.md) - yeet is an in-memory indexed static file webserver * [yeet serve local](yeet_serve_local.md) - Serve a local directory * [yeet serve s3](yeet_serve_s3.md) - Serve an S3 bucket +* [yeet serve s3-redis](yeet_serve_s3-redis.md) - Serve Redis-backed S3 buckets -###### Auto generated by spf13/cobra on 5-Jan-2022 +###### Auto generated by spf13/cobra on 7-Jan-2022 diff --git a/docs/yeet_serve_local.md b/docs/yeet_serve_local.md index 39c7e7d..aa8ff27 100644 --- a/docs/yeet_serve_local.md +++ b/docs/yeet_serve_local.md @@ -15,6 +15,8 @@ yeet serve local [flags] ### Options inherited from parent commands ``` + --bot-agents string Bot User-Agent header regex (default "bot|crawl|spider|external|meta|scrap|archive|discourse") + --bot-proxy string Bot proxy URL --expiry Use cache expiry --expiry-interval duration Interval between cache GC's (default 10m0s) --expiry-time duration Lifetime of a cache entry (default 1h0m0s) @@ -25,6 +27,8 @@ yeet serve local [flags] -p, --paths strings Paths to serve on the webserver (default [./www]) --port int Port to run the webserver on (default 8080) --quiet Do not log anything to console + --tls-cert string TLS Certificate file path + --tls-key string TLS Key file path --warmup Load all files into memory on startup -w, --watch Watch filesystem for changes ``` @@ -33,4 +37,4 @@ yeet serve local [flags] * [yeet serve](yeet_serve.md) - Serve files with yeet -###### Auto generated by spf13/cobra on 5-Jan-2022 +###### Auto generated by spf13/cobra on 7-Jan-2022 diff --git a/docs/yeet_serve_s3-redis.md b/docs/yeet_serve_s3-redis.md new file mode 100644 index 0000000..adc83c6 --- /dev/null +++ b/docs/yeet_serve_s3-redis.md @@ -0,0 +1,45 @@ +## yeet serve s3-redis + +Serve Redis-backed S3 buckets + +``` +yeet serve s3-redis [flags] +``` + +### Options + +``` + -a, --address string host:port address of Redis (default "localhost:6379") + --db int DB of Redis + -h, --help help for s3-redis + --network string The network type, either tcp or unix (default "tcp") + --pass string Password of Redis + --user string Username of Redis +``` + +### Options inherited from parent commands + +``` + --bot-agents string Bot User-Agent header regex (default "bot|crawl|spider|external|meta|scrap|archive|discourse") + --bot-proxy string Bot proxy URL + --expiry Use cache expiry + --expiry-interval duration Interval between cache GC's (default 10m0s) + --expiry-time duration Lifetime of a cache entry (default 1h0m0s) + --host string Hostname to bind the webserver + --index-file string The directory default index file (default "index.html") + --log string The log level to output (default "info") + --log-file string File to output logs to + -p, --paths strings Paths to serve on the webserver (default [./www]) + --port int Port to run the webserver on (default 8080) + --quiet Do not log anything to console + --tls-cert string TLS Certificate file path + --tls-key string TLS Key file path + --warmup Load all files into memory on startup + -w, --watch Watch filesystem for changes +``` + +### SEE ALSO + +* [yeet serve](yeet_serve.md) - Serve files with yeet + +###### Auto generated by spf13/cobra on 7-Jan-2022 diff --git a/docs/yeet_serve_s3.md b/docs/yeet_serve_s3.md index cceb28e..466e679 100644 --- a/docs/yeet_serve_s3.md +++ b/docs/yeet_serve_s3.md @@ -20,6 +20,8 @@ yeet serve s3 [flags] ### Options inherited from parent commands ``` + --bot-agents string Bot User-Agent header regex (default "bot|crawl|spider|external|meta|scrap|archive|discourse") + --bot-proxy string Bot proxy URL --expiry Use cache expiry --expiry-interval duration Interval between cache GC's (default 10m0s) --expiry-time duration Lifetime of a cache entry (default 1h0m0s) @@ -30,6 +32,8 @@ yeet serve s3 [flags] -p, --paths strings Paths to serve on the webserver (default [./www]) --port int Port to run the webserver on (default 8080) --quiet Do not log anything to console + --tls-cert string TLS Certificate file path + --tls-key string TLS Key file path --warmup Load all files into memory on startup -w, --watch Watch filesystem for changes ``` @@ -38,4 +42,4 @@ yeet serve s3 [flags] * [yeet serve](yeet_serve.md) - Serve files with yeet -###### Auto generated by spf13/cobra on 5-Jan-2022 +###### Auto generated by spf13/cobra on 7-Jan-2022 diff --git a/flat/s3.fbs b/flat/s3.fbs new file mode 100644 index 0000000..27b8fd5 --- /dev/null +++ b/flat/s3.fbs @@ -0,0 +1,10 @@ +namespace yeet; + +table S3 { + filter: string; + bucket: string; + key: string; + secret: string; + endpoint: string; + region: string; +} \ No newline at end of file diff --git a/flat/yeet/S3.go b/flat/yeet/S3.go new file mode 100644 index 0000000..b657c1a --- /dev/null +++ b/flat/yeet/S3.go @@ -0,0 +1,107 @@ +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package yeet + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type S3 struct { + _tab flatbuffers.Table +} + +func GetRootAsS3(buf []byte, offset flatbuffers.UOffsetT) *S3 { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &S3{} + x.Init(buf, n+offset) + return x +} + +func GetSizePrefixedRootAsS3(buf []byte, offset flatbuffers.UOffsetT) *S3 { + n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:]) + x := &S3{} + x.Init(buf, n+offset+flatbuffers.SizeUint32) + return x +} + +func (rcv *S3) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *S3) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *S3) Filter() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *S3) Bucket() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *S3) Key() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *S3) Secret() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *S3) Endpoint() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *S3) Region() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func S3Start(builder *flatbuffers.Builder) { + builder.StartObject(6) +} +func S3AddFilter(builder *flatbuffers.Builder, filter flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(filter), 0) +} +func S3AddBucket(builder *flatbuffers.Builder, bucket flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(bucket), 0) +} +func S3AddKey(builder *flatbuffers.Builder, key flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(key), 0) +} +func S3AddSecret(builder *flatbuffers.Builder, secret flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(secret), 0) +} +func S3AddEndpoint(builder *flatbuffers.Builder, endpoint flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(endpoint), 0) +} +func S3AddRegion(builder *flatbuffers.Builder, region flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(region), 0) +} +func S3End(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go.mod b/go.mod index e382e40..0a4f680 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,16 @@ module github.com/Vilsol/yeet go 1.17 require ( + github.com/MarvinJWendt/testza v0.2.12 github.com/aws/aws-sdk-go v1.42.23 - github.com/bits-and-blooms/bloom/v3 v3.1.0 github.com/cornelk/hashmap v1.0.1 github.com/fsnotify/fsnotify v1.5.1 + github.com/go-redis/redis/v8 v8.11.4 + github.com/google/flatbuffers v2.0.5+incompatible + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.26.1 + github.com/seiflotfy/cuckoofilter v0.0.0-20201222105146-bc6005554a0c github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.9.0 github.com/valyala/fasthttp v1.31.0 @@ -16,16 +20,26 @@ require ( require ( github.com/andybalholm/brotli v1.0.2 // indirect - github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/atomicgo/cursor v0.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dchest/siphash v1.1.0 // indirect + github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/gookit/color v1.4.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.13.4 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/magiconair/properties v1.8.5 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pterm/pterm v0.12.33 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/spf13/afero v1.6.0 // indirect @@ -34,7 +48,9 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect + github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect + golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.6 // indirect gopkg.in/ini.v1 v1.63.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 7fcc07a..5b2181f 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,12 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= +github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= +github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= +github.com/MarvinJWendt/testza v0.2.10/go.mod h1:pd+VWsoGUiFtq+hRKSU1Bktnn+DMCSrDrXDpX2bG66k= +github.com/MarvinJWendt/testza v0.2.12 h1:/PRp/BF+27t2ZxynTiqj0nyND5PbOtfJS0SuTuxmgeg= +github.com/MarvinJWendt/testza v0.2.12/go.mod h1:JOIegYyV7rX+7VZ9r77L/eH6CfJHHzXjB69adAhzZkI= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -53,16 +59,17 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/atomicgo/cursor v0.0.1 h1:xdogsqa6YYlLfM+GyClC/Lchf7aiMerFiZQn7soTOoU= +github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/aws/aws-sdk-go v1.42.23 h1:V0V5hqMEyVelgpu1e4gMPVCJ+KhmscdNxP/NWP1iCOA= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bloom/v3 v3.1.0 h1:o3Adl6bGuD9eZzMiLDepS5jqmoEAv/ZH+fFe/MH1quA= -github.com/bits-and-blooms/bloom/v3 v3.1.0/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -82,6 +89,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/siphash v1.1.0 h1:1Rs9eTUlZLPBEvV+2sTaM8O0NWn0ppbgqS7p11aWawI= github.com/dchest/siphash v1.1.0/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= +github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 h1:BS21ZUJ/B5X2UVUbczfmdWH7GapPWAhxcMsDnjJTU1E= +github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -92,6 +103,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -99,6 +111,9 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -134,6 +149,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v2.0.5+incompatible h1:ANsW0idDAXIY+mNHzIHxWRfabV2x5LUEEIIWcwsYgB8= +github.com/google/flatbuffers v2.0.5+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -170,6 +187,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/gookit/color v1.4.2 h1:tXy44JFSFkKnELV6WaMo/lLfu/meqITX3iAV52do7lk= +github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -201,6 +220,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -217,6 +237,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -234,6 +256,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -251,7 +275,20 @@ github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -264,6 +301,14 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI= +github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg= +github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE= +github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEejaWgXU= +github.com/pterm/pterm v0.12.33 h1:XiT50Pvdqn5O8FAiIqZMpXP6NkVEcmlUa+mkA1yWVCg= +github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -274,13 +319,13 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seiflotfy/cuckoofilter v0.0.0-20201222105146-bc6005554a0c h1:pqy40B3MQWYrza7YZXOXgl0Nf0QGFqrOC0BKae1UNAA= +github.com/seiflotfy/cuckoofilter v0.0.0-20201222105146-bc6005554a0c/go.mod h1:bR6DqgcAl1zTcOX8/pE2Qkj9XO00eCNqmKb7lXP8EAg= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -310,6 +355,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.31.0 h1:lrauRLII19afgCs2fnWRJ4M5IkV0lo2FqA61uGkNBfE= github.com/valyala/fasthttp v1.31.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -378,6 +425,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -401,6 +449,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -413,6 +462,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -447,6 +497,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -457,12 +508,14 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -485,6 +538,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -501,9 +555,14 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c h1:taxlMj0D/1sOAuv/CbSD+MMDof2vbyPTqz5FNYKpXt8= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -564,6 +623,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -708,15 +768,21 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/server/webserver.go b/server/webserver.go index 2540014..0ff987f 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -2,10 +2,15 @@ package server import ( "github.com/Vilsol/yeet/cache" + "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/spf13/viper" "github.com/valyala/fasthttp" + "github.com/valyala/fasthttp/fasthttpadaptor" "net" + "net/http/httputil" + "net/url" + "regexp" "strconv" ) @@ -21,13 +26,31 @@ func Run(c cache.Cache) error { return err } - log.Info().Msgf("Starting webserver on %s", address) + handler := ws.HandleFastHTTP + if viper.GetString("bot.proxy") != "" { + r, err := regexp.Compile(viper.GetString("bot.agents")) + if err != nil { + return errors.Wrap(err, "failed to compile bot proxy regex") + } - if err := fasthttp.Serve(ln, ws.HandleFastHTTP); err != nil { - return err + proxy, err := url.Parse(viper.GetString("bot.proxy")) + if err != nil { + return errors.Wrap(err, "failed to parse bot proxy url") + } + + reverseProxy := httputil.NewSingleHostReverseProxy(proxy) + proxyHandler := fasthttpadaptor.NewFastHTTPHandler(reverseProxy) + + handler = ws.HandleFastHTTPWithBotProxy(r, proxyHandler) + } + + if viper.GetString("tls.cert") != "" && viper.GetString("tls.key") != "" { + log.Info().Msgf("Starting webserver with TLS on %s", address) + return fasthttp.ServeTLS(ln, viper.GetString("tls.cert"), viper.GetString("tls.key"), handler) } - return nil + log.Info().Msgf("Starting webserver on %s", address) + return fasthttp.Serve(ln, handler) } type Webserver struct { @@ -42,3 +65,13 @@ func (h *Webserver) HandleFastHTTP(ctx *fasthttp.RequestCtx) { ctx.SetStatusCode(404) } } + +func (h *Webserver) HandleFastHTTPWithBotProxy(botHeaderRegex *regexp.Regexp, proxy fasthttp.RequestHandler) func(ctx *fasthttp.RequestCtx) { + return func(ctx *fasthttp.RequestCtx) { + if ctx.UserAgent() == nil || !botHeaderRegex.Match(ctx.UserAgent()) { + h.HandleFastHTTP(ctx) + } else { + proxy(ctx) + } + } +} diff --git a/source/local.go b/source/local.go index 227cf62..a9cd07d 100644 --- a/source/local.go +++ b/source/local.go @@ -2,7 +2,6 @@ package source import ( "github.com/Vilsol/yeet/utils" - "github.com/bits-and-blooms/bloom/v3" "github.com/fsnotify/fsnotify" "github.com/rs/zerolog/log" "github.com/spf13/viper" @@ -38,7 +37,6 @@ func (l Local) Get(path string, host []byte) *utils.StreamHijacker { func (l Local) IndexPath(dir string, f IndexFunc) (int64, int64, error) { totalSize := int64(0) totalCount := int64(0) - fileNames := make([]string, 0) if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -70,20 +68,11 @@ func (l Local) IndexPath(dir string, f IndexFunc) (int64, int64, error) { log.Trace().Msgf("Indexed: %s -> %s", cleanedPath, absPath) - fileNames = append(fileNames, cleanedPath) - return nil }); err != nil { return 0, 0, err } - filter := bloom.NewWithEstimates(uint(totalCount), 0.001) - for _, fileName := range fileNames { - filter.AddString(fileName) - } - - log.Trace().Msgf("Created bloom filter of size %d and hash count %d", filter.Cap(), filter.K()) - return totalSize, totalCount, nil } diff --git a/source/s3.go b/source/s3.go index 339e3d2..864d216 100644 --- a/source/s3.go +++ b/source/s3.go @@ -8,17 +8,13 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "mime" - "path/filepath" - "strings" ) var _ Source = (*S3)(nil) type S3 struct { - S3Client *s3.S3 - S3Session *session.Session - Bucket string + S3Client *s3.S3 + Bucket string } func NewS3(bucket string, key string, secret string, endpoint string, region string) (*S3, error) { @@ -38,28 +34,13 @@ func NewS3(bucket string, key string, secret string, endpoint string, region str s3Client := s3.New(newSession) return &S3{ - S3Client: s3Client, - S3Session: newSession, - Bucket: bucket, + S3Client: s3Client, + Bucket: bucket, }, nil } -func (s S3) Get(path string, host []byte) *utils.StreamHijacker { - cleanedKey := strings.TrimPrefix(path, "/") - - object, err := s.S3Client.GetObject(&s3.GetObjectInput{ - Bucket: aws.String(s.Bucket), - Key: aws.String(cleanedKey), - }) - - if err != nil { - log.Err(err).Msg("failed to get object") - return nil - } - - fileType := mime.TypeByExtension(filepath.Ext(filepath.Base(path))) - - return utils.NewStreamHijacker(int(*object.ContentLength), fileType, object.Body) +func (s S3) Get(path string, _ []byte) *utils.StreamHijacker { + return GetS3(s.S3Client, s.Bucket, path) } func (s S3) IndexPath(dirPath string, f IndexFunc) (int64, int64, error) { diff --git a/source/s3_redis.go b/source/s3_redis.go new file mode 100644 index 0000000..98582a3 --- /dev/null +++ b/source/s3_redis.go @@ -0,0 +1,113 @@ +package source + +import ( + "context" + "github.com/Vilsol/yeet/flat/yeet" + "github.com/Vilsol/yeet/utils" + "github.com/go-redis/redis/v8" + "github.com/patrickmn/go-cache" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/seiflotfy/cuckoofilter" + "time" +) + +const redisPrefix = "yeet:" + +var _ Source = (*S3Redis)(nil) + +type S3Redis struct { + RedisClient *redis.Client + CredentialCache *cache.Cache +} + +type S3Wrapper struct { + S3 + Filter *cuckoo.Filter +} + +func NewS3Redis(network string, address string, username string, password string, db int) (*S3Redis, error) { + rdb := redis.NewClient(&redis.Options{ + Network: network, + Addr: address, + Username: username, + Password: password, + DB: db, + }) + + if err := rdb.Ping(context.Background()).Err(); err != nil { + return nil, errors.Wrap(err, "failed to connect to redis") + } + + // TODO Configurable timeout and cleanup interval + return &S3Redis{ + RedisClient: rdb, + CredentialCache: cache.New(time.Second*10, time.Minute), + }, nil +} + +func (s S3Redis) Get(path string, host []byte) *utils.StreamHijacker { + var s3Wrapper *S3Wrapper + + if host == nil { + return nil + } + + if instance, ok := s.CredentialCache.Get(utils.ByteSliceToString(host)); ok { + s3Wrapper = instance.(*S3Wrapper) + } else { + get := s.RedisClient.Get(context.TODO(), redisPrefix+utils.ByteSliceToString(host)) + if get.Err() != nil { + if errors.Is(get.Err(), redis.Nil) { + log.Warn().Str("host", utils.ByteSliceToString(host)).Msg("no credentials found") + return nil + } + + log.Error().Err(get.Err()).Msg("failed to get credentials") + return nil + } + + s3Flat := yeet.GetRootAsS3(utils.UnsafeGetBytes(get.Val()), 0) + + s3Instance, err := NewS3( + utils.ByteSliceToString(s3Flat.Bucket()), + utils.ByteSliceToString(s3Flat.Key()), + utils.ByteSliceToString(s3Flat.Secret()), + utils.ByteSliceToString(s3Flat.Endpoint()), + utils.ByteSliceToString(s3Flat.Region()), + ) + + if err != nil { + log.Err(err).Msg("failed to create new S3 session") + return nil + } + + cf, err := cuckoo.Decode(s3Flat.Filter()) + if err != nil { + log.Err(err).Msg("failed to decode filter") + return nil + } + + s3Wrapper = &S3Wrapper{ + S3: *s3Instance, + Filter: cf, + } + + s.CredentialCache.Set(utils.ByteSliceToString(host), s3Wrapper, cache.DefaultExpiration) + } + + if s3Wrapper.Filter.Lookup(utils.UnsafeGetBytes(path)) { + return GetS3(s3Wrapper.S3Client, s3Wrapper.Bucket, path) + } + + return nil +} + +func (s S3Redis) IndexPath(_ string, _ IndexFunc) (int64, int64, error) { + // Indexing not supported for Redis-backed S3 + return 0, 0, nil +} + +func (s S3Redis) Watch() (<-chan WatchEvent, error) { + return nil, errors.New("s3-redis does not support watching") +} diff --git a/source/s3_utils.go b/source/s3_utils.go new file mode 100644 index 0000000..712fce3 --- /dev/null +++ b/source/s3_utils.go @@ -0,0 +1,29 @@ +package source + +import ( + "github.com/Vilsol/yeet/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/rs/zerolog/log" + "mime" + "path/filepath" + "strings" +) + +func GetS3(client *s3.S3, bucket string, path string) *utils.StreamHijacker { + cleanedKey := strings.TrimPrefix(path, "/") + + object, err := client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(cleanedKey), + }) + + if err != nil { + log.Err(err).Msg("failed to get object") + return nil + } + + fileType := mime.TypeByExtension(filepath.Ext(filepath.Base(path))) + + return utils.NewStreamHijacker(int(*object.ContentLength), fileType, object.Body) +} diff --git a/tests/s3_redis_test.go b/tests/s3_redis_test.go new file mode 100644 index 0000000..26d9fb7 --- /dev/null +++ b/tests/s3_redis_test.go @@ -0,0 +1,126 @@ +package tests + +import ( + "context" + "github.com/MarvinJWendt/testza" + "github.com/Vilsol/yeet/cache" + "github.com/Vilsol/yeet/flat/yeet" + "github.com/Vilsol/yeet/source" + "github.com/Vilsol/yeet/utils" + flatbuffers "github.com/google/flatbuffers/go" + "github.com/rs/zerolog" + cuckoo "github.com/seiflotfy/cuckoofilter" + "github.com/spf13/viper" + "io" + "os" + "testing" +) + +const host = "localhost:8080" + +func init() { + viper.Set("paths", []string{"."}) + zerolog.SetGlobalLevel(zerolog.FatalLevel) +} + +func BuildSample() []byte { + dir, _ := os.ReadDir("../docs/") + filterSize := utils.EstimateCuckooFilter(int64(len(dir))) + filter := cuckoo.NewFilter(filterSize) + + for _, entry := range dir { + filter.InsertUnique([]byte("/" + entry.Name())) + } + + builder := flatbuffers.NewBuilder(1024) + bucket := builder.CreateString("yeet") + endpoint := builder.CreateString("http://localhost:9000") + accessKey := builder.CreateString("minio") + secretKey := builder.CreateString("minio123") + region := builder.CreateString("us-west-002") + encodedFilter := builder.CreateByteString(filter.Encode()) + + yeet.S3Start(builder) + yeet.S3AddBucket(builder, bucket) + yeet.S3AddEndpoint(builder, endpoint) + yeet.S3AddKey(builder, accessKey) + yeet.S3AddSecret(builder, secretKey) + yeet.S3AddRegion(builder, region) + yeet.S3AddFilter(builder, encodedFilter) + s3 := yeet.S3End(builder) + builder.Finish(s3) + + return builder.FinishedBytes() +} + +func GetRedis() (*source.S3Redis, error) { + s, err := source.NewS3Redis("tcp", "localhost:6379", "", "", 0) + if err != nil { + return nil, err + } + + err = s.RedisClient.Set(context.Background(), "yeet:"+host, BuildSample(), 0).Err() + if err != nil { + return nil, err + } + + return s, nil +} + +func TestS3Redis(t *testing.T) { + s, err := GetRedis() + testza.AssertNil(t, err) + + c, err := cache.NewHashMapCache(s, true) + testza.AssertNil(t, err) + + dir, _ := os.ReadDir("../docs/") + for _, entry := range dir { + file, _ := os.ReadFile("../docs/" + entry.Name()) + + fileType, reader, size := c.Get([]byte("/"+entry.Name()), []byte(host)) + + testza.AssertEqual(t, "text/markdown; charset=utf-8", fileType) + testza.AssertNotNil(t, reader) + testza.AssertEqual(t, len(file), size) + + responseBody, err := io.ReadAll(reader) + + testza.AssertNil(t, err) + testza.AssertNil(t, reader.(io.Closer).Close()) + testza.AssertEqual(t, file, responseBody) + } + + fileType, reader, size := c.Get([]byte("/does-not-exist.md"), []byte(host)) + testza.AssertEqual(t, "", fileType) + testza.AssertNil(t, reader) + testza.AssertEqual(t, 0, size) + + fileType, reader, size = c.Get([]byte("/yeet.md"), []byte("does-not-exist")) + testza.AssertEqual(t, "", fileType) + testza.AssertNil(t, reader) + testza.AssertEqual(t, 0, size) +} + +func BenchmarkS3Redis(b *testing.B) { + s, err := GetRedis() + testza.AssertNil(b, err) + + c, err := cache.NewHashMapCache(s, true) + testza.AssertNil(b, err) + + hostBytes := []byte(host) + path := []byte("/yeet.md") + + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, reader, _ := c.Get(path, hostBytes) + io.ReadAll(reader) + if r, ok := reader.(io.Closer); ok { + r.Close() + } + } + }) +} diff --git a/tools.go b/tools.go index 5cbc8f8..ff62d39 100644 --- a/tools.go +++ b/tools.go @@ -12,6 +12,7 @@ import ( ) //go:generate go run -tags tools tools.go +//go:generate flatc --go -o flat flat/s3.fbs func main() { err := doc.GenMarkdownTree(cmd.RootCMD, "./docs/") diff --git a/utils/utils.go b/utils/utils.go index aa4b38b..46f0c06 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,6 +1,11 @@ package utils -import "fmt" +import ( + "fmt" + "math" + "reflect" + "unsafe" +) func ByteCountToHuman(b int64) string { const unit = 1000 @@ -15,13 +20,16 @@ func ByteCountToHuman(b int64) string { return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPEZY"[exp]) } -/* -function humanFileSize(size) { - if (size < 1024) return size + ' B' - let i = Math.floor(Math.log(size) / Math.log(1024)) - let num = (size / Math.pow(1024, i)) - let round = Math.round(num) - num = round < 10 ? num.toFixed(2) : round < 100 ? num.toFixed(1) : round - return `${num} ${'KMGTPEZY'[i-1]}B` +func EstimateCuckooFilter(elementCount int64) uint { + return uint(math.Ceil(-1 * float64(elementCount) * math.Log(0.03) / math.Pow(math.Log(2), 2))) +} + +func UnsafeGetBytes(s string) []byte { + return (*[0x7fff0000]byte)(unsafe.Pointer( + (*reflect.StringHeader)(unsafe.Pointer(&s)).Data), + )[:len(s):len(s)] +} + +func ByteSliceToString(bs []byte) string { + return *(*string)(unsafe.Pointer(&bs)) } -*/