Redis-backed S3, Cuckoo Filters, Bot proxy, TLS

.gitignore

@@ -111,4 +111,5 @@ modules.xml
 # End of https://www.gitignore.io/api/go,jetbrains+all
 
 www/
-dist/
+dist/
+testdata/

README.md

@@ -2,6 +2,16 @@
 
 CLI Usage: [Docs](./docs/yeet.md)
 
+## Features
+
+* Fast
+* 0 setup
+* Local fs support
+* S3 support
+* Redis-backed S3 support
+* Cuckoo filter for Redis-S3
+* File watching (local only)
+
 ## Docker
 
 ```

cache/hashmap.go

@@ -2,6 +2,7 @@ package cache
 
 import (
 	"github.com/Vilsol/yeet/source"
+	"github.com/Vilsol/yeet/utils"
 	"github.com/cornelk/hashmap"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
@@ -61,7 +62,7 @@ func (c *HashMapCache) Index() (int64, error) {
 					c.data.Set(event.CleanPath, instance)
 
 					if viper.GetBool("warmup") {
-						instance.Get(instance)
+						instance.Get(instance, nil)
 					}
 
 					log.Trace().Msgf("Added to cache: %s", event.CleanPath)
@@ -92,11 +93,25 @@ func (c *HashMapCache) Store(path []byte, host []byte, instance *commonInstance)
 func (c *HashMapCache) Get(path []byte, host []byte) (string, io.Reader, int) {
 	if !c.hosts {
 		if instance, ok := c.data.Get(path); ok {
-			return instance.(*commonInstance).Get(instance.(*commonInstance))
+			return instance.(*commonInstance).Get(instance.(*commonInstance), nil)
 		}
 	} else {
-		if instance, ok := c.data.Get(append(host, path...)); ok {
-			return instance.(*commonInstance).Get(instance.(*commonInstance))
+		key := append(host, path...)
+		instance, ok := c.data.Get(key)
+		if !ok {
+			pathStr := utils.ByteSliceToString(path)
+			instance = &commonInstance{
+				Instance: &CachedInstance{
+					RelativePath: pathStr,
+					AbsolutePath: pathStr,
+				},
+				Get: load(c),
+			}
+			c.data.Set(key, instance)
+		}
+
+		if instance != nil {
+			return instance.(*commonInstance).Get(instance.(*commonInstance), host)
 		}
 	}
 

cache/shared.go

@@ -9,9 +9,7 @@ import (
 	"github.com/spf13/viper"
 	"io"
 	"path"
-	"reflect"
 	"time"
-	"unsafe"
 )
 
 func indexBase(c Cache) (int64, error) {
@@ -25,10 +23,11 @@ func indexBase(c Cache) (int64, error) {
 		}
 
 		// Host is not used when indexing is supported
-		c.Store(unsafeGetBytes(cleanedPath), nil, instance)
+		c.Store(utils.UnsafeGetBytes(cleanedPath), nil, instance)
 
 		if viper.GetBool("warmup") {
-			instance.Get(instance)
+			// Host is not used when indexing is supported
+			instance.Get(instance, nil)
 			return int64(len(instance.Instance.Data))
 		}
 
@@ -58,17 +57,21 @@ func index(source source.Source, f source.IndexFunc) (int64, error) {
 	return totalSize, nil
 }
 
-func load(c Cache) func(*commonInstance) (string, io.Reader, int) {
-	return func(instance *commonInstance) (string, io.Reader, int) {
-		hijacker := c.Source().Get(instance.Instance.AbsolutePath, nil)
+func load(c Cache) func(*commonInstance, []byte) (string, io.Reader, int) {
+	return func(instance *commonInstance, host []byte) (string, io.Reader, int) {
+		hijacker := c.Source().Get(instance.Instance.AbsolutePath, host)
+
+		if hijacker == nil {
+			return "", nil, 0
+		}
 
 		log.Debug().Msgf("Loaded file [%d][%s]: %s", hijacker.Size, hijacker.FileType(), instance.Instance.AbsolutePath)
 
 		hijacker.OnClose = func(h *utils.StreamHijacker) {
 			instance.Instance.LoadTime = time.Now()
 			instance.Instance.Data = hijacker.Buffer
 			instance.Instance.ContentType = hijacker.FileType()
-			instance.Get = func(cache *commonInstance) (string, io.Reader, int) {
+			instance.Get = func(cache *commonInstance, _ []byte) (string, io.Reader, int) {
 				return cache.Instance.ContentType, bytes.NewReader(cache.Instance.Data), len(cache.Instance.Data)
 			}
 		}
@@ -94,13 +97,3 @@ func expiry(c Cache) {
 		}
 	}(c)
 }
-
-func unsafeGetBytes(s string) []byte {
-	return (*[0x7fff0000]byte)(unsafe.Pointer(
-		(*reflect.StringHeader)(unsafe.Pointer(&s)).Data),
-	)[:len(s):len(s)]
-}
-
-func byteSliceToString(bs []byte) string {
-	return *(*string)(unsafe.Pointer(&bs))
-}

cache/cache.go → cache/types.go

@@ -16,7 +16,7 @@ type CachedInstance struct {
 
 type commonInstance struct {
 	Instance *CachedInstance
-	Get      func(instance *commonInstance) (string, io.Reader, int)
+	Get      func(instance *commonInstance, host []byte) (string, io.Reader, int)
 }
 
 type KeyValue struct {

cmd/serve.go

@@ -7,6 +7,8 @@ import (
 	"time"
 )
 
+const botAgents = "bot|crawl|spider|external|meta|scrap|archive|discourse"
+
 func init() {
 	ServeCMD.PersistentFlags().String("host", "", "Hostname to bind the webserver")
 	ServeCMD.PersistentFlags().Int("port", 8080, "Port to run the webserver on")
@@ -22,6 +24,12 @@ func init() {
 	ServeCMD.PersistentFlags().StringSliceP("paths", "p", []string{"./www"}, "Paths to serve on the webserver")
 	ServeCMD.PersistentFlags().BoolP("watch", "w", false, "Watch filesystem for changes")
 
+	ServeCMD.PersistentFlags().String("tls-cert", "", "TLS Certificate file path")
+	ServeCMD.PersistentFlags().String("tls-key", "", "TLS Key file path")
+
+	ServeCMD.PersistentFlags().String("bot-proxy", "", "Bot proxy URL")
+	ServeCMD.PersistentFlags().String("bot-agents", botAgents, "Bot User-Agent header regex")
+
 	_ = viper.BindPFlag("paths", ServeCMD.PersistentFlags().Lookup("paths"))
 	_ = viper.BindPFlag("watch", ServeCMD.PersistentFlags().Lookup("watch"))
 
@@ -36,6 +44,12 @@ func init() {
 
 	_ = viper.BindPFlag("index.file", ServeCMD.PersistentFlags().Lookup("index-file"))
 
+	_ = viper.BindPFlag("tls.cert", ServeCMD.PersistentFlags().Lookup("tls-cert"))
+	_ = viper.BindPFlag("tls.key", ServeCMD.PersistentFlags().Lookup("tls-key"))
+
+	_ = viper.BindPFlag("bot.proxy", ServeCMD.PersistentFlags().Lookup("bot-proxy"))
+	_ = viper.BindPFlag("bot.agents", ServeCMD.PersistentFlags().Lookup("bot-agents"))
+
 	RootCMD.AddCommand(ServeCMD)
 }
 

cmd/serve/s3_redis.go

@@ -0,0 +1,63 @@
+package serve
+
+import (
+	"github.com/Vilsol/yeet/cache"
+	"github.com/Vilsol/yeet/cmd"
+	"github.com/Vilsol/yeet/server"
+	"github.com/Vilsol/yeet/source"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func init() {
+	s3RedisCmd.Flags().String("network", "tcp", "The network type, either tcp or unix")
+	s3RedisCmd.Flags().StringP("address", "a", "localhost:6379", "host:port address of Redis")
+	s3RedisCmd.Flags().String("user", "", "Username of Redis")
+	s3RedisCmd.Flags().String("pass", "", "Password of Redis")
+	s3RedisCmd.Flags().Int("db", 0, "DB of Redis")
+
+	_ = s3RedisCmd.MarkFlagRequired("address")
+
+	_ = viper.BindPFlag("network", s3RedisCmd.Flags().Lookup("network"))
+	_ = viper.BindPFlag("address", s3RedisCmd.Flags().Lookup("address"))
+	_ = viper.BindPFlag("username", s3RedisCmd.Flags().Lookup("username"))
+	_ = viper.BindPFlag("password", s3RedisCmd.Flags().Lookup("password"))
+	_ = viper.BindPFlag("db", s3RedisCmd.Flags().Lookup("db"))
+
+	cmd.ServeCMD.AddCommand(s3RedisCmd)
+}
+
+var s3RedisCmd = &cobra.Command{
+	Use:   "s3-redis",
+	Short: "Serve Redis-backed S3 buckets",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if viper.GetBool("watch") {
+			return errors.New("watch is not supported for s3-redis")
+		}
+
+		src, err := source.NewS3Redis(
+			viper.GetString("network"),
+			viper.GetString("address"),
+			viper.GetString("username"),
+			viper.GetString("password"),
+			viper.GetInt("db"),
+		)
+
+		if err != nil {
+			return err
+		}
+
+		c, err := cache.NewHashMapCache(src, true)
+
+		if err != nil {
+			return err
+		}
+
+		if _, err := c.Index(); err != nil {
+			return err
+		}
+
+		return server.Run(c)
+	},
+}

docs/yeet.md

@@ -15,4 +15,4 @@ yeet is an in-memory indexed static file webserver
 
 * [yeet serve](yeet_serve.md)	 - Serve files with yeet
 
-###### Auto generated by spf13/cobra on 5-Jan-2022
+###### Auto generated by spf13/cobra on 7-Jan-2022

docs/yeet_serve.md

@@ -5,6 +5,8 @@ Serve files with yeet
 ### Options
 
 ```
+      --bot-agents string          Bot User-Agent header regex (default "bot|crawl|spider|external|meta|scrap|archive|discourse")
+      --bot-proxy string           Bot proxy URL
       --expiry                     Use cache expiry
       --expiry-interval duration   Interval between cache GC's (default 10m0s)
       --expiry-time duration       Lifetime of a cache entry (default 1h0m0s)
@@ -13,6 +15,8 @@ Serve files with yeet
       --index-file string          The directory default index file (default "index.html")
   -p, --paths strings              Paths to serve on the webserver (default [./www])
       --port int                   Port to run the webserver on (default 8080)
+      --tls-cert string            TLS Certificate file path
+      --tls-key string             TLS Key file path
       --warmup                     Load all files into memory on startup
   -w, --watch                      Watch filesystem for changes
 ```
@@ -30,5 +34,6 @@ Serve files with yeet
 * [yeet](yeet.md)	 - yeet is an in-memory indexed static file webserver
 * [yeet serve local](yeet_serve_local.md)	 - Serve a local directory
 * [yeet serve s3](yeet_serve_s3.md)	 - Serve an S3 bucket
+* [yeet serve s3-redis](yeet_serve_s3-redis.md)	 - Serve Redis-backed S3 buckets
 
-###### Auto generated by spf13/cobra on 5-Jan-2022
+###### Auto generated by spf13/cobra on 7-Jan-2022

docs/yeet_serve_local.md

@@ -15,6 +15,8 @@ yeet serve local [flags]
 ### Options inherited from parent commands
 
 ```
+      --bot-agents string          Bot User-Agent header regex (default "bot|crawl|spider|external|meta|scrap|archive|discourse")
+      --bot-proxy string           Bot proxy URL
       --expiry                     Use cache expiry
       --expiry-interval duration   Interval between cache GC's (default 10m0s)
       --expiry-time duration       Lifetime of a cache entry (default 1h0m0s)
@@ -25,6 +27,8 @@ yeet serve local [flags]
   -p, --paths strings              Paths to serve on the webserver (default [./www])
       --port int                   Port to run the webserver on (default 8080)
       --quiet                      Do not log anything to console
+      --tls-cert string            TLS Certificate file path
+      --tls-key string             TLS Key file path
       --warmup                     Load all files into memory on startup
   -w, --watch                      Watch filesystem for changes
 ```
@@ -33,4 +37,4 @@ yeet serve local [flags]
 
 * [yeet serve](yeet_serve.md)	 - Serve files with yeet
 
-###### Auto generated by spf13/cobra on 5-Jan-2022
+###### Auto generated by spf13/cobra on 7-Jan-2022

docs/yeet_serve_s3-redis.md

@@ -0,0 +1,45 @@
+## yeet serve s3-redis
+
+Serve Redis-backed S3 buckets
+
+```
+yeet serve s3-redis [flags]
+```
+
+### Options
+
+```
+  -a, --address string   host:port address of Redis (default "localhost:6379")
+      --db int           DB of Redis
+  -h, --help             help for s3-redis
+      --network string   The network type, either tcp or unix (default "tcp")
+      --pass string      Password of Redis
+      --user string      Username of Redis
+```
+
+### Options inherited from parent commands
+
+```
+      --bot-agents string          Bot User-Agent header regex (default "bot|crawl|spider|external|meta|scrap|archive|discourse")
+      --bot-proxy string           Bot proxy URL
+      --expiry                     Use cache expiry
+      --expiry-interval duration   Interval between cache GC's (default 10m0s)
+      --expiry-time duration       Lifetime of a cache entry (default 1h0m0s)
+      --host string                Hostname to bind the webserver
+      --index-file string          The directory default index file (default "index.html")
+      --log string                 The log level to output (default "info")
+      --log-file string            File to output logs to
+  -p, --paths strings              Paths to serve on the webserver (default [./www])
+      --port int                   Port to run the webserver on (default 8080)
+      --quiet                      Do not log anything to console
+      --tls-cert string            TLS Certificate file path
+      --tls-key string             TLS Key file path
+      --warmup                     Load all files into memory on startup
+  -w, --watch                      Watch filesystem for changes
+```
+
+### SEE ALSO
+
+* [yeet serve](yeet_serve.md)	 - Serve files with yeet
+
+###### Auto generated by spf13/cobra on 7-Jan-2022

docs/yeet_serve_s3.md

@@ -20,6 +20,8 @@ yeet serve s3 [flags]
 ### Options inherited from parent commands
 
 ```
+      --bot-agents string          Bot User-Agent header regex (default "bot|crawl|spider|external|meta|scrap|archive|discourse")
+      --bot-proxy string           Bot proxy URL
       --expiry                     Use cache expiry
       --expiry-interval duration   Interval between cache GC's (default 10m0s)
       --expiry-time duration       Lifetime of a cache entry (default 1h0m0s)
@@ -30,6 +32,8 @@ yeet serve s3 [flags]
   -p, --paths strings              Paths to serve on the webserver (default [./www])
       --port int                   Port to run the webserver on (default 8080)
       --quiet                      Do not log anything to console
+      --tls-cert string            TLS Certificate file path
+      --tls-key string             TLS Key file path
       --warmup                     Load all files into memory on startup
   -w, --watch                      Watch filesystem for changes
 ```
@@ -38,4 +42,4 @@ yeet serve s3 [flags]
 
 * [yeet serve](yeet_serve.md)	 - Serve files with yeet
 
-###### Auto generated by spf13/cobra on 5-Jan-2022
+###### Auto generated by spf13/cobra on 7-Jan-2022

flat/s3.fbs

@@ -0,0 +1,10 @@
+namespace yeet;
+
+table S3 {
+    filter: string;
+    bucket: string;
+    key: string;
+    secret: string;
+    endpoint: string;
+    region: string;
+}