add uploader and streamer apps, add ristretto cache to session store

README.md

@@ -15,7 +15,7 @@ Uploaded mp4 files are pre-processed, so they could be streamed to dash clients.
 
 Also project uses:
 - PostgreSQL for video object storage and user storage
-- S3 for video data storage
+- S3 compatible storage for video data (compose project uses minio)
 
 ## Development
 

cmd/main.go

@@ -34,7 +34,7 @@ func main() {
 			AccessKey:  "admin",
 			SecretKey:  "password",
 			Bucket:     "vidi",
-			PathPrefix: "tests/",
+			URIPathPrefix: "tests/",
 			SSL:        false,
 			Logger:     logger,
 		})

cmd/streamer/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"os"
+
+	"github.com/adwski/vidi/internal/app/streamer"
+)
+
+func main() {
+	os.Exit(streamer.NewApp().Run())
+}

cmd/uploader/uploader.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"os"
+
+	"github.com/adwski/vidi/internal/app/uploader"
+)
+
+func main() {
+	os.Exit(uploader.NewApp().Run())
+}

cmd/userapi/main.go

@@ -3,7 +3,7 @@ package main
 import (
 	"os"
 
-	"github.com/adwski/vidi/internal/api/app/user"
+	"github.com/adwski/vidi/internal/app/user"
 )
 
 func main() {

cmd/videoapi/main.go

@@ -3,7 +3,7 @@ package main
 import (
 	"os"
 
-	"github.com/adwski/vidi/internal/api/app/video"
+	"github.com/adwski/vidi/internal/app/video"
 )
 
 func main() {

docker/compose/docker-compose.dev.yml

@@ -103,6 +103,56 @@ services:
       - postgres
       - redis
 
+  uploader:
+    logging: *logging
+    restart: unless-stopped
+    build:
+      context: ../..
+      dockerfile: docker/uploader.Dockerfile
+      target: dev
+    environment:
+      VIDI_API_PREFIX: /upload
+      VIDI_REDIS_DSN: redis://redis:6379/0
+      VIDI_VIDEO_API_ENDPOINT: http://videoapi:8080/api/video
+      VIDI_VIDEO_API_TOKEN: token # TODO add token
+      VIDI_SERVER_ADDRESS: ":8080"
+      VIDI_S3_UPLOAD_PREFIX: /upload
+      VIDI_S3_ENDPOINT: minio:9000
+      VIDI_S3_ACCESS_KEY: admin
+      VIDI_S3_SECRET_KEY: password
+      VIDI_S3_BUCKET: vidi
+    expose:
+      - 8080
+    networks:
+      - vidi
+    depends_on:
+      - minio
+      - redis
+
+  streamer:
+    logging: *logging
+    restart: unless-stopped
+    build:
+      context: ../..
+      dockerfile: docker/uploader.Dockerfile
+      target: dev
+    environment:
+      VIDI_API_PREFIX: /watch
+      VIDI_REDIS_DSN: redis://redis:6379/0
+      VIDI_SERVER_ADDRESS: ":8080"
+      VIDI_S3_WATCH_PREFIX: /videos
+      VIDI_S3_ENDPOINT: minio:9000
+      VIDI_S3_ACCESS_KEY: admin
+      VIDI_S3_SECRET_KEY: password
+      VIDI_S3_BUCKET: vidi
+    expose:
+      - 8080
+    networks:
+      - vidi
+    depends_on:
+      - minio
+      - redis
+
 volumes:
   minio-data: {}
   redis-data: {}

docker/compose/nginx.conf

@@ -13,6 +13,8 @@ http {
         # this makes nginx resolve endpoint at runtime
         set $userapi_endpont userapi:8080;
         set $videoapi_endpont videoapi:8080;
+        set $uploader_endpont uploader:8080;
+        set $streamer_endpont streamer:8080;
 
         # userapi location
         location /api/users {
@@ -26,6 +28,18 @@ http {
             break;
         }
 
+        # uploader location
+        location /upload {
+            proxy_pass http://$uploader_endpont;
+            break;
+        }
+
+        # streamer location
+        location /watch {
+            proxy_pass http://$streamer_endpont;
+            break;
+        }
+
         # test location for static files
         location / {
             root /var/www/output;

docker/streamer.Dockerfile

@@ -0,0 +1,31 @@
+FROM golang:1.21.4-bookworm as builder
+
+ENV CGO_ENABLED=0 \
+    GOOS=linux \
+    GO111MODULE=on \
+    GOARCH=amd64 \
+    GOPATH=/go
+
+ADD . /build
+
+WORKDIR /build
+
+RUN go mod download
+RUN <<EOF
+    go build -o streamer -ldflags '-d -w -s' -tags netgo -installsuffix netgo ./cmd/streamer/*.go
+    chmod +x /build/streamer
+EOF
+
+
+FROM builder as dev
+
+ENTRYPOINT ["/build/streamer"]
+
+
+FROM gcr.io/distroless/static as release
+
+WORKDIR /
+USER nonroot:nonroot
+COPY --from=builder /build/streamer /streamer
+
+ENTRYPOINT ["/streamer"]

docker/uploader.Dockerfile

@@ -0,0 +1,31 @@
+FROM golang:1.21.4-bookworm as builder
+
+ENV CGO_ENABLED=0 \
+    GOOS=linux \
+    GO111MODULE=on \
+    GOARCH=amd64 \
+    GOPATH=/go
+
+ADD . /build
+
+WORKDIR /build
+
+RUN go mod download
+RUN <<EOF
+    go build -o uploader -ldflags '-d -w -s' -tags netgo -installsuffix netgo ./cmd/uploader/*.go
+    chmod +x /build/uploader
+EOF
+
+
+FROM builder as dev
+
+ENTRYPOINT ["/build/uploader"]
+
+
+FROM gcr.io/distroless/static as release
+
+WORKDIR /
+USER nonroot:nonroot
+COPY --from=builder /build/uploader /uploader
+
+ENTRYPOINT ["/uploader"]

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Eyevinn/dash-mpd v0.10.0
 	github.com/Eyevinn/mp4ff v0.40.2
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/dgraph-io/ristretto v0.1.1
 	github.com/go-playground/validator/v10 v10.16.0
 	github.com/go-resty/resty/v2 v2.11.0
 	github.com/gofrs/uuid/v5 v5.0.0
@@ -36,6 +37,7 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -57,6 +59,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rs/xid v1.5.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect

go.sum

@@ -16,12 +16,17 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dhui/dktest v0.4.0 h1:z05UmuXZHO/bgj/ds2bGMBu8FI4WA+Ag/m3ghL+om7M=
@@ -34,6 +39,7 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -64,6 +70,8 @@ github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1
 github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-migrate/migrate/v4 v4.17.0 h1:rd40H3QXU0AA4IoLllFcEAEo9dYKRHYND2gB4p7xcaU=
 github.com/golang-migrate/migrate/v4 v4.17.0/go.mod h1:+Cp2mtLP4/aXDTKb9wmXYitdrNx2HGs45rbWAo6OsKM=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -168,6 +176,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ -222,6 +231,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -256,6 +266,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/api/user/auth/auth.go

@@ -77,13 +77,26 @@ func (a *Auth) NewTokenForUser(user *model.User) (string, error) {
 	return signedToken, nil
 }
 
-func (a *Auth) Middleware() echo.MiddlewareFunc {
+func (a *Auth) MiddlewareUser() echo.MiddlewareFunc {
 	return echojwt.WithConfig(echojwt.Config{
 		ContinueOnIgnoredError: false,
 		ContextKey:             SessionContextKey,
 		SigningKey:             a.secret,
 		SigningMethod:          echojwt.AlgorithmHS256,
-		TokenLookup:            "cookie:" + jwtCookieName + ",header:Authorization:Bearer",
+		TokenLookup:            "cookie:" + jwtCookieName,
+		NewClaimsFunc: func(c echo.Context) jwt.Claims {
+			return new(Claims)
+		},
+	})
+}
+
+func (a *Auth) MiddlewareService() echo.MiddlewareFunc {
+	return echojwt.WithConfig(echojwt.Config{
+		ContinueOnIgnoredError: false,
+		ContextKey:             SessionContextKey,
+		SigningKey:             a.secret,
+		SigningMethod:          echojwt.AlgorithmHS256,
+		TokenLookup:            "header:Authorization:Bearer",
 		NewClaimsFunc: func(c echo.Context) jwt.Claims {
 			return new(Claims)
 		},

internal/api/video/model/status.go

@@ -20,8 +20,6 @@ const (
 
 type Status int
 
-var GetStatusName = func(s Status) string { return statusNames[s] }
-
 var statusNames = map[Status]string{
 	VideoStatusError:      "error",
 	VideoStatusCreated:    "created",

internal/api/video/service.go

@@ -100,16 +100,17 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
 	e := middleware.GetEchoWithDefaultMiddleware()
 	// User zone
 	api := e.Group(apiPrefix) // /api/video
-	api.Use(authenticator.Middleware())
 
 	userAPI := api.Group("/user")
+	userAPI.Use(authenticator.MiddlewareUser())
 	userAPI.GET("/:id", svc.getVideo)
 	userAPI.POST("/", svc.createVideo)
 	userAPI.POST("/:id/watch", svc.watchVideo)
 	userAPI.DELETE("/:id", svc.deleteVideo)
 
 	// Service zone
 	serviceAPI := api.Group("/service")
+	serviceAPI.Use(authenticator.MiddlewareService())
 	serviceAPI.PUT("/:id/location/:location", svc.updateVideoLocation)
 	serviceAPI.PUT("/:id/status/:status", svc.updateVideoStatus)
 	serviceAPI.PUT("/:id", svc.updateVideo)