From 3aca86885924055b43a935a9d320ec74a8eb7104 Mon Sep 17 00:00:00 2001 From: Warren Nelson Date: Tue, 20 Feb 2018 23:53:32 -0500 Subject: [PATCH] Implement server /photos routing, GET/POST functionality (#11) * Add thumbnail functionality (#12) * Restructured server, added exif parsing, added fields to Photo * Fix exif value stripping of null character * Add IDs only response option * Pull Hotshots data dir from environment if exists --- .gitignore | 1 + config/config.go | 31 ++++++- server/helpers.go | 36 ++++++++ server/photo.go | 199 +++++++++++++++++++++++++++++++++++++++++++ server/request.go | 206 +++++++++++++++++++++++++++++++++++++++++++++ server/server.go | 67 ++++++++++++++- vendor/vendor.json | 96 +++++++++++++++++++++ 7 files changed, 631 insertions(+), 5 deletions(-) create mode 100644 server/helpers.go create mode 100644 server/photo.go create mode 100644 server/request.go diff --git a/.gitignore b/.gitignore index e96a8a5..e7990b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /vendor/*/ /hotshots +.idea diff --git a/config/config.go b/config/config.go index 5ca144b..df34505 100644 --- a/config/config.go +++ b/config/config.go @@ -1,11 +1,34 @@ package config +import "os" +import "path" + type Config struct { - ListenURL string + ListenURL string + PhotosDirectory string } func New() (*Config, error) { - return &Config{ - ListenURL: "127.0.0.1:8000", - }, nil + c := &Config{ + ListenURL: "127.0.0.1:8000", + PhotosDirectory: "/var/hotshots", + } + + dir, ok := os.LookupEnv("HOTSHOTS_DIR") + if ok { + c.PhotosDirectory = dir + } + + return c, nil +} + +func (c *Config) ImgFolder() string { + return path.Join(c.PhotosDirectory, "/img") +} +func (c *Config) ConfFolder() string { + return path.Join(c.PhotosDirectory, "/conf.d") +} + +func (c *Config) StormFile() string { + return path.Join(c.ConfFolder(), "/hotshot.db") } diff --git a/server/helpers.go b/server/helpers.go new file mode 100644 index 0000000..543f3d0 --- /dev/null +++ b/server/helpers.go @@ -0,0 +1,36 @@ +package server + +import ( + "encoding/json" + "math" + "net/http" +) + +func round(num float64) int { + return int(num + math.Copysign(0.5, num)) +} + +func toFixed(num float64, precision int) float64 { + output := math.Pow(10, float64(precision)) + return float64(round(num*output)) / output +} + +func WriteError(s string, status int, w http.ResponseWriter) { + w.WriteHeader(status) + v := ErrorResponse{ + Success: false, + Error: s, + } + WriteJsonResponse(v, w) +} + +func WriteJsonResponse(v interface{}, w http.ResponseWriter) { + js, err := json.Marshal(v) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(js) +} diff --git a/server/photo.go b/server/photo.go new file mode 100644 index 0000000..4ddfde0 --- /dev/null +++ b/server/photo.go @@ -0,0 +1,199 @@ +package server + +import ( + "crypto/sha1" + "errors" + "fmt" + "image" + "image/jpeg" + "io" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/kochman/hotshots/log" + "github.com/nfnt/resize" + "github.com/rwcarlsen/goexif/exif" + "github.com/rwcarlsen/goexif/mknote" +) + +/* + * Constants + */ + +const MaxWidth = 256 +const MaxHeight = 256 + +/* + * Data structs + */ + +type Photo struct { + ID string `storm:"id"` + UploadedAt time.Time `storm:"index"` + TakenAt time.Time `storm:"index"` + Width int `storm:"index"` + Height int `storm:"index"` + Megapixels float64 `storm:"index"` + Lat float64 + Long float64 + CamSerial string `storm:"index"` + CamMake string `storm:"index"` + CamModel string `storm:"index"` +} + +func NewPhoto(id string, r image.Rectangle, x exif.Exif) *Photo { + taken, err := x.DateTime() + if err != nil { + log.Error("Unable to find time for ", id) + } + + lat, long, err := x.LatLong() + if err != nil { + log.Info("Unable to find gps for ", id) + } + + cserial, err := x.Get(mknote.SerialNumber) + if err != nil { + log.Info("Unable to find serial for ", id) + } + + cmake, err := x.Get(exif.Make) + if err != nil { + log.Info("Unable to find make for ", id) + } + + cmodel, err := x.Get(exif.Model) + if err != nil { + log.Info("Unable to find model for ", id) + } + + mp := toFixed(float64(r.Dx())*float64(r.Dy())/1000000.0, 1) + + return &Photo{ + ID: id, + UploadedAt: time.Now(), + TakenAt: taken, + Width: r.Dx(), + Height: r.Dy(), + Megapixels: mp, + Lat: lat, + Long: long, + CamSerial: strings.TrimSuffix(string(cserial.Val), "\u0000"), + CamMake: strings.TrimSuffix(string(cmake.Val), "\u0000"), + CamModel: strings.TrimSuffix(string(cmodel.Val), "\u0000"), + } +} + +func ProcessPhoto(input io.Reader, id string, photoPath string, thumbPath string) (*exif.Exif, *image.Rectangle, error) { + saveErr := make(chan error) + defer close(saveErr) + + photor, photow := io.Pipe() + thumbr, thumbw := io.Pipe() + exifr, exifw := io.Pipe() + defer photor.Close() + defer thumbr.Close() + defer exifr.Close() + + go func() { + defer photow.Close() + defer thumbw.Close() + defer exifw.Close() + + mw := io.MultiWriter(photow, thumbw, exifw) + + if _, err := io.Copy(mw, input); err != nil { + log.Error(err) + return + } + + }() + + log.Info(fmt.Sprint("Saving image ", id)) + + var xif exif.Exif + var rect image.Rectangle + go SaveImage(photor, photoPath, saveErr) + go SaveThumb(&rect, thumbr, thumbPath, saveErr) + go GetExif(&xif, exifr, saveErr) + + wasErrorSaving := false + for saveProcesses := 0; saveProcesses < 3; saveProcesses++ { + err := <-saveErr + if err != nil { + log.Error(err) + wasErrorSaving = true + } + } + + if wasErrorSaving { + return nil, nil, errors.New("Unable to complete one of the photo processes") + } + + return &xif, &rect, nil +} + +func SaveImage(data io.Reader, path string, saveErr chan error) { + output, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0660) + if err != nil { + log.Error(err) + saveErr <- err + return + } + defer output.Close() + + if _, err := io.Copy(output, data); err != nil { + log.Error(err) + saveErr <- err + return + } + + saveErr <- nil +} + +func SaveThumb(r *image.Rectangle, data io.Reader, path string, saveErr chan error) { + img, err := jpeg.Decode(data) + io.Copy(ioutil.Discard, data) + if err != nil { + log.Error(err) + saveErr <- err + return + } + *r = img.Bounds() + resizedImg := resize.Thumbnail(MaxWidth, MaxHeight, img, resize.Bicubic) + output, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0660) + if err != nil { + log.Error(err) + saveErr <- err + return + } + defer output.Close() + + jpeg.Encode(output, resizedImg, nil) + + saveErr <- nil +} + +func GetExif(out *exif.Exif, data io.Reader, saveErr chan error) { + x, err := exif.Decode(data) + io.Copy(ioutil.Discard, data) + if err != nil { + log.Error(err) + saveErr <- err + return + } + *out = *x + + saveErr <- nil +} + +func GenPhotoID(f io.ReadSeeker) (string, error) { + digest := sha1.New() + if _, err := io.Copy(digest, f); err != nil { + return "", err + } + f.Seek(0, io.SeekStart) + return fmt.Sprintf("%x", digest.Sum(nil)), nil +} diff --git a/server/request.go b/server/request.go new file mode 100644 index 0000000..ebca751 --- /dev/null +++ b/server/request.go @@ -0,0 +1,206 @@ +package server + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path" + + "github.com/asdine/storm" + "github.com/go-chi/chi" + "github.com/kochman/hotshots/log" +) + +/* + * Response Structs + */ + +type GetPhotoIDsResponse struct { + Success bool + IDs []string +} + +type GetPhotosResponse struct { + Success bool + Photos []Photo +} + +type PostPhotoResponse struct { + Success bool + NewID string `json:",omitempty"` +} + +type GetPhotoMetadataResponse struct { + Success bool + Photo Photo +} + +type ErrorResponse struct { + Success bool + Error string +} + +type NotFoundResponse struct { + Success bool + Error string + URI string +} + +/* + * Handlers + */ + +func NotFound(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + v := NotFoundResponse{ + Success: false, + Error: "Requested URI not found", + URI: r.RequestURI, + } + WriteJsonResponse(v, w) +} + +func (s *Server) PhotoCtx(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + photoID := chi.URLParam(r, "pid") + var photo Photo + if err := s.db.One("ID", photoID, &photo); err != nil { + log.Info(err) + WriteError("Unable to find photo id", 404, w) + return + } + ctx := context.WithValue(r.Context(), "photo", photo) + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func (s *Server) GetPhotos(w http.ResponseWriter, r *http.Request) { + var photos []Photo + if err := s.db.All(&photos); err != nil { + log.Error(err) + WriteError("Unable to query photos", 500, w) + return + } + + v := GetPhotosResponse{ + Success: true, + Photos: photos, + } + WriteJsonResponse(v, w) +} + +func (s *Server) GetPhotoIDs(w http.ResponseWriter, r *http.Request) { + var photos []Photo + if err := s.db.All(&photos); err != nil { + log.Error(err) + WriteError("Unable to query photos", 500, w) + return + } + ids := make([]string, len(photos)) + for i, photo := range photos { + ids[i] = photo.ID + } + + v := GetPhotoIDsResponse{ + Success: true, + IDs: ids, + } + WriteJsonResponse(v, w) +} + +func (s *Server) PostPhoto(w http.ResponseWriter, r *http.Request) { + r.ParseMultipartForm(1 << 23) // 8M max memory + + input, _, err := r.FormFile("photo") + if err != nil { + log.Info(err) + WriteError("Unable to parse form value 'photo'", 400, w) + return + } + defer input.Close() + + id, err := GenPhotoID(input) + if err != nil { + log.Error(err) + WriteError("Unable to generate photo ID", 500, w) + return + } + + var existingPhoto Photo + if err := s.db.One("ID", id, &existingPhoto); err == nil { + log.Info(fmt.Sprintf("Photo %s already exists", id)) + WriteError("Photo already exists", 400, w) + return + } else if err != storm.ErrNotFound { + log.Error(err) + WriteError("Unable to query database", 500, w) + return + } + + photoPath := path.Join(s.cfg.ImgFolder(), fmt.Sprintf("%s.jpg", id)) + thumbPath := path.Join(s.cfg.ImgFolder(), fmt.Sprintf("%s-thumb.jpg", id)) + + xif, rect, err := ProcessPhoto(input, id, photoPath, thumbPath) + if err != nil { + log.Error("Failed to save image") + WriteError("Unable to save image", 500, w) + + os.Remove(photoPath) + os.Remove(thumbPath) + return + } + + newPhoto := NewPhoto(id, *rect, *xif) + + if err := s.db.Save(newPhoto); err != nil { + log.Error(err) + WriteError("Unable to save to database", 500, w) + + os.Remove(photoPath) + os.Remove(thumbPath) + return + } + + v := PostPhotoResponse{ + Success: true, + NewID: id, + } + WriteJsonResponse(v, w) +} + +func (s *Server) GetPhotoMetadata(w http.ResponseWriter, r *http.Request) { + v := GetPhotoMetadataResponse{ + Photo: r.Context().Value("photo").(Photo), + Success: true, + } + WriteJsonResponse(v, w) +} + +func (s *Server) GetPhoto(w http.ResponseWriter, r *http.Request) { + s.GetImage("%s.jpg", w, r) +} + +func (s *Server) GetThumbnail(w http.ResponseWriter, r *http.Request) { + s.GetImage("%s-thumb.jpg", w, r) +} + +func (s *Server) GetImage(imageFormat string, w http.ResponseWriter, r *http.Request) { + photo := r.Context().Value("photo").(Photo) + photoPath := path.Join(s.cfg.ImgFolder(), fmt.Sprintf(imageFormat, photo.ID)) + output, err := os.OpenFile(photoPath, os.O_RDONLY, 0660) + if err != nil { + log.Error(err) + WriteError("Unable to access internal storage of image", 500, w) + return + } + defer output.Close() + + w.Header().Set("Content-Type", "image/jpeg") + if _, err := io.Copy(w, output); err != nil { + log.Error(err) + WriteError("Unable to return image data", 500, w) + return + } +} diff --git a/server/server.go b/server/server.go index aa840f0..14ba10a 100644 --- a/server/server.go +++ b/server/server.go @@ -1,16 +1,26 @@ package server import ( + "errors" + "fmt" "net/http" + "os" + "github.com/asdine/storm" "github.com/go-chi/chi" - "github.com/kochman/hotshots/config" "github.com/kochman/hotshots/log" + "github.com/rwcarlsen/goexif/exif" + "github.com/rwcarlsen/goexif/mknote" + "golang.org/x/sys/unix" ) +/* + * Server struct + */ type Server struct { cfg config.Config + db *storm.DB handler http.Handler } @@ -19,11 +29,66 @@ func New(cfg *config.Config) (*Server, error) { cfg: *cfg, } + if err := s.Setup(); err != nil { + return nil, err + } + router := chi.NewRouter() + router.NotFound(NotFound) + router.Route("/photos", func(router chi.Router) { + router.Get("/", s.GetPhotos) + router.Post("/", s.PostPhoto) + router.Get("/ids", s.GetPhotoIDs) + router.Route("/{pid}", func(router chi.Router) { + router.Use(s.PhotoCtx) + router.Get("/image.jpg", s.GetPhoto) + router.Get("/thumb.jpg", s.GetThumbnail) + router.Get("/meta", s.GetPhotoMetadata) + }) + }) + s.handler = router return s, nil } +func (s *Server) Setup() error { + if !CanAccessDirectory(s) { + return errors.New(fmt.Sprintf("Directory %s not accessible", s.cfg.PhotosDirectory)) + } + + if _, err := os.Stat(s.cfg.ImgFolder()); err != nil { + os.Mkdir(s.cfg.ImgFolder(), 0775) + } + + if _, err := os.Stat(s.cfg.ConfFolder()); err != nil { + os.Mkdir(s.cfg.ConfFolder(), 0775) + } + + db, err := storm.Open(s.cfg.StormFile()) + if err != nil { + return err + } + s.db = db + + if err := s.db.Init(&Photo{}); err != nil { + return err + } + + exif.RegisterParsers(mknote.All...) + + return nil +} + +func CanAccessDirectory(serv *Server) bool { + stat, statErr := os.Stat(serv.cfg.PhotosDirectory) + if statErr != nil || !stat.IsDir() { + return false + } + readErr := unix.Access(serv.cfg.PhotosDirectory, unix.R_OK) + writeErr := unix.Access(serv.cfg.PhotosDirectory, unix.W_OK) + return readErr == nil && writeErr == nil +} + func (s *Server) Run() { if err := http.ListenAndServe(s.cfg.ListenURL, s.handler); err != nil { log.WithError(err).Error("Unable to serve") diff --git a/vendor/vendor.json b/vendor/vendor.json index 79082c1..a5c63fc 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -2,6 +2,48 @@ "comment": "", "ignore": "test", "package": [ + { + "checksumSHA1": "203+irJgy8JuQ66lCQXIFyhEe6A=", + "path": "github.com/asdine/storm", + "revision": "68fc73b635f890fe7ba2f3b15ce80c85b28a744f", + "revisionTime": "2018-01-06T14:43:59Z" + }, + { + "checksumSHA1": "KXHP3AdJVnDxab4J2mDknFi8SPQ=", + "path": "github.com/asdine/storm/codec", + "revision": "68fc73b635f890fe7ba2f3b15ce80c85b28a744f", + "revisionTime": "2018-01-06T14:43:59Z" + }, + { + "checksumSHA1": "8B9a1JXYnmmzo353mN/z9MgKa2M=", + "path": "github.com/asdine/storm/codec/json", + "revision": "68fc73b635f890fe7ba2f3b15ce80c85b28a744f", + "revisionTime": "2018-01-06T14:43:59Z" + }, + { + "checksumSHA1": "UohCA0ZSk6DO4wYTORS0NE58j1w=", + "path": "github.com/asdine/storm/index", + "revision": "68fc73b635f890fe7ba2f3b15ce80c85b28a744f", + "revisionTime": "2018-01-06T14:43:59Z" + }, + { + "checksumSHA1": "9EciDShF1A0nvE8x6FRw5Re9kzo=", + "path": "github.com/asdine/storm/internal", + "revision": "68fc73b635f890fe7ba2f3b15ce80c85b28a744f", + "revisionTime": "2018-01-06T14:43:59Z" + }, + { + "checksumSHA1": "xMrnqVXOk2L7vjtsMYfE73n+tKM=", + "path": "github.com/asdine/storm/q", + "revision": "68fc73b635f890fe7ba2f3b15ce80c85b28a744f", + "revisionTime": "2018-01-06T14:43:59Z" + }, + { + "checksumSHA1": "GeAYaNTJ/DTk2MGuzXHUAArWLD0=", + "path": "github.com/coreos/bbolt", + "revision": "b44cfbde695bad1a19cc09cf00ffb217ce98f038", + "revisionTime": "2018-02-14T19:29:54Z" + }, { "checksumSHA1": "YODbzvhUr0Tzp2/MkXflXph2hdY=", "path": "github.com/go-chi/chi", @@ -14,6 +56,42 @@ "revision": "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75", "revisionTime": "2014-10-17T20:07:13Z" }, + { + "checksumSHA1": "apvoHcksrwebj02FX32DEURlzlE=", + "path": "github.com/kochman/runner", + "revision": "307fc031d7792ec0c307f69a1d360c6f37650a7f", + "revisionTime": "2017-08-14T05:04:56Z" + }, + { + "checksumSHA1": "r5eQHkttko6kxroDEENXbmXKrSs=", + "path": "github.com/nfnt/resize", + "revision": "891127d8d1b52734debe1b3c3d7e747502b6c366", + "revisionTime": "2016-07-24T20:39:20Z" + }, + { + "checksumSHA1": "Q68FxXX04PtPDIG7C1RlSDhx/ko=", + "path": "github.com/rwcarlsen/goexif/exif", + "revision": "17202558c8d9c3fd047859f1a5e73fd9ae709187", + "revisionTime": "2018-01-10T18:11:40Z" + }, + { + "checksumSHA1": "smHMy/7tOLNYD7ghqesrqslA6Jg=", + "path": "github.com/rwcarlsen/goexif/mknote", + "revision": "17202558c8d9c3fd047859f1a5e73fd9ae709187", + "revisionTime": "2018-01-10T18:11:40Z" + }, + { + "checksumSHA1": "0+tTLlssYWyGuq+vQs4IiPIXJW4=", + "path": "github.com/rwcarlsen/goexif/tiff", + "revision": "17202558c8d9c3fd047859f1a5e73fd9ae709187", + "revisionTime": "2018-01-10T18:11:40Z" + }, + { + "checksumSHA1": "9y/iQh3/YtX31wAu7eJravtjTro=", + "path": "github.com/sirupsen/logrus", + "revision": "8c0189d9f6bbf301e5d055d34268156b317016af", + "revisionTime": "2018-02-13T14:31:10Z" + }, { "checksumSHA1": "bk8AIaMyTeY7CW85Zg2HDrZVSo0=", "path": "github.com/spf13/cobra", @@ -25,6 +103,24 @@ "path": "github.com/spf13/pflag", "revision": "6a877ebacf28c5fc79846f4fcd380a5d9872b997", "revisionTime": "2018-02-08T21:53:15Z" + }, + { + "checksumSHA1": "6U7dCaxxIMjf5V02iWgyAwppczw=", + "path": "golang.org/x/crypto/ssh/terminal", + "revision": "650f4a345ab4e5b245a3034b110ebc7299e68186", + "revisionTime": "2017-09-27T09:16:38Z" + }, + { + "checksumSHA1": "osb18zDjd7/RMAMUuN3qP+w0ewE=", + "path": "golang.org/x/sys/unix", + "revision": "37707fdb30a5b38865cfb95e5aab41707daec7fd", + "revisionTime": "2018-02-02T13:35:31Z" + }, + { + "checksumSHA1": "eQq+ZoTWPjyizS9XalhZwfGjQao=", + "path": "golang.org/x/sys/windows", + "revision": "37707fdb30a5b38865cfb95e5aab41707daec7fd", + "revisionTime": "2018-02-02T13:35:31Z" } ], "rootPath": "github.com/kochman/hotshots"