From f548b2c31cafc7f5f37e33310e4bb38f7608d7ed Mon Sep 17 00:00:00 2001 From: celogeek Date: Sun, 13 Feb 2022 18:51:38 +0100 Subject: [PATCH] prepare file upload --- .gitignore | 1 + cmd/photos-api/main.go | 74 ++++++++---------------------- internal/env/main.go | 72 +++++++++++++++++++++++++++++ internal/photos/api/album.go | 57 ----------------------- internal/photos/api/db.go | 30 ++++++++----- internal/photos/api/errors.go | 6 +++ internal/photos/api/file.go | 80 +++++++++++++++++++++++++++++++++ internal/photos/api/main.go | 35 ++++++--------- internal/photos/api/photo.go | 1 - internal/photos/models/album.go | 44 ------------------ internal/photos/models/file.go | 13 ++++++ internal/photos/models/photo.go | 15 ------- 12 files changed, 224 insertions(+), 204 deletions(-) create mode 100644 .gitignore create mode 100644 internal/env/main.go delete mode 100644 internal/photos/api/album.go create mode 100644 internal/photos/api/file.go delete mode 100644 internal/photos/api/photo.go delete mode 100644 internal/photos/models/album.go create mode 100644 internal/photos/models/file.go delete mode 100644 internal/photos/models/photo.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1269488 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +data diff --git a/cmd/photos-api/main.go b/cmd/photos-api/main.go index 8f572da..06b6f2e 100644 --- a/cmd/photos-api/main.go +++ b/cmd/photos-api/main.go @@ -4,83 +4,47 @@ import ( "flag" "fmt" "log" - "os" - "sort" - "strconv" - gomysql "github.com/go-sql-driver/mysql" + "github.com/go-sql-driver/mysql" + "gitlab.celogeek.com/photos/api/internal/env" "gitlab.celogeek.com/photos/api/internal/photos/api" ) var ( - mapEnv = map[string]string{} - mapCurrentEnv = map[string]string{} - listen = fmt.Sprintf("%s:%d", getEnv("LISTEN_HOST", "[::]"), getEnvUInt("LISTEN_PORT", 8000)) - mysqlAddr = fmt.Sprintf("%s:%d", getEnv("MYSQL_HOST", "localhost"), getEnvUInt("MYSQL_PORT", 3306)) - mysqlUser = getEnv("MYSQL_USER", "photos") - mysqlPassword = getEnv("MYSQL_PASSWD", "photos") - mysqlDatabase = getEnv("MYSQL_DB", "photos") + genv = env.New() + listen = fmt.Sprintf("%s:%d", genv.Get("LISTEN_HOST", "[::]"), genv.GetUInt("LISTEN_PORT", 8000)) + mysqlAddr = fmt.Sprintf("%s:%d", genv.Get("MYSQL_HOST", "localhost"), genv.GetUInt("MYSQL_PORT", 3306)) + mysqlUser = genv.Get("MYSQL_USER", "photos") + mysqlPassword = genv.Get("MYSQL_PASSWD", "photos") + mysqlDatabase = genv.Get("MYSQL_DB", "photos") getEnvConfig = false getCurrentEnvConfig = false + storePath = genv.Get("STORE_PATH", "./data") ) -func getEnv(key, fallback string) string { - mapEnv[key] = fallback - mapCurrentEnv[key] = fallback - if value, ok := os.LookupEnv(key); ok { - mapCurrentEnv[key] = value - return value - } - return fallback -} -func getEnvUInt(key string, fallback uint) uint { - mapEnv[key] = fmt.Sprint(fallback) - mapCurrentEnv[key] = fmt.Sprint(fallback) - if value, ok := os.LookupEnv(key); ok { - uvalue, err := strconv.Atoi(value) - if err != nil { - log.Fatal(err) - } - if uvalue <= 0 { - log.Fatalf("env %s need to greater than 0", key) - } - mapCurrentEnv[key] = fmt.Sprint(uvalue) - return uint(uvalue) - } - return fallback -} - -func GetOrderedKeys(o map[string]string) []string { - keys := make([]string, 0, len(o)) - for k := range o { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - func main() { - config := &api.ServiceConfig{Mysql: gomysql.NewConfig()} + config := &api.ServiceConfig{DB: mysql.NewConfig()} flag.StringVar(&config.Listen, "listen", listen, "Listen address") - flag.StringVar(&config.Mysql.Addr, "mysql-addr", mysqlAddr, "Mysql addr") - flag.StringVar(&config.Mysql.User, "mysql-user", mysqlUser, "Mysql user") - flag.StringVar(&config.Mysql.Passwd, "mysql-password", mysqlPassword, "Mysql password") - flag.StringVar(&config.Mysql.DBName, "mysql-database", mysqlDatabase, "Mysql database") + flag.StringVar(&config.DB.Addr, "mysql-addr", mysqlAddr, "Mysql addr") + flag.StringVar(&config.DB.User, "mysql-user", mysqlUser, "Mysql user") + flag.StringVar(&config.DB.Passwd, "mysql-password", mysqlPassword, "Mysql password") + flag.StringVar(&config.DB.DBName, "mysql-database", mysqlDatabase, "Mysql database") flag.BoolVar(&getEnvConfig, "generate-config", false, "Generate an env config") flag.BoolVar(&getCurrentEnvConfig, "generate-current-config", false, "Generate an env config with current env") + flag.StringVar(&config.StorePath, "file-path", storePath, "Store path for the files") flag.Parse() if getEnvConfig { - for _, k := range GetOrderedKeys(mapEnv) { - fmt.Printf("%s=%q\n", k, mapEnv[k]) + for _, r := range genv.Fallback() { + fmt.Printf("%s=%q\n", r.Key, r.Value) } return } if getCurrentEnvConfig { - for _, k := range GetOrderedKeys(mapCurrentEnv) { - fmt.Printf("%s=%q\n", k, mapCurrentEnv[k]) + for _, r := range genv.Current() { + fmt.Printf("%s=%q\n", r.Key, r.Value) } return } diff --git a/internal/env/main.go b/internal/env/main.go new file mode 100644 index 0000000..34a4033 --- /dev/null +++ b/internal/env/main.go @@ -0,0 +1,72 @@ +package env + +import ( + "fmt" + "log" + "os" + "sort" + "strconv" +) + +type Env struct { + fallback map[string]string + current map[string]string +} + +func New() *Env { + return &Env{ + fallback: map[string]string{}, + current: map[string]string{}, + } +} + +func (e *Env) Get(key, fallback string) string { + e.fallback[key] = fallback + e.current[key] = fallback + if value, ok := os.LookupEnv(key); ok { + e.current[key] = value + return value + } + return fallback +} + +func (e *Env) GetUInt(key string, fallback uint) uint { + e.fallback[key] = fmt.Sprint(fallback) + e.current[key] = fmt.Sprint(fallback) + if value, ok := os.LookupEnv(key); ok { + uvalue, err := strconv.Atoi(value) + if err != nil { + log.Fatal(err) + } + if uvalue <= 0 { + log.Fatalf("env %s need to greater than 0", key) + } + e.current[key] = fmt.Sprint(uvalue) + return uint(uvalue) + } + return fallback +} + +type Result struct { + Key string + Value string +} + +func (e *Env) sort(o map[string]string) []Result { + r := make([]Result, 0, len(o)) + for k := range o { + r = append(r, Result{k, o[k]}) + } + sort.Slice(r, func(i, j int) bool { + return r[i].Key < r[j].Key + }) + return r +} + +func (e *Env) Current() []Result { + return e.sort(e.current) +} + +func (e *Env) Fallback() []Result { + return e.sort(e.fallback) +} diff --git a/internal/photos/api/album.go b/internal/photos/api/album.go deleted file mode 100644 index 983a03d..0000000 --- a/internal/photos/api/album.go +++ /dev/null @@ -1,57 +0,0 @@ -package api - -import ( - "net/http" - - "github.com/gin-gonic/gin" - "gitlab.celogeek.com/photos/api/internal/photos/models" - "gopkg.in/validator.v2" - "gorm.io/gorm/clause" -) - -type AlbumCreateRequest struct { - Name string `validate:"min=1,max=255,regexp=^[^/]*$"` - Parent *uint32 -} - -func (s *Service) AlbumCreate(c *gin.Context) { - req := &AlbumCreateRequest{} - if err := c.ShouldBindJSON(req); err != nil { - s.Error(c, http.StatusBadRequest, err) - } - if err := validator.Validate(req); err != nil { - s.Error(c, http.StatusExpectationFailed, err) - return - } - - sess := s.CurrentSession(c) - - if req.Parent != nil { - var parentExists int64 - if err := s.DB.Model(&models.Album{}).Where("id = ?", req.Parent).Count(&parentExists).Error; err != nil { - s.Error(c, http.StatusInternalServerError, err) - return - } - if parentExists == 0 { - s.Error(c, http.StatusNotFound, ErrAlbumDontExists) - return - } - } - - album := &models.Album{ - Name: req.Name, - ParentId: req.Parent, - Author: sess.Account, - AuthorId: &sess.Account.ID, - } - - if err := s.DB.Debug().Omit(clause.Associations).Clauses(clause.Returning{}).Create(album).Error; err != nil { - s.Error(c, http.StatusConflict, err) - return - } - - c.JSON(http.StatusOK, gin.H{ - "status": "success", - "album": album, - }) -} diff --git a/internal/photos/api/db.go b/internal/photos/api/db.go index 16686af..83548ea 100644 --- a/internal/photos/api/db.go +++ b/internal/photos/api/db.go @@ -1,37 +1,47 @@ package api import ( + "log" + "os" "time" "gitlab.celogeek.com/photos/api/internal/photos/models" "gorm.io/driver/mysql" "gorm.io/gorm" + "gorm.io/gorm/logger" ) func (s *Service) Migrate() { tx := s.DB.Set("gorm:table_options", "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin") tx.AutoMigrate(&models.Account{}) tx.AutoMigrate(&models.Session{}) - tx.AutoMigrate(&models.Album{}) - tx.AutoMigrate(&models.Photo{}) + tx.AutoMigrate(&models.File{}) } func (s *Service) DBConfig() { - s.Config.Mysql.Params = map[string]string{ + s.Config.DB.Params = map[string]string{ "charset": "utf8mb4", } - s.Config.Mysql.Collation = "utf8mb4_bin" - s.Config.Mysql.ParseTime = true - s.Config.Mysql.Loc = time.UTC - s.Config.Mysql.Net = "tcp" - s.Config.Mysql.InterpolateParams = true + s.Config.DB.Collation = "utf8mb4_bin" + s.Config.DB.ParseTime = true + s.Config.DB.Loc = time.UTC + s.Config.DB.Net = "tcp" + s.Config.DB.InterpolateParams = true mysql.CreateClauses = []string{"INSERT", "VALUES", "ON CONFLICT", "RETURNING"} } func (s *Service) DBConnect() { - db, err := gorm.Open(mysql.Open(s.Config.Mysql.FormatDSN()), &gorm.Config{ - Logger: s.GormLogger, + db, err := gorm.Open(mysql.Open(s.Config.DB.FormatDSN()), &gorm.Config{ + Logger: logger.New( + log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer + logger.Config{ + SlowThreshold: time.Second, // Slow SQL threshold + LogLevel: logger.Error, // Log level + IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger + Colorful: true, // Disable color + }, + ), SkipDefaultTransaction: true, PrepareStmt: true, }) diff --git a/internal/photos/api/errors.go b/internal/photos/api/errors.go index 766dccc..f3f3549 100644 --- a/internal/photos/api/errors.go +++ b/internal/photos/api/errors.go @@ -25,6 +25,12 @@ var ( // Album ErrAlbumDontExists = errors.New("album doesn't exists") + + // Store + ErrStorePathNotADirectory = errors.New("store path is not a directory") + ErrStoreBadChecksum = errors.New("checksum should be sha1 in hex format") + ErrStoreBadChunkSize = errors.New("part file size should be 1MB max") + ErrStoreWrongPartChecksum = errors.New("part file wrong checksum") ) func (s *Service) Error(c *gin.Context, code int, err error) { diff --git a/internal/photos/api/file.go b/internal/photos/api/file.go new file mode 100644 index 0000000..29debb3 --- /dev/null +++ b/internal/photos/api/file.go @@ -0,0 +1,80 @@ +package api + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "io" + "net/http" + "os" + "path/filepath" + + "github.com/gin-gonic/gin" +) + +var CHUNK_SIZE int64 = 1 << 20 + +func (s *Service) PrepareStore() { + d, err := os.Stat(s.Config.StorePath) + if err != nil { + s.LogErr.Fatal("Store", err) + } + if !d.IsDir() { + s.LogErr.Fatal("Store", ErrStorePathNotADirectory) + } +} + +func (s *Service) FileCreate(c *gin.Context) { + var originalChecksum = c.Param("original_checksum") + c.JSON(http.StatusOK, gin.H{ + "original_checksum": originalChecksum, + }) +} + +func (s *Service) FileChunk(c *gin.Context) { + var ( + originalChecksum = c.Param("original_checksum") + part = c.Param("part") + partChecksum = c.Param("part_checksum") + ) + if len(originalChecksum) != 40 || len(partChecksum) != 40 { + s.Error(c, http.StatusBadRequest, ErrStoreBadChecksum) + return + } + + if c.Request.ContentLength > CHUNK_SIZE { + s.Error(c, http.StatusBadRequest, ErrStoreBadChunkSize) + return + } + + b := bytes.NewBuffer([]byte{}) + io.Copy(b, c.Request.Body) + c.Request.Body.Close() + + f, err := os.Create(filepath.Join(s.Config.StorePath, "test")) + if err != nil { + s.Error(c, http.StatusInternalServerError, err) + return + } + f.Write(b.Bytes()) + f.Close() + + sum := sha1.New() + sum.Write(b.Bytes()) + r := hex.EncodeToString(sum.Sum(nil)) + + if partChecksum != r { + s.Error(c, http.StatusBadRequest, ErrStoreWrongPartChecksum) + return + } + + c.JSON(http.StatusOK, gin.H{ + "original_checksum": originalChecksum, + "part": part, + "part_checksum": partChecksum, + "body": gin.H{ + "checksum": r, + "size": b.Len(), + }, + }) +} diff --git a/internal/photos/api/main.go b/internal/photos/api/main.go index 50ed71e..327f937 100644 --- a/internal/photos/api/main.go +++ b/internal/photos/api/main.go @@ -2,7 +2,6 @@ package api import ( "context" - "log" "math/rand" "net/http" "os" @@ -10,23 +9,22 @@ import ( "time" "github.com/gin-gonic/gin" - gomysql "github.com/go-sql-driver/mysql" + "github.com/go-sql-driver/mysql" "gorm.io/gorm" - gormlogger "gorm.io/gorm/logger" ) type Service struct { - Gin *gin.Engine - DB *gorm.DB - Config *ServiceConfig - LogOk *Logger - LogErr *Logger - GormLogger gormlogger.Interface + Gin *gin.Engine + DB *gorm.DB + Config *ServiceConfig + LogOk *Logger + LogErr *Logger } type ServiceConfig struct { - Listen string - Mysql *gomysql.Config + Listen string + DB *mysql.Config + StorePath string } func New(config *ServiceConfig) *Service { @@ -35,15 +33,6 @@ func New(config *ServiceConfig) *Service { Config: config, LogOk: &Logger{os.Stdout, "Photos"}, LogErr: &Logger{os.Stderr, "Photos"}, - GormLogger: gormlogger.New( - log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer - gormlogger.Config{ - SlowThreshold: time.Second, // Slow SQL threshold - LogLevel: gormlogger.Error, // Log level - IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger - Colorful: true, // Disable color - }, - ), } } @@ -64,9 +53,10 @@ func (s *Service) SetupRoutes() { }) }) - album := s.Gin.Group("/album") + album := s.Gin.Group("/file") album.Use(s.RequireSession) - album.POST("/", s.AlbumCreate) + album.POST("/:original_checksum", s.FileCreate) + album.POST("/:original_checksum/:part/:part_checksum", s.FileChunk) s.Gin.NoRoute(func(c *gin.Context) { s.Error(c, http.StatusNotFound, ErrReqNotFound) @@ -75,6 +65,7 @@ func (s *Service) SetupRoutes() { func (s *Service) Run() error { rand.Seed(time.Now().UnixNano()) + s.PrepareStore() s.SetupRoutes() s.SetupDB() go s.SessionCleaner() diff --git a/internal/photos/api/photo.go b/internal/photos/api/photo.go deleted file mode 100644 index 778f64e..0000000 --- a/internal/photos/api/photo.go +++ /dev/null @@ -1 +0,0 @@ -package api diff --git a/internal/photos/models/album.go b/internal/photos/models/album.go deleted file mode 100644 index 9aa49a4..0000000 --- a/internal/photos/models/album.go +++ /dev/null @@ -1,44 +0,0 @@ -package models - -import ( - "path/filepath" - "time" - - "gorm.io/gorm" -) - -type Album struct { - ID uint32 `gorm:"primary_key" json:"id"` - Name string `gorm:"not null" json:"name"` - Fullname string `gorm:"-" json:"fullname"` - Parent *Album `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"-"` - ParentId *uint32 `json:"parent_id"` - Author *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"` - AuthorId *uint32 `json:"-"` - Photos []*Photo `gorm:"many2many:album_photos" json:"-"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` -} - -func (a *Album) AfterFind(tx *gorm.DB) error { - a.ComputeFullpath(tx) - return nil -} - -func (a *Album) AfterCreate(tx *gorm.DB) error { - a.ComputeFullpath(tx) - return nil -} - -func (a *Album) ComputeFullpath(tx *gorm.DB) error { - if a.ParentId == nil { - a.Fullname = a.Name - } else { - parent := &Album{} - if err := tx.Session(&gorm.Session{NewDB: true}).Select("name", "parent_id").First(parent, "id = ?", a.ParentId).Error; err != nil { - return err - } - a.Fullname = filepath.Join(parent.Fullname, a.Name) - } - return nil -} diff --git a/internal/photos/models/file.go b/internal/photos/models/file.go new file mode 100644 index 0000000..4c8c217 --- /dev/null +++ b/internal/photos/models/file.go @@ -0,0 +1,13 @@ +package models + +import "time" + +type File struct { + ID uint32 `gorm:"primary_key" json:"id"` + Name string `gorm:"not null" json:"name"` + Checksum string `gorm:"unique;size:44;not null"` + Author *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"` + AuthorId *uint32 `json:"-"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/internal/photos/models/photo.go b/internal/photos/models/photo.go deleted file mode 100644 index 1eb3ea6..0000000 --- a/internal/photos/models/photo.go +++ /dev/null @@ -1,15 +0,0 @@ -package models - -import "time" - -type Photo struct { - ID uint32 `gorm:"primary_key"` - File string `gorm:"not null"` - Name string `gorm:"not null"` - Checksum string `gorm:"unique;size:44;not null"` - Author *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE"` - AuthorId uint32 - Albums []*Album `gorm:"many2many:album_photos"` - CreatedAt time.Time - UpdatedAt time.Time -}