prepare file upload

.gitignore

@@ -0,0 +1 @@
+data

cmd/photos-api/main.go

@@ -4,83 +4,47 @@ import (
 	"flag"
 	"fmt"
 	"log"
-	"os"
-	"sort"
-	"strconv"
 
-	gomysql "github.com/go-sql-driver/mysql"
+	"github.com/go-sql-driver/mysql"
+	"gitlab.celogeek.com/photos/api/internal/env"
 	"gitlab.celogeek.com/photos/api/internal/photos/api"
 )
 
 var (
-	mapEnv              = map[string]string{}
+	genv                = env.New()
-	mapCurrentEnv       = map[string]string{}
+	listen              = fmt.Sprintf("%s:%d", genv.Get("LISTEN_HOST", "[::]"), genv.GetUInt("LISTEN_PORT", 8000))
-	listen              = fmt.Sprintf("%s:%d", getEnv("LISTEN_HOST", "[::]"), getEnvUInt("LISTEN_PORT", 8000))
+	mysqlAddr           = fmt.Sprintf("%s:%d", genv.Get("MYSQL_HOST", "localhost"), genv.GetUInt("MYSQL_PORT", 3306))
-	mysqlAddr           = fmt.Sprintf("%s:%d", getEnv("MYSQL_HOST", "localhost"), getEnvUInt("MYSQL_PORT", 3306))
+	mysqlUser           = genv.Get("MYSQL_USER", "photos")
-	mysqlUser           = getEnv("MYSQL_USER", "photos")
+	mysqlPassword       = genv.Get("MYSQL_PASSWD", "photos")
-	mysqlPassword       = getEnv("MYSQL_PASSWD", "photos")
+	mysqlDatabase       = genv.Get("MYSQL_DB", "photos")
-	mysqlDatabase       = getEnv("MYSQL_DB", "photos")
 	getEnvConfig        = false
 	getCurrentEnvConfig = false
+	storePath           = genv.Get("STORE_PATH", "./data")
 )
 
-func getEnv(key, fallback string) string {
-	mapEnv[key] = fallback
-	mapCurrentEnv[key] = fallback
-	if value, ok := os.LookupEnv(key); ok {
-		mapCurrentEnv[key] = value
-		return value
-	}
-	return fallback
-}
-func getEnvUInt(key string, fallback uint) uint {
-	mapEnv[key] = fmt.Sprint(fallback)
-	mapCurrentEnv[key] = fmt.Sprint(fallback)
-	if value, ok := os.LookupEnv(key); ok {
-		uvalue, err := strconv.Atoi(value)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if uvalue <= 0 {
-			log.Fatalf("env %s need to greater than 0", key)
-		}
-		mapCurrentEnv[key] = fmt.Sprint(uvalue)
-		return uint(uvalue)
-	}
-	return fallback
-}
-
-func GetOrderedKeys(o map[string]string) []string {
-	keys := make([]string, 0, len(o))
-	for k := range o {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	return keys
-}
-
 func main() {
-	config := &api.ServiceConfig{Mysql: gomysql.NewConfig()}
+	config := &api.ServiceConfig{DB: mysql.NewConfig()}
 
 	flag.StringVar(&config.Listen, "listen", listen, "Listen address")
-	flag.StringVar(&config.Mysql.Addr, "mysql-addr", mysqlAddr, "Mysql addr")
+	flag.StringVar(&config.DB.Addr, "mysql-addr", mysqlAddr, "Mysql addr")
-	flag.StringVar(&config.Mysql.User, "mysql-user", mysqlUser, "Mysql user")
+	flag.StringVar(&config.DB.User, "mysql-user", mysqlUser, "Mysql user")
-	flag.StringVar(&config.Mysql.Passwd, "mysql-password", mysqlPassword, "Mysql password")
+	flag.StringVar(&config.DB.Passwd, "mysql-password", mysqlPassword, "Mysql password")
-	flag.StringVar(&config.Mysql.DBName, "mysql-database", mysqlDatabase, "Mysql database")
+	flag.StringVar(&config.DB.DBName, "mysql-database", mysqlDatabase, "Mysql database")
 	flag.BoolVar(&getEnvConfig, "generate-config", false, "Generate an env config")
 	flag.BoolVar(&getCurrentEnvConfig, "generate-current-config", false, "Generate an env config with current env")
+	flag.StringVar(&config.StorePath, "file-path", storePath, "Store path for the files")
 	flag.Parse()
 
 	if getEnvConfig {
-		for _, k := range GetOrderedKeys(mapEnv) {
+		for _, r := range genv.Fallback() {
-			fmt.Printf("%s=%q\n", k, mapEnv[k])
+			fmt.Printf("%s=%q\n", r.Key, r.Value)
 		}
 		return
 	}
 
 	if getCurrentEnvConfig {
-		for _, k := range GetOrderedKeys(mapCurrentEnv) {
+		for _, r := range genv.Current() {
-			fmt.Printf("%s=%q\n", k, mapCurrentEnv[k])
+			fmt.Printf("%s=%q\n", r.Key, r.Value)
 		}
 		return
 	}

internal/env/main.go

@@ -0,0 +1,72 @@
+package env
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strconv"
+)
+
+type Env struct {
+	fallback map[string]string
+	current  map[string]string
+}
+
+func New() *Env {
+	return &Env{
+		fallback: map[string]string{},
+		current:  map[string]string{},
+	}
+}
+
+func (e *Env) Get(key, fallback string) string {
+	e.fallback[key] = fallback
+	e.current[key] = fallback
+	if value, ok := os.LookupEnv(key); ok {
+		e.current[key] = value
+		return value
+	}
+	return fallback
+}
+
+func (e *Env) GetUInt(key string, fallback uint) uint {
+	e.fallback[key] = fmt.Sprint(fallback)
+	e.current[key] = fmt.Sprint(fallback)
+	if value, ok := os.LookupEnv(key); ok {
+		uvalue, err := strconv.Atoi(value)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if uvalue <= 0 {
+			log.Fatalf("env %s need to greater than 0", key)
+		}
+		e.current[key] = fmt.Sprint(uvalue)
+		return uint(uvalue)
+	}
+	return fallback
+}
+
+type Result struct {
+	Key   string
+	Value string
+}
+
+func (e *Env) sort(o map[string]string) []Result {
+	r := make([]Result, 0, len(o))
+	for k := range o {
+		r = append(r, Result{k, o[k]})
+	}
+	sort.Slice(r, func(i, j int) bool {
+		return r[i].Key < r[j].Key
+	})
+	return r
+}
+
+func (e *Env) Current() []Result {
+	return e.sort(e.current)
+}
+
+func (e *Env) Fallback() []Result {
+	return e.sort(e.fallback)
+}

internal/photos/api/album.go

@@ -1,57 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/gin-gonic/gin"
-	"gitlab.celogeek.com/photos/api/internal/photos/models"
-	"gopkg.in/validator.v2"
-	"gorm.io/gorm/clause"
-)
-
-type AlbumCreateRequest struct {
-	Name   string `validate:"min=1,max=255,regexp=^[^/]*$"`
-	Parent *uint32
-}
-
-func (s *Service) AlbumCreate(c *gin.Context) {
-	req := &AlbumCreateRequest{}
-	if err := c.ShouldBindJSON(req); err != nil {
-		s.Error(c, http.StatusBadRequest, err)
-	}
-	if err := validator.Validate(req); err != nil {
-		s.Error(c, http.StatusExpectationFailed, err)
-		return
-	}
-
-	sess := s.CurrentSession(c)
-
-	if req.Parent != nil {
-		var parentExists int64
-		if err := s.DB.Model(&models.Album{}).Where("id = ?", req.Parent).Count(&parentExists).Error; err != nil {
-			s.Error(c, http.StatusInternalServerError, err)
-			return
-		}
-		if parentExists == 0 {
-			s.Error(c, http.StatusNotFound, ErrAlbumDontExists)
-			return
-		}
-	}
-
-	album := &models.Album{
-		Name:     req.Name,
-		ParentId: req.Parent,
-		Author:   sess.Account,
-		AuthorId: &sess.Account.ID,
-	}
-
-	if err := s.DB.Debug().Omit(clause.Associations).Clauses(clause.Returning{}).Create(album).Error; err != nil {
-		s.Error(c, http.StatusConflict, err)
-		return
-	}
-
-	c.JSON(http.StatusOK, gin.H{
-		"status": "success",
-		"album":  album,
-	})
-}

internal/photos/api/db.go

@@ -1,37 +1,47 @@
 package api
 
 import (
+	"log"
+	"os"
 	"time"
 
 	"gitlab.celogeek.com/photos/api/internal/photos/models"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 )
 
 func (s *Service) Migrate() {
 	tx := s.DB.Set("gorm:table_options", "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")
 	tx.AutoMigrate(&models.Account{})
 	tx.AutoMigrate(&models.Session{})
-	tx.AutoMigrate(&models.Album{})
+	tx.AutoMigrate(&models.File{})
-	tx.AutoMigrate(&models.Photo{})
 }
 
 func (s *Service) DBConfig() {
-	s.Config.Mysql.Params = map[string]string{
+	s.Config.DB.Params = map[string]string{
 		"charset": "utf8mb4",
 	}
-	s.Config.Mysql.Collation = "utf8mb4_bin"
+	s.Config.DB.Collation = "utf8mb4_bin"
-	s.Config.Mysql.ParseTime = true
+	s.Config.DB.ParseTime = true
-	s.Config.Mysql.Loc = time.UTC
+	s.Config.DB.Loc = time.UTC
-	s.Config.Mysql.Net = "tcp"
+	s.Config.DB.Net = "tcp"
-	s.Config.Mysql.InterpolateParams = true
+	s.Config.DB.InterpolateParams = true
 
 	mysql.CreateClauses = []string{"INSERT", "VALUES", "ON CONFLICT", "RETURNING"}
 }
 
 func (s *Service) DBConnect() {
-	db, err := gorm.Open(mysql.Open(s.Config.Mysql.FormatDSN()), &gorm.Config{
+	db, err := gorm.Open(mysql.Open(s.Config.DB.FormatDSN()), &gorm.Config{
-		Logger:                 s.GormLogger,
+		Logger: logger.New(
+			log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer
+			logger.Config{
+				SlowThreshold:             time.Second,  // Slow SQL threshold
+				LogLevel:                  logger.Error, // Log level
+				IgnoreRecordNotFoundError: true,         // Ignore ErrRecordNotFound error for logger
+				Colorful:                  true,         // Disable color
+			},
+		),
 		SkipDefaultTransaction: true,
 		PrepareStmt:            true,
 	})

internal/photos/api/errors.go

@@ -25,6 +25,12 @@ var (
 
 	// Album
 	ErrAlbumDontExists = errors.New("album doesn't exists")
+
+	// Store
+	ErrStorePathNotADirectory = errors.New("store path is not a directory")
+	ErrStoreBadChecksum       = errors.New("checksum should be sha1 in hex format")
+	ErrStoreBadChunkSize      = errors.New("part file size should be 1MB max")
+	ErrStoreWrongPartChecksum = errors.New("part file wrong checksum")
 )
 
 func (s *Service) Error(c *gin.Context, code int, err error) {

internal/photos/api/file.go

@@ -0,0 +1,80 @@
+package api
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/gin-gonic/gin"
+)
+
+var CHUNK_SIZE int64 = 1 << 20
+
+func (s *Service) PrepareStore() {
+	d, err := os.Stat(s.Config.StorePath)
+	if err != nil {
+		s.LogErr.Fatal("Store", err)
+	}
+	if !d.IsDir() {
+		s.LogErr.Fatal("Store", ErrStorePathNotADirectory)
+	}
+}
+
+func (s *Service) FileCreate(c *gin.Context) {
+	var originalChecksum = c.Param("original_checksum")
+	c.JSON(http.StatusOK, gin.H{
+		"original_checksum": originalChecksum,
+	})
+}
+
+func (s *Service) FileChunk(c *gin.Context) {
+	var (
+		originalChecksum = c.Param("original_checksum")
+		part             = c.Param("part")
+		partChecksum     = c.Param("part_checksum")
+	)
+	if len(originalChecksum) != 40 || len(partChecksum) != 40 {
+		s.Error(c, http.StatusBadRequest, ErrStoreBadChecksum)
+		return
+	}
+
+	if c.Request.ContentLength > CHUNK_SIZE {
+		s.Error(c, http.StatusBadRequest, ErrStoreBadChunkSize)
+		return
+	}
+
+	b := bytes.NewBuffer([]byte{})
+	io.Copy(b, c.Request.Body)
+	c.Request.Body.Close()
+
+	f, err := os.Create(filepath.Join(s.Config.StorePath, "test"))
+	if err != nil {
+		s.Error(c, http.StatusInternalServerError, err)
+		return
+	}
+	f.Write(b.Bytes())
+	f.Close()
+
+	sum := sha1.New()
+	sum.Write(b.Bytes())
+	r := hex.EncodeToString(sum.Sum(nil))
+
+	if partChecksum != r {
+		s.Error(c, http.StatusBadRequest, ErrStoreWrongPartChecksum)
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"original_checksum": originalChecksum,
+		"part":              part,
+		"part_checksum":     partChecksum,
+		"body": gin.H{
+			"checksum": r,
+			"size":     b.Len(),
+		},
+	})
+}

internal/photos/api/main.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"context"
-	"log"
 	"math/rand"
 	"net/http"
 	"os"
@@ -10,23 +9,22 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	gomysql "github.com/go-sql-driver/mysql"
+	"github.com/go-sql-driver/mysql"
 	"gorm.io/gorm"
-	gormlogger "gorm.io/gorm/logger"
 )
 
 type Service struct {
-	Gin        *gin.Engine
+	Gin    *gin.Engine
-	DB         *gorm.DB
+	DB     *gorm.DB
-	Config     *ServiceConfig
+	Config *ServiceConfig
-	LogOk      *Logger
+	LogOk  *Logger
-	LogErr     *Logger
+	LogErr *Logger
-	GormLogger gormlogger.Interface
 }
 
 type ServiceConfig struct {
-	Listen string
+	Listen    string
-	Mysql  *gomysql.Config
+	DB        *mysql.Config
+	StorePath string
 }
 
 func New(config *ServiceConfig) *Service {
@@ -35,15 +33,6 @@ func New(config *ServiceConfig) *Service {
 		Config: config,
 		LogOk:  &Logger{os.Stdout, "Photos"},
 		LogErr: &Logger{os.Stderr, "Photos"},
-		GormLogger: gormlogger.New(
-			log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer
-			gormlogger.Config{
-				SlowThreshold:             time.Second,      // Slow SQL threshold
-				LogLevel:                  gormlogger.Error, // Log level
-				IgnoreRecordNotFoundError: true,             // Ignore ErrRecordNotFound error for logger
-				Colorful:                  true,             // Disable color
-			},
-		),
 	}
 }
 
@@ -64,9 +53,10 @@ func (s *Service) SetupRoutes() {
 		})
 	})
 
-	album := s.Gin.Group("/album")
+	album := s.Gin.Group("/file")
 	album.Use(s.RequireSession)
-	album.POST("/", s.AlbumCreate)
+	album.POST("/:original_checksum", s.FileCreate)
+	album.POST("/:original_checksum/:part/:part_checksum", s.FileChunk)
 
 	s.Gin.NoRoute(func(c *gin.Context) {
 		s.Error(c, http.StatusNotFound, ErrReqNotFound)
@@ -75,6 +65,7 @@ func (s *Service) SetupRoutes() {
 
 func (s *Service) Run() error {
 	rand.Seed(time.Now().UnixNano())
+	s.PrepareStore()
 	s.SetupRoutes()
 	s.SetupDB()
 	go s.SessionCleaner()

internal/photos/api/photo.go

@@ -1 +0,0 @@
-package api

internal/photos/models/album.go

@@ -1,44 +0,0 @@
-package models
-
-import (
-	"path/filepath"
-	"time"
-
-	"gorm.io/gorm"
-)
-
-type Album struct {
-	ID        uint32    `gorm:"primary_key" json:"id"`
-	Name      string    `gorm:"not null" json:"name"`
-	Fullname  string    `gorm:"-" json:"fullname"`
-	Parent    *Album    `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"-"`
-	ParentId  *uint32   `json:"parent_id"`
-	Author    *Account  `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"`
-	AuthorId  *uint32   `json:"-"`
-	Photos    []*Photo  `gorm:"many2many:album_photos" json:"-"`
-	CreatedAt time.Time `json:"created_at"`
-	UpdatedAt time.Time `json:"updated_at"`
-}
-
-func (a *Album) AfterFind(tx *gorm.DB) error {
-	a.ComputeFullpath(tx)
-	return nil
-}
-
-func (a *Album) AfterCreate(tx *gorm.DB) error {
-	a.ComputeFullpath(tx)
-	return nil
-}
-
-func (a *Album) ComputeFullpath(tx *gorm.DB) error {
-	if a.ParentId == nil {
-		a.Fullname = a.Name
-	} else {
-		parent := &Album{}
-		if err := tx.Session(&gorm.Session{NewDB: true}).Select("name", "parent_id").First(parent, "id = ?", a.ParentId).Error; err != nil {
-			return err
-		}
-		a.Fullname = filepath.Join(parent.Fullname, a.Name)
-	}
-	return nil
-}

internal/photos/models/file.go

@@ -0,0 +1,13 @@
+package models
+
+import "time"
+
+type File struct {
+	ID        uint32    `gorm:"primary_key" json:"id"`
+	Name      string    `gorm:"not null" json:"name"`
+	Checksum  string    `gorm:"unique;size:44;not null"`
+	Author    *Account  `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"`
+	AuthorId  *uint32   `json:"-"`
+	CreatedAt time.Time `json:"created_at"`
+	UpdatedAt time.Time `json:"updated_at"`
+}

internal/photos/models/photo.go

@@ -1,15 +0,0 @@
-package models
-
-import "time"
-
-type Photo struct {
-	ID        uint32   `gorm:"primary_key"`
-	File      string   `gorm:"not null"`
-	Name      string   `gorm:"not null"`
-	Checksum  string   `gorm:"unique;size:44;not null"`
-	Author    *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE"`
-	AuthorId  uint32
-	Albums    []*Album `gorm:"many2many:album_photos"`
-	CreatedAt time.Time
-	UpdatedAt time.Time
-}